From 92143952943e069f4fc5b9cc8c9ef45c526deb5e Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Fri, 25 Feb 2022 16:42:23 -0600
Subject: [PATCH 01/18] remove all submodules

The submodules system is being replaced with a command-line flag

    hx --fetch-grammars

Which shallow-clones grammar repositories at the given revision and

    hx --build-grammars

For building grammars separate of the initial compilation of helix.

Why remove submodules?

* Cloning helix in general takes a long time because of the submodules,
  especially when the submodules are not fetched as shallow
* Packaging is consistently painful no matter the package-manager
* It is quite difficult to devise a scheme where users can declare
  a desired set of grammars and implement it with submodules

This commit fully removes the existing tree-sitter submodules from
the tree (as well as the .gitmodules file which is no longer used).
---
 .gitmodules                                   | 240 ------------------
 helix-syntax/languages/tree-sitter-agda       |   1 -
 helix-syntax/languages/tree-sitter-bash       |   1 -
 helix-syntax/languages/tree-sitter-c          |   1 -
 helix-syntax/languages/tree-sitter-c-sharp    |   1 -
 helix-syntax/languages/tree-sitter-cmake      |   1 -
 helix-syntax/languages/tree-sitter-comment    |   1 -
 helix-syntax/languages/tree-sitter-cpp        |   1 -
 helix-syntax/languages/tree-sitter-css        |   1 -
 helix-syntax/languages/tree-sitter-dart       |   1 -
 helix-syntax/languages/tree-sitter-dockerfile |   1 -
 helix-syntax/languages/tree-sitter-elixir     |   1 -
 helix-syntax/languages/tree-sitter-elm        |   1 -
 helix-syntax/languages/tree-sitter-fish       |   1 -
 helix-syntax/languages/tree-sitter-git-commit |   1 -
 helix-syntax/languages/tree-sitter-git-config |   1 -
 helix-syntax/languages/tree-sitter-git-diff   |   1 -
 helix-syntax/languages/tree-sitter-git-rebase |   1 -
 helix-syntax/languages/tree-sitter-glsl       |   1 -
 helix-syntax/languages/tree-sitter-go         |   1 -
 helix-syntax/languages/tree-sitter-graphql    |   1 -
 helix-syntax/languages/tree-sitter-haskell    |   1 -
 helix-syntax/languages/tree-sitter-html       |   1 -
 helix-syntax/languages/tree-sitter-iex        |   1 -
 helix-syntax/languages/tree-sitter-java       |   1 -
 helix-syntax/languages/tree-sitter-javascript |   1 -
 helix-syntax/languages/tree-sitter-json       |   1 -
 helix-syntax/languages/tree-sitter-julia      |   1 -
 helix-syntax/languages/tree-sitter-latex      |   1 -
 helix-syntax/languages/tree-sitter-lean       |   1 -
 helix-syntax/languages/tree-sitter-ledger     |   1 -
 helix-syntax/languages/tree-sitter-llvm       |   1 -
 helix-syntax/languages/tree-sitter-llvm-mir   |   1 -
 helix-syntax/languages/tree-sitter-lua        |   1 -
 helix-syntax/languages/tree-sitter-make       |   1 -
 helix-syntax/languages/tree-sitter-markdown   |   1 -
 helix-syntax/languages/tree-sitter-nix        |   1 -
 helix-syntax/languages/tree-sitter-ocaml      |   1 -
 helix-syntax/languages/tree-sitter-perl       |   1 -
 helix-syntax/languages/tree-sitter-php        |   1 -
 helix-syntax/languages/tree-sitter-protobuf   |   1 -
 helix-syntax/languages/tree-sitter-python     |   1 -
 helix-syntax/languages/tree-sitter-regex      |   1 -
 helix-syntax/languages/tree-sitter-rescript   |   1 -
 helix-syntax/languages/tree-sitter-ruby       |   1 -
 helix-syntax/languages/tree-sitter-rust       |   1 -
 helix-syntax/languages/tree-sitter-scala      |   1 -
 helix-syntax/languages/tree-sitter-svelte     |   1 -
 helix-syntax/languages/tree-sitter-swift      |   1 -
 helix-syntax/languages/tree-sitter-tablegen   |   1 -
 helix-syntax/languages/tree-sitter-toml       |   1 -
 helix-syntax/languages/tree-sitter-tsq        |   1 -
 helix-syntax/languages/tree-sitter-twig       |   1 -
 helix-syntax/languages/tree-sitter-typescript |   1 -
 helix-syntax/languages/tree-sitter-vue        |   1 -
 helix-syntax/languages/tree-sitter-wgsl       |   1 -
 helix-syntax/languages/tree-sitter-yaml       |   1 -
 helix-syntax/languages/tree-sitter-zig        |   1 -
 58 files changed, 297 deletions(-)
 delete mode 100644 .gitmodules
 delete mode 160000 helix-syntax/languages/tree-sitter-agda
 delete mode 160000 helix-syntax/languages/tree-sitter-bash
 delete mode 160000 helix-syntax/languages/tree-sitter-c
 delete mode 160000 helix-syntax/languages/tree-sitter-c-sharp
 delete mode 160000 helix-syntax/languages/tree-sitter-cmake
 delete mode 160000 helix-syntax/languages/tree-sitter-comment
 delete mode 160000 helix-syntax/languages/tree-sitter-cpp
 delete mode 160000 helix-syntax/languages/tree-sitter-css
 delete mode 160000 helix-syntax/languages/tree-sitter-dart
 delete mode 160000 helix-syntax/languages/tree-sitter-dockerfile
 delete mode 160000 helix-syntax/languages/tree-sitter-elixir
 delete mode 160000 helix-syntax/languages/tree-sitter-elm
 delete mode 160000 helix-syntax/languages/tree-sitter-fish
 delete mode 160000 helix-syntax/languages/tree-sitter-git-commit
 delete mode 160000 helix-syntax/languages/tree-sitter-git-config
 delete mode 160000 helix-syntax/languages/tree-sitter-git-diff
 delete mode 160000 helix-syntax/languages/tree-sitter-git-rebase
 delete mode 160000 helix-syntax/languages/tree-sitter-glsl
 delete mode 160000 helix-syntax/languages/tree-sitter-go
 delete mode 160000 helix-syntax/languages/tree-sitter-graphql
 delete mode 160000 helix-syntax/languages/tree-sitter-haskell
 delete mode 160000 helix-syntax/languages/tree-sitter-html
 delete mode 160000 helix-syntax/languages/tree-sitter-iex
 delete mode 160000 helix-syntax/languages/tree-sitter-java
 delete mode 160000 helix-syntax/languages/tree-sitter-javascript
 delete mode 160000 helix-syntax/languages/tree-sitter-json
 delete mode 160000 helix-syntax/languages/tree-sitter-julia
 delete mode 160000 helix-syntax/languages/tree-sitter-latex
 delete mode 160000 helix-syntax/languages/tree-sitter-lean
 delete mode 160000 helix-syntax/languages/tree-sitter-ledger
 delete mode 160000 helix-syntax/languages/tree-sitter-llvm
 delete mode 160000 helix-syntax/languages/tree-sitter-llvm-mir
 delete mode 160000 helix-syntax/languages/tree-sitter-lua
 delete mode 160000 helix-syntax/languages/tree-sitter-make
 delete mode 160000 helix-syntax/languages/tree-sitter-markdown
 delete mode 160000 helix-syntax/languages/tree-sitter-nix
 delete mode 160000 helix-syntax/languages/tree-sitter-ocaml
 delete mode 160000 helix-syntax/languages/tree-sitter-perl
 delete mode 160000 helix-syntax/languages/tree-sitter-php
 delete mode 160000 helix-syntax/languages/tree-sitter-protobuf
 delete mode 160000 helix-syntax/languages/tree-sitter-python
 delete mode 160000 helix-syntax/languages/tree-sitter-regex
 delete mode 160000 helix-syntax/languages/tree-sitter-rescript
 delete mode 160000 helix-syntax/languages/tree-sitter-ruby
 delete mode 160000 helix-syntax/languages/tree-sitter-rust
 delete mode 160000 helix-syntax/languages/tree-sitter-scala
 delete mode 160000 helix-syntax/languages/tree-sitter-svelte
 delete mode 160000 helix-syntax/languages/tree-sitter-swift
 delete mode 160000 helix-syntax/languages/tree-sitter-tablegen
 delete mode 160000 helix-syntax/languages/tree-sitter-toml
 delete mode 160000 helix-syntax/languages/tree-sitter-tsq
 delete mode 160000 helix-syntax/languages/tree-sitter-twig
 delete mode 160000 helix-syntax/languages/tree-sitter-typescript
 delete mode 160000 helix-syntax/languages/tree-sitter-vue
 delete mode 160000 helix-syntax/languages/tree-sitter-wgsl
 delete mode 160000 helix-syntax/languages/tree-sitter-yaml
 delete mode 160000 helix-syntax/languages/tree-sitter-zig

diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index 6c0b1fe23a85..000000000000
--- a/.gitmodules
+++ /dev/null
@@ -1,240 +0,0 @@
-[submodule "helix-syntax/languages/tree-sitter-cpp"]
-	path = helix-syntax/languages/tree-sitter-cpp
-	url = https://github.com/tree-sitter/tree-sitter-cpp
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-javascript"]
-	path = helix-syntax/languages/tree-sitter-javascript
-	url = https://github.com/tree-sitter/tree-sitter-javascript
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-julia"]
-	path = helix-syntax/languages/tree-sitter-julia
-	url = https://github.com/tree-sitter/tree-sitter-julia
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-python"]
-	path = helix-syntax/languages/tree-sitter-python
-	url = https://github.com/tree-sitter/tree-sitter-python
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-typescript"]
-	path = helix-syntax/languages/tree-sitter-typescript
-	url = https://github.com/tree-sitter/tree-sitter-typescript
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-agda"]
-	path = helix-syntax/languages/tree-sitter-agda
-	url = https://github.com/tree-sitter/tree-sitter-agda
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-go"]
-	path = helix-syntax/languages/tree-sitter-go
-	url = https://github.com/tree-sitter/tree-sitter-go
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-ruby"]
-	path = helix-syntax/languages/tree-sitter-ruby
-	url = https://github.com/tree-sitter/tree-sitter-ruby
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-java"]
-	path = helix-syntax/languages/tree-sitter-java
-	url = https://github.com/tree-sitter/tree-sitter-java
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-php"]
-	path = helix-syntax/languages/tree-sitter-php
-	url = https://github.com/tree-sitter/tree-sitter-php
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-html"]
-	path = helix-syntax/languages/tree-sitter-html
-	url = https://github.com/tree-sitter/tree-sitter-html
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-scala"]
-	path = helix-syntax/languages/tree-sitter-scala
-	url = https://github.com/tree-sitter/tree-sitter-scala
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-bash"]
-	path = helix-syntax/languages/tree-sitter-bash
-	url = https://github.com/tree-sitter/tree-sitter-bash
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-rust"]
-	path = helix-syntax/languages/tree-sitter-rust
-	url = https://github.com/tree-sitter/tree-sitter-rust
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-json"]
-	path = helix-syntax/languages/tree-sitter-json
-	url = https://github.com/tree-sitter/tree-sitter-json
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-css"]
-	path = helix-syntax/languages/tree-sitter-css
-	url = https://github.com/tree-sitter/tree-sitter-css
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-c-sharp"]
-	path = helix-syntax/languages/tree-sitter-c-sharp
-	url = https://github.com/tree-sitter/tree-sitter-c-sharp
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-c"]
-	path = helix-syntax/languages/tree-sitter-c
-	url = https://github.com/tree-sitter/tree-sitter-c
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-haskell"]
-	path = helix-syntax/languages/tree-sitter-haskell
-	url = https://github.com/tree-sitter/tree-sitter-haskell
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-swift"]
-	path = helix-syntax/languages/tree-sitter-swift
-	url = https://github.com/tree-sitter/tree-sitter-swift
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-toml"]
-	path = helix-syntax/languages/tree-sitter-toml
-	url = https://github.com/ikatyang/tree-sitter-toml
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-elixir"]
-	path = helix-syntax/languages/tree-sitter-elixir
-	url = https://github.com/elixir-lang/tree-sitter-elixir
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-nix"]
-	path = helix-syntax/languages/tree-sitter-nix
-	url = https://github.com/cstrahan/tree-sitter-nix
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-latex"]
-	path = helix-syntax/languages/tree-sitter-latex
-	url = https://github.com/latex-lsp/tree-sitter-latex
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-ledger"]
-	path = helix-syntax/languages/tree-sitter-ledger
-	url = https://github.com/cbarrete/tree-sitter-ledger
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-protobuf"]
-	path = helix-syntax/languages/tree-sitter-protobuf
-	url = https://github.com/yusdacra/tree-sitter-protobuf.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-ocaml"]
-	path = helix-syntax/languages/tree-sitter-ocaml
-	url = https://github.com/tree-sitter/tree-sitter-ocaml
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-lua"]
-	path = helix-syntax/languages/tree-sitter-lua
-	url = https://github.com/nvim-treesitter/tree-sitter-lua
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-yaml"]
-	path = helix-syntax/languages/tree-sitter-yaml
-	url = https://github.com/ikatyang/tree-sitter-yaml
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-zig"]
-	path = helix-syntax/languages/tree-sitter-zig
-	url = https://github.com/maxxnino/tree-sitter-zig
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-svelte"]
-	path = helix-syntax/languages/tree-sitter-svelte
-	url = https://github.com/Himujjal/tree-sitter-svelte
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-vue"]
-	path = helix-syntax/languages/tree-sitter-vue
-	url = https://github.com/ikatyang/tree-sitter-vue
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-tsq"]
-	path = helix-syntax/languages/tree-sitter-tsq
-	url = https://github.com/tree-sitter/tree-sitter-tsq
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-cmake"]
-	path = helix-syntax/languages/tree-sitter-cmake
-	url = https://github.com/uyha/tree-sitter-cmake
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-glsl"]
-	path = helix-syntax/languages/tree-sitter-glsl
-	url = https://github.com/theHamsta/tree-sitter-glsl.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-perl"]
-	path = helix-syntax/languages/tree-sitter-perl
-	url = https://github.com/ganezdragon/tree-sitter-perl
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-comment"]
-	path = helix-syntax/languages/tree-sitter-comment
-	url = https://github.com/stsewd/tree-sitter-comment
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-wgsl"]
-	path = helix-syntax/languages/tree-sitter-wgsl
-	url = https://github.com/szebniok/tree-sitter-wgsl
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-llvm"]
-	path = helix-syntax/languages/tree-sitter-llvm
-	url = https://github.com/benwilliamgraham/tree-sitter-llvm
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-markdown"]
-	path = helix-syntax/languages/tree-sitter-markdown
-	url = https://github.com/MDeiml/tree-sitter-markdown
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-dart"]
-	path = helix-syntax/languages/tree-sitter-dart
-	url = https://github.com/UserNobody14/tree-sitter-dart.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-dockerfile"]
-	path = helix-syntax/languages/tree-sitter-dockerfile
-	url = https://github.com/camdencheek/tree-sitter-dockerfile.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-fish"]
-	path = helix-syntax/languages/tree-sitter-fish
-	url = https://github.com/ram02z/tree-sitter-fish
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-git-commit"]
-	path = helix-syntax/languages/tree-sitter-git-commit
-	url = https://github.com/the-mikedavis/tree-sitter-git-commit.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-llvm-mir"]
-	path = helix-syntax/languages/tree-sitter-llvm-mir
-	url = https://github.com/Flakebi/tree-sitter-llvm-mir.git
-    shallow = true
-[submodule "helix-syntax/languages/tree-sitter-git-diff"]
-	path = helix-syntax/languages/tree-sitter-git-diff
-	url = https://github.com/the-mikedavis/tree-sitter-git-diff.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-tablegen"]
-	path = helix-syntax/languages/tree-sitter-tablegen
-	url = https://github.com/Flakebi/tree-sitter-tablegen
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-git-rebase"]
-	path = helix-syntax/languages/tree-sitter-git-rebase
-	url = https://github.com/the-mikedavis/tree-sitter-git-rebase.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-lean"]
-	path = helix-syntax/languages/tree-sitter-lean
-	url = https://github.com/Julian/tree-sitter-lean
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-regex"]
-	path = helix-syntax/languages/tree-sitter-regex
-	url = https://github.com/tree-sitter/tree-sitter-regex.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-make"]
-	path = helix-syntax/languages/tree-sitter-make
-	url = https://github.com/alemuller/tree-sitter-make
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-git-config"]
-	path = helix-syntax/languages/tree-sitter-git-config
-	url = https://github.com/the-mikedavis/tree-sitter-git-config.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-graphql"]
-	path = helix-syntax/languages/tree-sitter-graphql
-	url = https://github.com/bkegley/tree-sitter-graphql
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-elm"]
-	path = helix-syntax/languages/tree-sitter-elm
-	url = https://github.com/elm-tooling/tree-sitter-elm
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-iex"]
-	path = helix-syntax/languages/tree-sitter-iex
-	url = https://github.com/elixir-lang/tree-sitter-iex
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-twig"]
-	path = helix-syntax/languages/tree-sitter-twig
-	url = https://github.com/eirabben/tree-sitter-twig.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-rescript"]
-	path = helix-syntax/languages/tree-sitter-rescript
-	url = https://github.com/jaredramirez/tree-sitter-rescript
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-erlang"]
-	path = helix-syntax/languages/tree-sitter-erlang
-	url = https://github.com/the-mikedavis/tree-sitter-erlang
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-kotlin"]
-	path = helix-syntax/languages/tree-sitter-kotlin
-	url = https://github.com/fwcd/tree-sitter-kotlin.git
-	shallow = true
-[submodule "helix-syntax/languages/tree-sitter-hcl"]
-	path = helix-syntax/languages/tree-sitter-hcl
-	url = https://github.com/MichaHoffmann/tree-sitter-hcl.git
-	shallow = true
diff --git a/helix-syntax/languages/tree-sitter-agda b/helix-syntax/languages/tree-sitter-agda
deleted file mode 160000
index ca69cdf485e9..000000000000
--- a/helix-syntax/languages/tree-sitter-agda
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit ca69cdf485e9ce2b2ef0991a720aa88d87d30231
diff --git a/helix-syntax/languages/tree-sitter-bash b/helix-syntax/languages/tree-sitter-bash
deleted file mode 160000
index a8eb5cb57c66..000000000000
--- a/helix-syntax/languages/tree-sitter-bash
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit a8eb5cb57c66f74c63ab950de081207cccf52017
diff --git a/helix-syntax/languages/tree-sitter-c b/helix-syntax/languages/tree-sitter-c
deleted file mode 160000
index f05e279aedde..000000000000
--- a/helix-syntax/languages/tree-sitter-c
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit f05e279aedde06a25801c3f2b2cc8ac17fac52ae
diff --git a/helix-syntax/languages/tree-sitter-c-sharp b/helix-syntax/languages/tree-sitter-c-sharp
deleted file mode 160000
index 53a65a908167..000000000000
--- a/helix-syntax/languages/tree-sitter-c-sharp
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 53a65a908167d6556e1fcdb67f1ee62aac101dda
diff --git a/helix-syntax/languages/tree-sitter-cmake b/helix-syntax/languages/tree-sitter-cmake
deleted file mode 160000
index f6616f1e417e..000000000000
--- a/helix-syntax/languages/tree-sitter-cmake
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit f6616f1e417ee8b62daf251aa1daa5d73781c596
diff --git a/helix-syntax/languages/tree-sitter-comment b/helix-syntax/languages/tree-sitter-comment
deleted file mode 160000
index 5dd3c62f1bbe..000000000000
--- a/helix-syntax/languages/tree-sitter-comment
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 5dd3c62f1bbe378b220fe16b317b85247898639e
diff --git a/helix-syntax/languages/tree-sitter-cpp b/helix-syntax/languages/tree-sitter-cpp
deleted file mode 160000
index e8dcc9d2b404..000000000000
--- a/helix-syntax/languages/tree-sitter-cpp
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit e8dcc9d2b404c542fd236ea5f7208f90be8a6e89
diff --git a/helix-syntax/languages/tree-sitter-css b/helix-syntax/languages/tree-sitter-css
deleted file mode 160000
index 94e10230939e..000000000000
--- a/helix-syntax/languages/tree-sitter-css
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 94e10230939e702b4fa3fa2cb5c3bc7173b95d07
diff --git a/helix-syntax/languages/tree-sitter-dart b/helix-syntax/languages/tree-sitter-dart
deleted file mode 160000
index 6a25376685d1..000000000000
--- a/helix-syntax/languages/tree-sitter-dart
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 6a25376685d1d47968c2cef06d4db8d84a70025e
diff --git a/helix-syntax/languages/tree-sitter-dockerfile b/helix-syntax/languages/tree-sitter-dockerfile
deleted file mode 160000
index 7af32bc04a66..000000000000
--- a/helix-syntax/languages/tree-sitter-dockerfile
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 7af32bc04a66ab196f5b9f92ac471f29372ae2ce
diff --git a/helix-syntax/languages/tree-sitter-elixir b/helix-syntax/languages/tree-sitter-elixir
deleted file mode 160000
index f5d7bda543da..000000000000
--- a/helix-syntax/languages/tree-sitter-elixir
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit f5d7bda543da788bd507b05bd722627dde66c9ec
diff --git a/helix-syntax/languages/tree-sitter-elm b/helix-syntax/languages/tree-sitter-elm
deleted file mode 160000
index bd50ccf66b42..000000000000
--- a/helix-syntax/languages/tree-sitter-elm
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit bd50ccf66b42c55252ac8efc1086af4ac6bab8cd
diff --git a/helix-syntax/languages/tree-sitter-fish b/helix-syntax/languages/tree-sitter-fish
deleted file mode 160000
index 04e54ab6585d..000000000000
--- a/helix-syntax/languages/tree-sitter-fish
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 04e54ab6585dfd4fee6ddfe5849af56f101b6d4f
diff --git a/helix-syntax/languages/tree-sitter-git-commit b/helix-syntax/languages/tree-sitter-git-commit
deleted file mode 160000
index 066e395e1107..000000000000
--- a/helix-syntax/languages/tree-sitter-git-commit
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 066e395e1107df17183cf3ae4230f1a1406cc972
diff --git a/helix-syntax/languages/tree-sitter-git-config b/helix-syntax/languages/tree-sitter-git-config
deleted file mode 160000
index 0e4f0baf90b5..000000000000
--- a/helix-syntax/languages/tree-sitter-git-config
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 0e4f0baf90b57e5aeb62dcdbf03062c6315d43ea
diff --git a/helix-syntax/languages/tree-sitter-git-diff b/helix-syntax/languages/tree-sitter-git-diff
deleted file mode 160000
index c12e6ecb5448..000000000000
--- a/helix-syntax/languages/tree-sitter-git-diff
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit c12e6ecb54485f764250556ffd7ccb18f8e2942b
diff --git a/helix-syntax/languages/tree-sitter-git-rebase b/helix-syntax/languages/tree-sitter-git-rebase
deleted file mode 160000
index 332dc528f270..000000000000
--- a/helix-syntax/languages/tree-sitter-git-rebase
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 332dc528f27044bc4427024dbb33e6941fc131f2
diff --git a/helix-syntax/languages/tree-sitter-glsl b/helix-syntax/languages/tree-sitter-glsl
deleted file mode 160000
index 88408ffc5e27..000000000000
--- a/helix-syntax/languages/tree-sitter-glsl
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 88408ffc5e27abcffced7010fc77396ae3636d7e
diff --git a/helix-syntax/languages/tree-sitter-go b/helix-syntax/languages/tree-sitter-go
deleted file mode 160000
index 0fa917a7022d..000000000000
--- a/helix-syntax/languages/tree-sitter-go
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 0fa917a7022d1cd2e9b779a6a8fc5dc7fad69c75
diff --git a/helix-syntax/languages/tree-sitter-graphql b/helix-syntax/languages/tree-sitter-graphql
deleted file mode 160000
index 5e66e961eee4..000000000000
--- a/helix-syntax/languages/tree-sitter-graphql
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 5e66e961eee421786bdda8495ed1db045e06b5fe
diff --git a/helix-syntax/languages/tree-sitter-haskell b/helix-syntax/languages/tree-sitter-haskell
deleted file mode 160000
index b6ec26f181dd..000000000000
--- a/helix-syntax/languages/tree-sitter-haskell
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit b6ec26f181dd059eedd506fa5fbeae1b8e5556c8
diff --git a/helix-syntax/languages/tree-sitter-html b/helix-syntax/languages/tree-sitter-html
deleted file mode 160000
index d93af487cc75..000000000000
--- a/helix-syntax/languages/tree-sitter-html
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit d93af487cc75120c89257195e6be46c999c6ba18
diff --git a/helix-syntax/languages/tree-sitter-iex b/helix-syntax/languages/tree-sitter-iex
deleted file mode 160000
index 3ec55082cf0b..000000000000
--- a/helix-syntax/languages/tree-sitter-iex
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 3ec55082cf0be015d03148be8edfdfa8c56e77f9
diff --git a/helix-syntax/languages/tree-sitter-java b/helix-syntax/languages/tree-sitter-java
deleted file mode 160000
index bd6186c24d5e..000000000000
--- a/helix-syntax/languages/tree-sitter-java
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit bd6186c24d5eb13b4623efac9d944dcc095c0dad
diff --git a/helix-syntax/languages/tree-sitter-javascript b/helix-syntax/languages/tree-sitter-javascript
deleted file mode 160000
index 4a95461c4761..000000000000
--- a/helix-syntax/languages/tree-sitter-javascript
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 4a95461c4761c624f2263725aca79eeaefd36cad
diff --git a/helix-syntax/languages/tree-sitter-json b/helix-syntax/languages/tree-sitter-json
deleted file mode 160000
index 65bceef69c3b..000000000000
--- a/helix-syntax/languages/tree-sitter-json
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 65bceef69c3b0f24c0b19ce67d79f57c96e90fcb
diff --git a/helix-syntax/languages/tree-sitter-julia b/helix-syntax/languages/tree-sitter-julia
deleted file mode 160000
index 12ea59726212..000000000000
--- a/helix-syntax/languages/tree-sitter-julia
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 12ea597262125fc22fd2e91aa953ac69b19c26ca
diff --git a/helix-syntax/languages/tree-sitter-latex b/helix-syntax/languages/tree-sitter-latex
deleted file mode 160000
index 7f720661de53..000000000000
--- a/helix-syntax/languages/tree-sitter-latex
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 7f720661de5316c0f8fee956526d4002fa1086d8
diff --git a/helix-syntax/languages/tree-sitter-lean b/helix-syntax/languages/tree-sitter-lean
deleted file mode 160000
index d98426109258..000000000000
--- a/helix-syntax/languages/tree-sitter-lean
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit d98426109258b266e1e92358c5f11716d2e8f638
diff --git a/helix-syntax/languages/tree-sitter-ledger b/helix-syntax/languages/tree-sitter-ledger
deleted file mode 160000
index 0cdeb0e51411..000000000000
--- a/helix-syntax/languages/tree-sitter-ledger
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 0cdeb0e51411a3ba5493662952c3039de08939ca
diff --git a/helix-syntax/languages/tree-sitter-llvm b/helix-syntax/languages/tree-sitter-llvm
deleted file mode 160000
index 3b213925b9c4..000000000000
--- a/helix-syntax/languages/tree-sitter-llvm
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 3b213925b9c4f42c1acfe2e10bfbb438d9c6834d
diff --git a/helix-syntax/languages/tree-sitter-llvm-mir b/helix-syntax/languages/tree-sitter-llvm-mir
deleted file mode 160000
index 06fabca19454..000000000000
--- a/helix-syntax/languages/tree-sitter-llvm-mir
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 06fabca19454b2dc00c1b211a7cb7ad0bc2585f1
diff --git a/helix-syntax/languages/tree-sitter-lua b/helix-syntax/languages/tree-sitter-lua
deleted file mode 160000
index 6f5d40190ec8..000000000000
--- a/helix-syntax/languages/tree-sitter-lua
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 6f5d40190ec8a0aa8c8410699353d820f4f7d7a6
diff --git a/helix-syntax/languages/tree-sitter-make b/helix-syntax/languages/tree-sitter-make
deleted file mode 160000
index a4b9187417d6..000000000000
--- a/helix-syntax/languages/tree-sitter-make
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit a4b9187417d6be349ee5fd4b6e77b4172c6827dd
diff --git a/helix-syntax/languages/tree-sitter-markdown b/helix-syntax/languages/tree-sitter-markdown
deleted file mode 160000
index ad8c32917a16..000000000000
--- a/helix-syntax/languages/tree-sitter-markdown
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit ad8c32917a16dfbb387d1da567bf0c3fb6fffde2
diff --git a/helix-syntax/languages/tree-sitter-nix b/helix-syntax/languages/tree-sitter-nix
deleted file mode 160000
index 50f38ceab667..000000000000
--- a/helix-syntax/languages/tree-sitter-nix
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 50f38ceab667f9d482640edfee803d74f4edeba5
diff --git a/helix-syntax/languages/tree-sitter-ocaml b/helix-syntax/languages/tree-sitter-ocaml
deleted file mode 160000
index 23d419ba4578..000000000000
--- a/helix-syntax/languages/tree-sitter-ocaml
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 23d419ba45789c5a47d31448061557716b02750a
diff --git a/helix-syntax/languages/tree-sitter-perl b/helix-syntax/languages/tree-sitter-perl
deleted file mode 160000
index 0ac2c6da562c..000000000000
--- a/helix-syntax/languages/tree-sitter-perl
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 0ac2c6da562c7a2c26ed7e8691d4a590f7e8b90a
diff --git a/helix-syntax/languages/tree-sitter-php b/helix-syntax/languages/tree-sitter-php
deleted file mode 160000
index 57f855461aee..000000000000
--- a/helix-syntax/languages/tree-sitter-php
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 57f855461aeeca73bd4218754fb26b5ac143f98f
diff --git a/helix-syntax/languages/tree-sitter-protobuf b/helix-syntax/languages/tree-sitter-protobuf
deleted file mode 160000
index 19c211a01434..000000000000
--- a/helix-syntax/languages/tree-sitter-protobuf
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 19c211a01434d9f03efff99f85e19f967591b175
diff --git a/helix-syntax/languages/tree-sitter-python b/helix-syntax/languages/tree-sitter-python
deleted file mode 160000
index d6210ceab11e..000000000000
--- a/helix-syntax/languages/tree-sitter-python
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit d6210ceab11e8d812d4ab59c07c81458ec6e5184
diff --git a/helix-syntax/languages/tree-sitter-regex b/helix-syntax/languages/tree-sitter-regex
deleted file mode 160000
index e1cfca3c7989..000000000000
--- a/helix-syntax/languages/tree-sitter-regex
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit e1cfca3c79896ff79842f057ea13e529b66af636
diff --git a/helix-syntax/languages/tree-sitter-rescript b/helix-syntax/languages/tree-sitter-rescript
deleted file mode 160000
index 761eb9126b65..000000000000
--- a/helix-syntax/languages/tree-sitter-rescript
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 761eb9126b65e078b1b5770ac296b4af8870f933
diff --git a/helix-syntax/languages/tree-sitter-ruby b/helix-syntax/languages/tree-sitter-ruby
deleted file mode 160000
index dfff673b41df..000000000000
--- a/helix-syntax/languages/tree-sitter-ruby
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit dfff673b41df7fadcbb609c6338f38da3cdd018e
diff --git a/helix-syntax/languages/tree-sitter-rust b/helix-syntax/languages/tree-sitter-rust
deleted file mode 160000
index a360da0a29a1..000000000000
--- a/helix-syntax/languages/tree-sitter-rust
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit a360da0a29a19c281d08295a35ecd0544d2da211
diff --git a/helix-syntax/languages/tree-sitter-scala b/helix-syntax/languages/tree-sitter-scala
deleted file mode 160000
index 0a3dd53a7fc4..000000000000
--- a/helix-syntax/languages/tree-sitter-scala
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 0a3dd53a7fc4b352a538397d054380aaa28be54c
diff --git a/helix-syntax/languages/tree-sitter-svelte b/helix-syntax/languages/tree-sitter-svelte
deleted file mode 160000
index 349a5984513b..000000000000
--- a/helix-syntax/languages/tree-sitter-svelte
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 349a5984513b4a4a9e143a6e746120c6ff6cf6ed
diff --git a/helix-syntax/languages/tree-sitter-swift b/helix-syntax/languages/tree-sitter-swift
deleted file mode 160000
index a22fa5e19bae..000000000000
--- a/helix-syntax/languages/tree-sitter-swift
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit a22fa5e19bae50098e2252ea96cba3aba43f4c58
diff --git a/helix-syntax/languages/tree-sitter-tablegen b/helix-syntax/languages/tree-sitter-tablegen
deleted file mode 160000
index 568dd8a93734..000000000000
--- a/helix-syntax/languages/tree-sitter-tablegen
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 568dd8a937347175fd58db83d4c4cdaeb6069bd2
diff --git a/helix-syntax/languages/tree-sitter-toml b/helix-syntax/languages/tree-sitter-toml
deleted file mode 160000
index 7cff70bbcbbc..000000000000
--- a/helix-syntax/languages/tree-sitter-toml
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 7cff70bbcbbc62001b465603ca1ea88edd668704
diff --git a/helix-syntax/languages/tree-sitter-tsq b/helix-syntax/languages/tree-sitter-tsq
deleted file mode 160000
index b665659d3238..000000000000
--- a/helix-syntax/languages/tree-sitter-tsq
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit b665659d3238e6036e22ed0e24935e60efb39415
diff --git a/helix-syntax/languages/tree-sitter-twig b/helix-syntax/languages/tree-sitter-twig
deleted file mode 160000
index b7444181fb38..000000000000
--- a/helix-syntax/languages/tree-sitter-twig
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit b7444181fb38e603e25ea8fcdac55f9492e49c27
diff --git a/helix-syntax/languages/tree-sitter-typescript b/helix-syntax/languages/tree-sitter-typescript
deleted file mode 160000
index 3e897ea5925f..000000000000
--- a/helix-syntax/languages/tree-sitter-typescript
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 3e897ea5925f037cfae2e551f8e6b12eec2a201a
diff --git a/helix-syntax/languages/tree-sitter-vue b/helix-syntax/languages/tree-sitter-vue
deleted file mode 160000
index 91fe2754796c..000000000000
--- a/helix-syntax/languages/tree-sitter-vue
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 91fe2754796cd8fba5f229505a23fa08f3546c06
diff --git a/helix-syntax/languages/tree-sitter-wgsl b/helix-syntax/languages/tree-sitter-wgsl
deleted file mode 160000
index f00ff52251ed..000000000000
--- a/helix-syntax/languages/tree-sitter-wgsl
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit f00ff52251edbd58f4d39c9c3204383253032c11
diff --git a/helix-syntax/languages/tree-sitter-yaml b/helix-syntax/languages/tree-sitter-yaml
deleted file mode 160000
index 0e36bed17176..000000000000
--- a/helix-syntax/languages/tree-sitter-yaml
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 0e36bed171768908f331ff7dff9d956bae016efb
diff --git a/helix-syntax/languages/tree-sitter-zig b/helix-syntax/languages/tree-sitter-zig
deleted file mode 160000
index 93331b8bd8b4..000000000000
--- a/helix-syntax/languages/tree-sitter-zig
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 93331b8bd8b4ebee2b575490b2758f16ad4e9f30

From 1f658621fbf5ef54752fa0ea5c1da993cf7727e9 Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Sun, 13 Feb 2022 12:11:44 -0600
Subject: [PATCH 02/18] add tree-sitter sources to languages.toml

Here we add syntax to the languages.toml languge

    [[grammar]]
    name = "<name>"
    source = { .. }

Which can be used to specify a tree-sitter grammar separately of
the language that defines it, and we make this distinction for
two reasons:

* In later commits, we will separate this code from helix-core
  and bring it to a new helix-loader crate. Using separate schemas
  for language and grammar configurations allows for a nice divide
  between the types needed to be declared in helix-loader and in
  helix-core/syntax

* Two different languages may use the same grammar. This is currently
  the case with llvm-mir-yaml and yaml. We could accomplish a config
  that works for this with just `[[languages]]`, but it gets a bit
  dicey with languages depending on one another. If you enable
  llvm-mir-yaml and disable yaml, does helix still need to fetch and
  build tree-sitter-yaml? It could be a matter of interpretation.
---
 helix-core/src/indent.rs |   1 +
 helix-core/src/syntax.rs |  29 ++++-
 languages.toml           | 273 +++++++++++++++++++++++++++++++++++----
 3 files changed, 275 insertions(+), 28 deletions(-)

diff --git a/helix-core/src/indent.rs b/helix-core/src/indent.rs
index 9a329d95716d..83b2be946fdf 100644
--- a/helix-core/src/indent.rs
+++ b/helix-core/src/indent.rs
@@ -444,6 +444,7 @@ where
                 debugger: None,
                 auto_pairs: None,
             }],
+            grammar: vec![],
         });
 
         // set runtime path so we can find the queries
diff --git a/helix-core/src/syntax.rs b/helix-core/src/syntax.rs
index c39e05843ec5..53d20da34bda 100644
--- a/helix-core/src/syntax.rs
+++ b/helix-core/src/syntax.rs
@@ -54,6 +54,7 @@ where
 #[serde(deny_unknown_fields)]
 pub struct Configuration {
     pub language: Vec<LanguageConfiguration>,
+    pub grammar: Vec<GrammarConfiguration>,
 }
 
 // largely based on tree-sitter/cli/src/loader.rs
@@ -239,6 +240,29 @@ pub struct IndentQuery {
     pub outdent: HashSet<String>,
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+pub struct GrammarConfiguration {
+    #[serde(rename = "name")]
+    pub grammar_id: String, // c-sharp, rust
+    pub source: GrammarSource,
+    pub path: Option<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+#[serde(untagged)]
+pub enum GrammarSource {
+    Local {
+        path: String,
+    },
+    Git {
+        #[serde(rename = "git")]
+        remote: String,
+        #[serde(rename = "rev")]
+        revision: String,
+    },
+}
+
 #[derive(Debug)]
 pub struct TextObjectQuery {
     pub query: Query,
@@ -2055,7 +2079,10 @@ mod test {
         .map(String::from)
         .collect();
 
-        let loader = Loader::new(Configuration { language: vec![] });
+        let loader = Loader::new(Configuration {
+            language: vec![],
+            grammar: vec![],
+        });
 
         let language = get_language(&crate::RUNTIME_DIR, "Rust").unwrap();
         let config = HighlightConfiguration::new(
diff --git a/languages.toml b/languages.toml
index 8d210b63cce3..18b45ea60909 100644
--- a/languages.toml
+++ b/languages.toml
@@ -45,6 +45,10 @@ request = "attach"
 completion = [ { name = "lldb connect url", default = "connect://localhost:3333" }, { name = "file", completion = "filename" }, "pid" ]
 args = { attachCommands = [ "platform select remote-gdb-server", "platform connect {0}", "file {1}", "attach {2}" ] }
 
+[[grammar]]
+name = "rust"
+source = { git = "https://github.com/tree-sitter/tree-sitter-rust", rev = "a360da0a29a19c281d08295a35ecd0544d2da211" }
+
 [[language]]
 name = "toml"
 scope = "source.toml"
@@ -52,9 +56,12 @@ injection-regex = "toml"
 file-types = ["toml"]
 roots = []
 comment-token = "#"
-
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "toml"
+source = { git = "https://github.com/ikatyang/tree-sitter-toml", rev = "7cff70bbcbbc62001b465603ca1ea88edd668704" }
+
 [[language]]
 name = "protobuf"
 scope = "source.proto"
@@ -62,9 +69,12 @@ injection-regex = "protobuf"
 file-types = ["proto"]
 roots = []
 comment-token = "//"
-
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "protobuf"
+source = { git = "https://github.com/yusdacra/tree-sitter-protobuf", rev = "19c211a01434d9f03efff99f85e19f967591b175" }
+
 [[language]]
 name = "elixir"
 scope = "source.elixir"
@@ -73,10 +83,13 @@ file-types = ["ex", "exs"]
 shebangs = ["elixir"]
 roots = []
 comment-token = "#"
-
 language-server = { command = "elixir-ls" }
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "elixir"
+source = { git = "https://github.com/elixir-lang/tree-sitter-elixir", rev = "f5d7bda543da788bd507b05bd722627dde66c9ec" }
+
 [[language]]
 name = "fish"
 scope = "source.fish"
@@ -85,9 +98,12 @@ file-types = ["fish"]
 shebangs = ["fish"]
 roots = []
 comment-token = "#"
-
 indent = { tab-width = 4, unit = "    " }
 
+[[grammar]]
+name = "fish"
+source = { git = "https://github.com/ram02z/tree-sitter-fish", rev = "04e54ab6585dfd4fee6ddfe5849af56f101b6d4f" }
+
 [[language]]
 name = "mint"
 scope = "source.mint"
@@ -96,7 +112,6 @@ file-types = ["mint"]
 shebangs = []
 roots = []
 comment-token = "//"
-
 language-server = { command = "mint", args = ["ls"] }
 indent = { tab-width = 2, unit = "  " }
 
@@ -106,9 +121,12 @@ scope = "source.json"
 injection-regex = "json"
 file-types = ["json"]
 roots = []
-
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "json"
+source = { git = "https://github.com/tree-sitter/tree-sitter-json", rev = "65bceef69c3b0f24c0b19ce67d79f57c96e90fcb" }
+
 [[language]]
 name = "c"
 scope = "source.c"
@@ -116,7 +134,6 @@ injection-regex = "c"
 file-types = ["c"] # TODO: ["h"]
 roots = []
 comment-token = "//"
-
 language-server = { command = "clangd" }
 indent = { tab-width = 2, unit = "  " }
 
@@ -143,6 +160,10 @@ request = "attach"
 completion = [ { name = "lldb connect url", default = "connect://localhost:3333" }, { name = "file", completion = "filename" }, "pid" ]
 args = { console = "internalConsole", attachCommands = [ "platform select remote-gdb-server", "platform connect {0}", "file {1}", "attach {2}" ] }
 
+[[grammar]]
+name = "c"
+source = { git = "https://github.com/tree-sitter/tree-sitter-c", rev = "f05e279aedde06a25801c3f2b2cc8ac17fac52ae" }
+
 [[language]]
 name = "cpp"
 scope = "source.cpp"
@@ -150,7 +171,6 @@ injection-regex = "cpp"
 file-types = ["cc", "hh", "cpp", "hpp", "h", "ipp", "tpp", "cxx", "hxx", "ixx", "txx", "ino"]
 roots = []
 comment-token = "//"
-
 language-server = { command = "clangd" }
 indent = { tab-width = 2, unit = "  " }
 
@@ -177,6 +197,10 @@ request = "attach"
 completion = [ { name = "lldb connect url", default = "connect://localhost:3333" }, { name = "file", completion = "filename" }, "pid" ]
 args = { console = "internalConsole", attachCommands = [ "platform select remote-gdb-server", "platform connect {0}", "file {1}", "attach {2}" ] }
 
+[[grammar]]
+name = "cpp"
+source = { git = "https://github.com/tree-sitter/tree-sitter-cpp", rev = "e8dcc9d2b404c542fd236ea5f7208f90be8a6e89" }
+
 [[language]]
 name = "c-sharp"
 scope = "source.csharp"
@@ -184,9 +208,12 @@ injection-regex = "c-?sharp"
 file-types = ["cs"]
 roots = []
 comment-token = "//"
-
 indent = { tab-width = 4, unit = "\t" }
 
+[[grammar]]
+name = "c-sharp"
+source = { git = "https://github.com/tree-sitter/tree-sitter-c-sharp", rev = "53a65a908167d6556e1fcdb67f1ee62aac101dda" }
+
 [[language]]
 name = "go"
 scope = "source.go"
@@ -195,7 +222,6 @@ file-types = ["go"]
 roots = ["Gopkg.toml", "go.mod"]
 auto-format = true
 comment-token = "//"
-
 language-server = { command = "gopls" }
 # TODO: gopls needs utf-8 offsets?
 indent = { tab-width = 4, unit = "\t" }
@@ -231,6 +257,10 @@ request = "attach"
 completion = [ "pid" ]
 args = { mode = "local", processId = "{0}" }
 
+[[grammar]]
+name = "go"
+source = { git = "https://github.com/tree-sitter/tree-sitter-go", rev = "0fa917a7022d1cd2e9b779a6a8fc5dc7fad69c75" }
+
 [[language]]
 name = "javascript"
 scope = "source.js"
@@ -240,7 +270,6 @@ shebangs = ["node"]
 roots = []
 comment-token = "//"
 # TODO: highlights-jsx, highlights-params
-
 language-server = { command = "typescript-language-server", args = ["--stdio"], language-id = "javascript" }
 indent = { tab-width = 2, unit = "  " }
 
@@ -256,6 +285,10 @@ request = "launch"
 completion = [ { name = "main", completion = "filename", default = "index.js" } ]
 args = { program = "{0}" }
 
+[[grammar]]
+name = "javascript"
+source = { git = "https://github.com/tree-sitter/tree-sitter-javascript", rev = "4a95461c4761c624f2263725aca79eeaefd36cad" }
+
 [[language]]
 name = "typescript"
 scope = "source.ts"
@@ -264,10 +297,14 @@ file-types = ["ts"]
 shebangs = []
 roots = []
 # TODO: highlights-jsx, highlights-params
-
 language-server = { command = "typescript-language-server", args = ["--stdio"], language-id = "typescript"}
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "typescript"
+source = { git = "https://github.com/tree-sitter/tree-sitter-typescript", rev = "3e897ea5925f037cfae2e551f8e6b12eec2a201a" }
+path = "typescript"
+
 [[language]]
 name = "tsx"
 scope = "source.tsx"
@@ -275,28 +312,38 @@ injection-regex = "^(tsx)$" # |typescript
 file-types = ["tsx"]
 roots = []
 # TODO: highlights-jsx, highlights-params
-
 language-server = { command = "typescript-language-server", args = ["--stdio"], language-id = "typescriptreact" }
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "tsx"
+source = { git = "https://github.com/tree-sitter/tree-sitter-typescript", rev = "3e897ea5925f037cfae2e551f8e6b12eec2a201a" }
+path = "tsx"
+
 [[language]]
 name = "css"
 scope = "source.css"
 injection-regex = "css"
 file-types = ["css", "scss"]
 roots = []
-
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "css"
+source = { git = "https://github.com/tree-sitter/tree-sitter-css", rev = "94e10230939e702b4fa3fa2cb5c3bc7173b95d07" }
+
 [[language]]
 name = "html"
 scope = "text.html.basic"
 injection-regex = "html"
 file-types = ["html"]
 roots = []
-
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "html"
+source = { git = "https://github.com/tree-sitter/tree-sitter-html", rev = "d93af487cc75120c89257195e6be46c999c6ba18" }
+
 [[language]]
 name = "python"
 scope = "source.python"
@@ -305,11 +352,14 @@ file-types = ["py"]
 shebangs = ["python"]
 roots = []
 comment-token = "#"
-
 language-server = { command = "pylsp" }
 # TODO: pyls needs utf-8 offsets
 indent = { tab-width = 4, unit = "    " }
 
+[[grammar]]
+name = "python"
+source = { git = "https://github.com/tree-sitter/tree-sitter-python", rev = "d6210ceab11e8d812d4ab59c07c81458ec6e5184" }
+
 [[language]]
 name = "nix"
 scope = "source.nix"
@@ -318,10 +368,13 @@ file-types = ["nix"]
 shebangs = []
 roots = []
 comment-token = "#"
-
 language-server = { command = "rnix-lsp" }
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "nix"
+source = { git = "https://github.com/cstrahan/tree-sitter-nix", rev = "50f38ceab667f9d482640edfee803d74f4edeba5" }
+
 [[language]]
 name = "ruby"
 scope = "source.ruby"
@@ -330,10 +383,13 @@ file-types = ["rb"]
 shebangs = ["ruby"]
 roots = []
 comment-token = "#"
-
 language-server = { command = "solargraph", args = ["stdio"] }
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "ruby"
+source = { git = "https://github.com/tree-sitter/tree-sitter-ruby", rev = "dfff673b41df7fadcbb609c6338f38da3cdd018e" }
+
 [[language]]
 name = "bash"
 scope = "source.bash"
@@ -342,10 +398,13 @@ file-types = ["sh", "bash", "zsh", ".bash_login", ".bash_logout", ".bash_profile
 shebangs = ["sh", "bash", "dash"]
 roots = []
 comment-token = "#"
-
 language-server = { command = "bash-language-server", args = ["start"] }
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "bash"
+source = { git = "https://github.com/tree-sitter/tree-sitter-bash", rev = "a8eb5cb57c66f74c63ab950de081207cccf52017" }
+
 [[language]]
 name = "php"
 scope = "source.php"
@@ -353,18 +412,24 @@ injection-regex = "php"
 file-types = ["php"]
 shebangs = ["php"]
 roots = []
-
 indent = { tab-width = 4, unit = "    " }
 
+[[grammar]]
+name = "php"
+source = { git = "https://github.com/tree-sitter/tree-sitter-php", rev = "57f855461aeeca73bd4218754fb26b5ac143f98f" }
+
 [[language]]
 name = "twig"
 scope = "source.twig"
 injection-regex = "twig"
 file-types = ["twig"]
 roots = []
-
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "twig"
+source = { git = "https://github.com/eirabben/tree-sitter-twig", rev = "b7444181fb38e603e25ea8fcdac55f9492e49c27" }
+
 [[language]]
 name = "latex"
 scope = "source.tex"
@@ -372,9 +437,12 @@ injection-regex = "tex"
 file-types = ["tex"]
 roots = []
 comment-token = "%"
-
 indent = { tab-width = 4, unit = "\t" }
 
+[[grammar]]
+name = "latex"
+source = { git = "https://github.com/latex-lsp/tree-sitter-latex", rev = "7f720661de5316c0f8fee956526d4002fa1086d8" }
+
 [[language]]
 name = "lean"
 scope = "source.lean"
@@ -383,9 +451,12 @@ file-types = ["lean"]
 roots = [ "lakefile.lean" ]
 comment-token = "--"
 language-server = { command = "lean", args = [ "--server" ] }
-
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "lean"
+source = { git = "https://github.com/Julian/tree-sitter-lean", rev = "d98426109258b266e1e92358c5f11716d2e8f638" }
+
 [[language]]
 name = "julia"
 scope = "source.julia"
@@ -410,6 +481,10 @@ language-server = { command = "julia", args = [
         ] }
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "julia"
+source = { git = "https://github.com/tree-sitter/tree-sitter-julia", rev = "12ea597262125fc22fd2e91aa953ac69b19c26ca" }
+
 [[language]]
 name = "java"
 scope = "source.java"
@@ -418,6 +493,10 @@ file-types = ["java"]
 roots = ["pom.xml"]
 indent = { tab-width = 4, unit = "    " }
 
+[[grammar]]
+name = "java"
+source = { git = "https://github.com/tree-sitter/tree-sitter-java", rev = "bd6186c24d5eb13b4623efac9d944dcc095c0dad" }
+
 [[language]]
 name = "ledger"
 scope = "source.ledger"
@@ -427,6 +506,10 @@ roots = []
 comment-token = ";"
 indent = { tab-width = 4, unit = "    " }
 
+[[grammar]]
+name = "ledger"
+source = { git = "https://github.com/cbarrete/tree-sitter-ledger", rev = "0cdeb0e51411a3ba5493662952c3039de08939ca" }
+
 [[language]]
 name = "ocaml"
 scope = "source.ocaml"
@@ -437,6 +520,11 @@ roots = []
 comment-token = "(**)"
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "ocaml"
+source = { git = "https://github.com/tree-sitter/tree-sitter-ocaml", rev = "23d419ba45789c5a47d31448061557716b02750a" }
+path = "ocaml"
+
 [[language]]
 name = "ocaml-interface"
 scope = "source.ocaml.interface"
@@ -446,6 +534,11 @@ roots = []
 comment-token = "(**)"
 indent = { tab-width = 2, unit = "  "}
 
+[[grammar]]
+name = "ocaml-interface"
+source = { git = "https://github.com/tree-sitter/tree-sitter-ocaml", rev = "23d419ba45789c5a47d31448061557716b02750a" }
+path = "interface"
+
 [[language]]
 name = "lua"
 scope = "source.lua"
@@ -455,6 +548,10 @@ roots = []
 comment-token = "--"
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "lua"
+source = { git = "https://github.com/nvim-treesitter/tree-sitter-lua", rev = "6f5d40190ec8a0aa8c8410699353d820f4f7d7a6" }
+
 [[language]]
 name = "svelte"
 scope = "source.svelte"
@@ -464,6 +561,9 @@ roots = []
 indent = { tab-width = 2, unit = "  " }
 language-server = { command = "svelteserver", args = ["--stdio"] }
 
+[[grammar]]
+name = "svelte"
+source = { git = "https://github.com/Himujjal/tree-sitter-svelte", rev = "349a5984513b4a4a9e143a6e746120c6ff6cf6ed" }
 
 [[language]]
 name = "vue"
@@ -473,6 +573,10 @@ file-types = ["vue"]
 roots = []
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "vue"
+source = { git = "https://github.com/ikatyang/tree-sitter-vue", rev = "91fe2754796cd8fba5f229505a23fa08f3546c06" }
+
 [[language]]
 name = "yaml"
 scope = "source.yaml"
@@ -482,6 +586,10 @@ comment-token = "#"
 indent = { tab-width = 2, unit = "  " }
 injection-regex = "yml|yaml"
 
+[[grammar]]
+name = "yaml"
+source = { git = "https://github.com/ikatyang/tree-sitter-yaml", rev = "0e36bed171768908f331ff7dff9d956bae016efb" }
+
 [[language]]
 name = "haskell"
 scope = "source.haskell"
@@ -492,6 +600,10 @@ comment-token = "--"
 language-server = { command = "haskell-language-server-wrapper", args = ["--lsp"] }
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "haskell"
+source = { git = "https://github.com/tree-sitter/tree-sitter-haskell", rev = "b6ec26f181dd059eedd506fa5fbeae1b8e5556c8" }
+
 [[language]]
 name = "zig"
 scope = "source.zig"
@@ -500,10 +612,13 @@ file-types = ["zig"]
 roots = ["build.zig"]
 auto-format = true
 comment-token = "//"
-
 language-server = { command = "zls" }
 indent = { tab-width = 4, unit = "    " }
 
+[[grammar]]
+name = "zig"
+source = { git = "https://github.com/maxxnino/tree-sitter-zig", rev = "93331b8bd8b4ebee2b575490b2758f16ad4e9f30" }
+
 [[language]]
 name = "prolog"
 scope = "source.prolog"
@@ -511,7 +626,6 @@ roots = []
 file-types = ["pl", "prolog"]
 shebangs = ["swipl"]
 comment-token = "%"
-
 language-server = { command = "swipl", args = [
     "-g", "use_module(library(lsp_server))",
     "-g", "lsp_server:main",
@@ -526,6 +640,10 @@ comment-token = ";"
 injection-regex = "tsq"
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "tsq"
+source = { git = "https://github.com/tree-sitter/tree-sitter-tsq", rev = "b665659d3238e6036e22ed0e24935e60efb39415" }
+
 [[language]]
 name = "cmake"
 scope = "source.cmake"
@@ -536,6 +654,10 @@ indent = { tab-width = 2, unit = "  " }
 language-server = { command = "cmake-language-server" }
 injection-regex = "cmake"
 
+[[grammar]]
+name = "cmake"
+source = { git = "https://github.com/uyha/tree-sitter-cmake", rev = "f6616f1e417ee8b62daf251aa1daa5d73781c596" }
+
 [[language]]
 name = "make"
 scope = "source.make"
@@ -544,6 +666,10 @@ roots =[]
 comment-token = "#"
 indent = { tab-width = 4, unit = "\t" }
 
+[[grammar]]
+name = "make"
+source = { git = "https://github.com/alemuller/tree-sitter-make", rev = "a4b9187417d6be349ee5fd4b6e77b4172c6827dd" }
+
 [[language]]
 name = "glsl"
 scope = "source.glsl"
@@ -553,6 +679,10 @@ comment-token = "//"
 indent = { tab-width = 4, unit = "    " }
 injection-regex = "glsl"
 
+[[grammar]]
+name = "glsl"
+source = { git = "https://github.com/theHamsta/tree-sitter-glsl", rev = "88408ffc5e27abcffced7010fc77396ae3636d7e" }
+
 [[language]]
 name = "perl"
 scope = "source.perl"
@@ -562,6 +692,10 @@ roots = []
 comment-token = "#"
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "perl"
+source = { git = "https://github.com/ganezdragon/tree-sitter-perl", rev = "0ac2c6da562c7a2c26ed7e8691d4a590f7e8b90a" }
+
 [[language]]
 name = "racket"
 scope = "source.rkt"
@@ -578,6 +712,10 @@ roots = []
 file-types = []
 injection-regex = "comment"
 
+[[grammar]]
+name = "comment"
+source = { git = "https://github.com/stsewd/tree-sitter-comment", rev = "5dd3c62f1bbe378b220fe16b317b85247898639e" }
+
 [[language]]
 name = "wgsl"
 scope = "source.wgsl"
@@ -586,6 +724,10 @@ roots = []
 comment-token = "//"
 indent = { tab-width = 4, unit = "    " }
 
+[[grammar]]
+name = "wgsl"
+source = { git = "https://github.com/szebniok/tree-sitter-wgsl", rev = "f00ff52251edbd58f4d39c9c3204383253032c11" }
+
 [[language]]
 name = "llvm"
 scope = "source.llvm"
@@ -595,6 +737,10 @@ comment-token = ";"
 indent = { tab-width = 2, unit = "  " }
 injection-regex = "llvm"
 
+[[grammar]]
+name = "llvm"
+source = { git = "https://github.com/benwilliamgraham/tree-sitter-llvm", rev = "3b213925b9c4f42c1acfe2e10bfbb438d9c6834d" }
+
 [[language]]
 name = "llvm-mir"
 scope = "source.llvm_mir"
@@ -604,9 +750,15 @@ comment-token = ";"
 indent = { tab-width = 2, unit = "  " }
 injection-regex = "mir"
 
+[[grammar]]
+name = "llvm-mir"
+source = { git = "https://github.com/Flakebi/tree-sitter-llvm-mir", rev = "06fabca19454b2dc00c1b211a7cb7ad0bc2585f1" }
+
 [[language]]
 name = "llvm-mir-yaml"
-tree-sitter-library = "yaml"
+# TODO allow languages to point to their grammar like so:
+#
+#     grammar = "yaml"
 scope = "source.yaml"
 roots = []
 file-types = ["mir"]
@@ -622,15 +774,22 @@ comment-token = "//"
 indent = { tab-width = 2, unit = "  " }
 injection-regex = "tablegen"
 
+[[grammar]]
+name = "tablegen"
+source = { git = "https://github.com/Flakebi/tree-sitter-tablegen", rev = "568dd8a937347175fd58db83d4c4cdaeb6069bd2" }
+
 [[language]]
 name = "markdown"
 scope = "source.md"
 injection-regex = "md|markdown"
 file-types = ["md"]
 roots = []
-
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "markdown"
+source = { git = "https://github.com/MDeiml/tree-sitter-markdown", rev = "ad8c32917a16dfbb387d1da567bf0c3fb6fffde2" }
+
 [[language]]
 name = "dart"
 scope = "source.dart"
@@ -641,6 +800,10 @@ comment-token = "//"
 language-server = { command = "dart", args = ["language-server", "--client-id=helix"] }
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "dart"
+source = { git = "https://github.com/UserNobody14/tree-sitter-dart", rev = "6a25376685d1d47968c2cef06d4db8d84a70025e" }
+
 [[language]]
 name = "scala"
 scope = "source.scala"
@@ -650,6 +813,10 @@ comment-token = "//"
 indent = { tab-width = 2, unit = "  " }
 language-server = { command = "metals" }
 
+[[grammar]]
+name = "scala"
+source = { git = "https://github.com/tree-sitter/tree-sitter-scala", rev = "0a3dd53a7fc4b352a538397d054380aaa28be54c" }
+
 [[language]]
 name = "dockerfile"
 scope = "source.dockerfile"
@@ -660,6 +827,10 @@ comment-token = "#"
 indent = { tab-width = 2, unit = "  " }
 language-server = { command = "docker-langserver", args = ["--stdio"] }
 
+[[grammar]]
+name = "dockerfile"
+source = { git = "https://github.com/camdencheek/tree-sitter-dockerfile", rev = "7af32bc04a66ab196f5b9f92ac471f29372ae2ce" }
+
 [[language]]
 name = "git-commit"
 scope = "git.commitmsg"
@@ -668,6 +839,10 @@ file-types = ["COMMIT_EDITMSG"]
 comment-token = "#"
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "git-commit"
+source = { git = "https://github.com/the-mikedavis/tree-sitter-git-commit", rev = "066e395e1107df17183cf3ae4230f1a1406cc972" }
+
 [[language]]
 name = "git-diff"
 scope = "source.diff"
@@ -677,6 +852,10 @@ injection-regex = "diff"
 comment-token = "#"
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "git-diff"
+source = { git = "https://github.com/the-mikedavis/tree-sitter-git-diff", rev = "c12e6ecb54485f764250556ffd7ccb18f8e2942b" }
+
 [[language]]
 name = "git-rebase"
 scope = "source.gitrebase"
@@ -686,6 +865,10 @@ injection-regex = "git-rebase"
 comment-token = "#"
 indent = { tab-width = 2, unit = " " }
 
+[[grammar]]
+name = "git-rebase"
+source = { git = "https://github.com/the-mikedavis/tree-sitter-git-rebase", rev = "332dc528f27044bc4427024dbb33e6941fc131f2" }
+
 [[language]]
 name = "regex"
 scope = "source.regex"
@@ -693,6 +876,10 @@ injection-regex = "regex"
 file-types = ["regex"]
 roots = []
 
+[[grammar]]
+name = "regex"
+source = { git = "https://github.com/tree-sitter/tree-sitter-regex", rev = "e1cfca3c79896ff79842f057ea13e529b66af636" }
+
 [[language]]
 name = "git-config"
 scope = "source.gitconfig"
@@ -703,6 +890,10 @@ injection-regex = "git-config"
 comment-token = "#"
 indent = { tab-width = 4, unit = "\t" }
 
+[[grammar]]
+name = "git-config"
+source = { git = "https://github.com/the-mikedavis/tree-sitter-git-config", rev = "0e4f0baf90b57e5aeb62dcdbf03062c6315d43ea" }
+
 [[language]]
 name = "graphql"
 scope = "source.graphql"
@@ -711,6 +902,10 @@ file-types = ["gql", "graphql"]
 roots = []
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "graphql"
+source = { git = "https://github.com/bkegley/tree-sitter-graphql", rev = "5e66e961eee421786bdda8495ed1db045e06b5fe" }
+
 [[language]]
 name = "elm"
 scope = "source.elm"
@@ -722,6 +917,10 @@ comment-token = "--"
 language-server = { command = "elm-language-server" }
 indent = { tab-width = 4, unit = "    " }
 
+[[grammar]]
+name = "elm"
+source = { git = "https://github.com/elm-tooling/tree-sitter-elm", rev = "bd50ccf66b42c55252ac8efc1086af4ac6bab8cd" }
+
 [[language]]
 name = "iex"
 scope = "source.iex"
@@ -729,6 +928,10 @@ injection-regex = "iex"
 file-types = ["iex"]
 roots = []
 
+[[grammar]]
+name = "iex"
+source = { git = "https://github.com/elixir-lang/tree-sitter-iex", rev = "39f20bb51f502e32058684e893c0c0b00bb2332c" }
+
 [[language]]
 name = "rescript"
 scope = "source.rescript"
@@ -740,6 +943,10 @@ comment-token = "//"
 language-server = { command = "rescript-language-server", args = ["--stdio"] }
 indent = { tab-width = 2, unit = "  " }
 
+[[grammar]]
+name = "rescript"
+source = { git = "https://github.com/jaredramirez/tree-sitter-rescript", rev = "789a171d9bcf73f6d76e67aca39ed14a75375b04" }
+
 [[language]]
 name = "erlang"
 scope = "source.erlang"
@@ -749,6 +956,10 @@ roots = ["rebar.config"]
 comment-token = "%%"
 indent = { tab-width = 4, unit = "    " }
 
+[[grammar]]
+name = "erlang"
+source = { git = "https://github.com/the-mikedavis/tree-sitter-erlang", rev = "86985bde399c5f40b00bc75f7ab70a6c69a5f9c3" }
+
 [[language]]
 name = "kotlin"
 scope = "source.kotlin"
@@ -758,6 +969,10 @@ comment-token = "//"
 indent = { tab-width = 4, unit = "    " }
 language-server = { command = "kotlin-language-server" } 
 
+[[grammar]]
+name = "kotlin"
+source = { git = "https://github.com/fwcd/tree-sitter-kotlin", rev = "a4f71eb9b8c9b19ded3e0e9470be4b1b77c2b569" }
+
 [[language]]
 name = "hcl"
 scope = "source.hcl"
@@ -768,3 +983,7 @@ comment-token = "#"
 indent = { tab-width = 2, unit = "  " }
 language-server = { command = "terraform-ls", args = ["serve"] }
 auto-format = true
+
+[[grammar]]
+name = "hcl"
+source = { git = "https://github.com/MichaHoffmann/tree-sitter-hcl", rev = "3cb7fc28247efbcb2973b97e71c78838ad98a583" }

From fbb88e5bca4298cbd5b546cc5d399f2aa8c06c24 Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Sun, 13 Feb 2022 10:42:18 -0600
Subject: [PATCH 03/18] migrate helix-syntax crate into helix-core and
 helix-term

helix-syntax mostly existed for the sake of the build task which
checks and compiles the submodules. Since we won't be relying on
that process anymore, it doesn't end up making much sense to have
a very thin crate just for some functions that we could port to
helix-core.

The remaining build-related code is moved to helix-term which will
be able to provide grammar builds through the --build-grammars CLI
flag.
---
 Cargo.lock                                    | 16 ++---
 Cargo.toml                                    |  5 --
 docs/architecture.md                          | 13 ++--
 helix-core/Cargo.toml                         |  5 +-
 helix-core/src/syntax.rs                      | 36 ++++++++++-
 helix-syntax/Cargo.toml                       | 21 -------
 helix-syntax/README.md                        | 13 ----
 helix-syntax/src/lib.rs                       | 31 ---------
 helix-term/Cargo.toml                         |  4 ++
 helix-term/build.rs                           |  5 ++
 .../build.rs => helix-term/src/grammars.rs    | 63 +++++--------------
 helix-term/src/lib.rs                         |  1 +
 12 files changed, 72 insertions(+), 141 deletions(-)
 delete mode 100644 helix-syntax/Cargo.toml
 delete mode 100644 helix-syntax/README.md
 delete mode 100644 helix-syntax/src/lib.rs
 rename helix-syntax/build.rs => helix-term/src/grammars.rs (74%)

diff --git a/Cargo.lock b/Cargo.lock
index ac49da51d889..469a586245f2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -358,11 +358,12 @@ dependencies = [
 name = "helix-core"
 version = "0.6.0"
 dependencies = [
+ "anyhow",
  "arc-swap",
  "chrono",
  "encoding_rs",
  "etcetera",
- "helix-syntax",
+ "libloading",
  "log",
  "once_cell",
  "quickcheck",
@@ -415,22 +416,12 @@ dependencies = [
  "which",
 ]
 
-[[package]]
-name = "helix-syntax"
-version = "0.6.0"
-dependencies = [
- "anyhow",
- "cc",
- "libloading",
- "threadpool",
- "tree-sitter",
-]
-
 [[package]]
 name = "helix-term"
 version = "0.6.0"
 dependencies = [
  "anyhow",
+ "cc",
  "chrono",
  "content_inspector",
  "crossterm",
@@ -454,6 +445,7 @@ dependencies = [
  "serde_json",
  "signal-hook",
  "signal-hook-tokio",
+ "threadpool",
  "tokio",
  "tokio-stream",
  "toml",
diff --git a/Cargo.toml b/Cargo.toml
index 31088f98a9d5..0847e6ba4461 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,7 +4,6 @@ members = [
   "helix-view",
   "helix-term",
   "helix-tui",
-  "helix-syntax",
   "helix-lsp",
   "helix-dap",
   "xtask",
@@ -14,10 +13,6 @@ default-members = [
   "helix-term"
 ]
 
-# Build helix-syntax in release mode to make the code path faster in development.
-# [profile.dev.package."helix-syntax"]
-# opt-level = 3
-
 [profile.dev]
 split-debuginfo = "unpacked"
 
diff --git a/docs/architecture.md b/docs/architecture.md
index 40e017454cc1..3c743eca2b7a 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -1,11 +1,10 @@
 
-| Crate        | Description                                            |
-| -----------  | -----------                                            |
-| helix-core   | Core editing primitives, functional.                   |
-| helix-syntax | Tree-sitter grammars                                   |
-| helix-lsp    | Language server client                                 |
-| helix-view   | UI abstractions for use in backends, imperative shell. |
-| helix-term   | Terminal UI                                            |
+| Crate        | Description                                             |
+| -----------  | -----------                                             |
+| helix-core   | Core editing primitives, functional.                    |
+| helix-lsp    | Language server client                                  |
+| helix-view   | UI abstractions for use in backends, imperative shell.  |
+| helix-term   | Terminal UI                                             |
 | helix-tui    | TUI primitives, forked from tui-rs, inspired by Cursive |
 
 
diff --git a/helix-core/Cargo.toml b/helix-core/Cargo.toml
index 6682c37f3568..5582d38b478d 100644
--- a/helix-core/Cargo.toml
+++ b/helix-core/Cargo.toml
@@ -13,8 +13,6 @@ include = ["src/**/*", "README.md"]
 [features]
 
 [dependencies]
-helix-syntax = { version = "0.6", path = "../helix-syntax" }
-
 ropey = "1.3"
 smallvec = "1.8"
 smartstring = "1.0.0"
@@ -40,5 +38,8 @@ encoding_rs = "0.8"
 
 chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
 
+libloading = "0.7"
+anyhow = "1"
+
 [dev-dependencies]
 quickcheck = { version = "1", default-features = false }
diff --git a/helix-core/src/syntax.rs b/helix-core/src/syntax.rs
index 53d20da34bda..0f7d224d39f6 100644
--- a/helix-core/src/syntax.rs
+++ b/helix-core/src/syntax.rs
@@ -7,7 +7,9 @@ use crate::{
     Rope, RopeSlice, Tendril,
 };
 
-pub use helix_syntax::get_language;
+use anyhow::{Context, Result};
+use libloading::{Library, Symbol};
+use tree_sitter::Language;
 
 use arc_swap::{ArcSwap, Guard};
 use slotmap::{DefaultKey as LayerId, HopSlotMap};
@@ -25,6 +27,34 @@ use std::{
 use once_cell::sync::{Lazy, OnceCell};
 use serde::{Deserialize, Serialize};
 
+#[cfg(unix)]
+pub const DYLIB_EXTENSION: &str = "so";
+
+#[cfg(windows)]
+pub const DYLIB_EXTENSION: &str = "dll";
+
+fn replace_dashes_with_underscores(name: &str) -> String {
+    name.replace('-', "_")
+}
+
+pub fn get_language(runtime_path: &std::path::Path, name: &str) -> Result<Language> {
+    let name = name.to_ascii_lowercase();
+    let mut library_path = runtime_path.join("grammars").join(&name);
+    library_path.set_extension(DYLIB_EXTENSION);
+
+    let library = unsafe { Library::new(&library_path) }
+        .with_context(|| format!("Error opening dynamic library {:?}", &library_path))?;
+    let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(&name));
+    let language = unsafe {
+        let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
+            .get(language_fn_name.as_bytes())
+            .with_context(|| format!("Failed to load symbol {}", language_fn_name))?;
+        language_fn()
+    };
+    std::mem::forget(library);
+    Ok(language)
+}
+
 fn deserialize_regex<'de, D>(deserializer: D) -> Result<Option<Regex>, D::Error>
 where
     D: serde::Deserializer<'de>,
@@ -426,7 +456,7 @@ impl LanguageConfiguration {
                 &injections_query,
                 &locals_query,
             )
-            .unwrap(); // TODO: avoid panic
+            .unwrap_or_else(|query_error| panic!("Could not parse queries for language {:?}. Are your grammars out of sync? Try running 'hx --fetch-grammars' and 'hx --build-grammars'. This query could not be parsed: {:?}", self.language_id, query_error));
 
             config.configure(scopes);
             Some(Arc::new(config))
@@ -2023,7 +2053,7 @@ mod test {
         );
 
         let loader = Loader::new(Configuration { language: vec![] });
-        let language = get_language(&crate::RUNTIME_DIR, "Rust").unwrap();
+        let language = get_language("Rust").unwrap();
 
         let query = Query::new(language, query_str).unwrap();
         let textobject = TextObjectQuery { query };
diff --git a/helix-syntax/Cargo.toml b/helix-syntax/Cargo.toml
deleted file mode 100644
index 855839be0022..000000000000
--- a/helix-syntax/Cargo.toml
+++ /dev/null
@@ -1,21 +0,0 @@
-[package]
-name = "helix-syntax"
-version = "0.6.0"
-authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
-edition = "2021"
-license = "MPL-2.0"
-description = "Tree-sitter grammars support"
-categories = ["editor"]
-repository = "https://github.com/helix-editor/helix"
-homepage = "https://helix-editor.com"
-include = ["src/**/*", "languages/**/*", "build.rs", "!**/docs/**/*", "!**/test/**/*", "!**/examples/**/*", "!**/build/**/*"]
-
-[dependencies]
-tree-sitter = "0.20"
-libloading = "0.7"
-anyhow = "1"
-
-[build-dependencies]
-cc = { version = "1" }
-threadpool = { version = "1.0" }
-anyhow = "1"
diff --git a/helix-syntax/README.md b/helix-syntax/README.md
deleted file mode 100644
index bba2197a3d2e..000000000000
--- a/helix-syntax/README.md
+++ /dev/null
@@ -1,13 +0,0 @@
-helix-syntax
-============
-
-Syntax highlighting for helix, (shallow) submodules resides here.
-
-Differences from nvim-treesitter
---------------------------------
-
-As the syntax are commonly ported from
-<https://github.com/nvim-treesitter/nvim-treesitter>.
-
-Note that we do not support the custom `#any-of` predicate which is
-supported by neovim so one needs to change it to `#match` with regex.
diff --git a/helix-syntax/src/lib.rs b/helix-syntax/src/lib.rs
deleted file mode 100644
index b0ec48d82791..000000000000
--- a/helix-syntax/src/lib.rs
+++ /dev/null
@@ -1,31 +0,0 @@
-use anyhow::{Context, Result};
-use libloading::{Library, Symbol};
-use tree_sitter::Language;
-
-fn replace_dashes_with_underscores(name: &str) -> String {
-    name.replace('-', "_")
-}
-#[cfg(unix)]
-const DYLIB_EXTENSION: &str = "so";
-
-#[cfg(windows)]
-const DYLIB_EXTENSION: &str = "dll";
-
-pub fn get_language(runtime_path: &std::path::Path, name: &str) -> Result<Language> {
-    let name = name.to_ascii_lowercase();
-    let mut library_path = runtime_path.join("grammars").join(&name);
-    // TODO: duplicated under build
-    library_path.set_extension(DYLIB_EXTENSION);
-
-    let library = unsafe { Library::new(&library_path) }
-        .with_context(|| format!("Error opening dynamic library {:?}", &library_path))?;
-    let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(&name));
-    let language = unsafe {
-        let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
-            .get(language_fn_name.as_bytes())
-            .with_context(|| format!("Failed to load symbol {}", language_fn_name))?;
-        language_fn()
-    };
-    std::mem::forget(library);
-    Ok(language)
-}
diff --git a/helix-term/Cargo.toml b/helix-term/Cargo.toml
index 9f7821f699d3..93d50d7e302d 100644
--- a/helix-term/Cargo.toml
+++ b/helix-term/Cargo.toml
@@ -66,5 +66,9 @@ grep-searcher = "0.1.8"
 # Remove once retain_mut lands in stable rust
 retain_mut = "0.1.7"
 
+# compiling grammars
+cc = { version = "1" }
+threadpool = { version = "1.0" }
+
 [target.'cfg(not(windows))'.dependencies]  # https://github.com/vorner/signal-hook/issues/100
 signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] }
diff --git a/helix-term/build.rs b/helix-term/build.rs
index b5d62b2856e5..7303041cdb75 100644
--- a/helix-term/build.rs
+++ b/helix-term/build.rs
@@ -14,5 +14,10 @@ fn main() {
         None => env!("CARGO_PKG_VERSION").into(),
     };
 
+    println!(
+        "cargo:rustc-env=BUILD_TARGET={}",
+        std::env::var("TARGET").unwrap()
+    );
+
     println!("cargo:rustc-env=VERSION_AND_GIT_HASH={}", version);
 }
diff --git a/helix-syntax/build.rs b/helix-term/src/grammars.rs
similarity index 74%
rename from helix-syntax/build.rs
rename to helix-term/src/grammars.rs
index fa8be8b380c2..6a4910a32a1c 100644
--- a/helix-syntax/build.rs
+++ b/helix-term/src/grammars.rs
@@ -6,11 +6,13 @@ use std::{
     process::Command,
 };
 
-use std::sync::mpsc::channel;
+use helix_core::syntax::DYLIB_EXTENSION;
 
-fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
+const BUILD_TARGET: &str = env!("BUILD_TARGET");
+
+pub fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
     let mut dirs = Vec::new();
-    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("languages");
+    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../helix-syntax/languages");
 
     for entry in fs::read_dir(path)? {
         let entry = entry?;
@@ -32,12 +34,6 @@ fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
     Ok(dirs)
 }
 
-#[cfg(unix)]
-const DYLIB_EXTENSION: &str = "so";
-
-#[cfg(windows)]
-const DYLIB_EXTENSION: &str = "dll";
-
 fn build_library(src_path: &Path, language: &str) -> Result<()> {
     let header_path = src_path;
     // let grammar_path = src_path.join("grammar.json");
@@ -65,7 +61,12 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
         return Ok(());
     }
     let mut config = cc::Build::new();
-    config.cpp(true).opt_level(2).cargo_metadata(false);
+    config
+        .cpp(true)
+        .opt_level(2)
+        .cargo_metadata(false)
+        .host(BUILD_TARGET)
+        .target(BUILD_TARGET);
     let compiler = config.get_compiler();
     let mut command = Command::new(compiler.path());
     command.current_dir(src_path);
@@ -148,9 +149,10 @@ fn mtime(path: &Path) -> Result<SystemTime> {
     Ok(fs::metadata(path)?.modified()?)
 }
 
-fn build_dir(dir: &str, language: &str) {
+pub fn build_dir(dir: &str, language: &str) {
     println!("Build language {}", language);
-    if PathBuf::from("languages")
+    if PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("../helix-syntax/languages")
         .join(dir)
         .read_dir()
         .unwrap()
@@ -158,49 +160,16 @@ fn build_dir(dir: &str, language: &str) {
         .is_none()
     {
         eprintln!(
-            "The directory {} is empty, you probably need to use 'git submodule update --init --recursive'?",
+            "The directory {} is empty, you probably need to use './scripts/grammars sync'?",
             dir
         );
         std::process::exit(1);
     }
 
     let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .join("languages")
+        .join("../helix-syntax/languages")
         .join(dir)
         .join("src");
 
     build_library(&path, language).unwrap();
 }
-
-fn main() {
-    let ignore = vec![
-        "tree-sitter-typescript".to_string(),
-        "tree-sitter-ocaml".to_string(),
-    ];
-    let dirs = collect_tree_sitter_dirs(&ignore).unwrap();
-
-    let mut n_jobs = 0;
-    let pool = threadpool::Builder::new().build(); // by going through the builder, it'll use num_cpus
-    let (tx, rx) = channel();
-
-    for dir in dirs {
-        let tx = tx.clone();
-        n_jobs += 1;
-
-        pool.execute(move || {
-            let language = &dir.strip_prefix("tree-sitter-").unwrap();
-            build_dir(&dir, language);
-
-            // report progress
-            tx.send(1).unwrap();
-        });
-    }
-    pool.join();
-    // drop(tx);
-    assert_eq!(rx.try_iter().sum::<usize>(), n_jobs);
-
-    build_dir("tree-sitter-typescript/tsx", "tsx");
-    build_dir("tree-sitter-typescript/typescript", "typescript");
-    build_dir("tree-sitter-ocaml/ocaml", "ocaml");
-    build_dir("tree-sitter-ocaml/interface", "ocaml-interface")
-}
diff --git a/helix-term/src/lib.rs b/helix-term/src/lib.rs
index fc8e934e1a7d..227479988594 100644
--- a/helix-term/src/lib.rs
+++ b/helix-term/src/lib.rs
@@ -7,6 +7,7 @@ pub mod commands;
 pub mod compositor;
 pub mod config;
 pub mod health;
+pub mod grammars;
 pub mod job;
 pub mod keymap;
 pub mod ui;

From c824ad43ff8b594c625e028e58c0737aa4e4c2ea Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Mon, 14 Feb 2022 11:24:28 -0600
Subject: [PATCH 04/18] rename tree_sitter_library in LanguageConfig to
 'grammar'

This is not strictly speaking necessary. tree_sitter_library was used by
just one grammar: llvm-mir-yaml, which uses the yaml grammar. This will
make the language more consistent, though. Each language can explicitly
say that they use Some(grammar), defaulting when None to the grammar that
has a grammar_id matching the language's language_id.
---
 helix-core/src/indent.rs | 2 +-
 helix-core/src/syntax.rs | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/helix-core/src/indent.rs b/helix-core/src/indent.rs
index 83b2be946fdf..ba02065c7ffb 100644
--- a/helix-core/src/indent.rs
+++ b/helix-core/src/indent.rs
@@ -433,7 +433,7 @@ where
                 comment_token: None,
                 auto_format: false,
                 diagnostic_severity: Severity::Warning,
-                tree_sitter_library: None,
+                grammar: None,
                 language_server: None,
                 indent: Some(IndentationConfiguration {
                     tab_width: 4,
diff --git a/helix-core/src/syntax.rs b/helix-core/src/syntax.rs
index 0f7d224d39f6..038fbfddf9d6 100644
--- a/helix-core/src/syntax.rs
+++ b/helix-core/src/syntax.rs
@@ -108,7 +108,7 @@ pub struct LanguageConfiguration {
     #[serde(default)]
     pub diagnostic_severity: Severity,
 
-    pub tree_sitter_library: Option<String>, // tree-sitter library name, defaults to language_id
+    pub grammar: Option<String>, // tree-sitter grammar name, defaults to language_id
 
     // content_regex
     #[serde(default, skip_serializing, deserialize_with = "deserialize_regex")]
@@ -444,9 +444,7 @@ impl LanguageConfiguration {
         } else {
             let language = get_language(
                 &crate::RUNTIME_DIR,
-                self.tree_sitter_library
-                    .as_deref()
-                    .unwrap_or(&self.language_id),
+                self.grammar.as_deref().unwrap_or(&self.language_id),
             )
             .map_err(|e| log::info!("{}", e))
             .ok()?;

From 77e7945f07b91c0322afbc96941644876c9ec341 Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Sun, 13 Feb 2022 20:08:11 -0600
Subject: [PATCH 05/18] add --fetch-grammars and --build-grammars CLI flags

---
 helix-term/src/args.rs |  4 ++++
 helix-term/src/main.rs | 18 ++++++++++--------
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/helix-term/src/args.rs b/helix-term/src/args.rs
index 11bad08c4b84..800765289151 100644
--- a/helix-term/src/args.rs
+++ b/helix-term/src/args.rs
@@ -9,6 +9,8 @@ pub struct Args {
     pub health: bool,
     pub health_arg: Option<String>,
     pub load_tutor: bool,
+    pub fetch_grammars: bool,
+    pub build_grammars: bool,
     pub verbosity: u64,
     pub files: Vec<(PathBuf, Position)>,
     pub edit_config: bool,
@@ -32,6 +34,8 @@ impl Args {
                     args.health = true;
                     args.health_arg = argv.next_if(|opt| !opt.starts_with('-'));
                 }
+                "--fetch-grammars" => args.fetch_grammars = true,
+                "--build-grammars" => args.build_grammars = true,
                 arg if arg.starts_with("--") => {
                     anyhow::bail!("unexpected double dash argument: {}", arg)
                 }
diff --git a/helix-term/src/main.rs b/helix-term/src/main.rs
index 607f1f59972e..4f73c9b45455 100644
--- a/helix-term/src/main.rs
+++ b/helix-term/src/main.rs
@@ -59,14 +59,16 @@ ARGS:
     <files>...    Sets the input file to use, position can also be specified via file[:row[:col]]
 
 FLAGS:
-    -h, --help       Prints help information
-    --edit-config    Opens the helix config file
-    --tutor          Loads the tutorial
-    --health [LANG]  Checks for potential errors in editor setup
-                     If given, checks for config errors in language LANG
-    -v               Increases logging verbosity each use for up to 3 times
-                     (default file: {})
-    -V, --version    Prints version information
+    -h, --help        Prints help information
+    --edit-config     Opens the helix config file
+    --tutor           Loads the tutorial
+    --health [LANG]   Checks for potential errors in editor setup
+                      If given, checks for config errors in language LANG
+    --fetch-grammars  Fetches tree-sitter grammars listed in languages.toml
+    --build-grammars  Builds tree-sitter grammars fetched with --fetch-grammars
+    -v                Increases logging verbosity each use for up to 3 times
+                      (default file: {})
+    -V, --version     Prints version information
 ",
         env!("CARGO_PKG_NAME"),
         env!("VERSION_AND_GIT_HASH"),

From 8dc1040ec04ed28c2c5229940ce15c91dcf72761 Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Sun, 13 Feb 2022 20:08:28 -0600
Subject: [PATCH 06/18] implement build_grammars and fetch_grammars

build_grammars adapts the functionality that previously came from
helix-syntax to be used at runtime from the command line flags.

fetch_grammars wraps command-line git to perform the same actions
previously done in the scripts in #1560.
---
 helix-term/src/grammars.rs | 205 ++++++++++++++++++++++++++++---------
 helix-term/src/main.rs     |  10 ++
 2 files changed, 167 insertions(+), 48 deletions(-)

diff --git a/helix-term/src/grammars.rs b/helix-term/src/grammars.rs
index 6a4910a32a1c..7a7a4c187eaf 100644
--- a/helix-term/src/grammars.rs
+++ b/helix-term/src/grammars.rs
@@ -4,39 +4,170 @@ use std::time::SystemTime;
 use std::{
     path::{Path, PathBuf},
     process::Command,
+    sync::mpsc::channel,
 };
 
-use helix_core::syntax::DYLIB_EXTENSION;
+use helix_core::syntax::{GrammarConfiguration, GrammarSource, DYLIB_EXTENSION};
 
 const BUILD_TARGET: &str = env!("BUILD_TARGET");
+const REMOTE_NAME: &str = "origin";
 
-pub fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
-    let mut dirs = Vec::new();
-    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../helix-syntax/languages");
+pub fn fetch_grammars() -> Result<()> {
+    run_parallel(get_grammar_configs(), fetch_grammar, "fetch")
+}
+
+pub fn build_grammars() -> Result<()> {
+    run_parallel(get_grammar_configs(), build_grammar, "build")
+}
+
+fn run_parallel<F>(grammars: Vec<GrammarConfiguration>, job: F, action: &'static str) -> Result<()>
+where
+    F: Fn(GrammarConfiguration) -> Result<()> + std::marker::Send + 'static + Copy,
+{
+    let mut n_jobs = 0;
+    let pool = threadpool::Builder::new().build();
+    let (tx, rx) = channel();
+
+    for grammar in grammars {
+        let tx = tx.clone();
+        n_jobs += 1;
+
+        pool.execute(move || {
+            let grammar_id = grammar.grammar_id.clone();
+            job(grammar).unwrap_or_else(|err| {
+                eprintln!("Failed to {} grammar '{}'\n{}", action, grammar_id, err)
+            });
+
+            // report progress
+            tx.send(1).unwrap();
+        });
+    }
+    pool.join();
+
+    if rx.try_iter().sum::<usize>() == n_jobs {
+        Ok(())
+    } else {
+        Err(anyhow!("Failed to {} some grammar(s).", action))
+    }
+}
 
-    for entry in fs::read_dir(path)? {
-        let entry = entry?;
-        let path = entry.path();
+fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
+    if let GrammarSource::Git { remote, revision } = grammar.source {
+        let grammar_dir = helix_core::runtime_dir()
+            .join("grammars/sources")
+            .join(grammar.grammar_id.clone());
 
-        if !entry.file_type()?.is_dir() {
-            continue;
+        fs::create_dir_all(grammar_dir.clone()).expect("Could not create grammar directory");
+
+        // create the grammar dir contains a git directory
+        if !grammar_dir.join(".git").is_dir() {
+            git(&grammar_dir, ["init"])?;
+        }
+
+        // ensure the remote matches the configured remote
+        if get_remote_url(&grammar_dir).map_or(true, |s| s.trim_end() != remote) {
+            set_remote(&grammar_dir, &remote)?;
         }
 
-        let dir = path.file_name().unwrap().to_str().unwrap().to_string();
+        // ensure the revision matches the configured revision
+        if get_revision(&grammar_dir).map_or(true, |s| s.trim_end() != revision) {
+            // Fetch the exact revision from the remote.
+            // Supported by server-side git since v2.5.0 (July 2015),
+            // enabled by default on major git hosts.
+            git(&grammar_dir, ["fetch", REMOTE_NAME, &revision])?;
+            git(&grammar_dir, ["checkout", &revision])?;
 
-        // filter ignores
-        if ignore.contains(&dir) {
-            continue;
+            println!(
+                "Grammar '{}' checked out at '{}'.",
+                grammar.grammar_id, revision
+            );
+            Ok(())
+        } else {
+            println!("Grammar '{}' is already up to date.", grammar.grammar_id);
+            Ok(())
         }
-        dirs.push(dir)
+    } else {
+        println!("Skipping local grammar '{}'", grammar.grammar_id);
+        Ok(())
     }
+}
 
-    Ok(dirs)
+// Sets the remote for a repository to the given URL, creating the remote if
+// it does not yet exist.
+fn set_remote(repository: &Path, remote_url: &str) -> Result<String> {
+    git(repository, ["remote", "set-url", REMOTE_NAME, remote_url])
+        .or_else(|_| git(repository, ["remote", "add", REMOTE_NAME, remote_url]))
 }
 
-fn build_library(src_path: &Path, language: &str) -> Result<()> {
+fn get_remote_url(repository: &Path) -> Option<String> {
+    git(repository, ["remote", "get-url", REMOTE_NAME]).ok()
+}
+
+fn get_revision(repository: &Path) -> Option<String> {
+    git(repository, ["rev-parse", "HEAD"]).ok()
+}
+
+// A wrapper around 'git' commands which returns stdout in success and a
+// helpful error message showing the command, stdout, and stderr in error.
+fn git<I, S>(repository: &Path, args: I) -> Result<String>
+where
+    I: IntoIterator<Item = S>,
+    S: AsRef<std::ffi::OsStr>,
+{
+    let output = Command::new("git")
+        .args(args)
+        .current_dir(repository)
+        .output()?;
+
+    if output.status.success() {
+        Ok(String::from_utf8_lossy(&output.stdout).into_owned())
+    } else {
+        // TODO: figure out how to display the git command using `args`
+        Err(anyhow!(
+            "Git command failed.\nStdout: {}\nStderr: {}",
+            String::from_utf8_lossy(&output.stdout),
+            String::from_utf8_lossy(&output.stderr),
+        ))
+    }
+}
+
+fn build_grammar(grammar: GrammarConfiguration) -> Result<()> {
+    let grammar_dir = if let GrammarSource::Local { ref path } = grammar.source {
+        PathBuf::from(path)
+    } else {
+        helix_core::runtime_dir()
+            .join("grammars/sources")
+            .join(grammar.grammar_id.clone())
+    };
+
+    grammar_dir.read_dir().with_context(|| {
+        format!(
+            "The directory {:?} is empty, you probably need to use 'hx --fetch-grammars'?",
+            grammar_dir
+        )
+    })?;
+
+    let path = match grammar.path {
+        Some(ref subpath) => grammar_dir.join(subpath),
+        None => grammar_dir,
+    }
+    .join("src");
+
+    build_tree_sitter_library(&path, grammar)
+}
+
+// Returns the set of grammar configurations the user requests.
+// Grammars are configured in the default and user `languages.toml` and are
+// merged. The `grammar_selection` key of the config is then used to filter
+// down all grammars into a subset of the user's choosing.
+fn get_grammar_configs() -> Vec<GrammarConfiguration> {
+    let config = helix_core::config::user_syntax_loader().expect("Could not parse languages.toml");
+
+    config.grammar
+}
+
+fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) -> Result<()> {
     let header_path = src_path;
-    // let grammar_path = src_path.join("grammar.json");
     let parser_path = src_path.join("parser.c");
     let mut scanner_path = src_path.join("scanner.c");
 
@@ -50,16 +181,20 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
             None
         }
     };
-    let parser_lib_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../runtime/grammars");
-    let mut library_path = parser_lib_path.join(language);
+    let parser_lib_path = helix_core::runtime_dir().join("../runtime/grammars");
+    let mut library_path = parser_lib_path.join(grammar.grammar_id.clone());
     library_path.set_extension(DYLIB_EXTENSION);
 
     let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
-        .with_context(|| "Failed to compare source and binary timestamps")?;
+        .context("Failed to compare source and binary timestamps")?;
 
     if !recompile {
+        println!("Grammar '{}' is already built.", grammar.grammar_id);
         return Ok(());
     }
+
+    println!("Building grammar '{}'", grammar.grammar_id);
+
     let mut config = cc::Build::new();
     config
         .cpp(true)
@@ -112,9 +247,7 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
         }
     }
 
-    let output = command
-        .output()
-        .with_context(|| "Failed to execute C compiler")?;
+    let output = command.output().context("Failed to execute C compiler")?;
     if !output.status.success() {
         return Err(anyhow!(
             "Parser compilation failed.\nStdout: {}\nStderr: {}",
@@ -125,6 +258,7 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
 
     Ok(())
 }
+
 fn needs_recompile(
     lib_path: &Path,
     parser_c_path: &Path,
@@ -148,28 +282,3 @@ fn needs_recompile(
 fn mtime(path: &Path) -> Result<SystemTime> {
     Ok(fs::metadata(path)?.modified()?)
 }
-
-pub fn build_dir(dir: &str, language: &str) {
-    println!("Build language {}", language);
-    if PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .join("../helix-syntax/languages")
-        .join(dir)
-        .read_dir()
-        .unwrap()
-        .next()
-        .is_none()
-    {
-        eprintln!(
-            "The directory {} is empty, you probably need to use './scripts/grammars sync'?",
-            dir
-        );
-        std::process::exit(1);
-    }
-
-    let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-        .join("../helix-syntax/languages")
-        .join(dir)
-        .join("src");
-
-    build_library(&path, language).unwrap();
-}
diff --git a/helix-term/src/main.rs b/helix-term/src/main.rs
index 4f73c9b45455..67f88b4e13dd 100644
--- a/helix-term/src/main.rs
+++ b/helix-term/src/main.rs
@@ -104,6 +104,16 @@ FLAGS:
         std::process::exit(0);
     }
 
+    if args.fetch_grammars {
+        helix_term::grammars::fetch_grammars()?;
+        return Ok(0);
+    }
+
+    if args.build_grammars {
+        helix_term::grammars::build_grammars()?;
+        return Ok(0);
+    }
+
     let conf_dir = helix_core::config_dir();
     if !conf_dir.exists() {
         std::fs::create_dir_all(&conf_dir).ok();

From c994379c0d0b1d112807a80651896413d7b070f8 Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Mon, 14 Feb 2022 17:23:35 -0600
Subject: [PATCH 07/18] replace all submodule documentation with flags
 documentation

---
 README.md                           |  7 ++-
 book/src/configuration.md           |  1 +
 book/src/guides/adding_languages.md | 97 +++++++++++++++++------------
 book/src/install.md                 |  8 ++-
 book/src/languages.md               | 20 +++++-
 helix-core/src/config.rs            |  2 +-
 6 files changed, 91 insertions(+), 44 deletions(-)

diff --git a/README.md b/README.md
index e8f72b6d8a7d..8e4e42d9e9fa 100644
--- a/README.md
+++ b/README.md
@@ -36,12 +36,14 @@ We provide packaging for various distributions, but here's a quick method to
 build from source.
 
 ```
-git clone --recurse-submodules --shallow-submodules -j8 https://github.com/helix-editor/helix
+git clone https://github.com/helix-editor/helix
 cd helix
 cargo install --path helix-term
+hx --fetch-grammars
+hx --build-grammars
 ```
 
-This will install the `hx` binary to `$HOME/.cargo/bin`.
+This will install the `hx` binary to `$HOME/.cargo/bin` and build tree-sitter grammars.
 
 Helix also needs its runtime files so make sure to copy/symlink the `runtime/` directory into the
 config directory (for example `~/.config/helix/runtime` on Linux/macOS, or `%AppData%/helix/runtime` on Windows).
@@ -56,6 +58,7 @@ that sets the variable to the install dir.
 [![Packaging status](https://repology.org/badge/vertical-allrepos/helix.svg)](https://repology.org/project/helix/versions)
 
 ## MacOS
+
 Helix can be installed on MacOS through homebrew via:
 
 ```
diff --git a/book/src/configuration.md b/book/src/configuration.md
index 2e4cab8ae7a9..2b29379eacda 100644
--- a/book/src/configuration.md
+++ b/book/src/configuration.md
@@ -131,6 +131,7 @@ Search specific options.
 ## LSP
 
 To display all language server messages in the status line add the following to your `config.toml`:
+
 ```toml
 [lsp]
 display-messages = true
diff --git a/book/src/guides/adding_languages.md b/book/src/guides/adding_languages.md
index 5844a48eeff0..51256288ff9c 100644
--- a/book/src/guides/adding_languages.md
+++ b/book/src/guides/adding_languages.md
@@ -1,45 +1,68 @@
 # Adding languages
 
-## Submodules
+## Language configuration
 
-To add a new language, you should first add a tree-sitter submodule. To do this,
-you can run the command
-```sh
-git submodule add -f <repository> helix-syntax/languages/tree-sitter-<name>
-```
-For example, to add tree-sitter-ocaml you would run
-```sh
-git submodule add -f https://github.com/tree-sitter/tree-sitter-ocaml helix-syntax/languages/tree-sitter-ocaml
+To add a new language, you need to add a `language` entry to the
+[`languages.toml`][languages.toml] found in the root of the repository;
+this `languages.toml` file is included at compilation time, and is
+distinct from the `languages.toml` file in the user's [configuration
+directory](../configuration.md).
+
+```toml
+[[language]]
+name = "mylang"
+scope = "scope.mylang"
+injection-regex = "^mylang$"
+file-types = ["mylang", "myl"]
+comment-token = "#"
+indent = { tab-width = 2, unit = "  " }
 ```
-Make sure the submodule is shallow by doing
-```sh
-git config -f .gitmodules submodule.helix-syntax/languages/tree-sitter-<name>.shallow true
+
+These are the available keys and descriptions for the file.
+
+| Key                   | Description                                                   |
+| ----                  | -----------                                                   |
+| `name`                | The name of the language                                      |
+| `scope`               | A string like `source.js` that identifies the language. Currently, we strive to match the scope names used by popular TextMate grammars and by the Linguist library. Usually `source.<name>` or `text.<name>` in case of markup languages |
+| `injection-regex`     | regex pattern that will be tested against a language name in order to determine whether this language should be used for a potential [language injection][treesitter-language-injection] site. |
+| `file-types`          | The filetypes of the language, for example `["yml", "yaml"]`. Extensions and full file names are supported.  |
+| `shebangs`            | The interpreters from the shebang line, for example `["sh", "bash"]` |
+| `roots`               | A set of marker files to look for when trying to find the workspace root. For example `Cargo.lock`, `yarn.lock` |
+| `auto-format`         | Whether to autoformat this language when saving               |
+| `diagnostic-severity` | Minimal severity of diagnostic for it to be displayed. (Allowed values: `Error`, `Warning`, `Info`, `Hint`) |
+| `comment-token`       | The token to use as a comment-token                           |
+| `indent`              | The indent to use. Has sub keys `tab-width` and `unit`        |
+| `config`              | Language server configuration                                 |
+| `grammar`             | The tree-sitter grammar to use (defaults to the value of `name`) |
+
+## Grammar configuration
+
+If a tree-sitter grammar is available for the language, add a new `grammar`
+entry to `languages.toml`.
+
+```toml
+[[grammar]]
+name = "mylang"
+source = { git = "https://github.com/example/mylang", rev = "a250c4582510ff34767ec3b7dcdd3c24e8c8aa68" }
 ```
 
-or you can manually add `shallow = true` to `.gitmodules`.
+Grammar configuration takes these keys:
 
-## languages.toml
+| Key      | Description                                                              |
+| ---      | -----------                                                              |
+| `name`   | The name of the tree-sitter grammar                                      |
+| `path`   | A path within the grammar directory which should be built. Some grammar repositories host multiple grammars (for example `tree-sitter-typescript` and `tree-sitter-ocaml`) in subdirectories. This key is used to point `hx --build-grammars` to the correct path for compilation. When ommitted, the root of the grammar directory is used |
+| `source` | The method of fetching the grammar - a table with a schema defined below |
 
-Next, you need to add the language to the [`languages.toml`][languages.toml] found in the root of
-the repository; this `languages.toml` file is included at compilation time, and
-is distinct from the `language.toml` file in the user's [configuration
-directory](../configuration.md).
+Where `source` is a table with either these keys when using a grammar from a
+git repository:
 
-These are the available keys and descriptions for the file.
+| Key   | Description                                               |
+| ---   | -----------                                               |
+| `git` | A git remote URL from which the grammar should be cloned  |
+| `rev` | The revision (commit hash or tag) which should be fetched |
 
-| Key                 | Description                                                   |
-| ----                | -----------                                                   |
-| name                | The name of the language                                      |
-| scope               | A string like `source.js` that identifies the language. Currently, we strive to match the scope names used by popular TextMate grammars and by the Linguist library. Usually `source.<name>` or `text.<name>` in case of markup languages |
-| injection-regex     | regex pattern that will be tested against a language name in order to determine whether this language should be used for a potential [language injection][treesitter-language-injection] site. |
-| file-types          | The filetypes of the language, for example `["yml", "yaml"]`  |
-| shebangs            | The interpreters from the shebang line, for example `["sh", "bash"]` |
-| roots               | A set of marker files to look for when trying to find the workspace root. For example `Cargo.lock`, `yarn.lock` |
-| auto-format         | Whether to autoformat this language when saving               |
-| diagnostic-severity | Minimal severity of diagnostic for it to be displayed. (Allowed values: `Error`, `Warning`, `Info`, `Hint`) |
-| comment-token       | The token to use as a comment-token                           |
-| indent              | The indent to use. Has sub keys `tab-width` and `unit`        |
-| config              | Language server configuration                                 |
+Or a `path` key with an absolute path to a locally available grammar directory.
 
 ## Queries
 
@@ -51,18 +74,14 @@ gives more info on how to write queries.
 
 > NOTE: When evaluating queries, the first matching query takes
 precedence, which is different from other editors like neovim where
-the last matching query supercedes the ones before it. See
+the last matching query supersedes the ones before it. See
 [this issue][neovim-query-precedence] for an example.
 
 ## Common Issues
 
-- If you get errors when building after switching branches, you may have to remove or update tree-sitter submodules. You can update submodules by running
-    ```sh
-    git submodule sync; git submodule update --init
-    ```
-- Make sure to not use the `--remote` flag. To remove submodules look inside the `.gitmodules` and remove directories that are not present inside of it.
+- If you get errors when running after switching branches, you may have to update the tree-sitter grammars. Run `hx --fetch-grammars` to fetch the grammars and `hx --build-grammars` to build any out-of-date grammars.
 
-- If a parser is segfaulting or you want to remove the parser, make sure to remove the submodule *and* the compiled parser in `runtime/grammar/<name>.so`
+- If a parser is segfaulting or you want to remove the parser, make sure to remove the compiled parser in `runtime/grammar/<name>.so`
 
 - The indents query is `indents.toml`, *not* `indents.scm`. See [this](https://github.com/helix-editor/helix/issues/114) issue for more information.
 
diff --git a/book/src/install.md b/book/src/install.md
index 8b7d8fa2d5b5..db42509c1841 100644
--- a/book/src/install.md
+++ b/book/src/install.md
@@ -44,7 +44,7 @@ sudo dnf install helix
 ## Build from source
 
 ```
-git clone --recurse-submodules --shallow-submodules -j8 https://github.com/helix-editor/helix
+git clone https://github.com/helix-editor/helix
 cd helix
 cargo install --path helix-term
 ```
@@ -54,3 +54,9 @@ This will install the `hx` binary to `$HOME/.cargo/bin`.
 Helix also needs it's runtime files so make sure to copy/symlink the `runtime/` directory into the
 config directory (for example `~/.config/helix/runtime` on Linux/macOS). This location can be overriden
 via the `HELIX_RUNTIME` environment variable.
+
+## Building tree-sitter grammars
+
+Tree-sitter grammars must be fetched and compiled if not pre-packaged.
+Fetch grammars with `hx --fetch-grammars` (requires `git`) and compile them
+with `hx --build-grammars` (requires a C compiler).
diff --git a/book/src/languages.md b/book/src/languages.md
index 4c4dc326d6eb..c5a7708faef5 100644
--- a/book/src/languages.md
+++ b/book/src/languages.md
@@ -4,10 +4,28 @@ Language-specific settings and settings for particular language servers can be c
 
 Changes made to the `languages.toml` file in a user's [configuration directory](./configuration.md) are merged with helix's defaults on start-up, such that a user's settings will take precedence over defaults in the event of a collision. For example, the default `languages.toml` sets rust's `auto-format` to `true`. If a user wants to disable auto-format, they can change the `languages.toml` in their [configuration directory](./configuration.md) to make the rust entry read like the example below; the new key/value pair `auto-format = false` will override the default when the two sets of settings are merged on start-up:
 
-```
+```toml
 # in <config_dir>/helix/languages.toml
 
 [[language]]
 name = "rust"
 auto-format = false
 ```
+
+## Tree-sitter grammars
+
+Tree-sitter grammars can also be configured in `languages.toml`:
+
+```toml
+# in <config_dir>/helix/languages.toml
+
+[[grammar]]
+name = "rust"
+source = { git = "https://github.com/tree-sitter/tree-sitter-rust", rev = "a250c4582510ff34767ec3b7dcdd3c24e8c8aa68" }
+
+[[grammar]]
+name = "c"
+source = { path = "/path/to/tree-sitter-c" }
+```
+
+If a user has a `languages.toml`, only the grammars in that `languages.toml` are evaluated when running `hx --fetch-grammars` and `hx --build-grammars`. Otherwise the [default `languages.toml`](https://github.com/helix-editor/helix/blob/master/languages.toml) is used.
diff --git a/helix-core/src/config.rs b/helix-core/src/config.rs
index 5bd16abde35e..d4ebee1f8bd9 100644
--- a/helix-core/src/config.rs
+++ b/helix-core/src/config.rs
@@ -25,7 +25,7 @@ pub fn user_lang_config() -> Result<toml::Value, toml::de::Error> {
 pub fn default_syntax_loader() -> crate::syntax::Configuration {
     default_lang_config()
         .try_into()
-        .expect("Could not serialize built-in language.toml")
+        .expect("Could not serialize built-in languages.toml")
 }
 /// Syntax configuration loader based on user configured languages.toml.
 pub fn user_syntax_loader() -> Result<crate::syntax::Configuration, toml::de::Error> {

From 333a59150a88a8acd5821db399d1ac68249ae8cd Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Mon, 14 Feb 2022 17:36:02 -0600
Subject: [PATCH 08/18] ensure rust grammar is available in CI

---
 .github/workflows/build.yml      | 30 ++++++++++++++++++++++--------
 .github/workflows/languages.toml | 26 ++++++++++++++++++++++++++
 .github/workflows/release.yml    | 22 ++++++++++++++--------
 helix-core/src/syntax.rs         | 12 ++++--------
 4 files changed, 66 insertions(+), 24 deletions(-)
 create mode 100644 .github/workflows/languages.toml

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 659c9c7896e7..e0eb5322b9f6 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -14,8 +14,6 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
-        with:
-          submodules: true
 
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
@@ -53,8 +51,6 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
-        with:
-          submodules: true
 
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
@@ -81,6 +77,28 @@ jobs:
           path: target
           key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
 
+      - name: Cache test tree-sitter grammar
+        uses: actions/cache@v2.1.7
+        with:
+          path: runtime/grammars
+          key: ${{ runner.os }}-v2-tree-sitter-grammars-${{ hashFiles('**/Cargo.lock') }}
+
+      - run: cp .github/workflows/languages.toml ./languages.toml
+
+      - name: Download test tree-sitter grammar
+        uses: actions-rs/cargo@v1
+        with:
+          use-cross: ${{ matrix.cross }}
+          command: run
+          args: -- --fetch-grammars
+
+      - name: Build test tree-sitter grammar
+        uses: actions-rs/cargo@v1
+        with:
+          use-cross: ${{ matrix.cross }}
+          command: run
+          args: -- --build-grammars
+
       - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:
@@ -97,8 +115,6 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
-        with:
-          submodules: true
 
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
@@ -144,8 +160,6 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
-        with:
-          submodules: true
 
       - name: Install stable toolchain
         uses: actions-rs/toolchain@v1
diff --git a/.github/workflows/languages.toml b/.github/workflows/languages.toml
new file mode 100644
index 000000000000..18cf71cf5ff7
--- /dev/null
+++ b/.github/workflows/languages.toml
@@ -0,0 +1,26 @@
+# This languages.toml is used for testing in CI.
+
+[[language]]
+name = "rust"
+scope = "source.rust"
+injection-regex = "rust"
+file-types = ["rs"]
+comment-token = "//"
+roots = ["Cargo.toml", "Cargo.lock"]
+indent = { tab-width = 4, unit = "    " }
+
+[[grammar]]
+name = "rust"
+source = { git = "https://github.com/tree-sitter/tree-sitter-rust", rev = "a360da0a29a19c281d08295a35ecd0544d2da211" }
+
+[[language]]
+name = "nix"
+scope = "source.nix"
+injection-regex = "nix"
+file-types = ["nix"]
+shebangs = []
+roots = []
+comment-token = "#"
+
+# A grammar entry is not necessary for this language - it is only used for
+# testing TOML merging behavior.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 1f9b6a326ba2..2a9ea80ebb7d 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -53,8 +53,6 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
-        with:
-          submodules: true
 
       - name: Install ${{ matrix.rust }} toolchain
         uses: actions-rs/toolchain@v1
@@ -64,6 +62,20 @@ jobs:
           target: ${{ matrix.target }}
           override: true
 
+      - name: Fetch tree-sitter grammars
+        uses: actions-rs/cargo@v1
+        with:
+          use-cross: ${{ matrix.cross }}
+          command: run
+          args: --release --locked --target ${{ matrix.target }} -- --fetch-grammars
+
+      - name: Build tree-sitter grammars
+        uses: actions-rs/cargo@v1
+        with:
+          use-cross: ${{ matrix.cross }}
+          command: run
+          args: --release --locked --target ${{ matrix.target }} -- --build-grammars
+
       - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:
@@ -114,14 +126,8 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
-        with:
-          submodules: false
 
       - uses: actions/download-artifact@v2
-        # with:
-        #   path: dist
-      # - run: ls -al ./dist
-      - run: ls -al bins-*
 
       - name: Calculate tag name
         run: |
diff --git a/helix-core/src/syntax.rs b/helix-core/src/syntax.rs
index 038fbfddf9d6..52239d100350 100644
--- a/helix-core/src/syntax.rs
+++ b/helix-core/src/syntax.rs
@@ -2115,14 +2115,10 @@ mod test {
         let language = get_language(&crate::RUNTIME_DIR, "Rust").unwrap();
         let config = HighlightConfiguration::new(
             language,
-            &std::fs::read_to_string(
-                "../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",
-            )
-            .unwrap(),
-            &std::fs::read_to_string(
-                "../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",
-            )
-            .unwrap(),
+            &std::fs::read_to_string("../runtime/grammars/sources/rust/queries/highlights.scm")
+                .unwrap(),
+            &std::fs::read_to_string("../runtime/grammars/sources/rust/queries/injections.scm")
+                .unwrap(),
             "", // locals.scm
         )
         .unwrap();

From c273cc591d015dd06edb553b0dca022a4fede5ba Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Mon, 14 Feb 2022 23:23:01 -0600
Subject: [PATCH 09/18] add 'use-grammars' to languages.toml

The vision with 'use-grammars' is to allow the long-requested feature
of being able to declare your own set of grammars that you would like.
A simple schema with only/except grammar names controls the list
of grammars that is fetched and built. It does not (yet) control which
grammars may be loaded at runtime if they already exist.
---
 book/src/languages.md      | 11 ++++++++++-
 helix-core/src/indent.rs   |  1 +
 helix-core/src/syntax.rs   | 12 +++++++++++-
 helix-term/src/grammars.rs | 16 ++++++++++++++--
 4 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/book/src/languages.md b/book/src/languages.md
index c5a7708faef5..3372a1202018 100644
--- a/book/src/languages.md
+++ b/book/src/languages.md
@@ -28,4 +28,13 @@ name = "c"
 source = { path = "/path/to/tree-sitter-c" }
 ```
 
-If a user has a `languages.toml`, only the grammars in that `languages.toml` are evaluated when running `hx --fetch-grammars` and `hx --build-grammars`. Otherwise the [default `languages.toml`](https://github.com/helix-editor/helix/blob/master/languages.toml) is used.
+You may use a top-level `use-grammars` key to control which grammars are fetched and built.
+
+```toml
+# Note: this key must come **before** the [[language]] and [[grammar]] sections
+use-grammars = { only = [ "rust", "c", "cpp" ] }
+# or
+use-grammars = { except = [ "yaml", "json" ] }
+```
+
+When omitted, all grammars are fetched and built.
diff --git a/helix-core/src/indent.rs b/helix-core/src/indent.rs
index ba02065c7ffb..ee9cbb165007 100644
--- a/helix-core/src/indent.rs
+++ b/helix-core/src/indent.rs
@@ -445,6 +445,7 @@ where
                 auto_pairs: None,
             }],
             grammar: vec![],
+            grammar_selection: None,
         });
 
         // set runtime path so we can find the queries
diff --git a/helix-core/src/syntax.rs b/helix-core/src/syntax.rs
index 52239d100350..28aa31f9926f 100644
--- a/helix-core/src/syntax.rs
+++ b/helix-core/src/syntax.rs
@@ -81,12 +81,21 @@ where
 }
 
 #[derive(Debug, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
+#[serde(rename_all = "kebab-case", deny_unknown_fields)]
 pub struct Configuration {
+    #[serde(rename = "use-grammars")]
+    pub grammar_selection: Option<GrammarSelection>,
     pub language: Vec<LanguageConfiguration>,
     pub grammar: Vec<GrammarConfiguration>,
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase", untagged)]
+pub enum GrammarSelection {
+    Only(HashSet<String>),
+    Except(HashSet<String>),
+}
+
 // largely based on tree-sitter/cli/src/loader.rs
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "kebab-case", deny_unknown_fields)]
@@ -2110,6 +2119,7 @@ mod test {
         let loader = Loader::new(Configuration {
             language: vec![],
             grammar: vec![],
+            grammar_selection: None,
         });
 
         let language = get_language(&crate::RUNTIME_DIR, "Rust").unwrap();
diff --git a/helix-term/src/grammars.rs b/helix-term/src/grammars.rs
index 7a7a4c187eaf..2e0be4bcbd98 100644
--- a/helix-term/src/grammars.rs
+++ b/helix-term/src/grammars.rs
@@ -7,7 +7,7 @@ use std::{
     sync::mpsc::channel,
 };
 
-use helix_core::syntax::{GrammarConfiguration, GrammarSource, DYLIB_EXTENSION};
+use helix_core::syntax::{GrammarConfiguration, GrammarSelection, GrammarSource, DYLIB_EXTENSION};
 
 const BUILD_TARGET: &str = env!("BUILD_TARGET");
 const REMOTE_NAME: &str = "origin";
@@ -163,7 +163,19 @@ fn build_grammar(grammar: GrammarConfiguration) -> Result<()> {
 fn get_grammar_configs() -> Vec<GrammarConfiguration> {
     let config = helix_core::config::user_syntax_loader().expect("Could not parse languages.toml");
 
-    config.grammar
+    match config.grammar_selection {
+        Some(GrammarSelection::Only(selections)) => config
+            .grammar
+            .into_iter()
+            .filter(|grammar| selections.contains(&grammar.grammar_id))
+            .collect(),
+        Some(GrammarSelection::Except(rejections)) => config
+            .grammar
+            .into_iter()
+            .filter(|grammar| !rejections.contains(&grammar.grammar_id))
+            .collect(),
+        None => config.grammar,
+    }
 }
 
 fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) -> Result<()> {

From 140c6da4b851892bc7f30f9b42c167cb15413c75 Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Wed, 16 Feb 2022 07:57:20 -0600
Subject: [PATCH 10/18] migrate grammar fetching/building code into
 helix-loader crate

This is a rather large refactor that moves most of the code for
loading, fetching, and building grammars into a new helix-loader
module. This works well with the [[grammars]] syntax for
languages.toml defined earlier: we only have to depend on the types
for GrammarConfiguration in helix-loader and can leave all the
[[language]] entries for helix-core.
---
 Cargo.lock                                    |  19 +-
 Cargo.toml                                    |   1 +
 book/src/guides/adding_languages.md           |  10 +-
 docs/architecture.md                          |  16 +-
 helix-core/Cargo.toml                         |   6 +-
 helix-core/src/config.rs                      |  27 +-
 helix-core/src/indent.rs                      |   2 -
 helix-core/src/lib.rs                         | 141 -----------
 helix-core/src/path.rs                        |   5 +-
 helix-core/src/syntax.rs                      |  91 +------
 helix-loader/Cargo.toml                       |  23 ++
 helix-loader/build.rs                         |   6 +
 .../src/grammar.rs                            | 230 ++++++++++++------
 helix-loader/src/lib.rs                       | 161 ++++++++++++
 helix-term/Cargo.toml                         |   1 +
 helix-term/build.rs                           |   5 -
 helix-term/src/application.rs                 |   6 +-
 helix-term/src/commands/typed.rs              |   2 +-
 helix-term/src/health.rs                      |  14 +-
 helix-term/src/lib.rs                         |   1 -
 helix-term/src/main.rs                        |  10 +-
 helix-term/src/ui/mod.rs                      |   4 +-
 languages.toml                                |  12 +-
 23 files changed, 419 insertions(+), 374 deletions(-)
 create mode 100644 helix-loader/Cargo.toml
 create mode 100644 helix-loader/build.rs
 rename helix-term/src/grammars.rs => helix-loader/src/grammar.rs (58%)
 create mode 100644 helix-loader/src/lib.rs

diff --git a/Cargo.lock b/Cargo.lock
index 469a586245f2..de4b2279a885 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -358,12 +358,11 @@ dependencies = [
 name = "helix-core"
 version = "0.6.0"
 dependencies = [
- "anyhow",
  "arc-swap",
  "chrono",
  "encoding_rs",
  "etcetera",
- "libloading",
+ "helix-loader",
  "log",
  "once_cell",
  "quickcheck",
@@ -397,6 +396,21 @@ dependencies = [
  "which",
 ]
 
+[[package]]
+name = "helix-loader"
+version = "0.6.0"
+dependencies = [
+ "anyhow",
+ "cc",
+ "etcetera",
+ "libloading",
+ "once_cell",
+ "serde",
+ "threadpool",
+ "toml",
+ "tree-sitter",
+]
+
 [[package]]
 name = "helix-lsp"
 version = "0.6.0"
@@ -432,6 +446,7 @@ dependencies = [
  "grep-searcher",
  "helix-core",
  "helix-dap",
+ "helix-loader",
  "helix-lsp",
  "helix-tui",
  "helix-view",
diff --git a/Cargo.toml b/Cargo.toml
index 0847e6ba4461..780811f7802c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,6 +6,7 @@ members = [
   "helix-tui",
   "helix-lsp",
   "helix-dap",
+  "helix-loader",
   "xtask",
 ]
 
diff --git a/book/src/guides/adding_languages.md b/book/src/guides/adding_languages.md
index 51256288ff9c..e000770c62e9 100644
--- a/book/src/guides/adding_languages.md
+++ b/book/src/guides/adding_languages.md
@@ -51,16 +51,16 @@ Grammar configuration takes these keys:
 | Key      | Description                                                              |
 | ---      | -----------                                                              |
 | `name`   | The name of the tree-sitter grammar                                      |
-| `path`   | A path within the grammar directory which should be built. Some grammar repositories host multiple grammars (for example `tree-sitter-typescript` and `tree-sitter-ocaml`) in subdirectories. This key is used to point `hx --build-grammars` to the correct path for compilation. When ommitted, the root of the grammar directory is used |
 | `source` | The method of fetching the grammar - a table with a schema defined below |
 
 Where `source` is a table with either these keys when using a grammar from a
 git repository:
 
-| Key   | Description                                               |
-| ---   | -----------                                               |
-| `git` | A git remote URL from which the grammar should be cloned  |
-| `rev` | The revision (commit hash or tag) which should be fetched |
+| Key    | Description                                               |
+| ---    | -----------                                               |
+| `git`  | A git remote URL from which the grammar should be cloned  |
+| `rev`  | The revision (commit hash or tag) which should be fetched |
+| `subpath` | A path within the grammar directory which should be built. Some grammar repositories host multiple grammars (for example `tree-sitter-typescript` and `tree-sitter-ocaml`) in subdirectories. This key is used to point `hx --build-grammars` to the correct path for compilation. When omitted, the root of repository is used |
 
 Or a `path` key with an absolute path to a locally available grammar directory.
 
diff --git a/docs/architecture.md b/docs/architecture.md
index 3c743eca2b7a..33624aac2b6d 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -1,11 +1,13 @@
 
-| Crate        | Description                                             |
-| -----------  | -----------                                             |
-| helix-core   | Core editing primitives, functional.                    |
-| helix-lsp    | Language server client                                  |
-| helix-view   | UI abstractions for use in backends, imperative shell.  |
-| helix-term   | Terminal UI                                             |
-| helix-tui    | TUI primitives, forked from tui-rs, inspired by Cursive |
+| Crate        | Description                                                      |
+| -----------  | -----------                                                      |
+| helix-core   | Core editing primitives, functional.                             |
+| helix-lsp    | Language server client                                           |
+| helix-dap    | Debug Adapter Protocol (DAP) client                              |
+| helix-loader | Functions for building, fetching, and loading external resources |
+| helix-view   | UI abstractions for use in backends, imperative shell.           |
+| helix-term   | Terminal UI                                                      |
+| helix-tui    | TUI primitives, forked from tui-rs, inspired by Cursive          |
 
 
 This document contains a high-level overview of Helix internals.
diff --git a/helix-core/Cargo.toml b/helix-core/Cargo.toml
index 5582d38b478d..8152da57474b 100644
--- a/helix-core/Cargo.toml
+++ b/helix-core/Cargo.toml
@@ -13,6 +13,8 @@ include = ["src/**/*", "README.md"]
 [features]
 
 [dependencies]
+helix-loader = { version = "0.6", path = "../helix-loader" }
+
 ropey = "1.3"
 smallvec = "1.8"
 smartstring = "1.0.0"
@@ -33,13 +35,11 @@ toml = "0.5"
 
 similar = "2.1"
 
-etcetera = "0.3"
 encoding_rs = "0.8"
 
 chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
 
-libloading = "0.7"
-anyhow = "1"
+etcetera = "0.3"
 
 [dev-dependencies]
 quickcheck = { version = "1", default-features = false }
diff --git a/helix-core/src/config.rs b/helix-core/src/config.rs
index d4ebee1f8bd9..f399850e61e3 100644
--- a/helix-core/src/config.rs
+++ b/helix-core/src/config.rs
@@ -1,33 +1,10 @@
-use crate::merge_toml_values;
-
-/// Default bultin-in languages.toml.
-pub fn default_lang_config() -> toml::Value {
-    toml::from_slice(include_bytes!("../../languages.toml"))
-        .expect("Could not parse bultin-in languages.toml to valid toml")
-}
-
-/// User configured languages.toml file, merged with the default config.
-pub fn user_lang_config() -> Result<toml::Value, toml::de::Error> {
-    let def_lang_conf = default_lang_config();
-    let data = std::fs::read(crate::config_dir().join("languages.toml"));
-    let user_lang_conf = match data {
-        Ok(raw) => {
-            let value = toml::from_slice(&raw)?;
-            merge_toml_values(def_lang_conf, value)
-        }
-        Err(_) => def_lang_conf,
-    };
-
-    Ok(user_lang_conf)
-}
-
 /// Syntax configuration loader based on built-in languages.toml.
 pub fn default_syntax_loader() -> crate::syntax::Configuration {
-    default_lang_config()
+    helix_loader::default_lang_config()
         .try_into()
         .expect("Could not serialize built-in languages.toml")
 }
 /// Syntax configuration loader based on user configured languages.toml.
 pub fn user_syntax_loader() -> Result<crate::syntax::Configuration, toml::de::Error> {
-    user_lang_config()?.try_into()
+    helix_loader::user_lang_config()?.try_into()
 }
diff --git a/helix-core/src/indent.rs b/helix-core/src/indent.rs
index ee9cbb165007..30f4a3405d9c 100644
--- a/helix-core/src/indent.rs
+++ b/helix-core/src/indent.rs
@@ -444,8 +444,6 @@ where
                 debugger: None,
                 auto_pairs: None,
             }],
-            grammar: vec![],
-            grammar_selection: None,
         });
 
         // set runtime path so we can find the queries
diff --git a/helix-core/src/lib.rs b/helix-core/src/lib.rs
index c3a349c1276d..1f43c26677bb 100644
--- a/helix-core/src/lib.rs
+++ b/helix-core/src/lib.rs
@@ -33,9 +33,6 @@ pub mod unicode {
     pub use unicode_width as width;
 }
 
-static RUNTIME_DIR: once_cell::sync::Lazy<std::path::PathBuf> =
-    once_cell::sync::Lazy::new(runtime_dir);
-
 pub fn find_first_non_whitespace_char(line: RopeSlice) -> Option<usize> {
     line.chars().position(|ch| !ch.is_whitespace())
 }
@@ -85,144 +82,6 @@ pub fn find_root(root: Option<&str>, root_markers: &[String]) -> Option<std::pat
     }
 }
 
-pub fn runtime_dir() -> std::path::PathBuf {
-    if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
-        return dir.into();
-    }
-
-    const RT_DIR: &str = "runtime";
-    let conf_dir = config_dir().join(RT_DIR);
-    if conf_dir.exists() {
-        return conf_dir;
-    }
-
-    if let Ok(dir) = std::env::var("CARGO_MANIFEST_DIR") {
-        // this is the directory of the crate being run by cargo, we need the workspace path so we take the parent
-        return std::path::PathBuf::from(dir).parent().unwrap().join(RT_DIR);
-    }
-
-    // fallback to location of the executable being run
-    std::env::current_exe()
-        .ok()
-        .and_then(|path| path.parent().map(|path| path.to_path_buf().join(RT_DIR)))
-        .unwrap()
-}
-
-pub fn config_dir() -> std::path::PathBuf {
-    // TODO: allow env var override
-    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
-    let mut path = strategy.config_dir();
-    path.push("helix");
-    path
-}
-
-pub fn cache_dir() -> std::path::PathBuf {
-    // TODO: allow env var override
-    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
-    let mut path = strategy.cache_dir();
-    path.push("helix");
-    path
-}
-
-pub fn config_file() -> std::path::PathBuf {
-    config_dir().join("config.toml")
-}
-
-pub fn lang_config_file() -> std::path::PathBuf {
-    config_dir().join("languages.toml")
-}
-
-pub fn log_file() -> std::path::PathBuf {
-    cache_dir().join("helix.log")
-}
-
-// right overrides left
-pub fn merge_toml_values(left: toml::Value, right: toml::Value) -> toml::Value {
-    use toml::Value;
-
-    fn get_name(v: &Value) -> Option<&str> {
-        v.get("name").and_then(Value::as_str)
-    }
-
-    match (left, right) {
-        (Value::Array(mut left_items), Value::Array(right_items)) => {
-            left_items.reserve(right_items.len());
-            for rvalue in right_items {
-                let lvalue = get_name(&rvalue)
-                    .and_then(|rname| left_items.iter().position(|v| get_name(v) == Some(rname)))
-                    .map(|lpos| left_items.remove(lpos));
-                let mvalue = match lvalue {
-                    Some(lvalue) => merge_toml_values(lvalue, rvalue),
-                    None => rvalue,
-                };
-                left_items.push(mvalue);
-            }
-            Value::Array(left_items)
-        }
-        (Value::Table(mut left_map), Value::Table(right_map)) => {
-            for (rname, rvalue) in right_map {
-                match left_map.remove(&rname) {
-                    Some(lvalue) => {
-                        let merged_value = merge_toml_values(lvalue, rvalue);
-                        left_map.insert(rname, merged_value);
-                    }
-                    None => {
-                        left_map.insert(rname, rvalue);
-                    }
-                }
-            }
-            Value::Table(left_map)
-        }
-        // Catch everything else we didn't handle, and use the right value
-        (_, value) => value,
-    }
-}
-
-#[cfg(test)]
-mod merge_toml_tests {
-    use super::merge_toml_values;
-
-    #[test]
-    fn language_tomls() {
-        use toml::Value;
-
-        const USER: &str = "
-        [[language]]
-        name = \"nix\"
-        test = \"bbb\"
-        indent = { tab-width = 4, unit = \"    \", test = \"aaa\" }
-        ";
-
-        let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
-            .expect("Couldn't parse built-in languages config");
-        let user: Value = toml::from_str(USER).unwrap();
-
-        let merged = merge_toml_values(base, user);
-        let languages = merged.get("language").unwrap().as_array().unwrap();
-        let nix = languages
-            .iter()
-            .find(|v| v.get("name").unwrap().as_str().unwrap() == "nix")
-            .unwrap();
-        let nix_indent = nix.get("indent").unwrap();
-
-        // We changed tab-width and unit in indent so check them if they are the new values
-        assert_eq!(
-            nix_indent.get("tab-width").unwrap().as_integer().unwrap(),
-            4
-        );
-        assert_eq!(nix_indent.get("unit").unwrap().as_str().unwrap(), "    ");
-        // We added a new keys, so check them
-        assert_eq!(nix.get("test").unwrap().as_str().unwrap(), "bbb");
-        assert_eq!(nix_indent.get("test").unwrap().as_str().unwrap(), "aaa");
-        // We didn't change comment-token so it should be same
-        assert_eq!(nix.get("comment-token").unwrap().as_str().unwrap(), "#");
-    }
-}
-
-pub use etcetera::home_dir;
-
-use etcetera::base_strategy::{choose_base_strategy, BaseStrategy};
-
 pub use ropey::{Rope, RopeBuilder, RopeSlice};
 
 // pub use tendril::StrTendril as Tendril;
diff --git a/helix-core/src/path.rs b/helix-core/src/path.rs
index a66444651500..e0c3bef65b05 100644
--- a/helix-core/src/path.rs
+++ b/helix-core/src/path.rs
@@ -1,9 +1,10 @@
+use etcetera::home_dir;
 use std::path::{Component, Path, PathBuf};
 
 /// Replaces users home directory from `path` with tilde `~` if the directory
 /// is available, otherwise returns the path unchanged.
 pub fn fold_home_dir(path: &Path) -> PathBuf {
-    if let Ok(home) = super::home_dir() {
+    if let Ok(home) = home_dir() {
         if path.starts_with(&home) {
             // it's ok to unwrap, the path starts with home dir
             return PathBuf::from("~").join(path.strip_prefix(&home).unwrap());
@@ -20,7 +21,7 @@ pub fn expand_tilde(path: &Path) -> PathBuf {
     let mut components = path.components().peekable();
     if let Some(Component::Normal(c)) = components.peek() {
         if c == &"~" {
-            if let Ok(home) = super::home_dir() {
+            if let Ok(home) = home_dir() {
                 // it's ok to unwrap, the path starts with `~`
                 return home.join(path.strip_prefix("~").unwrap());
             }
diff --git a/helix-core/src/syntax.rs b/helix-core/src/syntax.rs
index 28aa31f9926f..6ae46d4ff237 100644
--- a/helix-core/src/syntax.rs
+++ b/helix-core/src/syntax.rs
@@ -7,10 +7,6 @@ use crate::{
     Rope, RopeSlice, Tendril,
 };
 
-use anyhow::{Context, Result};
-use libloading::{Library, Symbol};
-use tree_sitter::Language;
-
 use arc_swap::{ArcSwap, Guard};
 use slotmap::{DefaultKey as LayerId, HopSlotMap};
 
@@ -27,33 +23,7 @@ use std::{
 use once_cell::sync::{Lazy, OnceCell};
 use serde::{Deserialize, Serialize};
 
-#[cfg(unix)]
-pub const DYLIB_EXTENSION: &str = "so";
-
-#[cfg(windows)]
-pub const DYLIB_EXTENSION: &str = "dll";
-
-fn replace_dashes_with_underscores(name: &str) -> String {
-    name.replace('-', "_")
-}
-
-pub fn get_language(runtime_path: &std::path::Path, name: &str) -> Result<Language> {
-    let name = name.to_ascii_lowercase();
-    let mut library_path = runtime_path.join("grammars").join(&name);
-    library_path.set_extension(DYLIB_EXTENSION);
-
-    let library = unsafe { Library::new(&library_path) }
-        .with_context(|| format!("Error opening dynamic library {:?}", &library_path))?;
-    let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(&name));
-    let language = unsafe {
-        let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
-            .get(language_fn_name.as_bytes())
-            .with_context(|| format!("Failed to load symbol {}", language_fn_name))?;
-        language_fn()
-    };
-    std::mem::forget(library);
-    Ok(language)
-}
+use helix_loader::grammar::{get_language, load_runtime_file};
 
 fn deserialize_regex<'de, D>(deserializer: D) -> Result<Option<Regex>, D::Error>
 where
@@ -81,19 +51,8 @@ where
 }
 
 #[derive(Debug, Serialize, Deserialize)]
-#[serde(rename_all = "kebab-case", deny_unknown_fields)]
 pub struct Configuration {
-    #[serde(rename = "use-grammars")]
-    pub grammar_selection: Option<GrammarSelection>,
     pub language: Vec<LanguageConfiguration>,
-    pub grammar: Vec<GrammarConfiguration>,
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(rename_all = "lowercase", untagged)]
-pub enum GrammarSelection {
-    Only(HashSet<String>),
-    Except(HashSet<String>),
 }
 
 // largely based on tree-sitter/cli/src/loader.rs
@@ -279,29 +238,6 @@ pub struct IndentQuery {
     pub outdent: HashSet<String>,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
-pub struct GrammarConfiguration {
-    #[serde(rename = "name")]
-    pub grammar_id: String, // c-sharp, rust
-    pub source: GrammarSource,
-    pub path: Option<String>,
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(rename_all = "lowercase")]
-#[serde(untagged)]
-pub enum GrammarSource {
-    Local {
-        path: String,
-    },
-    Git {
-        #[serde(rename = "git")]
-        remote: String,
-        #[serde(rename = "rev")]
-        revision: String,
-    },
-}
-
 #[derive(Debug)]
 pub struct TextObjectQuery {
     pub query: Query,
@@ -398,14 +334,6 @@ impl TextObjectQuery {
     }
 }
 
-pub fn load_runtime_file(language: &str, filename: &str) -> Result<String, std::io::Error> {
-    let path = crate::RUNTIME_DIR
-        .join("queries")
-        .join(language)
-        .join(filename);
-    std::fs::read_to_string(&path)
-}
-
 fn read_query(language: &str, filename: &str) -> String {
     static INHERITS_REGEX: Lazy<Regex> =
         Lazy::new(|| Regex::new(r";+\s*inherits\s*:?\s*([a-z_,()]+)\s*").unwrap());
@@ -451,12 +379,9 @@ impl LanguageConfiguration {
         if highlights_query.is_empty() {
             None
         } else {
-            let language = get_language(
-                &crate::RUNTIME_DIR,
-                self.grammar.as_deref().unwrap_or(&self.language_id),
-            )
-            .map_err(|e| log::info!("{}", e))
-            .ok()?;
+            let language = get_language(self.grammar.as_deref().unwrap_or(&self.language_id))
+                .map_err(|e| log::info!("{}", e))
+                .ok()?;
             let config = HighlightConfiguration::new(
                 language,
                 &highlights_query,
@@ -2116,13 +2041,9 @@ mod test {
         .map(String::from)
         .collect();
 
-        let loader = Loader::new(Configuration {
-            language: vec![],
-            grammar: vec![],
-            grammar_selection: None,
-        });
+        let loader = Loader::new(Configuration { language: vec![] });
 
-        let language = get_language(&crate::RUNTIME_DIR, "Rust").unwrap();
+        let language = get_language("Rust").unwrap();
         let config = HighlightConfiguration::new(
             language,
             &std::fs::read_to_string("../runtime/grammars/sources/rust/queries/highlights.scm")
diff --git a/helix-loader/Cargo.toml b/helix-loader/Cargo.toml
new file mode 100644
index 000000000000..21b37333afc8
--- /dev/null
+++ b/helix-loader/Cargo.toml
@@ -0,0 +1,23 @@
+[package]
+name = "helix-loader"
+version = "0.6.0"
+description = "A post-modern text editor."
+authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
+edition = "2021"
+license = "MPL-2.0"
+categories = ["editor"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+
+[dependencies]
+anyhow = "1"
+serde = { version = "1.0", features = ["derive"] }
+toml = "0.5"
+etcetera = "0.3"
+tree-sitter = "0.20"
+libloading = "0.7"
+once_cell = "1.9"
+
+# cloning/compiling tree-sitter grammars
+cc = { version = "1" }
+threadpool = { version = "1.0" }
diff --git a/helix-loader/build.rs b/helix-loader/build.rs
new file mode 100644
index 000000000000..e0ebd1c48f17
--- /dev/null
+++ b/helix-loader/build.rs
@@ -0,0 +1,6 @@
+fn main() {
+    println!(
+        "cargo:rustc-env=BUILD_TARGET={}",
+        std::env::var("TARGET").unwrap()
+    );
+}
diff --git a/helix-term/src/grammars.rs b/helix-loader/src/grammar.rs
similarity index 58%
rename from helix-term/src/grammars.rs
rename to helix-loader/src/grammar.rs
index 2e0be4bcbd98..61ef464fca01 100644
--- a/helix-term/src/grammars.rs
+++ b/helix-loader/src/grammar.rs
@@ -1,63 +1,155 @@
 use anyhow::{anyhow, Context, Result};
+use libloading::{Library, Symbol};
+use serde::{Deserialize, Serialize};
 use std::fs;
 use std::time::SystemTime;
 use std::{
+    collections::HashSet,
     path::{Path, PathBuf},
     process::Command,
     sync::mpsc::channel,
 };
+use tree_sitter::Language;
 
-use helix_core::syntax::{GrammarConfiguration, GrammarSelection, GrammarSource, DYLIB_EXTENSION};
+#[cfg(unix)]
+const DYLIB_EXTENSION: &str = "so";
+
+#[cfg(windows)]
+const DYLIB_EXTENSION: &str = "dll";
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Configuration {
+    #[serde(rename = "use-grammars")]
+    pub grammar_selection: Option<GrammarSelection>,
+    pub grammar: Vec<GrammarConfiguration>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase", untagged)]
+pub enum GrammarSelection {
+    Only(HashSet<String>),
+    Except(HashSet<String>),
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct GrammarConfiguration {
+    #[serde(rename = "name")]
+    pub grammar_id: String,
+    pub source: GrammarSource,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase", untagged)]
+pub enum GrammarSource {
+    Local {
+        path: String,
+    },
+    Git {
+        #[serde(rename = "git")]
+        remote: String,
+        #[serde(rename = "rev")]
+        revision: String,
+        subpath: Option<String>,
+    },
+}
 
 const BUILD_TARGET: &str = env!("BUILD_TARGET");
 const REMOTE_NAME: &str = "origin";
 
+pub fn get_language(name: &str) -> Result<Language> {
+    let name = name.to_ascii_lowercase();
+    let mut library_path = crate::runtime_dir().join("grammars").join(&name);
+    library_path.set_extension(DYLIB_EXTENSION);
+
+    let library = unsafe { Library::new(&library_path) }
+        .with_context(|| format!("Error opening dynamic library {library_path:?}"))?;
+    let language_fn_name = format!("tree_sitter_{}", name.replace('-', "_"));
+    let language = unsafe {
+        let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
+            .get(language_fn_name.as_bytes())
+            .with_context(|| format!("Failed to load symbol {language_fn_name}"))?;
+        language_fn()
+    };
+    std::mem::forget(library);
+    Ok(language)
+}
+
 pub fn fetch_grammars() -> Result<()> {
-    run_parallel(get_grammar_configs(), fetch_grammar, "fetch")
+    run_parallel(get_grammar_configs()?, fetch_grammar, "fetch")
 }
 
 pub fn build_grammars() -> Result<()> {
-    run_parallel(get_grammar_configs(), build_grammar, "build")
+    run_parallel(get_grammar_configs()?, build_grammar, "build")
+}
+
+// Returns the set of grammar configurations the user requests.
+// Grammars are configured in the default and user `languages.toml` and are
+// merged. The `grammar_selection` key of the config is then used to filter
+// down all grammars into a subset of the user's choosing.
+fn get_grammar_configs() -> Result<Vec<GrammarConfiguration>> {
+    let config: Configuration = crate::user_lang_config()
+        .context("Could not parse languages.toml")?
+        .try_into()?;
+
+    let grammars = match config.grammar_selection {
+        Some(GrammarSelection::Only(selections)) => config
+            .grammar
+            .into_iter()
+            .filter(|grammar| selections.contains(&grammar.grammar_id))
+            .collect(),
+        Some(GrammarSelection::Except(rejections)) => config
+            .grammar
+            .into_iter()
+            .filter(|grammar| !rejections.contains(&grammar.grammar_id))
+            .collect(),
+        None => config.grammar,
+    };
+
+    Ok(grammars)
 }
 
 fn run_parallel<F>(grammars: Vec<GrammarConfiguration>, job: F, action: &'static str) -> Result<()>
 where
     F: Fn(GrammarConfiguration) -> Result<()> + std::marker::Send + 'static + Copy,
 {
-    let mut n_jobs = 0;
     let pool = threadpool::Builder::new().build();
     let (tx, rx) = channel();
 
     for grammar in grammars {
         let tx = tx.clone();
-        n_jobs += 1;
 
         pool.execute(move || {
-            let grammar_id = grammar.grammar_id.clone();
-            job(grammar).unwrap_or_else(|err| {
-                eprintln!("Failed to {} grammar '{}'\n{}", action, grammar_id, err)
-            });
-
-            // report progress
-            tx.send(1).unwrap();
+            tx.send(job(grammar)).unwrap();
         });
     }
     pool.join();
 
-    if rx.try_iter().sum::<usize>() == n_jobs {
-        Ok(())
+    // TODO: print all failures instead of the first one found.
+    if let Some(failure) = rx.try_iter().find_map(|result| result.err()) {
+        Err(anyhow!(
+            "Failed to {} some grammar(s).\n{}",
+            action,
+            failure
+        ))
     } else {
-        Err(anyhow!("Failed to {} some grammar(s).", action))
+        Ok(())
     }
 }
 
 fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
-    if let GrammarSource::Git { remote, revision } = grammar.source {
-        let grammar_dir = helix_core::runtime_dir()
+    if let GrammarSource::Git {
+        remote, revision, ..
+    } = grammar.source
+    {
+        let grammar_dir = crate::runtime_dir()
             .join("grammars/sources")
-            .join(grammar.grammar_id.clone());
+            .join(&grammar.grammar_id);
 
-        fs::create_dir_all(grammar_dir.clone()).expect("Could not create grammar directory");
+        fs::create_dir_all(&grammar_dir).context(format!(
+            "Could not create grammar directory {:?}",
+            grammar_dir
+        ))?;
 
         // create the grammar dir contains a git directory
         if !grammar_dir.join(".git").is_dir() {
@@ -65,12 +157,12 @@ fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
         }
 
         // ensure the remote matches the configured remote
-        if get_remote_url(&grammar_dir).map_or(true, |s| s.trim_end() != remote) {
+        if get_remote_url(&grammar_dir).map_or(true, |s| s != remote) {
             set_remote(&grammar_dir, &remote)?;
         }
 
         // ensure the revision matches the configured revision
-        if get_revision(&grammar_dir).map_or(true, |s| s.trim_end() != revision) {
+        if get_revision(&grammar_dir).map_or(true, |s| s != revision) {
             // Fetch the exact revision from the remote.
             // Supported by server-side git since v2.5.0 (July 2015),
             // enabled by default on major git hosts.
@@ -94,33 +186,38 @@ fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
 
 // Sets the remote for a repository to the given URL, creating the remote if
 // it does not yet exist.
-fn set_remote(repository: &Path, remote_url: &str) -> Result<String> {
-    git(repository, ["remote", "set-url", REMOTE_NAME, remote_url])
-        .or_else(|_| git(repository, ["remote", "add", REMOTE_NAME, remote_url]))
+fn set_remote(repository_dir: &Path, remote_url: &str) -> Result<String> {
+    git(
+        repository_dir,
+        ["remote", "set-url", REMOTE_NAME, remote_url],
+    )
+    .or_else(|_| git(repository_dir, ["remote", "add", REMOTE_NAME, remote_url]))
 }
 
-fn get_remote_url(repository: &Path) -> Option<String> {
-    git(repository, ["remote", "get-url", REMOTE_NAME]).ok()
+fn get_remote_url(repository_dir: &Path) -> Option<String> {
+    git(repository_dir, ["remote", "get-url", REMOTE_NAME]).ok()
 }
 
-fn get_revision(repository: &Path) -> Option<String> {
-    git(repository, ["rev-parse", "HEAD"]).ok()
+fn get_revision(repository_dir: &Path) -> Option<String> {
+    git(repository_dir, ["rev-parse", "HEAD"]).ok()
 }
 
 // A wrapper around 'git' commands which returns stdout in success and a
 // helpful error message showing the command, stdout, and stderr in error.
-fn git<I, S>(repository: &Path, args: I) -> Result<String>
+fn git<I, S>(repository_dir: &Path, args: I) -> Result<String>
 where
     I: IntoIterator<Item = S>,
     S: AsRef<std::ffi::OsStr>,
 {
     let output = Command::new("git")
         .args(args)
-        .current_dir(repository)
+        .current_dir(repository_dir)
         .output()?;
 
     if output.status.success() {
-        Ok(String::from_utf8_lossy(&output.stdout).into_owned())
+        Ok(String::from_utf8_lossy(&output.stdout)
+            .trim_end()
+            .to_owned())
     } else {
         // TODO: figure out how to display the git command using `args`
         Err(anyhow!(
@@ -132,52 +229,37 @@ where
 }
 
 fn build_grammar(grammar: GrammarConfiguration) -> Result<()> {
-    let grammar_dir = if let GrammarSource::Local { ref path } = grammar.source {
-        PathBuf::from(path)
+    println!("{:#?}", grammar);
+    let grammar_dir = if let GrammarSource::Local { path } = &grammar.source {
+        PathBuf::from(&path)
     } else {
-        helix_core::runtime_dir()
+        crate::runtime_dir()
             .join("grammars/sources")
-            .join(grammar.grammar_id.clone())
+            .join(&grammar.grammar_id)
     };
 
-    grammar_dir.read_dir().with_context(|| {
-        format!(
-            "The directory {:?} is empty, you probably need to use 'hx --fetch-grammars'?",
-            grammar_dir
-        )
+    let grammar_dir_entries = grammar_dir.read_dir().with_context(|| {
+        format!("Failed to read directory {grammar_dir:?}. Did you use 'hx --fetch-grammars'?")
     })?;
 
-    let path = match grammar.path {
-        Some(ref subpath) => grammar_dir.join(subpath),
-        None => grammar_dir,
+    if grammar_dir_entries.count() == 0 {
+        return Err(anyhow!(
+            "Directory {grammar_dir:?} is empty. Did you use 'hx --fetch-grammars'?"
+        ));
+    };
+
+    let path = match &grammar.source {
+        GrammarSource::Git {
+            subpath: Some(subpath),
+            ..
+        } => grammar_dir.join(subpath),
+        _ => grammar_dir,
     }
     .join("src");
 
     build_tree_sitter_library(&path, grammar)
 }
 
-// Returns the set of grammar configurations the user requests.
-// Grammars are configured in the default and user `languages.toml` and are
-// merged. The `grammar_selection` key of the config is then used to filter
-// down all grammars into a subset of the user's choosing.
-fn get_grammar_configs() -> Vec<GrammarConfiguration> {
-    let config = helix_core::config::user_syntax_loader().expect("Could not parse languages.toml");
-
-    match config.grammar_selection {
-        Some(GrammarSelection::Only(selections)) => config
-            .grammar
-            .into_iter()
-            .filter(|grammar| selections.contains(&grammar.grammar_id))
-            .collect(),
-        Some(GrammarSelection::Except(rejections)) => config
-            .grammar
-            .into_iter()
-            .filter(|grammar| !rejections.contains(&grammar.grammar_id))
-            .collect(),
-        None => config.grammar,
-    }
-}
-
 fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) -> Result<()> {
     let header_path = src_path;
     let parser_path = src_path.join("parser.c");
@@ -193,8 +275,8 @@ fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) ->
             None
         }
     };
-    let parser_lib_path = helix_core::runtime_dir().join("../runtime/grammars");
-    let mut library_path = parser_lib_path.join(grammar.grammar_id.clone());
+    let parser_lib_path = crate::runtime_dir().join("grammars");
+    let mut library_path = parser_lib_path.join(&grammar.grammar_id);
     library_path.set_extension(DYLIB_EXTENSION);
 
     let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
@@ -210,7 +292,7 @@ fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) ->
     let mut config = cc::Build::new();
     config
         .cpp(true)
-        .opt_level(2)
+        .opt_level(3)
         .cargo_metadata(false)
         .host(BUILD_TARGET)
         .target(BUILD_TARGET);
@@ -245,7 +327,7 @@ fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) ->
             .arg(header_path)
             .arg("-o")
             .arg(&library_path)
-            .arg("-O2");
+            .arg("-O3");
         if let Some(scanner_path) = scanner_path.as_ref() {
             if scanner_path.extension() == Some("c".as_ref()) {
                 command.arg("-xc").arg("-std=c99").arg(scanner_path);
@@ -294,3 +376,13 @@ fn needs_recompile(
 fn mtime(path: &Path) -> Result<SystemTime> {
     Ok(fs::metadata(path)?.modified()?)
 }
+
+/// Gives the contents of a file from a language's `runtime/queries/<lang>`
+/// directory
+pub fn load_runtime_file(language: &str, filename: &str) -> Result<String, std::io::Error> {
+    let path = crate::RUNTIME_DIR
+        .join("queries")
+        .join(language)
+        .join(filename);
+    std::fs::read_to_string(&path)
+}
diff --git a/helix-loader/src/lib.rs b/helix-loader/src/lib.rs
new file mode 100644
index 000000000000..a2c4d96f0688
--- /dev/null
+++ b/helix-loader/src/lib.rs
@@ -0,0 +1,161 @@
+pub mod grammar;
+
+use etcetera::base_strategy::{choose_base_strategy, BaseStrategy};
+
+pub static RUNTIME_DIR: once_cell::sync::Lazy<std::path::PathBuf> =
+    once_cell::sync::Lazy::new(runtime_dir);
+
+pub fn runtime_dir() -> std::path::PathBuf {
+    if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
+        return dir.into();
+    }
+
+    const RT_DIR: &str = "runtime";
+    let conf_dir = config_dir().join(RT_DIR);
+    if conf_dir.exists() {
+        return conf_dir;
+    }
+
+    if let Ok(dir) = std::env::var("CARGO_MANIFEST_DIR") {
+        // this is the directory of the crate being run by cargo, we need the workspace path so we take the parent
+        return std::path::PathBuf::from(dir).parent().unwrap().join(RT_DIR);
+    }
+
+    // fallback to location of the executable being run
+    std::env::current_exe()
+        .ok()
+        .and_then(|path| path.parent().map(|path| path.to_path_buf().join(RT_DIR)))
+        .unwrap()
+}
+
+pub fn config_dir() -> std::path::PathBuf {
+    // TODO: allow env var override
+    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
+    let mut path = strategy.config_dir();
+    path.push("helix");
+    path
+}
+
+pub fn cache_dir() -> std::path::PathBuf {
+    // TODO: allow env var override
+    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
+    let mut path = strategy.cache_dir();
+    path.push("helix");
+    path
+}
+
+pub fn config_file() -> std::path::PathBuf {
+    config_dir().join("config.toml")
+}
+
+pub fn lang_config_file() -> std::path::PathBuf {
+    config_dir().join("languages.toml")
+}
+
+pub fn log_file() -> std::path::PathBuf {
+    cache_dir().join("helix.log")
+}
+
+/// Default bultin-in languages.toml.
+pub fn default_lang_config() -> toml::Value {
+    toml::from_slice(include_bytes!("../../languages.toml"))
+        .expect("Could not parse bultin-in languages.toml to valid toml")
+}
+
+/// User configured languages.toml file, merged with the default config.
+pub fn user_lang_config() -> Result<toml::Value, toml::de::Error> {
+    let def_lang_conf = default_lang_config();
+    let data = std::fs::read(crate::config_dir().join("languages.toml"));
+    let user_lang_conf = match data {
+        Ok(raw) => {
+            let value = toml::from_slice(&raw)?;
+            merge_toml_values(def_lang_conf, value)
+        }
+        Err(_) => def_lang_conf,
+    };
+
+    Ok(user_lang_conf)
+}
+
+// right overrides left
+pub fn merge_toml_values(left: toml::Value, right: toml::Value) -> toml::Value {
+    use toml::Value;
+
+    fn get_name(v: &Value) -> Option<&str> {
+        v.get("name").and_then(Value::as_str)
+    }
+
+    match (left, right) {
+        (Value::Array(mut left_items), Value::Array(right_items)) => {
+            left_items.reserve(right_items.len());
+            for rvalue in right_items {
+                let lvalue = get_name(&rvalue)
+                    .and_then(|rname| left_items.iter().position(|v| get_name(v) == Some(rname)))
+                    .map(|lpos| left_items.remove(lpos));
+                let mvalue = match lvalue {
+                    Some(lvalue) => merge_toml_values(lvalue, rvalue),
+                    None => rvalue,
+                };
+                left_items.push(mvalue);
+            }
+            Value::Array(left_items)
+        }
+        (Value::Table(mut left_map), Value::Table(right_map)) => {
+            for (rname, rvalue) in right_map {
+                match left_map.remove(&rname) {
+                    Some(lvalue) => {
+                        let merged_value = merge_toml_values(lvalue, rvalue);
+                        left_map.insert(rname, merged_value);
+                    }
+                    None => {
+                        left_map.insert(rname, rvalue);
+                    }
+                }
+            }
+            Value::Table(left_map)
+        }
+        // Catch everything else we didn't handle, and use the right value
+        (_, value) => value,
+    }
+}
+
+#[cfg(test)]
+mod merge_toml_tests {
+    use super::merge_toml_values;
+
+    #[test]
+    fn language_tomls() {
+        use toml::Value;
+
+        const USER: &str = "
+        [[language]]
+        name = \"nix\"
+        test = \"bbb\"
+        indent = { tab-width = 4, unit = \"    \", test = \"aaa\" }
+        ";
+
+        let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
+            .expect("Couldn't parse built-in languages config");
+        let user: Value = toml::from_str(USER).unwrap();
+
+        let merged = merge_toml_values(base, user);
+        let languages = merged.get("language").unwrap().as_array().unwrap();
+        let nix = languages
+            .iter()
+            .find(|v| v.get("name").unwrap().as_str().unwrap() == "nix")
+            .unwrap();
+        let nix_indent = nix.get("indent").unwrap();
+
+        // We changed tab-width and unit in indent so check them if they are the new values
+        assert_eq!(
+            nix_indent.get("tab-width").unwrap().as_integer().unwrap(),
+            4
+        );
+        assert_eq!(nix_indent.get("unit").unwrap().as_str().unwrap(), "    ");
+        // We added a new keys, so check them
+        assert_eq!(nix.get("test").unwrap().as_str().unwrap(), "bbb");
+        assert_eq!(nix_indent.get("test").unwrap().as_str().unwrap(), "aaa");
+        // We didn't change comment-token so it should be same
+        assert_eq!(nix.get("comment-token").unwrap().as_str().unwrap(), "#");
+    }
+}
diff --git a/helix-term/Cargo.toml b/helix-term/Cargo.toml
index 93d50d7e302d..86d72561a842 100644
--- a/helix-term/Cargo.toml
+++ b/helix-term/Cargo.toml
@@ -26,6 +26,7 @@ helix-core = { version = "0.6", path = "../helix-core" }
 helix-view = { version = "0.6", path = "../helix-view" }
 helix-lsp = { version = "0.6", path = "../helix-lsp" }
 helix-dap = { version = "0.6", path = "../helix-dap" }
+helix-loader = { version = "0.6", path = "../helix-loader" }
 
 anyhow = "1"
 once_cell = "1.10"
diff --git a/helix-term/build.rs b/helix-term/build.rs
index 7303041cdb75..b5d62b2856e5 100644
--- a/helix-term/build.rs
+++ b/helix-term/build.rs
@@ -14,10 +14,5 @@ fn main() {
         None => env!("CARGO_PKG_VERSION").into(),
     };
 
-    println!(
-        "cargo:rustc-env=BUILD_TARGET={}",
-        std::env::var("TARGET").unwrap()
-    );
-
     println!("cargo:rustc-env=VERSION_AND_GIT_HASH={}", version);
 }
diff --git a/helix-term/src/application.rs b/helix-term/src/application.rs
index e885bc49bd26..269ce13d15a8 100644
--- a/helix-term/src/application.rs
+++ b/helix-term/src/application.rs
@@ -61,10 +61,10 @@ impl Application {
         let mut compositor = Compositor::new()?;
         let size = compositor.size();
 
-        let conf_dir = helix_core::config_dir();
+        let conf_dir = helix_loader::config_dir();
 
         let theme_loader =
-            std::sync::Arc::new(theme::Loader::new(&conf_dir, &helix_core::runtime_dir()));
+            std::sync::Arc::new(theme::Loader::new(&conf_dir, &helix_loader::runtime_dir()));
 
         let true_color = config.editor.true_color || crate::true_color();
         let theme = config
@@ -109,7 +109,7 @@ impl Application {
         compositor.push(editor_view);
 
         if args.load_tutor {
-            let path = helix_core::runtime_dir().join("tutor.txt");
+            let path = helix_loader::runtime_dir().join("tutor.txt");
             editor.open(path, Action::VerticalSplit)?;
             // Unset path to prevent accidentally saving to the original tutor file.
             doc_mut!(editor).set_path(None)?;
diff --git a/helix-term/src/commands/typed.rs b/helix-term/src/commands/typed.rs
index 4cc996d65798..3301d1486bd9 100644
--- a/helix-term/src/commands/typed.rs
+++ b/helix-term/src/commands/typed.rs
@@ -828,7 +828,7 @@ fn tutor(
     _args: &[Cow<str>],
     _event: PromptEvent,
 ) -> anyhow::Result<()> {
-    let path = helix_core::runtime_dir().join("tutor.txt");
+    let path = helix_loader::runtime_dir().join("tutor.txt");
     cx.editor.open(path, Action::Replace)?;
     // Unset path to prevent accidentally saving to the original tutor file.
     doc_mut!(cx.editor).set_path(None)?;
diff --git a/helix-term/src/health.rs b/helix-term/src/health.rs
index 5ef20d934cfe..f13d35f09c27 100644
--- a/helix-term/src/health.rs
+++ b/helix-term/src/health.rs
@@ -1,8 +1,6 @@
 use crossterm::style::{Color, Print, Stylize};
-use helix_core::{
-    config::{default_syntax_loader, user_syntax_loader},
-    syntax::load_runtime_file,
-};
+use helix_core::config::{default_syntax_loader, user_syntax_loader};
+use helix_loader::grammar::load_runtime_file;
 
 #[derive(Copy, Clone)]
 pub enum TsFeature {
@@ -43,10 +41,10 @@ impl TsFeature {
 
 /// Display general diagnostics.
 pub fn general() {
-    let config_file = helix_core::config_file();
-    let lang_file = helix_core::lang_config_file();
-    let log_file = helix_core::log_file();
-    let rt_dir = helix_core::runtime_dir();
+    let config_file = helix_loader::config_file();
+    let lang_file = helix_loader::lang_config_file();
+    let log_file = helix_loader::log_file();
+    let rt_dir = helix_loader::runtime_dir();
 
     if config_file.exists() {
         println!("Config file: {}", config_file.display());
diff --git a/helix-term/src/lib.rs b/helix-term/src/lib.rs
index 227479988594..fc8e934e1a7d 100644
--- a/helix-term/src/lib.rs
+++ b/helix-term/src/lib.rs
@@ -7,7 +7,6 @@ pub mod commands;
 pub mod compositor;
 pub mod config;
 pub mod health;
-pub mod grammars;
 pub mod job;
 pub mod keymap;
 pub mod ui;
diff --git a/helix-term/src/main.rs b/helix-term/src/main.rs
index 67f88b4e13dd..a69e121b1549 100644
--- a/helix-term/src/main.rs
+++ b/helix-term/src/main.rs
@@ -40,7 +40,7 @@ fn main() -> Result<()> {
 
 #[tokio::main]
 async fn main_impl() -> Result<i32> {
-    let logpath = helix_core::log_file();
+    let logpath = helix_loader::log_file();
     let parent = logpath.parent().unwrap();
     if !parent.exists() {
         std::fs::create_dir_all(parent).ok();
@@ -105,21 +105,21 @@ FLAGS:
     }
 
     if args.fetch_grammars {
-        helix_term::grammars::fetch_grammars()?;
+        helix_loader::grammar::fetch_grammars()?;
         return Ok(0);
     }
 
     if args.build_grammars {
-        helix_term::grammars::build_grammars()?;
+        helix_loader::grammar::build_grammars()?;
         return Ok(0);
     }
 
-    let conf_dir = helix_core::config_dir();
+    let conf_dir = helix_loader::config_dir();
     if !conf_dir.exists() {
         std::fs::create_dir_all(&conf_dir).ok();
     }
 
-    let config = match std::fs::read_to_string(helix_core::config_file()) {
+    let config = match std::fs::read_to_string(helix_loader::config_file()) {
         Ok(config) => toml::from_str(&config)
             .map(merge_keys)
             .unwrap_or_else(|err| {
diff --git a/helix-term/src/ui/mod.rs b/helix-term/src/ui/mod.rs
index d46de2d39422..6299a473967c 100644
--- a/helix-term/src/ui/mod.rs
+++ b/helix-term/src/ui/mod.rs
@@ -218,9 +218,9 @@ pub mod completers {
     }
 
     pub fn theme(_editor: &Editor, input: &str) -> Vec<Completion> {
-        let mut names = theme::Loader::read_names(&helix_core::runtime_dir().join("themes"));
+        let mut names = theme::Loader::read_names(&helix_loader::runtime_dir().join("themes"));
         names.extend(theme::Loader::read_names(
-            &helix_core::config_dir().join("themes"),
+            &helix_loader::config_dir().join("themes"),
         ));
         names.push("default".into());
         names.push("base16_default".into());
diff --git a/languages.toml b/languages.toml
index 18b45ea60909..1818201121da 100644
--- a/languages.toml
+++ b/languages.toml
@@ -302,8 +302,7 @@ indent = { tab-width = 2, unit = "  " }
 
 [[grammar]]
 name = "typescript"
-source = { git = "https://github.com/tree-sitter/tree-sitter-typescript", rev = "3e897ea5925f037cfae2e551f8e6b12eec2a201a" }
-path = "typescript"
+source = { git = "https://github.com/tree-sitter/tree-sitter-typescript", rev = "3e897ea5925f037cfae2e551f8e6b12eec2a201a", subpath = "typescript" }
 
 [[language]]
 name = "tsx"
@@ -317,8 +316,7 @@ indent = { tab-width = 2, unit = "  " }
 
 [[grammar]]
 name = "tsx"
-source = { git = "https://github.com/tree-sitter/tree-sitter-typescript", rev = "3e897ea5925f037cfae2e551f8e6b12eec2a201a" }
-path = "tsx"
+source = { git = "https://github.com/tree-sitter/tree-sitter-typescript", rev = "3e897ea5925f037cfae2e551f8e6b12eec2a201a", subpath = "tsx" }
 
 [[language]]
 name = "css"
@@ -522,8 +520,7 @@ indent = { tab-width = 2, unit = "  " }
 
 [[grammar]]
 name = "ocaml"
-source = { git = "https://github.com/tree-sitter/tree-sitter-ocaml", rev = "23d419ba45789c5a47d31448061557716b02750a" }
-path = "ocaml"
+source = { git = "https://github.com/tree-sitter/tree-sitter-ocaml", rev = "23d419ba45789c5a47d31448061557716b02750a", subpath = "ocaml" }
 
 [[language]]
 name = "ocaml-interface"
@@ -536,8 +533,7 @@ indent = { tab-width = 2, unit = "  "}
 
 [[grammar]]
 name = "ocaml-interface"
-source = { git = "https://github.com/tree-sitter/tree-sitter-ocaml", rev = "23d419ba45789c5a47d31448061557716b02750a" }
-path = "interface"
+source = { git = "https://github.com/tree-sitter/tree-sitter-ocaml", rev = "23d419ba45789c5a47d31448061557716b02750a", subpath = "interface" }
 
 [[language]]
 name = "lua"

From 7b7143f09b4ef4adb494fdbe0997710653b0ab48 Mon Sep 17 00:00:00 2001
From: Skyler Hawthorne <skyler@dead10ck.com>
Date: Sat, 19 Feb 2022 00:45:22 -0500
Subject: [PATCH 11/18] fix context in error

---
 helix-loader/src/grammar.rs | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/helix-loader/src/grammar.rs b/helix-loader/src/grammar.rs
index 61ef464fca01..4fe7fd04c9cd 100644
--- a/helix-loader/src/grammar.rs
+++ b/helix-loader/src/grammar.rs
@@ -123,18 +123,14 @@ where
             tx.send(job(grammar)).unwrap();
         });
     }
-    pool.join();
+
+    drop(tx);
 
     // TODO: print all failures instead of the first one found.
-    if let Some(failure) = rx.try_iter().find_map(|result| result.err()) {
-        Err(anyhow!(
-            "Failed to {} some grammar(s).\n{}",
-            action,
-            failure
-        ))
-    } else {
-        Ok(())
-    }
+    rx.iter()
+        .find(|result| result.is_err())
+        .map(|err| err.with_context(|| format!("Failed to {} some grammar(s)", action)))
+        .unwrap_or(Ok(()))
 }
 
 fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {

From b73352570ab530becd303e15a0f2baaf84f5eb4d Mon Sep 17 00:00:00 2001
From: Skyler Hawthorne <skyler@dead10ck.com>
Date: Sat, 19 Feb 2022 00:52:36 -0500
Subject: [PATCH 12/18] shallow clone

---
 helix-loader/src/grammar.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/helix-loader/src/grammar.rs b/helix-loader/src/grammar.rs
index 4fe7fd04c9cd..a0aa9583a84a 100644
--- a/helix-loader/src/grammar.rs
+++ b/helix-loader/src/grammar.rs
@@ -129,7 +129,7 @@ where
     // TODO: print all failures instead of the first one found.
     rx.iter()
         .find(|result| result.is_err())
-        .map(|err| err.with_context(|| format!("Failed to {} some grammar(s)", action)))
+        .map(|err| err.with_context(|| format!("Failed to {action} some grammar(s)")))
         .unwrap_or(Ok(()))
 }
 
@@ -162,7 +162,10 @@ fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
             // Fetch the exact revision from the remote.
             // Supported by server-side git since v2.5.0 (July 2015),
             // enabled by default on major git hosts.
-            git(&grammar_dir, ["fetch", REMOTE_NAME, &revision])?;
+            git(
+                &grammar_dir,
+                ["fetch", "--depth", "1", REMOTE_NAME, &revision],
+            )?;
             git(&grammar_dir, ["checkout", &revision])?;
 
             println!(

From 3a21d618d4f57041d49bf6c7f4661db59644d4d9 Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Sat, 19 Feb 2022 04:51:14 -0600
Subject: [PATCH 13/18] only fetch git-sourced grammars

This is a bit of a micro-optimization: in the current setup we waste
a thread in the pool for a local grammar only to println! a message
saying we're skipping fetching because it's a local grammar.
---
 helix-loader/src/grammar.rs | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/helix-loader/src/grammar.rs b/helix-loader/src/grammar.rs
index a0aa9583a84a..b7a3cfcfccb4 100644
--- a/helix-loader/src/grammar.rs
+++ b/helix-loader/src/grammar.rs
@@ -76,7 +76,11 @@ pub fn get_language(name: &str) -> Result<Language> {
 }
 
 pub fn fetch_grammars() -> Result<()> {
-    run_parallel(get_grammar_configs()?, fetch_grammar, "fetch")
+    // We do not need to fetch local grammars.
+    let mut grammars = get_grammar_configs()?;
+    grammars.retain(|grammar| !matches!(grammar.source, GrammarSource::Local { .. }));
+
+    run_parallel(grammars, fetch_grammar, "fetch")
 }
 
 pub fn build_grammars() -> Result<()> {
@@ -172,15 +176,12 @@ fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
                 "Grammar '{}' checked out at '{}'.",
                 grammar.grammar_id, revision
             );
-            Ok(())
         } else {
             println!("Grammar '{}' is already up to date.", grammar.grammar_id);
-            Ok(())
         }
-    } else {
-        println!("Skipping local grammar '{}'", grammar.grammar_id);
-        Ok(())
     }
+
+    Ok(())
 }
 
 // Sets the remote for a repository to the given URL, creating the remote if

From c2b6a0b7769f7c80b174198b29effff140dfb8f5 Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Mon, 21 Feb 2022 06:39:23 -0600
Subject: [PATCH 14/18] fetch and compile tree-sitter grammars in helix-term
 build

This restores much of the behavior that existed before this PR:
helix will build the grammars when compiling. The difference is that
now fetching is also done during the build phase and is done much
more quickly - both shallow and in parallel.
---
 .github/workflows/build.yml   | 21 ++++-----------------
 .github/workflows/release.yml | 14 --------------
 flake.nix                     |  2 ++
 helix-term/Cargo.toml         |  7 +++----
 helix-term/build.rs           |  8 ++++++++
 5 files changed, 17 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e0eb5322b9f6..776c8651d2ca 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -77,27 +77,14 @@ jobs:
           path: target
           key: ${{ runner.os }}-v2-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
 
+      - name: Copy minimal languages config
+        run: cp .github/workflows/languages.toml ./languages.toml
+
       - name: Cache test tree-sitter grammar
         uses: actions/cache@v2.1.7
         with:
           path: runtime/grammars
-          key: ${{ runner.os }}-v2-tree-sitter-grammars-${{ hashFiles('**/Cargo.lock') }}
-
-      - run: cp .github/workflows/languages.toml ./languages.toml
-
-      - name: Download test tree-sitter grammar
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: ${{ matrix.cross }}
-          command: run
-          args: -- --fetch-grammars
-
-      - name: Build test tree-sitter grammar
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: ${{ matrix.cross }}
-          command: run
-          args: -- --build-grammars
+          key: ${{ runner.os }}-v2-tree-sitter-grammars-${{ hashFiles('languages.toml') }}
 
       - name: Run cargo test
         uses: actions-rs/cargo@v1
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 2a9ea80ebb7d..2d0b86e2f5cc 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -62,20 +62,6 @@ jobs:
           target: ${{ matrix.target }}
           override: true
 
-      - name: Fetch tree-sitter grammars
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: ${{ matrix.cross }}
-          command: run
-          args: --release --locked --target ${{ matrix.target }} -- --fetch-grammars
-
-      - name: Build tree-sitter grammars
-        uses: actions-rs/cargo@v1
-        with:
-          use-cross: ${{ matrix.cross }}
-          command: run
-          args: --release --locked --target ${{ matrix.target }} -- --build-grammars
-
       - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:
diff --git a/flake.nix b/flake.nix
index 71825431f212..c4838ffd4410 100644
--- a/flake.nix
+++ b/flake.nix
@@ -35,6 +35,8 @@
       overrides = {
         crateOverrides = common: _: rec {
           helix-term = prev: {
+            # disable fetching and building of tree-sitter grammars in the helix-term build.rs
+            HELIX_DISABLE_AUTO_GRAMMAR_BUILD = "1";
             buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
             nativeBuildInputs = (prev.nativeBuildInputs or [ ]) ++ [ common.pkgs.makeWrapper ];
             preConfigure = ''
diff --git a/helix-term/Cargo.toml b/helix-term/Cargo.toml
index 86d72561a842..48365743fa00 100644
--- a/helix-term/Cargo.toml
+++ b/helix-term/Cargo.toml
@@ -67,9 +67,8 @@ grep-searcher = "0.1.8"
 # Remove once retain_mut lands in stable rust
 retain_mut = "0.1.7"
 
-# compiling grammars
-cc = { version = "1" }
-threadpool = { version = "1.0" }
-
 [target.'cfg(not(windows))'.dependencies]  # https://github.com/vorner/signal-hook/issues/100
 signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] }
+
+[build-dependencies]
+helix-loader = { version = "0.6", path = "../helix-loader" }
diff --git a/helix-term/build.rs b/helix-term/build.rs
index b5d62b2856e5..526cdc41311a 100644
--- a/helix-term/build.rs
+++ b/helix-term/build.rs
@@ -1,3 +1,4 @@
+use helix_loader::grammar::{build_grammars, fetch_grammars};
 use std::borrow::Cow;
 use std::process::Command;
 
@@ -14,5 +15,12 @@ fn main() {
         None => env!("CARGO_PKG_VERSION").into(),
     };
 
+    if std::env::var("HELIX_DISABLE_AUTO_GRAMMAR_BUILD").is_err() {
+        fetch_grammars().expect("Failed to fetch tree-sitter grammars");
+        build_grammars().expect("Failed to compile tree-sitter grammars");
+    }
+
+    println!("cargo:rerun-if-changed=../runtime/grammars/");
+
     println!("cargo:rustc-env=VERSION_AND_GIT_HASH={}", version);
 }

From 5b4aec78b02e78565193be8bc3d08fdf531e6ef7 Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Mon, 14 Feb 2022 21:50:08 -0600
Subject: [PATCH 15/18] fetch and build grammars with nix in flake

This commit replaces the out-of-date builder in the flake which relied
on submodules for fetching and the compiler for building. Now we
disable fetching and building explicitly with the environment variable
and then use builtins.fetchGit and a derivation mostly derived from
upstream to compile the grammars.

Anecdotally, this is still quite slow as builtins.fetchGit does not
seem to do shallow clones. I'm not sure I see a way around it though
without recording sha256s, which seems cumbersome.
---
 flake.nix    | 46 +++++++++++++++------------
 grammars.nix | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 113 insertions(+), 20 deletions(-)
 create mode 100644 grammars.nix

diff --git a/flake.nix b/flake.nix
index c4838ffd4410..e1cd50360b8d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -33,26 +33,32 @@
         package = "helix";
       };
       overrides = {
-        crateOverrides = common: _: rec {
-          helix-term = prev: {
-            # disable fetching and building of tree-sitter grammars in the helix-term build.rs
-            HELIX_DISABLE_AUTO_GRAMMAR_BUILD = "1";
-            buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
-            nativeBuildInputs = (prev.nativeBuildInputs or [ ]) ++ [ common.pkgs.makeWrapper ];
-            preConfigure = ''
-              ${prev.preConfigure}
-              rm -r helix-syntax/languages
-              ln -s ${helix}/helix-syntax/languages helix-syntax/languages
-              ln -s "$PWD/helix-syntax/languages" languages
-              mkdir -p runtime/grammars
-            '';
-            postInstall = ''
-              ${prev.postInstall or ""}
-              mkdir -p $out/lib
-              cp -r runtime $out/lib
-              wrapProgram "$out/bin/hx" --set HELIX_RUNTIME "$out/lib/runtime"
-            '';
-          };
+        crateOverrides = common: _: {
+          helix-term = prev:
+            let
+              inherit (common) pkgs;
+              grammars = pkgs.callPackage ./grammars.nix { };
+              runtimeDir = pkgs.runCommand "helix-runtime" { } ''
+                mkdir -p $out
+                ln -s ${common.root}/runtime/* $out
+                rm -r $out/grammars
+                ln -s ${grammars} $out/grammars
+              '';
+            in
+            {
+              # disable fetching and building of tree-sitter grammars in the helix-term build.rs
+              HELIX_DISABLE_AUTO_GRAMMAR_BUILD = "1";
+              # link languages and theme toml files since helix-term expects them (for tests)
+              preConfigure = "ln -s ${common.root}/{languages.toml,theme.toml,base16_theme.toml} ..";
+              buildInputs = (prev.buildInputs or [ ]) ++ [ common.cCompiler.cc.lib ];
+              nativeBuildInputs = [ pkgs.makeWrapper ];
+
+              postFixup = ''
+                if [ -f "$out/bin/hx" ]; then
+                  wrapProgram "$out/bin/hx" --set HELIX_RUNTIME "${runtimeDir}"
+                fi
+              '';
+            };
         };
         shell = common: prev: {
           packages = prev.packages ++ (with common.pkgs; [ lld_13 lldb cargo-tarpaulin cargo-flamegraph ]);
diff --git a/grammars.nix b/grammars.nix
new file mode 100644
index 000000000000..6dbc05c87332
--- /dev/null
+++ b/grammars.nix
@@ -0,0 +1,87 @@
+{ stdenv, lib, runCommand, yj }:
+let
+  # HACK: nix < 2.6 has a bug in the toml parser, so we convert to JSON
+  # before parsing
+  languages-json = runCommand "languages-toml-to-json" { } ''
+    ${yj}/bin/yj -t < ${./languages.toml} > $out
+  '';
+  languagesConfig =
+    builtins.fromJSON (builtins.readFile (builtins.toPath languages-json));
+  isGitGrammar = (grammar:
+    builtins.hasAttr "source" grammar && builtins.hasAttr "git" grammar.source
+    && builtins.hasAttr "rev" grammar.source);
+  gitGrammars = builtins.filter isGitGrammar languagesConfig.grammar;
+  buildGrammar = grammar:
+    let
+      source = builtins.fetchGit {
+        url = grammar.source.git;
+        rev = grammar.source.rev;
+        allRefs = true;
+      };
+    in stdenv.mkDerivation rec {
+      # see https://github.com/NixOS/nixpkgs/blob/fbdd1a7c0bc29af5325e0d7dd70e804a972eb465/pkgs/development/tools/parsing/tree-sitter/grammar.nix
+
+      pname = "helix-tree-sitter-${grammar.name}";
+      version = grammar.source.rev;
+
+      src = if builtins.hasAttr "subpath" grammar.source then
+        "${source}/${grammar.source.subpath}"
+      else
+        source;
+
+      dontUnpack = true;
+      dontConfigure = true;
+
+      FLAGS = [
+        "-I${src}/src"
+        "-g"
+        "-O3"
+        "-fPIC"
+        "-fno-exceptions"
+        "-Wl,-z,relro,-z,now"
+      ];
+
+      NAME = grammar.name;
+
+      buildPhase = ''
+        runHook preBuild
+
+        if [[ -e "$src/src/scanner.cc" ]]; then
+          $CXX -c "$src/src/scanner.cc" -o scanner.o $FLAGS
+        elif [[ -e "$src/src/scanner.c" ]]; then
+          $CC -c "$src/src/scanner.c" -o scanner.o $FLAGS
+        fi
+
+        $CC -c "$src/src/parser.c" -o parser.o $FLAGS
+        $CXX -shared -o $NAME.so *.o
+
+        ls -al
+
+        runHook postBuild
+      '';
+
+      installPhase = ''
+        runHook preInstall
+        mkdir $out
+        mv $NAME.so $out/
+        runHook postInstall
+      '';
+
+      # Strip failed on darwin: strip: error: symbols referenced by indirect symbol table entries that can't be stripped
+      fixupPhase = lib.optionalString stdenv.isLinux ''
+        runHook preFixup
+        $STRIP $out/$NAME.so
+        runHook postFixup
+      '';
+    };
+  builtGrammars = builtins.map (grammar: {
+    inherit (grammar) name;
+    artifact = buildGrammar grammar;
+  }) gitGrammars;
+  grammarLinks = builtins.map (grammar:
+    "ln -s ${grammar.artifact}/${grammar.name}.so $out/${grammar.name}.so")
+    builtGrammars;
+in runCommand "consolidated-helix-grammars" { } ''
+  mkdir -p $out
+  ${builtins.concatStringsSep "\n" grammarLinks}
+''

From c05b6b3a29ef918d286dbd89285fa07d8af26d0e Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Tue, 8 Mar 2022 00:13:15 -0600
Subject: [PATCH 16/18] rename '--fetch/build-grammars' flags into '--grammar
 fetch/build'

The old flags were a bit long. --grammar is also aliased to -g to make
it even easier.
---
 Cargo.lock                          |  2 --
 README.md                           |  4 ++--
 book/src/guides/adding_languages.md |  4 ++--
 book/src/install.md                 |  4 ++--
 helix-core/src/syntax.rs            |  2 +-
 helix-loader/src/grammar.rs         |  4 ++--
 helix-term/src/args.rs              |  9 +++++++--
 helix-term/src/main.rs              | 19 +++++++++----------
 8 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index de4b2279a885..0e9f9765adb3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -435,7 +435,6 @@ name = "helix-term"
 version = "0.6.0"
 dependencies = [
  "anyhow",
- "cc",
  "chrono",
  "content_inspector",
  "crossterm",
@@ -460,7 +459,6 @@ dependencies = [
  "serde_json",
  "signal-hook",
  "signal-hook-tokio",
- "threadpool",
  "tokio",
  "tokio-stream",
  "toml",
diff --git a/README.md b/README.md
index 8e4e42d9e9fa..4052d9411b4e 100644
--- a/README.md
+++ b/README.md
@@ -39,8 +39,8 @@ build from source.
 git clone https://github.com/helix-editor/helix
 cd helix
 cargo install --path helix-term
-hx --fetch-grammars
-hx --build-grammars
+hx --grammar fetch
+hx --grammar build
 ```
 
 This will install the `hx` binary to `$HOME/.cargo/bin` and build tree-sitter grammars.
diff --git a/book/src/guides/adding_languages.md b/book/src/guides/adding_languages.md
index e000770c62e9..e5fa456c60be 100644
--- a/book/src/guides/adding_languages.md
+++ b/book/src/guides/adding_languages.md
@@ -60,7 +60,7 @@ git repository:
 | ---    | -----------                                               |
 | `git`  | A git remote URL from which the grammar should be cloned  |
 | `rev`  | The revision (commit hash or tag) which should be fetched |
-| `subpath` | A path within the grammar directory which should be built. Some grammar repositories host multiple grammars (for example `tree-sitter-typescript` and `tree-sitter-ocaml`) in subdirectories. This key is used to point `hx --build-grammars` to the correct path for compilation. When omitted, the root of repository is used |
+| `subpath` | A path within the grammar directory which should be built. Some grammar repositories host multiple grammars (for example `tree-sitter-typescript` and `tree-sitter-ocaml`) in subdirectories. This key is used to point `hx --grammar build` to the correct path for compilation. When omitted, the root of repository is used |
 
 Or a `path` key with an absolute path to a locally available grammar directory.
 
@@ -79,7 +79,7 @@ the last matching query supersedes the ones before it. See
 
 ## Common Issues
 
-- If you get errors when running after switching branches, you may have to update the tree-sitter grammars. Run `hx --fetch-grammars` to fetch the grammars and `hx --build-grammars` to build any out-of-date grammars.
+- If you get errors when running after switching branches, you may have to update the tree-sitter grammars. Run `hx --grammar fetch` to fetch the grammars and `hx --grammar build` to build any out-of-date grammars.
 
 - If a parser is segfaulting or you want to remove the parser, make sure to remove the compiled parser in `runtime/grammar/<name>.so`
 
diff --git a/book/src/install.md b/book/src/install.md
index db42509c1841..b3d42aaf4808 100644
--- a/book/src/install.md
+++ b/book/src/install.md
@@ -58,5 +58,5 @@ via the `HELIX_RUNTIME` environment variable.
 ## Building tree-sitter grammars
 
 Tree-sitter grammars must be fetched and compiled if not pre-packaged.
-Fetch grammars with `hx --fetch-grammars` (requires `git`) and compile them
-with `hx --build-grammars` (requires a C compiler).
+Fetch grammars with `hx --grammar fetch` (requires `git`) and compile them
+with `hx --grammar build` (requires a C compiler).
diff --git a/helix-core/src/syntax.rs b/helix-core/src/syntax.rs
index 6ae46d4ff237..3b2d56d12640 100644
--- a/helix-core/src/syntax.rs
+++ b/helix-core/src/syntax.rs
@@ -388,7 +388,7 @@ impl LanguageConfiguration {
                 &injections_query,
                 &locals_query,
             )
-            .unwrap_or_else(|query_error| panic!("Could not parse queries for language {:?}. Are your grammars out of sync? Try running 'hx --fetch-grammars' and 'hx --build-grammars'. This query could not be parsed: {:?}", self.language_id, query_error));
+            .unwrap_or_else(|query_error| panic!("Could not parse queries for language {:?}. Are your grammars out of sync? Try running 'hx --grammar build' and 'hx --grammar build'. This query could not be parsed: {:?}", self.language_id, query_error));
 
             config.configure(scopes);
             Some(Arc::new(config))
diff --git a/helix-loader/src/grammar.rs b/helix-loader/src/grammar.rs
index b7a3cfcfccb4..dc327498238d 100644
--- a/helix-loader/src/grammar.rs
+++ b/helix-loader/src/grammar.rs
@@ -239,12 +239,12 @@ fn build_grammar(grammar: GrammarConfiguration) -> Result<()> {
     };
 
     let grammar_dir_entries = grammar_dir.read_dir().with_context(|| {
-        format!("Failed to read directory {grammar_dir:?}. Did you use 'hx --fetch-grammars'?")
+        format!("Failed to read directory {grammar_dir:?}. Did you use 'hx --grammar fetch'?")
     })?;
 
     if grammar_dir_entries.count() == 0 {
         return Err(anyhow!(
-            "Directory {grammar_dir:?} is empty. Did you use 'hx --fetch-grammars'?"
+            "Directory {grammar_dir:?} is empty. Did you use 'hx --grammar fetch'?"
         ));
     };
 
diff --git a/helix-term/src/args.rs b/helix-term/src/args.rs
index 800765289151..e0f0af00f55d 100644
--- a/helix-term/src/args.rs
+++ b/helix-term/src/args.rs
@@ -34,8 +34,13 @@ impl Args {
                     args.health = true;
                     args.health_arg = argv.next_if(|opt| !opt.starts_with('-'));
                 }
-                "--fetch-grammars" => args.fetch_grammars = true,
-                "--build-grammars" => args.build_grammars = true,
+                "-g" | "--grammar" => match argv.next().as_deref() {
+                    Some("fetch") => args.fetch_grammars = true,
+                    Some("build") => args.build_grammars = true,
+                    _ => {
+                        anyhow::bail!("--grammar must be followed by either 'fetch' or 'build'")
+                    }
+                },
                 arg if arg.starts_with("--") => {
                     anyhow::bail!("unexpected double dash argument: {}", arg)
                 }
diff --git a/helix-term/src/main.rs b/helix-term/src/main.rs
index a69e121b1549..6511e0046bad 100644
--- a/helix-term/src/main.rs
+++ b/helix-term/src/main.rs
@@ -59,16 +59,15 @@ ARGS:
     <files>...    Sets the input file to use, position can also be specified via file[:row[:col]]
 
 FLAGS:
-    -h, --help        Prints help information
-    --edit-config     Opens the helix config file
-    --tutor           Loads the tutorial
-    --health [LANG]   Checks for potential errors in editor setup
-                      If given, checks for config errors in language LANG
-    --fetch-grammars  Fetches tree-sitter grammars listed in languages.toml
-    --build-grammars  Builds tree-sitter grammars fetched with --fetch-grammars
-    -v                Increases logging verbosity each use for up to 3 times
-                      (default file: {})
-    -V, --version     Prints version information
+    -h, --help                     Prints help information
+    --edit-config                  Opens the helix config file
+    --tutor                        Loads the tutorial
+    --health [LANG]                Checks for potential errors in editor setup
+                                   If given, checks for config errors in language LANG
+    -g, --grammars {{fetch|build}}   Fetches or builds tree-sitter grammars listed in languages.toml
+    -v                             Increases logging verbosity each use for up to 3 times
+                                   (default file: {})
+    -V, --version                  Prints version information
 ",
         env!("CARGO_PKG_NAME"),
         env!("VERSION_AND_GIT_HASH"),

From af0383dc068d0afdf584c756dfd7c046ff50041a Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Wed, 9 Mar 2022 08:34:52 -0600
Subject: [PATCH 17/18] flake: use builtins.fetchTree to shallow-clone grammar
 repos

Here we perform a shallow fetch using builtins.fetchTree. In order
to make this work, we need to specify the `ref' for any repository
that doesn't have `master' as its default branch (I'm not sure why
this limitation exists since we don't need this when performing
the shallow fetch in `--grammar build')

This `ref' field is ignored by helix, so I have left it undocumented
for now, but I could be open to documenting it.
---
 flake.lock     | 49 +++++++++++++++----------------------------------
 flake.nix      | 10 +---------
 grammars.nix   |  6 ++++--
 languages.toml |  4 ++--
 4 files changed, 22 insertions(+), 47 deletions(-)

diff --git a/flake.lock b/flake.lock
index f061c80fbeaf..40a87eb55fd0 100644
--- a/flake.lock
+++ b/flake.lock
@@ -25,11 +25,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1646322147,
-        "narHash": "sha256-XwrdjThHPq/APV7B6mXJwYvN/3RmsjX1W4zPgjvCp0A=",
+        "lastModified": 1646667754,
+        "narHash": "sha256-LahZHvCC3UVzGQ55iWDRZkuDssXl1rYgqgScrPV9S38=",
         "owner": "numtide",
         "repo": "devshell",
-        "rev": "2cc45675b223a35ca1d8af6383752c3d4b66f484",
+        "rev": "59fbe1dfc0de8c3332957c16998a7d16dff365d8",
         "type": "github"
       },
       "original": {
@@ -75,11 +75,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1646647374,
-        "narHash": "sha256-sFGoE9LbHfP5t8NGcNkX4sPVq9kp8+ae8ODnW5eOkzo=",
+        "lastModified": 1646710334,
+        "narHash": "sha256-eLBcDgcbOUfeH4k6SEW5a5v0PTp2KNCn+5ZXIoWGYww=",
         "owner": "nix-community",
         "repo": "dream2nix",
-        "rev": "76412363073ea1252700eb48901849cf8480e138",
+        "rev": "5dcfbfd3b60ce0208b894c1bdea00e2bdf80ca6a",
         "type": "github"
       },
       "original": {
@@ -119,24 +119,6 @@
         "type": "github"
       }
     },
-    "helix": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1646500939,
-        "narHash": "sha256-M0QvrRuluDkBWUfd5CYwvj9WtgalKlm5stJUAwnhrwI=",
-        "ref": "master",
-        "rev": "7633c5acd30258fc9caca926bfaa264d07d508ec",
-        "revCount": 2438,
-        "submodules": true,
-        "type": "git",
-        "url": "https://github.com/helix-editor/helix.git"
-      },
-      "original": {
-        "submodules": true,
-        "type": "git",
-        "url": "https://github.com/helix-editor/helix.git"
-      }
-    },
     "nixCargoIntegration": {
       "inputs": {
         "devshell": "devshell",
@@ -149,11 +131,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1646681124,
-        "narHash": "sha256-1ytR1z6RyBbxhk0LiO18QBH6Mu1JfAEWuUVBdbD4Bw8=",
+        "lastModified": 1646766572,
+        "narHash": "sha256-DV3+zxvAIKsMHsHedJKYFsracvFyLKpFQqurUBR86oY=",
         "owner": "yusdacra",
         "repo": "nix-cargo-integration",
-        "rev": "3b08d21177cecd5d69bdf76400a6145a021d1e49",
+        "rev": "3a3f47f43ba486b7554164a698c8dfc5a38624ce",
         "type": "github"
       },
       "original": {
@@ -164,11 +146,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1646254136,
-        "narHash": "sha256-8nQx02tTzgYO21BP/dy5BCRopE8OwE8Drsw98j+Qoaw=",
+        "lastModified": 1646497237,
+        "narHash": "sha256-Ccpot1h/rV8MgcngDp5OrdmLTMaUTbStZTR5/sI7zW0=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "3e072546ea98db00c2364b81491b893673267827",
+        "rev": "062a0c5437b68f950b081bbfc8a699d57a4ee026",
         "type": "github"
       },
       "original": {
@@ -180,7 +162,6 @@
     },
     "root": {
       "inputs": {
-        "helix": "helix",
         "nixCargoIntegration": "nixCargoIntegration",
         "nixpkgs": "nixpkgs",
         "rust-overlay": "rust-overlay"
@@ -194,11 +175,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1642128126,
-        "narHash": "sha256-av8JUACdrTfQYl/ftZJvKpZEmZfa0avCq7tt5Usdoq0=",
+        "lastModified": 1646792695,
+        "narHash": "sha256-2drCXIKIQnJMlTZbcCfuHZAh+iPcdlRkCqtZnA6MHLY=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "ce4ef6f2d74f2b68f7547df1de22d1b0037ce4ad",
+        "rev": "7f599870402c8d2a5806086c8ee0f2d92b175c54",
         "type": "github"
       },
       "original": {
diff --git a/flake.nix b/flake.nix
index e1cd50360b8d..38ba9fd0c555 100644
--- a/flake.nix
+++ b/flake.nix
@@ -12,17 +12,9 @@
       inputs.nixpkgs.follows = "nixpkgs";
       inputs.rustOverlay.follows = "rust-overlay";
     };
-    # NOTE: the flake looks like it is hanging when it pulls this input because
-    # the submodules take a long time to clone. This will be fixed in #1659.
-    helix = {
-      url = "https://github.com/helix-editor/helix.git";
-      type = "git";
-      submodules = true;
-      flake = false;
-    };
   };
 
-  outputs = inputs@{ nixCargoIntegration, helix, ... }:
+  outputs = inputs@{ nixCargoIntegration, ... }:
     nixCargoIntegration.lib.makeOutputs {
       root = ./.;
       renameOutputs = { "helix-term" = "helix"; };
diff --git a/grammars.nix b/grammars.nix
index 6dbc05c87332..bd2c95cc1dba 100644
--- a/grammars.nix
+++ b/grammars.nix
@@ -13,10 +13,12 @@ let
   gitGrammars = builtins.filter isGitGrammar languagesConfig.grammar;
   buildGrammar = grammar:
     let
-      source = builtins.fetchGit {
+      source = builtins.fetchTree {
+        type = "git";
         url = grammar.source.git;
         rev = grammar.source.rev;
-        allRefs = true;
+        ref = grammar.source.ref or "HEAD";
+        shallow = true;
       };
     in stdenv.mkDerivation rec {
       # see https://github.com/NixOS/nixpkgs/blob/fbdd1a7c0bc29af5325e0d7dd70e804a972eb465/pkgs/development/tools/parsing/tree-sitter/grammar.nix
diff --git a/languages.toml b/languages.toml
index 1818201121da..b0ed4b340966 100644
--- a/languages.toml
+++ b/languages.toml
@@ -73,7 +73,7 @@ indent = { tab-width = 2, unit = "  " }
 
 [[grammar]]
 name = "protobuf"
-source = { git = "https://github.com/yusdacra/tree-sitter-protobuf", rev = "19c211a01434d9f03efff99f85e19f967591b175" }
+source = { git = "https://github.com/yusdacra/tree-sitter-protobuf", rev = "19c211a01434d9f03efff99f85e19f967591b175"}
 
 [[language]]
 name = "elixir"
@@ -88,7 +88,7 @@ indent = { tab-width = 2, unit = "  " }
 
 [[grammar]]
 name = "elixir"
-source = { git = "https://github.com/elixir-lang/tree-sitter-elixir", rev = "f5d7bda543da788bd507b05bd722627dde66c9ec" }
+source = { git = "https://github.com/elixir-lang/tree-sitter-elixir", rev = "f5d7bda543da788bd507b05bd722627dde66c9ec"  }
 
 [[language]]
 name = "fish"

From d0f4646c0295290d9781ea898c189122f07fa20c Mon Sep 17 00:00:00 2001
From: Michael Davis <mcarsondavis@gmail.com>
Date: Thu, 10 Mar 2022 01:54:57 -0600
Subject: [PATCH 18/18] update revision for tree-sitter-rescript

Looks like this was rebased a few hours ago and now the 789a171
revision no longer exists.
---
 languages.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/languages.toml b/languages.toml
index b0ed4b340966..4c2bb1c412e9 100644
--- a/languages.toml
+++ b/languages.toml
@@ -941,7 +941,7 @@ indent = { tab-width = 2, unit = "  " }
 
 [[grammar]]
 name = "rescript"
-source = { git = "https://github.com/jaredramirez/tree-sitter-rescript", rev = "789a171d9bcf73f6d76e67aca39ed14a75375b04" }
+source = { git = "https://github.com/jaredramirez/tree-sitter-rescript", rev = "4cd7ba91696886fdaca086fb32b5fd8cc294a129" }
 
 [[language]]
 name = "erlang"