Skip to content

Commit

Permalink
implement build_grammars and fetch_grammars
Browse files Browse the repository at this point in the history 
build_grammars adapts the functionality that previously came from
helix-syntax to be used at runtime from the command line flags.

fetch_grammars wraps command-line git to perform the same actions
previously done in the scripts in helix-editor#1560.
  • Loading branch information
@the-mikedavis
the-mikedavis committed Mar 8, 2022
1 parent 3a1958e commit 28dd5dd
Show file tree
Hide file tree
Showing 2 changed files with 167 additions and 48 deletions.
205 changes: 157 additions & 48 deletions helix-term/src/grammars.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,39 +4,170 @@ use std::time::SystemTime;
use std::{
path::{Path, PathBuf},
process::Command,
sync::mpsc::channel,
};

use helix_core::syntax::DYLIB_EXTENSION;
use helix_core::syntax::{GrammarConfiguration, GrammarSource, DYLIB_EXTENSION};

const BUILD_TARGET: &str = env!("BUILD_TARGET");
const REMOTE_NAME: &str = "origin";

pub fn collect_tree_sitter_dirs(ignore: &[String]) -> Result<Vec<String>> {
let mut dirs = Vec::new();
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../helix-syntax/languages");
pub fn fetch_grammars() -> Result<()> {
run_parallel(get_grammar_configs(), fetch_grammar, "fetch")
}

pub fn build_grammars() -> Result<()> {
run_parallel(get_grammar_configs(), build_grammar, "build")
}

fn run_parallel<F>(grammars: Vec<GrammarConfiguration>, job: F, action: &'static str) -> Result<()>
where
F: Fn(GrammarConfiguration) -> Result<()> + std::marker::Send + 'static + Copy,
{
let mut n_jobs = 0;
let pool = threadpool::Builder::new().build();
let (tx, rx) = channel();

for grammar in grammars {
let tx = tx.clone();
n_jobs += 1;

pool.execute(move || {
let grammar_id = grammar.grammar_id.clone();
job(grammar).unwrap_or_else(|err| {
eprintln!("Failed to {} grammar '{}'\n{}", action, grammar_id, err)
});

// report progress
tx.send(1).unwrap();
});
}
pool.join();

if rx.try_iter().sum::<usize>() == n_jobs {
Ok(())
} else {
Err(anyhow!("Failed to {} some grammar(s).", action))
}
}

for entry in fs::read_dir(path)? {
let entry = entry?;
let path = entry.path();
fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
if let GrammarSource::Git { remote, revision } = grammar.source {
let grammar_dir = helix_core::runtime_dir()
.join("grammars/sources")
.join(grammar.grammar_id.clone());

if !entry.file_type()?.is_dir() {
continue;
fs::create_dir_all(grammar_dir.clone()).expect("Could not create grammar directory");

// create the grammar dir contains a git directory
if !grammar_dir.join(".git").is_dir() {
git(&grammar_dir, ["init"])?;
}

// ensure the remote matches the configured remote
if get_remote_url(&grammar_dir).map_or(true, |s| s.trim_end() != remote) {
set_remote(&grammar_dir, &remote)?;
}

let dir = path.file_name().unwrap().to_str().unwrap().to_string();
// ensure the revision matches the configured revision
if get_revision(&grammar_dir).map_or(true, |s| s.trim_end() != revision) {
// Fetch the exact revision from the remote.
// Supported by server-side git since v2.5.0 (July 2015),
// enabled by default on major git hosts.
git(&grammar_dir, ["fetch", REMOTE_NAME, &revision])?;
git(&grammar_dir, ["checkout", &revision])?;

// filter ignores
if ignore.contains(&dir) {
continue;
println!(
"Grammar '{}' checked out at '{}'.",
grammar.grammar_id, revision
);
Ok(())
} else {
println!("Grammar '{}' is already up to date.", grammar.grammar_id);
Ok(())
}
dirs.push(dir)
} else {
println!("Skipping local grammar '{}'", grammar.grammar_id);
Ok(())
}
}

Ok(dirs)
// Sets the remote for a repository to the given URL, creating the remote if
// it does not yet exist.
fn set_remote(repository: &Path, remote_url: &str) -> Result<String> {
git(repository, ["remote", "set-url", REMOTE_NAME, remote_url])
.or_else(|_| git(repository, ["remote", "add", REMOTE_NAME, remote_url]))
}

fn build_library(src_path: &Path, language: &str) -> Result<()> {
fn get_remote_url(repository: &Path) -> Option<String> {
git(repository, ["remote", "get-url", REMOTE_NAME]).ok()
}

fn get_revision(repository: &Path) -> Option<String> {
git(repository, ["rev-parse", "HEAD"]).ok()
}

// A wrapper around 'git' commands which returns stdout in success and a
// helpful error message showing the command, stdout, and stderr in error.
fn git<I, S>(repository: &Path, args: I) -> Result<String>
where
I: IntoIterator<Item = S>,
S: AsRef<std::ffi::OsStr>,
{
let output = Command::new("git")
.args(args)
.current_dir(repository)
.output()?;

if output.status.success() {
Ok(String::from_utf8_lossy(&output.stdout).into_owned())
} else {
// TODO: figure out how to display the git command using `args`
Err(anyhow!(
"Git command failed.\nStdout: {}\nStderr: {}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
))
}
}

fn build_grammar(grammar: GrammarConfiguration) -> Result<()> {
let grammar_dir = if let GrammarSource::Local { ref path } = grammar.source {
PathBuf::from(path)
} else {
helix_core::runtime_dir()
.join("grammars/sources")
.join(grammar.grammar_id.clone())
};

grammar_dir.read_dir().with_context(|| {
format!(
"The directory {:?} is empty, you probably need to use 'hx --fetch-grammars'?",
grammar_dir
)
})?;

let path = match grammar.path {
Some(ref subpath) => grammar_dir.join(subpath),
None => grammar_dir,
}
.join("src");

build_tree_sitter_library(&path, grammar)
}

// Returns the set of grammar configurations the user requests.
// Grammars are configured in the default and user `languages.toml` and are
// merged. The `grammar_selection` key of the config is then used to filter
// down all grammars into a subset of the user's choosing.
fn get_grammar_configs() -> Vec<GrammarConfiguration> {
let config = helix_core::config::user_syntax_loader().expect("Could not parse languages.toml");

config.grammar
}

fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) -> Result<()> {
let header_path = src_path;
// let grammar_path = src_path.join("grammar.json");
let parser_path = src_path.join("parser.c");
let mut scanner_path = src_path.join("scanner.c");

Expand All @@ -50,16 +181,20 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
None
}
};
let parser_lib_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../runtime/grammars");
let mut library_path = parser_lib_path.join(language);
let parser_lib_path = helix_core::runtime_dir().join("../runtime/grammars");
let mut library_path = parser_lib_path.join(grammar.grammar_id.clone());
library_path.set_extension(DYLIB_EXTENSION);

let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
.with_context(|| "Failed to compare source and binary timestamps")?;
.context("Failed to compare source and binary timestamps")?;

if !recompile {
println!("Grammar '{}' is already built.", grammar.grammar_id);
return Ok(());
}

println!("Building grammar '{}'", grammar.grammar_id);

let mut config = cc::Build::new();
config
.cpp(true)
Expand Down Expand Up @@ -112,9 +247,7 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {
}
}

let output = command
.output()
.with_context(|| "Failed to execute C compiler")?;
let output = command.output().context("Failed to execute C compiler")?;
if !output.status.success() {
return Err(anyhow!(
"Parser compilation failed.\nStdout: {}\nStderr: {}",
Expand All @@ -125,6 +258,7 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> {

Ok(())
}

fn needs_recompile(
lib_path: &Path,
parser_c_path: &Path,
Expand All @@ -148,28 +282,3 @@ fn needs_recompile(
fn mtime(path: &Path) -> Result<SystemTime> {
Ok(fs::metadata(path)?.modified()?)
}

pub fn build_dir(dir: &str, language: &str) {
println!("Build language {}", language);
if PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../helix-syntax/languages")
.join(dir)
.read_dir()
.unwrap()
.next()
.is_none()
{
eprintln!(
"The directory {} is empty, you probably need to use './scripts/grammars sync'?",
dir
);
std::process::exit(1);
}

let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../helix-syntax/languages")
.join(dir)
.join("src");

build_library(&path, language).unwrap();
}
10 changes: 10 additions & 0 deletions helix-term/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,16 @@ FLAGS:
std::process::exit(0);
}

if args.fetch_grammars {
helix_term::grammars::fetch_grammars()?;
return Ok(0);
}

if args.build_grammars {
helix_term::grammars::build_grammars()?;
return Ok(0);
}

let conf_dir = helix_core::config_dir();
if !conf_dir.exists() {
std::fs::create_dir_all(&conf_dir).ok();
Expand Down

0 comments on commit 28dd5dd

Please sign in to comment.