From e1ad17a0c50484c518dc19e171bd12d3dfd820c0 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Sun, 13 Feb 2022 20:08:28 -0600 Subject: [PATCH] implement build_grammars and fetch_grammars build_grammars adapts the functionality that previously came from helix-syntax to be used at runtime from the command line flags. fetch_grammars wraps command-line git to perform the same actions previously done in the scripts in #1560. --- helix-term/src/grammars.rs | 205 ++++++++++++++++++++++++++++--------- helix-term/src/main.rs | 10 ++ 2 files changed, 167 insertions(+), 48 deletions(-) diff --git a/helix-term/src/grammars.rs b/helix-term/src/grammars.rs index 6a4910a32a1c..7a7a4c187eaf 100644 --- a/helix-term/src/grammars.rs +++ b/helix-term/src/grammars.rs @@ -4,39 +4,170 @@ use std::time::SystemTime; use std::{ path::{Path, PathBuf}, process::Command, + sync::mpsc::channel, }; -use helix_core::syntax::DYLIB_EXTENSION; +use helix_core::syntax::{GrammarConfiguration, GrammarSource, DYLIB_EXTENSION}; const BUILD_TARGET: &str = env!("BUILD_TARGET"); +const REMOTE_NAME: &str = "origin"; -pub fn collect_tree_sitter_dirs(ignore: &[String]) -> Result> { - let mut dirs = Vec::new(); - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../helix-syntax/languages"); +pub fn fetch_grammars() -> Result<()> { + run_parallel(get_grammar_configs(), fetch_grammar, "fetch") +} + +pub fn build_grammars() -> Result<()> { + run_parallel(get_grammar_configs(), build_grammar, "build") +} + +fn run_parallel(grammars: Vec, job: F, action: &'static str) -> Result<()> +where + F: Fn(GrammarConfiguration) -> Result<()> + std::marker::Send + 'static + Copy, +{ + let mut n_jobs = 0; + let pool = threadpool::Builder::new().build(); + let (tx, rx) = channel(); + + for grammar in grammars { + let tx = tx.clone(); + n_jobs += 1; + + pool.execute(move || { + let grammar_id = grammar.grammar_id.clone(); + job(grammar).unwrap_or_else(|err| { + eprintln!("Failed to {} grammar '{}'\n{}", action, grammar_id, err) + }); + + // report progress + tx.send(1).unwrap(); + }); + } + pool.join(); + + if rx.try_iter().sum::() == n_jobs { + Ok(()) + } else { + Err(anyhow!("Failed to {} some grammar(s).", action)) + } +} - for entry in fs::read_dir(path)? { - let entry = entry?; - let path = entry.path(); +fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> { + if let GrammarSource::Git { remote, revision } = grammar.source { + let grammar_dir = helix_core::runtime_dir() + .join("grammars/sources") + .join(grammar.grammar_id.clone()); - if !entry.file_type()?.is_dir() { - continue; + fs::create_dir_all(grammar_dir.clone()).expect("Could not create grammar directory"); + + // create the grammar dir contains a git directory + if !grammar_dir.join(".git").is_dir() { + git(&grammar_dir, ["init"])?; + } + + // ensure the remote matches the configured remote + if get_remote_url(&grammar_dir).map_or(true, |s| s.trim_end() != remote) { + set_remote(&grammar_dir, &remote)?; } - let dir = path.file_name().unwrap().to_str().unwrap().to_string(); + // ensure the revision matches the configured revision + if get_revision(&grammar_dir).map_or(true, |s| s.trim_end() != revision) { + // Fetch the exact revision from the remote. + // Supported by server-side git since v2.5.0 (July 2015), + // enabled by default on major git hosts. + git(&grammar_dir, ["fetch", REMOTE_NAME, &revision])?; + git(&grammar_dir, ["checkout", &revision])?; - // filter ignores - if ignore.contains(&dir) { - continue; + println!( + "Grammar '{}' checked out at '{}'.", + grammar.grammar_id, revision + ); + Ok(()) + } else { + println!("Grammar '{}' is already up to date.", grammar.grammar_id); + Ok(()) } - dirs.push(dir) + } else { + println!("Skipping local grammar '{}'", grammar.grammar_id); + Ok(()) } +} - Ok(dirs) +// Sets the remote for a repository to the given URL, creating the remote if +// it does not yet exist. +fn set_remote(repository: &Path, remote_url: &str) -> Result { + git(repository, ["remote", "set-url", REMOTE_NAME, remote_url]) + .or_else(|_| git(repository, ["remote", "add", REMOTE_NAME, remote_url])) } -fn build_library(src_path: &Path, language: &str) -> Result<()> { +fn get_remote_url(repository: &Path) -> Option { + git(repository, ["remote", "get-url", REMOTE_NAME]).ok() +} + +fn get_revision(repository: &Path) -> Option { + git(repository, ["rev-parse", "HEAD"]).ok() +} + +// A wrapper around 'git' commands which returns stdout in success and a +// helpful error message showing the command, stdout, and stderr in error. +fn git(repository: &Path, args: I) -> Result +where + I: IntoIterator, + S: AsRef, +{ + let output = Command::new("git") + .args(args) + .current_dir(repository) + .output()?; + + if output.status.success() { + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) + } else { + // TODO: figure out how to display the git command using `args` + Err(anyhow!( + "Git command failed.\nStdout: {}\nStderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + )) + } +} + +fn build_grammar(grammar: GrammarConfiguration) -> Result<()> { + let grammar_dir = if let GrammarSource::Local { ref path } = grammar.source { + PathBuf::from(path) + } else { + helix_core::runtime_dir() + .join("grammars/sources") + .join(grammar.grammar_id.clone()) + }; + + grammar_dir.read_dir().with_context(|| { + format!( + "The directory {:?} is empty, you probably need to use 'hx --fetch-grammars'?", + grammar_dir + ) + })?; + + let path = match grammar.path { + Some(ref subpath) => grammar_dir.join(subpath), + None => grammar_dir, + } + .join("src"); + + build_tree_sitter_library(&path, grammar) +} + +// Returns the set of grammar configurations the user requests. +// Grammars are configured in the default and user `languages.toml` and are +// merged. The `grammar_selection` key of the config is then used to filter +// down all grammars into a subset of the user's choosing. +fn get_grammar_configs() -> Vec { + let config = helix_core::config::user_syntax_loader().expect("Could not parse languages.toml"); + + config.grammar +} + +fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) -> Result<()> { let header_path = src_path; - // let grammar_path = src_path.join("grammar.json"); let parser_path = src_path.join("parser.c"); let mut scanner_path = src_path.join("scanner.c"); @@ -50,16 +181,20 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> { None } }; - let parser_lib_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../runtime/grammars"); - let mut library_path = parser_lib_path.join(language); + let parser_lib_path = helix_core::runtime_dir().join("../runtime/grammars"); + let mut library_path = parser_lib_path.join(grammar.grammar_id.clone()); library_path.set_extension(DYLIB_EXTENSION); let recompile = needs_recompile(&library_path, &parser_path, &scanner_path) - .with_context(|| "Failed to compare source and binary timestamps")?; + .context("Failed to compare source and binary timestamps")?; if !recompile { + println!("Grammar '{}' is already built.", grammar.grammar_id); return Ok(()); } + + println!("Building grammar '{}'", grammar.grammar_id); + let mut config = cc::Build::new(); config .cpp(true) @@ -112,9 +247,7 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> { } } - let output = command - .output() - .with_context(|| "Failed to execute C compiler")?; + let output = command.output().context("Failed to execute C compiler")?; if !output.status.success() { return Err(anyhow!( "Parser compilation failed.\nStdout: {}\nStderr: {}", @@ -125,6 +258,7 @@ fn build_library(src_path: &Path, language: &str) -> Result<()> { Ok(()) } + fn needs_recompile( lib_path: &Path, parser_c_path: &Path, @@ -148,28 +282,3 @@ fn needs_recompile( fn mtime(path: &Path) -> Result { Ok(fs::metadata(path)?.modified()?) } - -pub fn build_dir(dir: &str, language: &str) { - println!("Build language {}", language); - if PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../helix-syntax/languages") - .join(dir) - .read_dir() - .unwrap() - .next() - .is_none() - { - eprintln!( - "The directory {} is empty, you probably need to use './scripts/grammars sync'?", - dir - ); - std::process::exit(1); - } - - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../helix-syntax/languages") - .join(dir) - .join("src"); - - build_library(&path, language).unwrap(); -} diff --git a/helix-term/src/main.rs b/helix-term/src/main.rs index 0760e204229a..ce3266f2a4a1 100644 --- a/helix-term/src/main.rs +++ b/helix-term/src/main.rs @@ -103,6 +103,16 @@ FLAGS: std::process::exit(0); } + if args.fetch_grammars { + helix_term::grammars::fetch_grammars()?; + return Ok(0); + } + + if args.build_grammars { + helix_term::grammars::build_grammars()?; + return Ok(0); + } + let conf_dir = helix_core::config_dir(); if !conf_dir.exists() { std::fs::create_dir_all(&conf_dir).ok();