diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 2221c1204..687a04e85 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -18,11 +18,13 @@ jobs: - name: Install Foundry uses: foundry-rs/foundry-toolchain@v1 + - uses: dtolnay/rust-toolchain@nightly - name: Update run: | - git config --global user.email "update@update.com" - git config --global user.name "update" + git config --global user.name "GitHub Actions Bot" + git config --global user.email "<>" + git config --global url."https://github.com/".insteadOf "git@github.com:" ./scripts/gen_output.sh - name: Create pull request diff --git a/scripts/gen_output/help.py b/scripts/gen_output/help.py deleted file mode 100755 index daf26c322..000000000 --- a/scripts/gen_output/help.py +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import re -import subprocess -import sys -from os import makedirs, path - -HELP_KEY = "help" -SECTION_START = "" -SECTION_END = "" -SECTION_RE = rf"\s*{SECTION_START}.*?{SECTION_END}" - -README = """\ -# CLI Reference - - - -Automatically-generated CLI reference from `--help` output. - -{{#include ./SUMMARY.md}} -""" - - -def main(): - args = parse_args(sys.argv[1:]) - for cmd in args.commands: - if cmd.find(" ") >= 0: - raise Exception(f"subcommands are not allowed: {cmd}") - makedirs(args.out_dir, exist_ok=True) - - output = {} - - # Iterate over all commands and their subcommands. - cmd_iter = [[cmd] for cmd in args.commands] - for cmd in cmd_iter: - subcmds, stdout = get_entry(cmd) - if args.verbose and len(subcmds) > 0: - eprint(f"Found subcommands for \"{' '.join(cmd)}\": {subcmds}") - - # Add entry to output map, e.g. `output["cmd"]["subcmd"]["help"] = "..."`. - e = output - for arg in cmd: - tmp = e.get(arg) - if not tmp: - e[arg] = {} - tmp = e[arg] - e = tmp - e[HELP_KEY] = stdout - - # Append subcommands. - for subcmd in subcmds: - cmd_iter.append(cmd + [subcmd]) - - # Generate markdown files. - summary = "" - root_summary = "" - for cmd, obj in output.items(): - cmd_markdown(args.out_dir, cmd, obj) - - root_path = path.relpath(args.out_dir, args.root_dir) - summary += cmd_summary("", cmd, obj, 0) - summary += "\n" - - root_summary += cmd_summary(root_path, cmd, obj, args.root_indentation) - root_summary += "\n" - with open(path.join(args.out_dir, "SUMMARY.md"), "w") as f: - f.write(summary) - - # Generate README.md. - if args.readme: - with open(path.join(args.out_dir, "README.md"), "w") as f: - f.write(README) - - if args.root_summary: - update_root_summary(args.root_dir, root_summary) - - -def parse_args(args: list[str]): - """Parses command line arguments.""" - parser = argparse.ArgumentParser( - description="Generate markdown files from help output of commands" - ) - parser.add_argument("--root-dir", default=".", help="Root directory") - parser.add_argument( - "--root-indentation", - default=0, - type=int, - help="Indentation for the root SUMMARY.md file", - ) - parser.add_argument("--out-dir", help="Output directory") - parser.add_argument( - "--readme", - action="store_true", - help="Whether to add a README.md file", - ) - parser.add_argument( - "--root-summary", - action="store_true", - help="Whether to update the root SUMMARY.md file", - ) - parser.add_argument( - "commands", - nargs="+", - help="Command to generate markdown for. Can be a subcommand.", - ) - parser.add_argument( - "--verbose", "-v", action="store_true", help="Print verbose output" - ) - return parser.parse_known_args(args)[0] - - -def get_entry(cmd: list[str]): - """Returns the subcommands and help output for a command.""" - env = os.environ.copy() - env["NO_COLOR"] = "1" - env["COLUMNS"] = "100" - env["LINES"] = "10000" - output = subprocess.run(cmd + ["--help"], capture_output=True, env=env) - if output.returncode != 0: - stderr = output.stderr.decode("utf-8") - raise Exception(f"Command \"{' '.join(cmd)}\" failed:\n{stderr}") - stdout = output.stdout.decode("utf-8") - subcmds = parse_sub_commands(stdout) - return subcmds, stdout - - -def parse_sub_commands(s: str): - """Returns a list of subcommands from the help output of a command.""" - idx = s.find("Commands:") - if idx < 0: - return [] - s = s[idx:] - - idx = s.find("Options:") - if idx < 0: - return [] - s = s[:idx] - - idx = s.find("Arguments:") - if idx >= 0: - s = s[:idx] - - subcmds = s.splitlines()[1:] - subcmds = filter( - lambda x: x.strip() != "" and x.startswith(" ") and x[2] != " ", subcmds - ) - subcmds = map(lambda x: x.strip().split(" ")[0], subcmds) - subcmds = filter(lambda x: x != "help", subcmds) - return list(subcmds) - - -def cmd_markdown(out_dir: str, cmd: str, obj: object): - """Writes the markdown for a command and its subcommands to out_dir.""" - - def rec(cmd: list[str], obj: object): - out = "" - out += f"# {' '.join(cmd)}\n\n" - out += help_markdown(cmd, obj[HELP_KEY]) - out_path = out_dir - for arg in cmd: - out_path = path.join(out_path, arg) - makedirs(path.dirname(out_path), exist_ok=True) - with open(f"{out_path}.md", "w") as f: - f.write(out) - - for k, v in obj.items(): - if k == HELP_KEY: - continue - rec(cmd + [k], v) - - rec([command_name(cmd)], obj) - - -def help_markdown(cmd: list[str], s: str): - """Returns the markdown for a command's help output.""" - cmd[0] = command_name(cmd[0]) - description, s = parse_description(s) - return f"""\ -{description} - -```bash -$ {' '.join(cmd)} --help -{s.strip()} -```""" - - -def parse_description(s: str): - """Splits the help output into a description and the rest.""" - idx = s.find("Usage:") - if idx < 0: - return "", s - lines = s[:idx].strip().splitlines() - if len(lines) == 0: - return "", s - return lines[0].strip(), s[idx:] - - -def cmd_summary(md_root: str, cmd: str, obj: object, indent: int): - """Returns the summary for a command and its subcommands.""" - - def rec(cmd: list[str], obj: object, indent: int): - nonlocal out - cmd_s = " ".join(cmd) - cmd_path = cmd_s.replace(" ", "/") - if md_root != "": - cmd_path = f"{md_root}/{cmd_path}" - out += f"{' ' * indent}- [`{cmd_s}`](./{cmd_path}.md)\n" - - for k, v in obj.items(): - if k == HELP_KEY: - continue - rec(cmd + [k], v, indent + 2) - - out = "" - rec([command_name(cmd)], obj, indent) - return out - - -def update_root_summary(root_dir: str, root_summary: str): - """Replaces the CLI_REFERENCE section in the root SUMMARY.md file.""" - summary_file = path.join(root_dir, "SUMMARY.md") - - with open(summary_file, "r") as f: - real_root_summary = f.read() - - if not re.search(SECTION_RE, real_root_summary, flags=re.DOTALL): - raise Exception( - f"Could not find CLI_REFERENCE section in {summary_file}. " - "Please add the following section to the file:\n" - f"{SECTION_START}\n{SECTION_END}" - ) - - last_line = re.findall(f".*{SECTION_END}", real_root_summary)[0] - root_summary_s = root_summary.rstrip().replace("\n\n", "\n") - replace_with = f" {SECTION_START}\n{root_summary_s}\n{last_line}" - - real_root_summary = re.sub( - SECTION_RE, replace_with, real_root_summary, flags=re.DOTALL - ) - root_summary_file = path.join(root_dir, "SUMMARY.md") - with open(root_summary_file, "w") as f: - f.write(real_root_summary) - - -def eprint(*args, **kwargs): - """Prints to stderr.""" - print(*args, file=sys.stderr, **kwargs) - - -def command_name(cmd: str): - """Returns the name of a command.""" - return cmd.split("/")[-1] - - -if __name__ == "__main__": - main() diff --git a/scripts/gen_output/help.rs b/scripts/gen_output/help.rs new file mode 100755 index 000000000..fc63fdac3 --- /dev/null +++ b/scripts/gen_output/help.rs @@ -0,0 +1,357 @@ +#!/usr/bin/env -S cargo +nightly -Zscript +--- +[package] +edition = "2021" + +[dependencies] +clap = { version = "4", features = ["derive"] } +pathdiff = "0.2" +regex = "1" +--- +use clap::Parser; +use regex::Regex; +use std::borrow::Cow; +use std::fs::{self, File}; +use std::io::{self, Write}; +use std::iter::once; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::str; +use std::sync::LazyLock; +use std::{fmt, process}; + +const SECTION_START: &str = ""; +const SECTION_END: &str = ""; +const README: &str = r#"# CLI Reference + + + +Automatically-generated CLI reference from `--help` output. + +{{#include ./SUMMARY.md}} +"#; +const TRIM_LINE_END_MARKDOWN: bool = false; + +/// Lazy static regex to avoid recompiling the same regex pattern multiple times. +macro_rules! regex { + ($re:expr) => {{ + static RE: LazyLock = + LazyLock::new(|| Regex::new($re).expect("Failed to compile regex pattern")); + &*RE + }}; +} + +/// Generate markdown files from help output of commands +#[derive(Parser, Debug)] +#[command(about, long_about = None)] +struct Args { + /// Root directory + #[arg(long, default_value_t = String::from("."))] + root_dir: String, + + /// Indentation for the root SUMMARY.md file + #[arg(long, default_value_t = 2)] + root_indentation: usize, + + /// Output directory + #[arg(long)] + out_dir: PathBuf, + + /// Whether to add a README.md file + #[arg(long)] + readme: bool, + + /// Whether to update the root SUMMARY.md file + #[arg(long)] + root_summary: bool, + + /// Print verbose output + #[arg(short, long)] + verbose: bool, + + /// Commands to generate markdown for. + #[arg(required = true, num_args = 1..)] + commands: Vec, +} + +fn write_file(file_path: &Path, content: &str) -> io::Result<()> { + let content = if TRIM_LINE_END_MARKDOWN { + content + .lines() + .map(|line| line.trim_end()) + .collect::>() + .join("\n") + } else { + content.to_string() + }; + fs::write(file_path, content) +} + +fn main() -> io::Result<()> { + let args = Args::parse(); + debug_assert!(args.commands.len() >= 1); + + let out_dir = args.out_dir; + fs::create_dir_all(&out_dir)?; + + let mut todo_iter: Vec = args + .commands + .iter() + .rev() // reverse to keep the order (pop) + .map(Cmd::new) + .collect(); + let mut output = Vec::new(); + + // Iterate over all commands and their subcommands. + while let Some(cmd) = todo_iter.pop() { + let (new_subcmds, stdout) = get_entry(&cmd)?; + if args.verbose && !new_subcmds.is_empty() { + println!( + "Found subcommands for \"{}\": {:?}", + cmd.command_name(), + new_subcmds + ); + } + // Add new subcommands to todo_iter (so that they are processed in the correct order). + for subcmd in new_subcmds.into_iter().rev() { + let new_subcmds: Vec<_> = cmd + .subcommands + .iter() + .cloned() + .chain(once(subcmd)) + .collect(); + + todo_iter.push(Cmd { + cmd: cmd.cmd, + subcommands: new_subcmds, + }); + } + output.push((cmd, stdout)); + } + + // Generate markdown files. + for (cmd, stdout) in &output { + cmd_markdown(&out_dir, cmd, stdout)?; + } + + // Generate SUMMARY.md. + let summary: String = output + .iter() + .map(|(cmd, _)| cmd_summary(None, cmd, 0)) + .chain(once("\n".to_string())) + .collect(); + + write_file(&out_dir.clone().join("SUMMARY.md"), &summary)?; + + // Generate README.md. + if args.readme { + let path = &out_dir.join("README.md"); + if args.verbose { + println!("Writing README.md to \"{}\"", path.to_string_lossy()); + } + write_file(path, README)?; + } + + // Generate root SUMMARY.md. + if args.root_summary { + let root_summary: String = output + .iter() + .map(|(cmd, _)| { + let root_path = pathdiff::diff_paths(&out_dir, &args.root_dir); + cmd_summary(root_path, cmd, args.root_indentation) + }) + .collect(); + + let path = Path::new(args.root_dir.as_str()); + if args.verbose { + println!("Updating root summary in \"{}\"", path.to_string_lossy()); + } + update_root_summary(path, &root_summary)?; + } + + Ok(()) +} + +/// Returns the subcommands and help output for a command. +fn get_entry(cmd: &Cmd) -> io::Result<(Vec, String)> { + let output = Command::new(cmd.cmd) + .args(&cmd.subcommands) + .arg("--help") + .env("NO_COLOR", "1") + .env("COLUMNS", "100") + .env("LINES", "10000") + .stdout(Stdio::piped()) + .output()?; + + if !output.status.success() { + let stderr = str::from_utf8(&output.stderr).unwrap_or("Failed to parse stderr as UTF-8"); + return Err(io::Error::new( + io::ErrorKind::Other, + format!("Command \"{}\" failed:\n{}", cmd, stderr), + )); + } + + let stdout = str::from_utf8(&output.stdout) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))? + .to_string(); + + // Parse subcommands from the help output + let subcmds = parse_sub_commands(&stdout); + + Ok((subcmds, stdout)) +} + +/// Returns a list of subcommands from the help output of a command. +fn parse_sub_commands(s: &str) -> Vec { + // This regex matches lines starting with two spaces, followed by the subcommand name. + let re = regex!(r"^ (\S+)"); + + s.split("Commands:") + .nth(1) // Get the part after "Commands:" + .map(|commands_section| { + commands_section + .lines() + .take_while(|line| !line.starts_with("Options:") && !line.starts_with("Arguments:")) + .filter_map(|line| { + re.captures(line) + .and_then(|cap| cap.get(1).map(|m| m.as_str().to_string())) + }) + .filter(|cmd| cmd != "help") + .map(String::from) + .collect() + }) + .unwrap_or_default() // Return an empty Vec if "Commands:" was not found +} + +/// Writes the markdown for a command to out_dir. +fn cmd_markdown(out_dir: &Path, cmd: &Cmd, stdout: &str) -> io::Result<()> { + let out = format!("# {}\n\n{}", cmd, help_markdown(cmd, stdout)); + + let out_path = out_dir.join(cmd.to_string().replace(" ", "/")); + fs::create_dir_all(out_path.parent().unwrap())?; + write_file(&out_path.with_extension("md"), &out)?; + + Ok(()) +} + +/// Returns the markdown for a command's help output. +fn help_markdown(cmd: &Cmd, stdout: &str) -> String { + let (description, s) = parse_description(stdout); + format!( + "{}\n\n```bash\n$ {} --help\n{}\n```", + description, + cmd, + preprocess_help(s.trim()) + ) +} + +/// Splits the help output into a description and the rest. +fn parse_description(s: &str) -> (&str, &str) { + match s.find("Usage:") { + Some(idx) => { + let description = s[..idx].trim().lines().next().unwrap_or(""); + (description, &s[idx..]) + } + None => ("", s), + } +} + +/// Returns the summary for a command and its subcommands. +fn cmd_summary(md_root: Option, cmd: &Cmd, indent: usize) -> String { + let cmd_s = cmd.to_string(); + let cmd_path = cmd_s.replace(" ", "/"); + let full_cmd_path = match md_root { + None => cmd_path, + Some(md_root) => format!("{}/{}", md_root.to_string_lossy(), cmd_path), + }; + let indent_string = " ".repeat(indent + (cmd.subcommands.len() * 2)); + format!("{}- [`{}`](./{}.md)\n", indent_string, cmd_s, full_cmd_path) +} + +/// Replaces the CLI_REFERENCE section in the root SUMMARY.md file. +fn update_root_summary(root_dir: &Path, root_summary: &str) -> io::Result<()> { + let summary_file = root_dir.join("SUMMARY.md"); + let original_summary_content = fs::read_to_string(&summary_file)?; + + let section_re = regex!(&format!(r"(?s)\s*{SECTION_START}.*?{SECTION_END}")); + if !section_re.is_match(&original_summary_content) { + eprintln!( + "Could not find CLI_REFERENCE section in {}. Please add the following section to the file:\n{}\n... CLI Reference goes here ...\n\n{}", + summary_file.display(), + SECTION_START, + SECTION_END + ); + process::exit(1); + } + + let section_end_re = regex!(&format!(r".*{SECTION_END}")); + let last_line = section_end_re + .find(&original_summary_content) + .map(|m| m.as_str().to_string()) + .expect("Could not extract last line of CLI_REFERENCE section"); + + let root_summary_s = root_summary.trim_end().replace("\n\n", "\n"); + let replace_with = format!(" {}\n{}\n{}", SECTION_START, root_summary_s, last_line); + + let new_root_summary = section_re + .replace(&original_summary_content, replace_with.as_str()) + .to_string(); + + let mut root_summary_file = File::create(&summary_file)?; + root_summary_file.write_all(new_root_summary.as_bytes()) +} + +/// Preprocesses the help output of a command. +fn preprocess_help(s: &str) -> Cow<'_, str> { + static REPLACEMENTS: LazyLock> = LazyLock::new(|| { + let patterns: &[(&str, &str)] = &[ + // No patterns yet + ]; + patterns + .iter() + .map(|&(re, replace_with)| (Regex::new(re).expect(re), replace_with)) + .collect() + }); + + let mut s = Cow::Borrowed(s); + for (re, replacement) in REPLACEMENTS.iter() { + if let Cow::Owned(result) = re.replace_all(&s, *replacement) { + s = Cow::Owned(result); + } + } + s +} + +#[derive(Hash, Debug, PartialEq, Eq)] +struct Cmd<'a> { + /// path to binary (e.g. ./target/debug/reth) + cmd: &'a Path, + /// subcommands (e.g. [db, stats]) + subcommands: Vec, +} + +impl<'a> Cmd<'a> { + fn command_name(&self) -> &str { + self.cmd + .file_name() + .and_then(|os_str| os_str.to_str()) + .expect("Expect valid command") + } + + fn new(cmd: &'a PathBuf) -> Self { + Self { + cmd, + subcommands: Vec::new(), + } + } +} + +impl<'a> fmt::Display for Cmd<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.command_name())?; + if !self.subcommands.is_empty() { + write!(f, " {}", self.subcommands.join(" "))?; + } + Ok(()) + } +} diff --git a/scripts/gen_output/help.sh b/scripts/gen_output/help.sh index 24b9b37cb..6ae57c5de 100644 --- a/scripts/gen_output/help.sh +++ b/scripts/gen_output/help.sh @@ -9,11 +9,12 @@ gen_help() { done cmd=( - "$SCRIPTS/gen_output/help.py" + "$SCRIPTS/gen_output/help.rs" --root-dir "$ROOT/src/" --root-summary --root-indentation 4 --readme + --verbose --out-dir "$ROOT/src/reference/cli/" "${bins[@]}" ) diff --git a/src/reference/cli/README.md b/src/reference/cli/README.md index 041d3cfa6..278fcc2fe 100644 --- a/src/reference/cli/README.md +++ b/src/reference/cli/README.md @@ -1,6 +1,6 @@ # CLI Reference - + Automatically-generated CLI reference from `--help` output. diff --git a/src/reference/cli/SUMMARY.md b/src/reference/cli/SUMMARY.md index 44c95fd88..05437c22f 100644 --- a/src/reference/cli/SUMMARY.md +++ b/src/reference/cli/SUMMARY.md @@ -46,7 +46,6 @@ - [`forge verify-bytecode`](./forge/verify-bytecode.md) - [`forge verify-check`](./forge/verify-check.md) - [`forge verify-contract`](./forge/verify-contract.md) - - [`cast`](./cast.md) - [`cast 4byte`](./cast/4byte.md) - [`cast 4byte-decode`](./cast/4byte-decode.md) @@ -151,11 +150,9 @@ - [`cast wallet list`](./cast/wallet/list.md) - [`cast wallet private-key`](./cast/wallet/private-key.md) - [`cast wallet decrypt-keystore`](./cast/wallet/decrypt-keystore.md) - - [`anvil`](./anvil.md) - [`anvil completions`](./anvil/completions.md) - [`anvil generate-fig-spec`](./anvil/generate-fig-spec.md) - - [`chisel`](./chisel.md) - [`chisel list`](./chisel/list.md) - [`chisel load`](./chisel/load.md)