From 4f8a526e23cf16223bd06e391807ae5fb7f18913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Quentin?= Date: Tue, 26 Nov 2024 15:36:40 +0100 Subject: [PATCH] Add external processor support (#705) * Add external processor support * CHANGELOG.md * Clippy * Address review comments --- CHANGELOG.md | 1 + cargo-espflash/src/main.rs | 4 +- espflash/src/bin/espflash.rs | 4 +- espflash/src/cli/mod.rs | 10 +- .../src/cli/monitor/external_processors.rs | 162 ++++++++++++++++++ espflash/src/cli/monitor/mod.rs | 11 +- 6 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 espflash/src/cli/monitor/external_processors.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index f65e5af7..fd0a1b93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Allow `partition_table_offset` to be specified in the config file. (for #699) +- Support external log-processors (#705) ### Changed diff --git a/cargo-espflash/src/main.rs b/cargo-espflash/src/main.rs index 26bc3aa2..1c1e3633 100644 --- a/cargo-espflash/src/main.rs +++ b/cargo-espflash/src/main.rs @@ -323,7 +323,7 @@ fn flash(args: FlashArgs, config: &Config) -> Result<()> { build(&args.build_args, &cargo_config, chip).wrap_err("Failed to build project")?; // Read the ELF data from the build path and load it to the target. - let elf_data = fs::read(build_ctx.artifact_path).into_diagnostic()?; + let elf_data = fs::read(build_ctx.artifact_path.clone()).into_diagnostic()?; print_board_info(&mut flasher)?; @@ -368,6 +368,8 @@ fn flash(args: FlashArgs, config: &Config) -> Result<()> { args.flash_args.monitor_baud.unwrap_or(default_baud), args.flash_args.log_format, true, + args.flash_args.processors, + Some(build_ctx.artifact_path), ) } else { Ok(()) diff --git a/espflash/src/bin/espflash.rs b/espflash/src/bin/espflash.rs index 6dd80a5e..f117b9ce 100644 --- a/espflash/src/bin/espflash.rs +++ b/espflash/src/bin/espflash.rs @@ -30,7 +30,7 @@ pub struct Cli { subcommand: Commands, /// Do not check for updates - #[clap(short, long, global = true, action)] + #[clap(short = 'S', long, global = true, action)] skip_update_check: bool, } @@ -303,6 +303,8 @@ fn flash(args: FlashArgs, config: &Config) -> Result<()> { args.flash_args.monitor_baud.unwrap_or(default_baud), args.flash_args.log_format, true, + args.flash_args.processors, + Some(args.image), ) } else { Ok(()) diff --git a/espflash/src/cli/mod.rs b/espflash/src/cli/mod.rs index 5166aced..b4825ce9 100644 --- a/espflash/src/cli/mod.rs +++ b/espflash/src/cli/mod.rs @@ -156,6 +156,9 @@ pub struct FlashArgs { pub no_skip: bool, #[clap(flatten)] pub image: ImageArgs, + /// External log processors to use (comma separated executables) + #[arg(long, requires = "monitor")] + pub processors: Option, } /// Operations for partitions tables @@ -262,6 +265,9 @@ pub struct MonitorArgs { /// Logging format. #[arg(long, short = 'L', default_value = "serial", requires = "elf")] pub log_format: LogFormat, + /// External log processors to use (comma separated executables) + #[arg(long)] + processors: Option, } #[derive(Debug, Args)] @@ -418,7 +424,7 @@ pub fn serial_monitor(args: MonitorArgs, config: &Config) -> Result<()> { let mut flasher = connect(&args.connect_args, config, true, true)?; let pid = flasher.get_usb_pid()?; - let elf = if let Some(elf_path) = args.elf { + let elf = if let Some(elf_path) = args.elf.clone() { let path = fs::canonicalize(elf_path).into_diagnostic()?; let data = fs::read(path).into_diagnostic()?; @@ -447,6 +453,8 @@ pub fn serial_monitor(args: MonitorArgs, config: &Config) -> Result<()> { args.connect_args.baud.unwrap_or(default_baud), args.log_format, !args.non_interactive, + args.processors, + args.elf, ) } diff --git a/espflash/src/cli/monitor/external_processors.rs b/espflash/src/cli/monitor/external_processors.rs new file mode 100644 index 00000000..844a4f76 --- /dev/null +++ b/espflash/src/cli/monitor/external_processors.rs @@ -0,0 +1,162 @@ +#![allow(clippy::needless_doctest_main)] +//! External processor support +//! +//! Via the command line argument `--processors` you can instruct espflash to run external executables to pre-process +//! the logs received from the target. Multiple processors are supported by separating them via `,`. Processors are executed in the specified order. +//! +//! You can use full-qualified paths or run an executable which is already in the search path. +//! +//! A processors reads from stdin and output to stdout. Be aware this runs before further processing by espflash. +//! i.e. addresses are not resolved and when using `defmt` you will see encoded data. +//! +//! Additionally be aware that you might receive chunked data which is not always split at valid UTF character boundaries. +//! +//! The executable will get the path of the ELF file as the first argument if available. +//! +//! Example processor which turns some letters into uppercase +//! ```rust,no-run +//! use std::io::{stdin, stdout, Read, Write}; +//! +//! fn main() { +//! let args: Vec = std::env::args().collect(); +//! println!("ELF file: {:?}", args[1]); +//! +//! let mut buf = [0u8; 1024]; +//! loop { +//! if let Ok(len) = stdin().read(&mut buf) { +//! for b in &mut buf[..len] { +//! *b = if b"abdfeo".contains(b) { +//! b.to_ascii_uppercase() +//! } else { +//! *b +//! }; +//! } +//! +//! stdout().write(&buf[..len]).unwrap(); +//! stdout().flush().unwrap(); +//! } else { +//! // ignored +//! } +//! } +//! } +//! ``` + +use std::{ + fmt::Display, + io::{Read, Write}, + path::PathBuf, + process::{Child, ChildStdin, Stdio}, + sync::mpsc, +}; + +use miette::Diagnostic; + +#[derive(Debug)] +pub struct Error { + executable: String, +} + +impl Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Failed to launch '{}'", self.executable) + } +} + +impl std::error::Error for Error {} + +impl Diagnostic for Error {} + +struct Processor { + rx: mpsc::Receiver, + stdin: ChildStdin, + child: Child, +} + +impl Processor { + pub fn new(child: Child) -> Self { + let mut child = child; + let (tx, rx) = mpsc::channel::(); + + let mut stdout = child.stdout.take().unwrap(); + let stdin = child.stdin.take().unwrap(); + + std::thread::spawn(move || { + let mut buffer = [0u8; 1024]; + loop { + if let Ok(len) = stdout.read(&mut buffer) { + for b in &buffer[..len] { + if tx.send(*b).is_err() { + break; + } + } + } + } + }); + + Self { rx, stdin, child } + } + + pub fn try_receive(&mut self) -> Vec { + let mut res = Vec::new(); + while let Ok(b) = self.rx.try_recv() { + res.push(b); + } + res + } + + pub fn send(&mut self, data: Vec) { + let _ignored = self.stdin.write(&data).ok(); + } +} + +impl Drop for Processor { + fn drop(&mut self) { + self.child.kill().unwrap(); + } +} + +pub struct ExternalProcessors { + processors: Vec, +} + +impl ExternalProcessors { + pub fn new(processors: Option, elf: Option) -> Result { + let mut args = Vec::new(); + + if let Some(elf) = elf { + args.push(elf.as_os_str().to_str().unwrap().to_string()); + }; + + let mut spawned = Vec::new(); + if let Some(processors) = processors { + for processor in processors.split(",") { + let processor = std::process::Command::new(processor) + .args(args.clone()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .map_err(|_| Error { + executable: processor.to_string(), + })?; + spawned.push(Processor::new(processor)); + } + } + + Ok(Self { + processors: spawned, + }) + } + + pub fn process(&mut self, read: &[u8]) -> Vec { + let mut buffer = Vec::new(); + buffer.extend_from_slice(read); + + for processor in &mut self.processors { + processor.send(buffer); + buffer = processor.try_receive(); + } + + buffer + } +} diff --git a/espflash/src/cli/monitor/mod.rs b/espflash/src/cli/monitor/mod.rs index 1d733639..562be438 100644 --- a/espflash/src/cli/monitor/mod.rs +++ b/espflash/src/cli/monitor/mod.rs @@ -12,6 +12,7 @@ use std::{ io::{stdout, ErrorKind, Read, Write}, + path::PathBuf, time::Duration, }; @@ -20,6 +21,7 @@ use crossterm::{ event::{poll, read, Event, KeyCode, KeyEvent, KeyModifiers}, terminal::{disable_raw_mode, enable_raw_mode}, }; +use external_processors::ExternalProcessors; use log::error; use miette::{IntoDiagnostic, Result}; #[cfg(feature = "serialport")] @@ -31,6 +33,7 @@ use crate::{ connection::{reset::reset_after_flash, Port}, }; +pub mod external_processors; pub mod parser; mod line_endings; @@ -66,6 +69,7 @@ impl Drop for RawModeGuard { } /// Open a serial monitor on the given serial port, using the given input parser. +#[allow(clippy::too_many_arguments)] pub fn monitor( mut serial: Port, elf: Option<&[u8]>, @@ -73,6 +77,8 @@ pub fn monitor( baud: u32, log_format: LogFormat, interactive_mode: bool, + processors: Option, + elf_file: Option, ) -> miette::Result<()> { if interactive_mode { println!("Commands:"); @@ -101,6 +107,8 @@ pub fn monitor( LogFormat::Serial => Box::new(parser::serial::Serial), }; + let mut external_processors = ExternalProcessors::new(processors, elf_file)?; + let mut buff = [0; 1024]; loop { let read_count = match serial.read(&mut buff) { @@ -110,7 +118,8 @@ pub fn monitor( err => err.into_diagnostic(), }?; - parser.feed(&buff[0..read_count], &mut stdout); + let processed = external_processors.process(&buff[0..read_count]); + parser.feed(&processed, &mut stdout); // Don't forget to flush the writer! stdout.flush().ok();