From 6046cacfd0a3408951a1a07e80e15ec0b2acbf9b Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Fri, 5 Apr 2024 20:09:31 +0200 Subject: [PATCH 01/12] fix(shell.nix): explicitly use rust tools from rustup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It’s just a better environment that way … sigh --- shell.nix | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/shell.nix b/shell.nix index 609b875b..ce20a18b 100644 --- a/shell.nix +++ b/shell.nix @@ -48,10 +48,8 @@ let CARGO_INSTALL_ROOT = "${LORRI_ROOT}/.cargo"; buildInputs = [ - pkgs.cargo - pkgs.rustc - pkgs.rustfmt - pkgs.rustPackages.clippy + # please use rustup to install rust, setting it up via nix is a bother + pkgs.rustup pkgs.git pkgs.direnv pkgs.crate2nix @@ -80,10 +78,7 @@ in pkgs.mkShell ( { name = "lorri"; - buildInputs = buildInputs - ++ pkgs.lib.optionals isDevelopmentShell [ - pkgs.rust-analyzer - ]; + inherit buildInputs; inherit BUILD_REV_COUNT RUN_TIME_CLOSURE; From adfc561cf17e40d5f768c04432e39f7a0d1cd0ef Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Sun, 14 Apr 2024 14:25:33 +0200 Subject: [PATCH 02/12] refact(ops/stream_events): directly construct json value Instead of having complex generic types internally that we map over, just create the json in a macro. Removes `ProjectAdded` from `Reason`, because it was not used anywhere. Changes the daemon format, so make sure to run the daemon with this new lorri version or you get serialization errors. --- src/build_loop.rs | 71 +++-------------------------- src/builder.rs | 17 ++----- src/lib.rs | 18 +++++++- src/ops.rs | 71 ++++++++++++++--------------- src/project.rs | 6 +-- tests/integration/direnvtestcase.rs | 2 +- 6 files changed, 62 insertions(+), 123 deletions(-) diff --git a/src/build_loop.rs b/src/build_loop.rs index f68b3030..711f55e7 100644 --- a/src/build_loop.rs +++ b/src/build_loop.rs @@ -17,7 +17,7 @@ use std::path::PathBuf; /// Build events that can happen. /// Abstracting over its internal to make different serialize instances possible. #[derive(Clone, Debug, Serialize, Deserialize)] -pub enum EventI { +pub enum Event { /// Demarks a stream of events from recent history becoming live SectionEnd, /// A build has started @@ -32,7 +32,7 @@ pub enum EventI { /// The shell.nix file for the building project nix_file: NixFile, /// the output paths of the build - rooted_output_paths: OutputPath, + rooted_output_paths: builder::OutputPath, }, /// A build command returned a failing exit status Failure { @@ -43,75 +43,16 @@ pub enum EventI { }, } -/// Builder events sent back over `BuildLoop.tx`. -pub type Event = EventI, BuildError>; - -impl EventI { - /// Map over the inner types. - pub fn map( - self, - nix_file_f: F, - reason_f: G, - output_paths_f: H, - build_error_f: I, - ) -> EventI - where - F: Fn(NixFile) -> NixFile2, - G: Fn(Reason) -> Reason2, - H: Fn(OutputPath) -> OutputPaths2, - I: Fn(BuildError) -> BuildError2, - { - use EventI::*; - match self { - SectionEnd => SectionEnd, - Started { nix_file, reason } => Started { - nix_file: nix_file_f(nix_file), - reason: reason_f(reason), - }, - Completed { - nix_file, - rooted_output_paths, - } => Completed { - nix_file: nix_file_f(nix_file), - rooted_output_paths: output_paths_f(rooted_output_paths), - }, - Failure { nix_file, failure } => Failure { - nix_file: nix_file_f(nix_file), - failure: build_error_f(failure), - }, - } - } -} - /// Description of the project change that triggered a build. #[derive(Clone, Debug, Serialize, Deserialize)] -pub enum ReasonI { - /// When a project is presented to Lorri to track, it's built for this reason. - ProjectAdded(NixFile), - /// When a ping is received. +pub enum Reason { + /// We received a ping, requesting us to reevaluate and maybe build the project PingReceived, /// When there is a filesystem change, the first changed file is recorded, /// along with a count of other filesystem events. FilesChanged(Vec), } -impl ReasonI { - /// Map over the inner types. - pub fn map(self, nix_file_f: F) -> ReasonI - where - F: Fn(NixFile) -> NixFile2, - { - use ReasonI::*; - match self { - ProjectAdded(nix_file) => ProjectAdded(nix_file_f(nix_file)), - PingReceived => PingReceived, - FilesChanged(vec) => FilesChanged(vec), - } - } -} - -type Reason = ReasonI; - /// The BuildLoop repeatedly builds the Nix expression in /// `project` each time a source file influencing /// a previous build changes. @@ -315,7 +256,7 @@ impl<'a> BuildLoop<'a> { /// /// This will create GC roots and expand the file watch list for /// the evaluation. - pub fn once(&mut self) -> Result, BuildError> { + pub fn once(&mut self) -> Result { let nix_file = self.project.nix_file.clone(); let cas = self.project.cas.clone(); let extra_nix_options = self.extra_nix_options.clone(); @@ -331,7 +272,7 @@ impl<'a> BuildLoop<'a> { fn handle_run_result( &mut self, run_result: Result, - ) -> Result, BuildError> { + ) -> Result { let run_result = run_result?; let paths = run_result.referenced_paths; diff --git a/src/builder.rs b/src/builder.rs index b652ea32..8227a3b6 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -10,6 +10,7 @@ use crate::cas::ContentAddressable; use crate::nix::{options::NixOptions, StorePath}; use crate::osstrlines; +use crate::project::RootPath; use crate::watch::WatchPathBuf; use crate::{DrvFile, NixFile}; use regex::Regex; @@ -475,21 +476,9 @@ where /// Output path generated by `logged-evaluation.nix` #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct OutputPath { +pub struct OutputPath { /// Shell path modified to work as a gc root - pub shell_gc_root: T, -} - -impl OutputPath { - /// map over the inner type. - pub fn map(self, f: F) -> OutputPath - where - F: Fn(T) -> T2, - { - OutputPath { - shell_gc_root: f(self.shell_gc_root), - } - } + pub shell_gc_root: RootPath, } #[cfg(test)] diff --git a/src/lib.rs b/src/lib.rs index 3a36527a..7318b0fd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -108,6 +108,11 @@ impl AbsPathBuf { self.0.display() } + /// Print a path to a json string, assuming it is UTF-8, converting any non-utf codeblocks to replacement characters + pub fn to_json_value(&self) -> serde_json::Value { + path_to_json_string(self.0.as_path()) + } + /// Joins a path to the end of this absolute path. /// If the path is absolute, it will replace this absolute path. pub fn join>(&self, pb: P) -> Self { @@ -140,13 +145,22 @@ impl NixFile { pub fn as_absolute_path(&self) -> &Path { self.0.as_path() } -} -impl NixFile { /// `display` the path. pub fn display(&self) -> std::path::Display { self.0.display() } + + /// Print a path to a json string, assuming it is UTF-8, converting any non-utf codeblocks to replacement characters + pub fn to_json_value(&self) -> serde_json::Value { + path_to_json_string(self.0.as_path()) + } +} + +/// Print a path to a json string, assuming it is UTF-8, converting any non-utf codeblocks to replacement characters +fn path_to_json_string(p: &Path) -> serde_json::Value { + let s = p.as_os_str().to_string_lossy().into_owned(); + serde_json::json!(s) } impl From for NixFile { diff --git a/src/ops.rs b/src/ops.rs index 0a1ddafb..134626fa 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -4,7 +4,8 @@ mod direnv; pub mod error; use crate::build_loop::BuildLoop; -use crate::build_loop::{Event, EventI, ReasonI}; +use crate::build_loop::Event; +use crate::build_loop::Reason; use crate::builder::OutputPath; use crate::cas::ContentAddressable; use crate::changelog; @@ -20,6 +21,7 @@ use crate::nix::options::NixOptions; use crate::nix::CallOpts; use crate::ops::direnv::{DirenvVersion, MIN_DIRENV_VERSION}; use crate::ops::error::{ExitAs, ExitError, ExitErrorType}; +use crate::path_to_json_string; use crate::project::{NixGcRootUserDir, Project}; use crate::run_async::Async; use crate::socket::path::SocketPath; @@ -45,6 +47,7 @@ use std::{fmt::Debug, fs::remove_dir_all}; use anyhow::Context; use crossbeam_channel as chan; +use serde_json::json; use slog::{debug, info, warn}; use thiserror::Error; @@ -633,30 +636,6 @@ impl FromStr for EventKind { } } -// These types are just transparent newtype wrappers to implement a different serde class and JsonEncode - -/// For now use the EventI structure, in the future we might want to split it off. -/// At least it will show us that we need to change something here if we change it -/// and it relates to this interface. -#[derive(Serialize)] -#[serde(transparent)] -struct StreamEvent(EventI); - -/// Nix files are encoded as strings -#[derive(Serialize)] -#[serde(transparent)] -struct StreamNixFile(String); - -/// Same here, the reason contains a nix file which has to be converted to a string. -#[derive(Serialize)] -#[serde(transparent)] -struct StreamReason(ReasonI); - -/// And same here, OutputPaths are GcRoots and have to be converted as well. -#[derive(Serialize)] -#[serde(transparent)] -struct StreamOutputPath(OutputPath); - /// Just expose the error message for now. #[derive(Serialize)] struct StreamBuildError { @@ -717,21 +696,37 @@ pub fn op_stream_events( } ev => match (snapshot_done, &kind) { (_, EventKind::All) | (false, EventKind::Snapshot) | (true, EventKind::Live) => { - fn nix_file_string(nix_file: NixFile) -> String { - nix_file.display().to_string() - } + let json: serde_json::Value = match ev { + Event::SectionEnd => json!({"SectionEnd":{}}), + Event::Started { nix_file, reason } => + json!({ + "Started": { + "nix_file": nix_file.to_json_value(), + "reason": match reason { + Reason::PingReceived => json!({"PingReceived": {}}), + Reason::FilesChanged(files) => json!({"FilesChanged": files.iter().map(|p| path_to_json_string(p)).collect::>()}) + } + } + }), + Event::Completed { nix_file, rooted_output_paths } => json!({ + "Completed": { + "nix_file": nix_file.to_json_value(), + "rooted_output_paths": { + "shell_gc_root": rooted_output_paths.shell_gc_root.0.to_json_value() + } + } + }), + Event::Failure { nix_file, failure } => json!({ + "Failure": { + "nix_file": nix_file.to_json_value(), + "failure": { "message": format!("{}", failure) } + } + }), + }; + serde_json::to_writer( std::io::stdout(), - &StreamEvent(ev.map( - |nix_file| StreamNixFile(nix_file_string(nix_file)), - |reason| StreamReason(reason.map(nix_file_string)), - |output_path| { - StreamOutputPath(output_path.map(|o| o.display().to_string())) - }, - |build_error| StreamBuildError { - message: format!("{}", build_error), - }, - )), + &json ) .expect("couldn't serialize event"); writeln!(std::io::stdout()).expect("couldn't serialize event"); diff --git a/src/project.rs b/src/project.rs index 90e3aebc..fc70e335 100644 --- a/src/project.rs +++ b/src/project.rs @@ -90,7 +90,7 @@ impl Project { } /// Return the filesystem paths for these roots. - pub fn root_paths(&self) -> OutputPath { + pub fn root_paths(&self) -> OutputPath { OutputPath { shell_gc_root: RootPath(self.shell_gc_root()), } @@ -105,7 +105,7 @@ impl Project { path: RootedPath, nix_gc_root_user_dir: NixGcRootUserDir, logger: &slog::Logger, - ) -> Result, AddRootError> + ) -> Result where { let store_path = &path.path; @@ -158,7 +158,7 @@ impl RootPath { } } -impl OutputPath { +impl OutputPath { /// Check whether all all GC roots exist. pub fn all_exist(&self) -> bool { let crate::builder::OutputPath { shell_gc_root } = self; diff --git a/tests/integration/direnvtestcase.rs b/tests/integration/direnvtestcase.rs index 57ad9553..7ce3f041 100644 --- a/tests/integration/direnvtestcase.rs +++ b/tests/integration/direnvtestcase.rs @@ -51,7 +51,7 @@ impl DirenvTestCase { } /// Execute the build loop one time - pub fn evaluate(&mut self) -> Result, BuildError> { + pub fn evaluate(&mut self) -> Result { let username = project::Username::from_env_var().unwrap(); BuildLoop::new( &self.project, From ba27237417f176599a6715401f04ca8225fad5b9 Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Sun, 14 Apr 2024 14:34:45 +0200 Subject: [PATCH 03/12] refact(builder): run -> instantiate_and_build --- src/build_loop.rs | 4 ++-- src/builder.rs | 6 +++--- src/ops.rs | 2 +- tests/shell/main.rs | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/build_loop.rs b/src/build_loop.rs index 711f55e7..116dc328 100644 --- a/src/build_loop.rs +++ b/src/build_loop.rs @@ -248,7 +248,7 @@ impl<'a> BuildLoop<'a> { let extra_nix_options = self.extra_nix_options.clone(); let logger2 = self.logger.clone(); crate::run_async::Async::run(&self.logger, move || { - builder::run(&nix_file, &cas, &extra_nix_options, &logger2) + builder::instantiate_and_build(&nix_file, &cas, &extra_nix_options, &logger2) }) } @@ -263,7 +263,7 @@ impl<'a> BuildLoop<'a> { let logger2 = self.logger.clone(); self.handle_run_result( crate::run_async::Async::run(&self.logger, move || { - builder::run(&nix_file, &cas, &extra_nix_options, &logger2) + builder::instantiate_and_build(&nix_file, &cas, &extra_nix_options, &logger2) }) .block(), ) diff --git a/src/builder.rs b/src/builder.rs index 8227a3b6..cfebcf6a 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -391,7 +391,7 @@ pub struct RunResult { /// /// Instruments the nix file to gain extra information, /// which is valuable even if the build fails. -pub fn run( +pub fn instantiate_and_build( root_nix_file: &NixFile, cas: &ContentAddressable, extra_nix_options: &NixOptions, @@ -573,7 +573,7 @@ in {} print!("{}", nix_drv); // build, because instantiate doesn’t return the build output (obviously …) - run( + instantiate_and_build( &crate::NixFile::from(cas.file_from_string(&nix_drv)?), &cas, &NixOptions::empty(), @@ -594,7 +594,7 @@ in {} &format!("dep = {};", drv("dep", r#"args = [ "-c" "exit 1" ];"#)), ))?); - if let Err(BuildError::Exit { .. }) = run( + if let Err(BuildError::Exit { .. }) = instantiate_and_build( &d, &cas, &NixOptions::empty(), diff --git a/src/ops.rs b/src/ops.rs index 134626fa..2b684611 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -441,7 +441,7 @@ fn build_root( }); // TODO: add the ability to pass extra_nix_options to shell - let run_result = builder::run( + let run_result = builder::instantiate_and_build( &project.nix_file, &project.cas, &crate::nix::options::NixOptions::empty(), diff --git a/tests/shell/main.rs b/tests/shell/main.rs index 7f686737..8e7d5b24 100644 --- a/tests/shell/main.rs +++ b/tests/shell/main.rs @@ -84,7 +84,7 @@ fn project(name: &str, cache_dir: &AbsPathBuf) -> Project { fn build(project: &Project, logger: &slog::Logger) -> PathBuf { project .create_roots( - builder::run( + builder::instantiate_and_build( &project.nix_file, &project.cas, &NixOptions::empty(), From 6630b36a40377b6b03bae72cc88b0ee0638bbc62 Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Sun, 14 Apr 2024 17:22:00 +0200 Subject: [PATCH 04/12] feat(thread): Add `race()` function for racing two threads --- src/thread.rs | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/src/thread.rs b/src/thread.rs index 05c7cd98..584b5c2e 100644 --- a/src/thread.rs +++ b/src/thread.rs @@ -7,6 +7,7 @@ //! we don't block joining one thread while another thread has panicked //! already. +use chan::Receiver; use crossbeam_channel as chan; use slog::debug; use std::any::Any; @@ -14,6 +15,8 @@ use std::collections::HashMap; use std::thread; use std::thread::ThreadId; +use crate::Never; + struct Thread { name: String, join_handle: thread::JoinHandle<()>, @@ -133,9 +136,121 @@ impl Pool { // The thread died successfully Cause::Natural(Ok(())) => {} // The thread didn’t panic, it returned an error, so we return early + // TODO: this will not join all threads?? Why is the return here? Sounds wrong. Cause::Natural(Err(err)) => return Err(err), Cause::Paniced(panic) => std::panic::resume_unwind(panic), } } } } + +/// This value should be returned from a racing thread when the stop signal has been received. Do not construct (TODO: how to prevent other modules from constructing?) +#[derive(Debug)] +pub struct StopReceived(); + +/// Race two closures. +/// +/// Each closure is passed a channel that is sent a single message when the other closure has finished. +/// Ideally, the closure then stops, but it’s up to the closure, we cannot interrupt threads (cooperative multitasking). +#[allow(private_bounds)] +pub fn race(logger: &slog::Logger, first: F, second: G) -> Res +where + F: FnOnce(Receiver) -> Result, + F: std::panic::UnwindSafe, + F: Send + 'static, + G: FnOnce(Receiver) -> Result, + G: std::panic::UnwindSafe, + G: Send + 'static, + Res: Send + 'static, +{ + let (one_res_tx, one_res_rx) = chan::bounded(1); + let (two_res_tx, two_res_rx) = chan::bounded(1); + let (one_stop_tx, one_stop_rx) = chan::bounded::(1); + let (two_stop_tx, two_stop_rx) = chan::bounded::(1); + let mut thread: Pool = Pool::new(logger.clone()); + thread.spawn("racing thread 1", move || { + let res = first(one_stop_rx); + match one_res_tx.try_send(res) { + Ok(()) => Ok(()), + Err(err) => panic!("unable to send the first racing result, because the channel was disconnected (should never happen): {:?}", err) + } + }).expect("unable to spawn racing thread, should not happen"); + thread.spawn("racing thread 2", move || { + let res = second(two_stop_rx); + match two_res_tx.try_send(res) { + Ok(()) => Ok(()), + Err(err) => panic!("unable to send the second racing result, because the channel was disconnected (should never happen): {:?}", err) + } + }).expect("unable to spawn racing thread, should not happen"); + + let res = chan::select! { + recv(one_res_rx) -> res => res, + recv(two_res_rx) -> res => res, + } + .expect("Could not receive async results"); + // ask the channels to stop + let _ = one_stop_tx.try_send(StopReceived()); + let _ = two_stop_tx.try_send(StopReceived()); + // join both threads + thread.join_all_or_panic().unwrap_or_else(|n| n.never()); + res.expect("Should never receive a stop message, because we already got the result from the other thread.") +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[test] + fn test_race() { + // one stops when requested, two will ignore the stop command entirely but send a msg after 10ms + let mk = |timeout1, timeout2, two_msg_tx: Option>| { + race( + &crate::logging::test_logger(), + move |stop| { + chan::select! { + recv(stop) -> s => { + println!("1: received request to stop, stopping"); + Err(s.unwrap()) + }, + recv(chan::after(timeout1)) -> _ => { + println!("1: timed out, returning result"); + Ok("1: myresult") + } + } + }, + move |_stop| { + chan::select! { + recv(chan::after(timeout2)) -> _ => { + println!("2: timed out, returning result"); + Ok("2: otherresult") + } + recv(chan::after(Duration::from_millis(10))) -> _ => { + match two_msg_tx { + Some(tx) => tx.send(()).unwrap(), + None => {} + } + Ok("hit the message timeout") + } + } + }, + ) + }; + let one = mk(Duration::from_millis(5), Duration::from_millis(10), None); + assert_eq!(one, "1: myresult", "one was shorter"); + let two = mk(Duration::from_millis(100), Duration::from_millis(5), None); + assert_eq!(two, "2: otherresult", "two was shorter"); + + let (two_msg_tx, two_msg_rx) = chan::bounded::<()>(1); + let three = mk( + Duration::from_millis(5), + Duration::from_millis(1000), + Some(two_msg_tx), + ); + assert_eq!(three, "1: myresult", "one finishes, while two still runs"); + two_msg_rx + .recv_timeout(Duration::from_millis(30)) + .expect("We expect the other thread to send a message after the first one won the race") + } +} From d6f7d49a66975095236227c38841e4a5a3ea9d37 Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Sun, 14 Apr 2024 17:22:21 +0200 Subject: [PATCH 05/12] refact(ops/shell): Rephrase Arc code in terms of `thread::race()` This new function makes it easy to race two threads, instead of syncing on shared state. This also simplifies the duration code, since all we are doing is waiting on timeouts and the stop channel. --- src/ops.rs | 86 +++++++++++++++++++++++++++++------------------------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/src/ops.rs b/src/ops.rs index 2b684611..3a35d2cf 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -36,12 +36,10 @@ use std::path::Path; use std::path::PathBuf; use std::process::Command; use std::str::FromStr; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; use std::time::Duration; use std::time::Instant; use std::{collections::HashSet, fs::File, time::SystemTime}; -use std::{env, fs, io, thread}; +use std::{env, fs, io}; use std::{fmt::Debug, fs::remove_dir_all}; use anyhow::Context; @@ -411,44 +409,54 @@ fn build_root( nix_gc_root_user_dir: NixGcRootUserDir, logger: &slog::Logger, ) -> Result { - let building = Arc::new(AtomicBool::new(true)); - let building_clone = building.clone(); let logger2 = logger.clone(); - let progress_thread = Async::run(logger, move || { - // Keep track of the start time to display a hint to the user that they can use `--cached`, - // but only if a cached version of the environment exists - let mut start = if cached { Some(Instant::now()) } else { None }; - - eprint!("lorri: building environment"); - while building_clone.load(Ordering::SeqCst) { - // Show `--cached` hint once after some time has passed - if let Some(start_time) = start { - if start_time.elapsed() >= Duration::from_millis(10_000) { - eprintln!( - "\nHint: you can use `lorri shell --cached` to use the most recent \ - environment that was built successfully." - ); - start = None; // Don't show the hint again - } - } - thread::sleep(Duration::from_millis(500)); - - // Indicate progress - eprint!("."); - io::stderr().flush().expect("couldn’t flush‽"); - } - eprintln!(". done"); - }); + let project2 = project.clone(); - // TODO: add the ability to pass extra_nix_options to shell - let run_result = builder::instantiate_and_build( - &project.nix_file, - &project.cas, - &crate::nix::options::NixOptions::empty(), - &logger2, + let run_result = crate::thread::race( + logger, + move |_ignored_stop| { + Ok(builder::instantiate_and_build( + &project2.nix_file, + &project2.cas, + &crate::nix::options::NixOptions::empty(), + &logger2, + )) + }, + // display a a progress bar on stderr while the shell is loading + move |stop| { + // Keep track of the start time to display a hint to the user that they can use `--cached`, + // but only if a cached version of the environment exists + let hint_time = Instant::now() + Duration::from_secs(10); + eprint!("lorri: building environment"); + loop { + let now = Instant::now(); + + let show_hint_after = if cached && hint_time > now { + chan::after(hint_time - now) + } else { + chan::never() + }; + + chan::select!( + recv(stop) -> stop => { + eprintln!(". done"); + return Err(stop.unwrap()); + }, + recv(show_hint_after) -> _ => { + eprintln!( + "\nHint: you can use `lorri shell --cached` to use the most recent \ + environment that was built successfully." + ); + }, + recv(chan::after(Duration::from_millis(500))) -> _ => { + // Indicate progress + eprint!("."); + io::stderr().flush().expect("couldn’t flush‽"); + } + ); + } + }, ); - building.store(false, Ordering::SeqCst); - progress_thread.block(); let run_result = run_result .map_err(|e| { @@ -470,7 +478,7 @@ fn build_root( .result; Ok(project - .create_roots(run_result, nix_gc_root_user_dir, &logger2) + .create_roots(run_result, nix_gc_root_user_dir, &logger) .map_err(|e| { ExitError::temporary(anyhow::Error::new(e).context("rooting the environment failed")) })? From cfe2f6cf8fffb6fd1fee3b11fcc978866a820cb5 Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Sun, 14 Apr 2024 17:25:49 +0200 Subject: [PATCH 06/12] fix(ops/shell): shorten hint time to 3s --- src/ops.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ops.rs b/src/ops.rs index 3a35d2cf..e8634a06 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -426,7 +426,7 @@ fn build_root( move |stop| { // Keep track of the start time to display a hint to the user that they can use `--cached`, // but only if a cached version of the environment exists - let hint_time = Instant::now() + Duration::from_secs(10); + let hint_time = Instant::now() + Duration::from_secs(3); eprint!("lorri: building environment"); loop { let now = Instant::now(); From e6aad7a6798f46f75a5f35d837dc29035c1655b3 Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Sun, 14 Apr 2024 17:27:59 +0200 Subject: [PATCH 07/12] refact(ops/shell): inline cached_root --- src/ops.rs | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/ops.rs b/src/ops.rs index e8634a06..1c7b841e 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -367,7 +367,15 @@ pub fn op_shell( })?; let username = project::Username::from_env_var().map_err(ExitError::environment_problem)?; let nix_gc_root_user_dir = project::NixGcRootUserDir::get_or_create(&username)?; - let cached = cached_root(&project); + let cached = { + if !project.root_paths().all_exist() { + Err(ExitError::temporary(anyhow::anyhow!( + "project has not previously been built successfully", + ))) + } else { + Ok(project.root_paths().shell_gc_root.0.as_path().to_owned()) + } + }; let mut bash_cmd = bash_cmd( if opts.cached { cached? @@ -488,17 +496,6 @@ fn build_root( .to_owned()) } -fn cached_root(project: &Project) -> Result { - let root_paths = project.root_paths(); - if !root_paths.all_exist() { - Err(ExitError::temporary(anyhow::anyhow!( - "project has not previously been built successfully", - ))) - } else { - Ok(root_paths.shell_gc_root.0.as_path().to_owned()) - } -} - /// Instantiates a `Command` to start bash. pub fn bash_cmd( project_root: PathBuf, From 47b1f9156ea19a1973a6684a694c0d22c96bd029 Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Sun, 14 Apr 2024 18:07:32 +0200 Subject: [PATCH 08/12] refact(ops/gc): generate json via macro Instead of using serde, defining a clear API boundary (preventing breakage on refactors). --- src/main.rs | 2 +- src/ops.rs | 25 ++++++++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/main.rs b/src/main.rs index 8373c3fc..d383abb2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -118,7 +118,7 @@ fn run_command(logger: &slog::Logger, opts: Arguments) -> Result<(), ExitError> let (project, _logger) = with_project(&nix_file)?; ops::op_info(&paths, project, &logger) } - Command::Gc(opts) => ops::gc(logger, opts), + Command::Gc(opts) => ops::op_gc(logger, opts), Command::Direnv(opts) => { let (project, logger) = with_project(&opts.nix_file)?; ops::op_direnv( diff --git a/src/ops.rs b/src/ops.rs index 1c7b841e..981e064a 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -25,6 +25,7 @@ use crate::path_to_json_string; use crate::project::{NixGcRootUserDir, Project}; use crate::run_async::Async; use crate::socket::path::SocketPath; +use crate::AbsPathBuf; use crate::NixFile; use crate::VERSION_BUILD_REV; use crate::{builder, project}; @@ -46,6 +47,7 @@ use anyhow::Context; use crossbeam_channel as chan; use serde_json::json; +use serde_json::Value; use slog::{debug, info, warn}; use thiserror::Error; @@ -961,7 +963,7 @@ fn main_run_once( /// Represents a gc root along with some metadata, used for json output of lorri gc info struct GcRootInfo { /// directory where root is stored - gc_dir: PathBuf, + gc_dir: AbsPathBuf, /// nix file from which the root originates. If None, then the root is considered dead. nix_file: Option, /// timestamp of the last build @@ -985,7 +987,7 @@ fn list_roots(logger: &slog::Logger) -> Result, ExitError> { ); continue; } - let gc_dir = entry.path(); + let gc_dir = AbsPathBuf::new(entry.path()).expect("entry.path() should always be absolute"); let gc_root_dir = gc_dir.join("gc_root"); if !std::fs::metadata(&gc_root_dir).map_or(false, |m| m.is_dir()) { debug!( @@ -1035,13 +1037,26 @@ struct RemovalStatus { } /// Print or remove gc roots depending on cli options. -pub fn gc(logger: &slog::Logger, opts: crate::cli::GcOptions) -> Result<(), ExitError> { +pub fn op_gc(logger: &slog::Logger, opts: crate::cli::GcOptions) -> Result<(), ExitError> { let infos = list_roots(logger)?; match opts.action { cli::GcSubcommand::Info => { if opts.json { - serde_json::to_writer(std::io::stdout(), &infos) - .expect("could not serialize gc roots"); + serde_json::to_writer( + std::io::stdout(), + &infos + .iter() + .map(|info| { + json!({ + "gc_dir": info.gc_dir.to_json_value(), + "nix_file": info.nix_file.as_ref().map_or(Value::Null, |n| path_to_json_string(&n)), + "timestamp": info.timestamp, + "alive": info.alive + }) + }) + .collect::>(), + ) + .expect("could not serialize gc roots"); } else { for info in infos { let target = match info.nix_file { From 2208ddc6f44eb6ddcab00a2f233cdbdc5710a3fc Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Sun, 14 Apr 2024 18:54:11 +0200 Subject: [PATCH 09/12] refact(*): Remove ContentAddressable from Project & pass manually MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We only pass the `cas` through the `Project` constructor, never actually using it in the project `impl`!. This means we don’t need to burden `Project` with the cas, since it’s only used by the builder in the end. Insead, put it on the build loop for now. Funnily enough, `StartUserShell` does not need the project init anymore, cause it only wanted the `cas` from it. --- src/build_loop.rs | 9 +++++++-- src/daemon.rs | 4 +++- src/main.rs | 11 ++++------- src/ops.rs | 30 +++++++++++++++++++++-------- src/project.rs | 11 +---------- tests/integration/direnvtestcase.rs | 5 ++++- tests/shell/main.rs | 27 ++++++++++---------------- 7 files changed, 51 insertions(+), 46 deletions(-) diff --git a/src/build_loop.rs b/src/build_loop.rs index 116dc328..fde3e590 100644 --- a/src/build_loop.rs +++ b/src/build_loop.rs @@ -2,6 +2,7 @@ //! evaluate and build a given Nix file. use crate::builder::{self, BuildError}; +use crate::cas::ContentAddressable; use crate::daemon::LoopHandlerEvent; use crate::nix::options::NixOptions; use crate::pathreduction::reduce_paths; @@ -67,6 +68,8 @@ pub struct BuildLoop<'a> { /// Watches all input files for changes. /// As new input files are discovered, they are added to the watchlist. watch: Watch, + /// Content addressable store to save static files in. + cas: ContentAddressable, nix_gc_root_user_dir: project::NixGcRootUserDir, logger: slog::Logger, } @@ -109,6 +112,7 @@ impl<'a> BuildLoop<'a> { project: &'a Project, extra_nix_options: NixOptions, nix_gc_root_user_dir: project::NixGcRootUserDir, + cas: ContentAddressable, logger: slog::Logger, ) -> anyhow::Result> { let mut watch = Watch::try_new(logger.clone()).map_err(|err| anyhow!(err))?; @@ -128,6 +132,7 @@ impl<'a> BuildLoop<'a> { extra_nix_options, watch, nix_gc_root_user_dir, + cas, logger, }) } @@ -244,7 +249,7 @@ impl<'a> BuildLoop<'a> { /// Start an actual build, asynchronously. fn start_build(&self) -> Async> { let nix_file = self.project.nix_file.clone(); - let cas = self.project.cas.clone(); + let cas = self.cas.clone(); let extra_nix_options = self.extra_nix_options.clone(); let logger2 = self.logger.clone(); crate::run_async::Async::run(&self.logger, move || { @@ -258,7 +263,7 @@ impl<'a> BuildLoop<'a> { /// the evaluation. pub fn once(&mut self) -> Result { let nix_file = self.project.nix_file.clone(); - let cas = self.project.cas.clone(); + let cas = self.cas.clone(); let extra_nix_options = self.extra_nix_options.clone(); let logger2 = self.logger.clone(); self.handle_run_result( diff --git a/src/daemon.rs b/src/daemon.rs index 685981d2..205b0911 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -187,7 +187,7 @@ impl Daemon { // For each build instruction, add the corresponding file // to the watch list. for IndicateActivity { nix_file, rebuild } in rx_activity { - let project = crate::project::Project::new(nix_file, gc_root_dir, cas.clone()) + let project = crate::project::Project::new(nix_file, gc_root_dir) // TODO: the project needs to create its gc root dir .unwrap(); @@ -215,6 +215,7 @@ impl Daemon { let nix_gc_root_user_dir = nix_gc_root_user_dir.clone(); let logger = logger.clone(); let logger2 = logger.clone(); + let cas2 = cas.clone(); // TODO: how to use the pool here? // We cannot just spawn new threads once messages come in, // because then then pool objects is stuck in this loop @@ -228,6 +229,7 @@ impl Daemon { &project, extra_nix_options, nix_gc_root_user_dir, + cas2, logger, ) { Ok(mut build_loop) => { diff --git a/src/main.rs b/src/main.rs index d383abb2..49a84a12 100644 --- a/src/main.rs +++ b/src/main.rs @@ -87,7 +87,7 @@ fn find_nix_file(shellfile: &Path) -> Result { } fn create_project(paths: &constants::Paths, shell_nix: NixFile) -> Result { - Project::new(shell_nix, paths.gc_root_dir(), paths.cas_store().clone()).map_err(|err| { + Project::new(shell_nix, paths.gc_root_dir()).map_err(|err| { ExitError::temporary(anyhow::anyhow!(err).context("Could not set up project paths")) }) } @@ -130,12 +130,12 @@ fn run_command(logger: &slog::Logger, opts: Arguments) -> Result<(), ExitError> } Command::Shell(opts) => { let (project, logger) = with_project(&opts.nix_file)?; - ops::op_shell(project, opts, &logger) + ops::op_shell(project, paths.cas_store(), opts, &logger) } Command::Watch(opts) => { let (project, logger) = with_project(&opts.nix_file)?; - ops::op_watch(project, opts, &logger) + ops::op_watch(project, paths.cas_store(), opts, &logger) } Command::Daemon(opts) => { install_signal_handler(); @@ -149,10 +149,7 @@ fn run_command(logger: &slog::Logger, opts: Arguments) -> Result<(), ExitError> let nix_file = find_nix_file(&opts.nix_file)?; ops::op_ping(&paths, nix_file, logger) } - Internal_::StartUserShell_(opts) => { - let (project, _logger) = with_project(&opts.nix_file)?; - ops::op_start_user_shell(project, opts) - } + Internal_::StartUserShell_(opts) => ops::op_start_user_shell(paths.cas_store(), opts), Internal_::StreamEvents_(se) => ops::op_stream_events(&paths, se.kind, logger), }, } diff --git a/src/ops.rs b/src/ops.rs index 981e064a..58c09f16 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -356,6 +356,7 @@ pub fn op_ping(paths: &Paths, nix_file: NixFile, logger: &slog::Logger) -> Resul /// way the prompt looks. pub fn op_shell( project: Project, + cas: &ContentAddressable, opts: ShellOptions, logger: &slog::Logger, ) -> Result<(), ExitError> { @@ -382,9 +383,9 @@ pub fn op_shell( if opts.cached { cached? } else { - build_root(&project, cached.is_ok(), nix_gc_root_user_dir, logger)? + build_root(&project, cached.is_ok(), nix_gc_root_user_dir, cas, logger)? }, - &project.cas, + cas, logger, )?; @@ -417,17 +418,19 @@ fn build_root( project: &Project, cached: bool, nix_gc_root_user_dir: NixGcRootUserDir, + cas: &ContentAddressable, logger: &slog::Logger, ) -> Result { let logger2 = logger.clone(); let project2 = project.clone(); + let cas2 = cas.clone(); let run_result = crate::thread::race( logger, move |_ignored_stop| { Ok(builder::instantiate_and_build( &project2.nix_file, - &project2.cas, + &cas2, &crate::nix::options::NixOptions::empty(), &logger2, )) @@ -535,7 +538,7 @@ EVALUATION_ROOT="{}" /// /// See the documentation for `crate::ops::shell`. pub fn op_start_user_shell( - project: Project, + cas: &ContentAddressable, opts: StartUserShellOptions_, ) -> Result<(), ExitError> { // This temporary directory will not be cleaned up by lorri because we exec into the shell @@ -543,7 +546,7 @@ pub fn op_start_user_shell( // lorri creates in this directory are only a few hundred bytes long; (2) the directory will be // cleaned up on reboot or whenever the OS decides to purge temporary directories. let tempdir = tempfile::tempdir().expect("failed to create temporary directory"); - let e = shell_cmd(opts.shell_path.as_ref(), &project.cas, tempdir.path()).exec(); + let e = shell_cmd(opts.shell_path.as_ref(), cas, tempdir.path()).exec(); // 'exec' will never return on success, so if we get here, we know something has gone wrong. panic!("failed to exec into '{}': {}", opts.shell_path.display(), e); @@ -917,21 +920,23 @@ pub fn op_upgrade( /// details. pub fn op_watch( project: Project, + cas: &ContentAddressable, opts: WatchOptions, logger: &slog::Logger, ) -> Result<(), ExitError> { let username = project::Username::from_env_var().map_err(ExitError::temporary)?; let nix_gc_root_user_dir = project::NixGcRootUserDir::get_or_create(&username)?; if opts.once { - main_run_once(project, nix_gc_root_user_dir, logger) + main_run_once(project, nix_gc_root_user_dir, cas, logger) } else { - main_run_forever(project, nix_gc_root_user_dir, logger) + main_run_forever(project, nix_gc_root_user_dir, cas, logger) } } fn main_run_once( project: Project, nix_gc_root_user_dir: NixGcRootUserDir, + cas: &ContentAddressable, logger: &slog::Logger, ) -> Result<(), ExitError> { // TODO: add the ability to pass extra_nix_options to watch @@ -939,6 +944,7 @@ fn main_run_once( &project, NixOptions::empty(), nix_gc_root_user_dir, + cas.clone(), logger.clone(), ) .map_err(ExitError::temporary)?; @@ -1143,15 +1149,23 @@ pub fn op_gc(logger: &slog::Logger, opts: crate::cli::GcOptions) -> Result<(), E fn main_run_forever( project: Project, nix_gc_root_user_dir: NixGcRootUserDir, + cas: &ContentAddressable, logger: &slog::Logger, ) -> Result<(), ExitError> { let (tx_build_results, rx_build_results) = chan::unbounded(); let (tx_ping, rx_ping) = chan::unbounded(); let logger2 = logger.clone(); + let cas2 = cas.clone(); // TODO: add the ability to pass extra_nix_options to watch let build_thread = { Async::run(logger, move || { - match BuildLoop::new(&project, NixOptions::empty(), nix_gc_root_user_dir, logger2) { + match BuildLoop::new( + &project, + NixOptions::empty(), + nix_gc_root_user_dir, + cas2, + logger2, + ) { Ok(mut bl) => bl.forever(tx_build_results, rx_ping).never(), Err(e) => Err(ExitError::temporary(e)), } diff --git a/src/project.rs b/src/project.rs index fc70e335..5d28fd97 100644 --- a/src/project.rs +++ b/src/project.rs @@ -4,7 +4,6 @@ use slog::debug; use thiserror::Error; use crate::builder::{OutputPath, RootedPath}; -use crate::cas::ContentAddressable; use crate::ops::error::ExitError; use crate::{AbsPathBuf, NixFile}; use std::ffi::{CString, OsString}; @@ -24,20 +23,13 @@ pub struct Project { /// Hash of the nix file’s absolute path. hash: String, - - /// Content-addressable store to save static files in - pub cas: ContentAddressable, } impl Project { /// Construct a `Project` from nix file path /// and the base GC root directory /// (as returned by `Paths.gc_root_dir()`), - pub fn new( - nix_file: NixFile, - gc_root_dir: &AbsPathBuf, - cas: ContentAddressable, - ) -> std::io::Result { + pub fn new(nix_file: NixFile, gc_root_dir: &AbsPathBuf) -> std::io::Result { let hash = format!( "{:x}", md5::compute(nix_file.as_absolute_path().as_os_str().as_bytes()) @@ -74,7 +66,6 @@ impl Project { nix_file, gc_root_path: project_gc_root, hash, - cas, }) } diff --git a/tests/integration/direnvtestcase.rs b/tests/integration/direnvtestcase.rs index 7ce3f041..5469ee31 100644 --- a/tests/integration/direnvtestcase.rs +++ b/tests/integration/direnvtestcase.rs @@ -25,6 +25,7 @@ pub struct DirenvTestCase { #[allow(dead_code)] pub cachedir: TempDir, project: Project, + cas: ContentAddressable, logger: slog::Logger, } @@ -40,12 +41,13 @@ impl DirenvTestCase { let shell_file = NixFile::from(AbsPathBuf::new(test_root.join("shell.nix")).unwrap()); let cas = ContentAddressable::new(cachedir.join("cas").to_owned()).unwrap(); - let project = Project::new(shell_file.clone(), &cachedir.join("gc_roots"), cas).unwrap(); + let project = Project::new(shell_file.clone(), &cachedir.join("gc_roots")).unwrap(); DirenvTestCase { projectdir, cachedir: cachedir_tmp, project, + cas, logger: lorri::logging::test_logger(), } } @@ -57,6 +59,7 @@ impl DirenvTestCase { &self.project, NixOptions::empty(), project::NixGcRootUserDir::get_or_create(&username).unwrap(), + self.cas.clone(), self.logger.clone(), ) .expect("could not set up build loop") diff --git a/tests/shell/main.rs b/tests/shell/main.rs index 8e7d5b24..e2ab5895 100644 --- a/tests/shell/main.rs +++ b/tests/shell/main.rs @@ -25,10 +25,8 @@ fn cargo_bin(name: &str) -> PathBuf { #[test] fn loads_env() { let tempdir = tempfile::tempdir().expect("tempfile::tempdir() failed us!"); - let project = project( - "loads_env", - &lorri::AbsPathBuf::new(tempdir.path().to_owned()).unwrap(), - ); + let cache_dir = &lorri::AbsPathBuf::new(tempdir.path().to_owned()).unwrap(); + let project = project("loads_env", cache_dir); // Launch as a real user let res = Command::new(cargo_bin("lorri")) @@ -48,8 +46,11 @@ fn loads_env() { assert!(res.status.success(), "lorri shell command failed"); let logger = lorri::logging::test_logger(); + let cas_dir = cache_dir.join("cas").to_owned(); + fs::create_dir_all(&cas_dir).expect("failed to create CAS directory"); + let cas = ContentAddressable::new(cas_dir).unwrap(); - let output = ops::bash_cmd(build(&project, &logger), &project.cas, &logger) + let output = ops::bash_cmd(build(&project, &cas, &logger), &cas, &logger) .unwrap() .args(["-c", "echo $MY_ENV_VAR"]) .output() @@ -71,27 +72,19 @@ fn project(name: &str, cache_dir: &AbsPathBuf) -> Project { name, ])) .expect("CARGO_MANIFEST_DIR was not absolute"); - let cas_dir = cache_dir.join("cas").to_owned(); - fs::create_dir_all(&cas_dir).expect("failed to create CAS directory"); Project::new( NixFile::from(test_root.join("shell.nix")), &cache_dir.join("gc_roots"), - ContentAddressable::new(cas_dir).unwrap(), ) .unwrap() } -fn build(project: &Project, logger: &slog::Logger) -> PathBuf { +fn build(project: &Project, cas: &ContentAddressable, logger: &slog::Logger) -> PathBuf { project .create_roots( - builder::instantiate_and_build( - &project.nix_file, - &project.cas, - &NixOptions::empty(), - logger, - ) - .unwrap() - .result, + builder::instantiate_and_build(&project.nix_file, cas, &NixOptions::empty(), logger) + .unwrap() + .result, project::NixGcRootUserDir::get_or_create(&project::Username::from_env_var().unwrap()) .unwrap(), logger, From 3258e3e1746b8af80d12db2b67c379e0b829e3e4 Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Mon, 15 Apr 2024 00:02:12 +0200 Subject: [PATCH 10/12] =?UTF-8?q?refact(ops/op=5Fgc):=20don=E2=80=99t=20qu?= =?UTF-8?q?it=20on=20first=20error=20for=20non-json?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collect errors and continue deleting when json is not on. --- Cargo.lock | 16 +++++++++++++ Cargo.nix | 39 +++++++++++++++++++++++++++++++ Cargo.toml | 7 ++++++ src/ops.rs | 68 ++++++++++++++++++++++++++++++++---------------------- 4 files changed, 102 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6f3b027..4e3d28f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -244,6 +244,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "either" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" + [[package]] name = "errno" version = "0.3.8" @@ -435,6 +441,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -503,6 +518,7 @@ dependencies = [ "directories", "fastrand 1.9.0", "human-panic", + "itertools", "lazy_static", "md5", "nix 0.20.2", diff --git a/Cargo.nix b/Cargo.nix index 50a922ca..f51b430f 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -715,6 +715,20 @@ rec { ]; }; + "either" = rec { + crateName = "either"; + version = "1.11.0"; + edition = "2018"; + sha256 = "18l0cwyw18syl8b52syv6balql8mnwfyhihjqqllx5pms93iqz54"; + authors = [ + "bluss" + ]; + features = { + "default" = [ "use_std" ]; + "serde" = [ "dep:serde" ]; + }; + resolvedDefaultFeatures = [ "use_std" ]; + }; "errno" = rec { crateName = "errno"; version = "0.3.8"; @@ -1220,6 +1234,27 @@ rec { ]; }; + "itertools" = rec { + crateName = "itertools"; + version = "0.12.1"; + edition = "2018"; + sha256 = "0s95jbb3ndj1lvfxyq5wanc0fm0r6hg6q4ngb92qlfdxvci10ads"; + authors = [ + "bluss" + ]; + dependencies = [ + { + name = "either"; + packageId = "either"; + usesDefaultFeatures = false; + } + ]; + features = { + "default" = [ "use_std" ]; + "use_std" = [ "use_alloc" "either/use_std" ]; + }; + resolvedDefaultFeatures = [ "default" "use_alloc" "use_std" ]; + }; "itoa" = rec { crateName = "itoa"; version = "1.0.11"; @@ -1417,6 +1452,10 @@ rec { name = "human-panic"; packageId = "human-panic"; } + { + name = "itertools"; + packageId = "itertools"; + } { name = "lazy_static"; packageId = "lazy_static"; diff --git a/Cargo.toml b/Cargo.toml index c22c7da1..81bdf06d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,11 @@ thiserror = "1.0" # TODO: update to 0.3 structopt.version = "0.2" + +# TODO: update to 0.3 structopt.default-features = false + +# TODO: update to 0.3 structopt.features = [ # "default", "suggestions", @@ -55,11 +59,14 @@ lazy_static = "1.4.0" md5 = "0.7.0" vec1 = ">= 1.1.0, <1.7.0" human-panic = { path = "vendor/human-panic" } +itertools = "0.12.1" [dev-dependencies] # 1.0.0 requires at least rust 1.50 proptest.version = "0.10.1" +# 1.0.0 requires at least rust 1.50 proptest.default-features = false +# 1.0.0 requires at least rust 1.50 proptest.features = [ "std", # reenable if proptest kills the test runner diff --git a/src/ops.rs b/src/ops.rs index 58c09f16..b8a9821b 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -46,6 +46,7 @@ use std::{fmt::Debug, fs::remove_dir_all}; use anyhow::Context; use crossbeam_channel as chan; +use itertools::Itertools; use serde_json::json; use serde_json::Value; use slog::{debug, info, warn}; @@ -1033,15 +1034,6 @@ fn list_roots(logger: &slog::Logger) -> Result, ExitError> { Ok(res) } -#[derive(Serialize)] -/// How removing a gc root went, for json ouput -struct RemovalStatus { - /// What error was encountered, or success - error: Option, - /// The root we tried to remove - root: GcRootInfo, -} - /// Print or remove gc roots depending on cli options. pub fn op_gc(logger: &slog::Logger, opts: crate::cli::GcOptions) -> Result<(), ExitError> { let infos = list_roots(logger)?; @@ -1109,35 +1101,47 @@ pub fn op_gc(logger: &slog::Logger, opts: crate::cli::GcOptions) -> Result<(), E }) .collect(); let mut result = Vec::new(); - let n = to_remove.len(); for info in to_remove { match remove_dir_all(&info.gc_dir) { Ok(_) => { - if opts.json { - result.push(RemovalStatus { - error: None, - root: info, - }); - } + result.push(Ok(info)); } Err(e) => { - if opts.json { - result.push(RemovalStatus { - error: Some(e.to_string()), - root: info, - }); - } else { - eprintln!("Error: could not remove {}: {}", info.gc_dir.display(), e); - } + result.push(Err((info, e.to_string()))); } } } if opts.json { - serde_json::to_writer(std::io::stdout(), &result) - .expect("failed to serialize result"); + let res = result + .into_iter() + .map(|r| match r { + Err((info, err)) => json!({ + // Error, if any + "error": err, + // The root we tried to remove + "root": info + }), + Ok(info) => json!({ + "error": null, + "root": info + }), + }) + .collect::>(); + serde_json::to_writer(std::io::stdout(), &res).expect("failed to serialize result"); } else { - println!("Removed {} gc roots.", n); - if n > 0 { + let (ok, err): (Vec<_>, Vec<_>) = result.into_iter().partition_result(); + println!("Removed {} gc roots.", ok.len()); + if err.len() > 0 { + for (info, e) in err { + warn!( + logger, + "Failed to remove gc root: {}: {}", + info.gc_dir.display(), + e + ) + } + } + if ok.len() > 0 { println!("Remember to run nix-collect-garbage to actually free space."); } } @@ -1145,6 +1149,14 @@ pub fn op_gc(logger: &slog::Logger, opts: crate::cli::GcOptions) -> Result<(), E } Ok(()) } +#[derive(Serialize)] +/// How removing a gc root went, for json ouput +struct RemovalStatus { + /// What error was encountered, or success + error: Option, + /// The root we tried to remove + root: GcRootInfo, +} fn main_run_forever( project: Project, From e00176277865993d729a8085540328f2f5a0d23c Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Mon, 15 Apr 2024 00:15:05 +0200 Subject: [PATCH 11/12] feat(ops/op_gc): add --dry-run for `gc rm` --- src/cli.rs | 3 ++ src/ops.rs | 132 ++++++++++++++++++++++++++++++----------------------- 2 files changed, 79 insertions(+), 56 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index aad3e583..b30fddfc 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -181,6 +181,9 @@ pub enum GcSubcommand { /// Also delete the root of projects that were last built before this amount of time, e.g. 30d. #[structopt(long = "older-than", parse(try_from_str = "human_friendly_duration"))] older_than: Option, + /// Only print which gc roots would be deleted. No `--json` output yet. + #[structopt(long = "dry-run")] + dry_run: bool, }, } diff --git a/src/ops.rs b/src/ops.rs index b8a9821b..b39f0615 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -979,6 +979,30 @@ struct GcRootInfo { alive: bool, } +impl GcRootInfo { + fn format_pretty_oneline(&self) -> String { + let target = match &self.nix_file { + Some(p) => p.display().to_string(), + None => "(?)".to_owned(), + }; + let age = match self.timestamp.elapsed() { + Err(_) => "future".to_owned(), + Ok(d) => { + let days = d.as_secs() / (24 * 60 * 60); + format!("{} days ago", days) + } + }; + let alive = if self.alive { "" } else { "[dead]" }; + format!( + "{} -> {} {} ({})", + self.gc_dir.display(), + target, + alive, + age + ) + } +} + /// Returns a list of existing gc roots along with some metadata fn list_roots(logger: &slog::Logger) -> Result, ExitError> { let paths = crate::ops::get_paths()?; @@ -1057,25 +1081,7 @@ pub fn op_gc(logger: &slog::Logger, opts: crate::cli::GcOptions) -> Result<(), E .expect("could not serialize gc roots"); } else { for info in infos { - let target = match info.nix_file { - Some(p) => p.display().to_string(), - None => "(?)".to_owned(), - }; - let age = match info.timestamp.elapsed() { - Err(_) => "future".to_owned(), - Ok(d) => { - let days = d.as_secs() / (24 * 60 * 60); - format!("{} days ago", days) - } - }; - let alive = if info.alive { "" } else { "[dead]" }; - println!( - "{} -> {} {} ({})", - info.gc_dir.display(), - target, - alive, - age - ); + println!("{}", info.format_pretty_oneline()); } } } @@ -1083,6 +1089,7 @@ pub fn op_gc(logger: &slog::Logger, opts: crate::cli::GcOptions) -> Result<(), E shell_file, all, older_than, + dry_run, } => { let files_to_remove: HashSet = shell_file.into_iter().collect(); let to_remove: Vec = infos @@ -1101,54 +1108,67 @@ pub fn op_gc(logger: &slog::Logger, opts: crate::cli::GcOptions) -> Result<(), E }) .collect(); let mut result = Vec::new(); - for info in to_remove { - match remove_dir_all(&info.gc_dir) { - Ok(_) => { - result.push(Ok(info)); - } - Err(e) => { - result.push(Err((info, e.to_string()))); + if dry_run { + if to_remove.len() > 0 { + println!("--dry-run: Would delete the following GC roots:"); + for info in to_remove { + println!("{}", info.format_pretty_oneline()); } + } else { + println!("--dry-run: Would not delete any GC roots"); } - } - if opts.json { - let res = result - .into_iter() - .map(|r| match r { - Err((info, err)) => json!({ - // Error, if any - "error": err, - // The root we tried to remove - "root": info - }), - Ok(info) => json!({ - "error": null, - "root": info - }), - }) - .collect::>(); - serde_json::to_writer(std::io::stdout(), &res).expect("failed to serialize result"); } else { - let (ok, err): (Vec<_>, Vec<_>) = result.into_iter().partition_result(); - println!("Removed {} gc roots.", ok.len()); - if err.len() > 0 { - for (info, e) in err { - warn!( - logger, - "Failed to remove gc root: {}: {}", - info.gc_dir.display(), - e - ) + for info in to_remove { + match remove_dir_all(&info.gc_dir) { + Ok(_) => { + result.push(Ok(info)); + } + Err(e) => { + result.push(Err((info, e.to_string()))); + } } } - if ok.len() > 0 { - println!("Remember to run nix-collect-garbage to actually free space."); + if opts.json { + let res = result + .into_iter() + .map(|r| match r { + Err((info, err)) => json!({ + // Error, if any + "error": err, + // The root we tried to remove + "root": info + }), + Ok(info) => json!({ + "error": null, + "root": info + }), + }) + .collect::>(); + serde_json::to_writer(std::io::stdout(), &res) + .expect("failed to serialize result"); + } else { + let (ok, err): (Vec<_>, Vec<_>) = result.into_iter().partition_result(); + println!("Removed {} gc roots.", ok.len()); + if err.len() > 0 { + for (info, e) in err { + warn!( + logger, + "Failed to remove gc root: {}: {}", + info.gc_dir.display(), + e + ) + } + } + if ok.len() > 0 { + println!("Remember to run nix-collect-garbage to actually free space."); + } } } } } Ok(()) } + #[derive(Serialize)] /// How removing a gc root went, for json ouput struct RemovalStatus { From 9249133a806111be833fadd88f4f9eec41f7f3e2 Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Fri, 19 Apr 2024 11:29:20 +0200 Subject: [PATCH 12/12] refact(ops): remove remaining json Serialize instances --- src/ops.rs | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/ops.rs b/src/ops.rs index b39f0615..e5f0e57d 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -647,12 +647,6 @@ impl FromStr for EventKind { } } -/// Just expose the error message for now. -#[derive(Serialize)] -struct StreamBuildError { - message: String, -} - /// Run to output a stream of build events in a machine-parseable form. /// /// See the documentation for lorri::cli::Command::StreamEvents_ for more @@ -966,7 +960,6 @@ fn main_run_once( } } -#[derive(Serialize)] /// Represents a gc root along with some metadata, used for json output of lorri gc info struct GcRootInfo { /// directory where root is stored @@ -1136,11 +1129,23 @@ pub fn op_gc(logger: &slog::Logger, opts: crate::cli::GcOptions) -> Result<(), E // Error, if any "error": err, // The root we tried to remove - "root": info + "root": { + "gc_dir": info.gc_dir, + "nix_file": info.nix_file.map(|p| path_to_json_string(&p)), + // we use the Serialize instance for SystemTime + "timestamp": info.timestamp, + "alive": info.alive + } }), Ok(info) => json!({ "error": null, - "root": info + "root": { + "gc_dir": info.gc_dir, + "nix_file": info.nix_file.map(|p| path_to_json_string(&p)), + // we use the Serialize instance for SystemTime + "timestamp": info.timestamp, + "alive": info.alive + } }), }) .collect::>(); @@ -1169,15 +1174,6 @@ pub fn op_gc(logger: &slog::Logger, opts: crate::cli::GcOptions) -> Result<(), E Ok(()) } -#[derive(Serialize)] -/// How removing a gc root went, for json ouput -struct RemovalStatus { - /// What error was encountered, or success - error: Option, - /// The root we tried to remove - root: GcRootInfo, -} - fn main_run_forever( project: Project, nix_gc_root_user_dir: NixGcRootUserDir,