From 5be6620bd97f9077f8a5039a8741b5c08771e198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 29 Feb 2024 21:02:08 +0100 Subject: [PATCH 1/3] Allow garbage-collecting whithin a closure Make `nix store gc` accept installable arguments. If these are provided, then the gc will only happen within the closure of these installables and ignore any path outside, even if it's dead. `nix store gc foo` is morally equivalent to ```bash for validPath in $(nix path-info --recursive foo); do nix store delete "$validPath" || true done ``` --- src/libcmd/command.hh | 1 - src/libstore/gc-store.hh | 7 ++++--- src/libstore/gc.cc | 18 ++++++++++++++---- src/nix/store-gc.cc | 24 ++++++++++++++++++++++-- src/nix/store-gc.md | 11 ++++++++++- tests/functional/gc-closure.sh | 21 +++++++++++++++++++++ tests/functional/local.mk | 1 + 7 files changed, 72 insertions(+), 11 deletions(-) create mode 100644 tests/functional/gc-closure.sh diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 4a72627ed4d..084a66d4af2 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -169,7 +169,6 @@ struct RawInstallablesCommand : virtual Args, SourceExprCommand void run(ref store) override; - // FIXME make const after `CmdRepl`'s override is fixed up virtual void applyDefaultInstallables(std::vector & rawInstallables); bool readFromStdIn = false; diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index ab1059fb1ec..58518036621 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -23,8 +23,8 @@ struct GCOptions * * - `gcDeleteDead`: actually delete the latter set. * - * - `gcDeleteSpecific`: delete the paths listed in - * `pathsToDelete`, insofar as they are not reachable. + * - `gcDeleteSpecific`: delete all the paths, and fail if one of them + * isn't dead. */ typedef enum { gcReturnLive, @@ -44,7 +44,8 @@ struct GCOptions bool ignoreLiveness{false}; /** - * For `gcDeleteSpecific`, the paths to delete. + * The paths from which to delete. + * If empty, and `action` is not `gcDeleteSpecific`, act on the whole store. */ StorePathSet pathsToDelete; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index cb820e2d500..51726fa002b 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -475,6 +475,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) bool gcKeepOutputs = settings.gcKeepOutputs; bool gcKeepDerivations = settings.gcKeepDerivations; + if (options.action == GCOptions::gcDeleteSpecific && options.pathsToDelete.empty()) { + // This violates the convention that an empty `pathsToDelete` corresponds + // to the whole store, but deleting the whole store doesn't make sense, + // and `nix-store --delete` is a valid command that deletes nothing, so + // we need to keep it as-it-is. + return; + } + StorePathSet roots, dead, alive; struct Shared @@ -732,7 +740,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) return markAlive(); } - if (options.action == GCOptions::gcDeleteSpecific + if (!options.pathsToDelete.empty() && !options.pathsToDelete.count(*path)) return; @@ -790,11 +798,13 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Either delete all garbage paths, or just the specified paths (for gcDeleteSpecific). */ - if (options.action == GCOptions::gcDeleteSpecific) { + if (!options.pathsToDelete.empty()) { for (auto & i : options.pathsToDelete) { - deleteReferrersClosure(i); - if (!dead.count(i)) + if (shouldDelete) { + deleteReferrersClosure(i); + } + if (options.action == GCOptions::gcDeleteSpecific && !dead.count(i)) throw Error( "Cannot delete path '%1%' since it is still alive. " "To find out why, use: " diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index 8b9b5d1642a..283471994d2 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -7,7 +7,7 @@ using namespace nix; -struct CmdStoreGC : StoreCommand, MixDryRun +struct CmdStoreGC : InstallablesCommand, MixDryRun { GCOptions options; @@ -33,11 +33,31 @@ struct CmdStoreGC : StoreCommand, MixDryRun ; } - void run(ref store) override + // Don't add a default installable if none is specified so that + // `nix store gc` runs a full gc + void applyDefaultInstallables(std::vector & rawInstallables) override { + } + + void run(ref store, Installables && installables) override { auto & gcStore = require(*store); options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; + + // Add the closure of the installables to the set of paths to delete. + // If there's no installable specified, this will leave an empty set + // of paths to delete, which means the whole store will be gc-ed. + StorePathSet closureRoots; + for (auto & i : installables) { + try { + auto installableOutPath = Installable::toStorePath(getEvalStore(), store, Realise::Derivation, OperateOn::Output, i); + if (store->isValidPath(installableOutPath)) { + closureRoots.insert(installableOutPath); + } + } catch (MissingRealisation &) { + } + } + store->computeFSClosure(closureRoots, options.pathsToDelete); GCResults results; PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); gcStore.collectGarbage(options, results); diff --git a/src/nix/store-gc.md b/src/nix/store-gc.md index 956b3c8727a..e0957f9298a 100644 --- a/src/nix/store-gc.md +++ b/src/nix/store-gc.md @@ -14,8 +14,17 @@ R""( # nix store gc --max 1G ``` +* Delete the unreachable paths in the closure of the current development shell + + ```console + # nix store gc .#devShells.default + ``` + # Description -This command deletes unreachable paths in the Nix store. +This command deletes unreachable paths from the Nix store. + +If called with no argument, it will delete all the unreachable paths from the store. +If called with an installable argument, it will delete the unreachable paths whithin the closure of that argument. )"" diff --git a/tests/functional/gc-closure.sh b/tests/functional/gc-closure.sh new file mode 100644 index 00000000000..e392cad1460 --- /dev/null +++ b/tests/functional/gc-closure.sh @@ -0,0 +1,21 @@ +source common.sh + +nix_gc_closure() { + clearStore + nix build -f dependencies.nix input0_drv --out-link $TEST_ROOT/gc-root + input0=$(realpath $TEST_ROOT/gc-root) + input1=$(nix build -f dependencies.nix input1_drv --no-link --print-out-paths) + input2=$(nix build -f dependencies.nix input2_drv --no-link --print-out-paths) + top=$(nix build -f dependencies.nix --no-link --print-out-paths) + somthing_else=$(nix store add-path ./dependencies.nix) + + # Check that nix store gc is best-effort (doesn't fail when some paths in the closure are alive) + nix store gc "$top" + [[ ! -e "$top" ]] || fail "top should have been deleted" + [[ -e "$input0" ]] || fail "input0 is a gc root, shouldn't have been deleted" + [[ ! -e "$input2" ]] || fail "input2 is not a gc root and is part of top's closure, it should have been deleted" + [[ -e "$input1" ]] || fail "input1 is not ins the closure of top, it shouldn't have been deleted" + [[ -e "$somthing_else" ]] || fail "somthing_else is not in the closure of top, it shouldn't have been deleted" +} + +nix_gc_closure diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 18eb887cdd1..7a6f43d49b9 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -50,6 +50,7 @@ nix_tests = \ hash-convert.sh \ hash-path.sh \ gc-non-blocking.sh \ + gc-closure.sh \ check.sh \ nix-shell.sh \ check-refs.sh \ From 045f0fcded3d9d1f48b2a64b18d6bd55fec168d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 1 Mar 2024 06:43:32 +0100 Subject: [PATCH 2/3] Ensure best-effort compatibility with older daemon version Fallback to a full GC if the daemon doesn't support partial ones --- src/libstore/remote-store.cc | 7 +++++++ src/libstore/worker-protocol.hh | 2 +- tests/functional/gc-closure.sh | 22 ++++++++++++++++------ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8dfe8addab7..3ab12fd1fc0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -851,6 +851,13 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) { auto conn(getConnection()); + if ( + options.action != GCOptions::gcDeleteSpecific && + ! options.pathsToDelete.empty() && + GET_PROTOCOL_MINOR(conn->daemonVersion) < 38) { + warn("Your daemon version is too old to support garbage collecting a closure, falling back to a full gc"); + } + conn->to << WorkerProto::Op::CollectGarbage << options.action; WorkerProto::write(*this, *conn, options.pathsToDelete); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 91d277b7705..af42b4dcb0a 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -11,7 +11,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 37) +#define PROTOCOL_VERSION (1 << 8 | 38) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/tests/functional/gc-closure.sh b/tests/functional/gc-closure.sh index e392cad1460..e63f4369348 100644 --- a/tests/functional/gc-closure.sh +++ b/tests/functional/gc-closure.sh @@ -9,13 +9,23 @@ nix_gc_closure() { top=$(nix build -f dependencies.nix --no-link --print-out-paths) somthing_else=$(nix store add-path ./dependencies.nix) - # Check that nix store gc is best-effort (doesn't fail when some paths in the closure are alive) nix store gc "$top" - [[ ! -e "$top" ]] || fail "top should have been deleted" - [[ -e "$input0" ]] || fail "input0 is a gc root, shouldn't have been deleted" - [[ ! -e "$input2" ]] || fail "input2 is not a gc root and is part of top's closure, it should have been deleted" - [[ -e "$input1" ]] || fail "input1 is not ins the closure of top, it shouldn't have been deleted" - [[ -e "$somthing_else" ]] || fail "somthing_else is not in the closure of top, it shouldn't have been deleted" + + if isDaemonNewer "2.21.0pre20240229"; then + # Check that nix store gc is best-effort (doesn't fail when some paths in the closure are alive) + [[ ! -e "$top" ]] || fail "top should have been deleted" + [[ -e "$input0" ]] || fail "input0 is a gc root, shouldn't have been deleted" + [[ ! -e "$input2" ]] || fail "input2 is not a gc root and is part of top's closure, it should have been deleted" + [[ -e "$input1" ]] || fail "input1 is not ins the closure of top, it shouldn't have been deleted" + [[ -e "$somthing_else" ]] || fail "somthing_else is not in the closure of top, it shouldn't have been deleted" + else + # If the daemon is too old to handle closure gc, fallback to a full GC + [[ ! -e "$top" ]] || fail "top should have been deleted" + [[ -e "$input0" ]] || fail "input0 is a gc root, shouldn't have been deleted" + [[ ! -e "$input2" ]] || fail "input2 is not a gc root and is part of top's closure, it should have been deleted" + [[ ! -e "$input1" ]] || fail "input1 is not a gc root, it should have been deleted" + [[ ! -e "$somthing_else" ]] || fail "somthing_else is not a gc root, it should have been deleted" + fi } nix_gc_closure From 8db988fd59f7779a16fd4ee0197e80c7db1f6154 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 1 Mar 2024 06:44:23 +0100 Subject: [PATCH 3/3] Add a release note for the closure GC --- doc/manual/rl-next/closure-gc.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/manual/rl-next/closure-gc.md diff --git a/doc/manual/rl-next/closure-gc.md b/doc/manual/rl-next/closure-gc.md new file mode 100644 index 00000000000..0b7424394d8 --- /dev/null +++ b/doc/manual/rl-next/closure-gc.md @@ -0,0 +1,8 @@ +--- +synopsis: "`nix store gc` can now collect garbage whithin a closure" +issues: 7239 +prs: 8417 +--- + +`nix store gc` can now be called with an installable argument, in which case it +will only collect the dead paths that are part of the closure of its argument.