From d30ea9eaf372d80034503e0fb6f522225fc53e32 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Thu, 18 May 2023 15:44:51 +0400 Subject: [PATCH] Refactor GCOptions and GCAction, add recursive flag --- src/libstore/daemon.cc | 26 ++- src/libstore/gc-store.hh | 81 ++++---- src/libstore/gc.cc | 173 ++++++++++-------- src/libstore/remote-store.cc | 36 +++- .../nix-collect-garbage.cc | 2 +- src/nix-store/nix-store.cc | 19 +- src/nix/store-delete.cc | 16 +- src/nix/store-gc.cc | 7 +- 8 files changed, 213 insertions(+), 147 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ad3dee1a27c6..a5bc03e8edb5 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -712,18 +712,36 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::CollectGarbage: { GCOptions options; - options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); - from >> options.ignoreLiveness >> options.maxFreed; + int action; + bool ignoreLiveness; + StorePathSet pathsToDelete; + bool skipAlive {false}; + action = readInt(from); + pathsToDelete = WorkerProto::Serialise::read(*store, from); + from >> ignoreLiveness >> options.maxFreed; // obsolete fields readInt(from); readInt(from); readInt(from); + if (GET_PROTOCOL_MINOR(clientVersion) >= 31) { + from >> recursive; + }; + + switch (action) { + case 0: + options.action = GCReturn::Live; break; + case 1: + options.action = GCReturn::Dead; break; + case 2: + options.action = GCDelete { .pathsToDelete = std::nullopt, .ignoreLiveness = ignoreLiveness }; break; + case 3: + options.action = GCDelete { .pathsToDelete = GCPathsToDelete { .paths = pathsToDelete, .skipAlive = skipAlive }, .ignoreLiveness = ignoreLiveness, }; break; + }; GCResults results; logger->startWork(); - if (options.ignoreLiveness) + if (ignoreLiveness) throw Error("you are not allowed to ignore liveness"); auto & gcStore = require(*store); gcStore.collectGarbage(options, results); diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index 2c26c65c42ca..f6ed53b7539e 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -9,51 +9,45 @@ namespace nix { typedef std::unordered_map> Roots; +/** + * Return either live (reachable) or dead (unreachable) paths + */ +enum class GCReturn { Live, Dead }; + +/** + * Set of paths to delete, and whether to skip paths which are alive + */ +struct GCPathsToDelete { + StorePathSet paths; + bool skipAlive; +}; -struct GCOptions -{ - /** - * Garbage collector operation: - * - * - `gcReturnLive`: return the set of paths reachable from - * (i.e. in the closure of) the roots. - * - * - `gcReturnDead`: return the set of paths not reachable from - * the roots. - * - * - `gcDeleteDead`: actually delete the latter set. - * - * - `gcDeleteSpecific`: delete the paths listed in - * `pathsToDelete`, insofar as they are not reachable. - */ - typedef enum { - gcReturnLive, - gcReturnDead, - gcDeleteDead, - gcDeleteSpecific, - } GCAction; - - GCAction action{gcDeleteDead}; - - /** - * If `ignoreLiveness` is set, then reachability from the roots is - * ignored (dangerous!). However, the paths must still be - * unreferenced *within* the store (i.e., there can be no other - * store paths that depend on them). - */ - bool ignoreLiveness{false}; - - /** - * For `gcDeleteSpecific`, the paths to delete. - */ - StorePathSet pathsToDelete; - - /** - * Stop after at least `maxFreed` bytes have been freed. - */ - uint64_t maxFreed{std::numeric_limits::max()}; +/** + Delete either a given set of paths, or all dead paths + */ +struct GCDelete { + /* Delete this set, or all dead paths if it is std::nullopt */ + std::optional pathsToDelete; + /* If `ignoreLiveness' is set, then reachability from the roots is + ignored (dangerous!). However, the paths must still be + unreferenced *within* the store (i.e., there can be no other + store paths that depend on them). */ + bool ignoreLiveness{false}; }; +/** + * Garbage collection action: either return paths, or delete them + */ +using GCAction = std::variant; + +/** + * Options for the garbage collector + */ +struct GCOptions { + GCAction action; + /* Stop after at least `maxFreed' bytes have been freed. */ + uint64_t maxFreed{std::numeric_limits::max()}; +}; struct GCResults { @@ -64,8 +58,7 @@ struct GCResults PathSet paths; /** - * For `gcReturnDead`, `gcDeleteDead` and `gcDeleteSpecific`, the - * number of bytes that would be or was freed. + * For `GCDelete', the number of bytes that would be or was freed. */ uint64_t bytesFreed = 0; }; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index efa725283d3c..cac20047a04f 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -367,6 +367,8 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { UncheckedRoots unchecked; + std::string pids = std::to_string(getpid()); + auto procDir = AutoCloseDir{opendir("/proc")}; if (procDir) { struct dirent * ent; @@ -375,7 +377,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); - if (std::regex_match(ent->d_name, digitsRegex)) { + if (std::regex_match(ent->d_name, digitsRegex) && strcmp(pids.c_str(), ent->d_name)) { try { readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); @@ -469,7 +471,7 @@ struct GCLimitReached { }; void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { - bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; + bool shouldDelete = std::holds_alternative(options.action); bool gcKeepOutputs = settings.gcKeepOutputs; bool gcKeepDerivations = settings.gcKeepDerivations; @@ -494,10 +496,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) consequences if `keep-outputs' or `keep-derivations' are true (the garbage collector will recurse into deleting the outputs or derivers, respectively). So disable them. */ - if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { - gcKeepOutputs = false; - gcKeepDerivations = false; - } + std::visit(overloaded{[&](GCDelete del){ + if (del.pathsToDelete.has_value() && del.ignoreLiveness) { + gcKeepOutputs = false; + gcKeepDerivations = false; + } + }, [](GCReturn arg){}}, options.action); if (shouldDelete) deletePath(reservedPath); @@ -617,8 +621,16 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) permanent roots cannot increase now. */ printInfo("finding garbage collector roots..."); Roots rootMap; - if (!options.ignoreLiveness) - findRootsNoTemp(rootMap, true); + + std::visit(overloaded{ + [&](GCDelete del) { + if (!del.ignoreLiveness) + findRootsNoTemp(rootMap, true); + }, + [&](GCReturn r){ + findRootsNoTemp(rootMap, true); + }}, + options.action); for (auto & i : rootMap) roots.insert(i.first); @@ -721,9 +733,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) return markAlive(); } - if (options.action == GCOptions::gcDeleteSpecific - && !options.pathsToDelete.count(*path)) - return; + if (const GCDelete * del = std::get_if(&options.action)) + if (del->pathsToDelete.has_value() && !del->pathsToDelete->paths.count(*path)) + return; { auto hashPart = std::string(path->hashPart()); @@ -781,27 +793,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (auto p = getEnv("_NIX_TEST_GC_SYNC")) readFile(*p); - /* Either delete all garbage paths, or just the specified - paths (for gcDeleteSpecific). */ - if (options.action == GCOptions::gcDeleteSpecific || options.action == GCOptions::gcDeleteDead) { - - for (auto & i : options.pathsToDelete) { - deleteReferrersClosure(i); - if (options.action == GCOptions::gcDeleteSpecific && !dead.count(i)) - throw Error( - "Cannot delete path '%1%' since it is still alive. " - "To find out why, use: " - "nix-store --query --roots", - printStorePath(i)); - } - - } else if (options.maxFreed > 0) { - - if (shouldDelete) - printInfo("deleting garbage..."); - else - printInfo("determining live/dead paths..."); - + auto findOrDeleteRoots = [&](){ try { AutoCloseDir dir(opendir(realStoreDir.get().c_str())); if (!dir) throw SysError("opening directory '%1%'", realStoreDir); @@ -825,65 +817,87 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } } catch (GCLimitReached & e) { } - } + }; - if (options.action == GCOptions::gcReturnLive) { - for (auto & i : alive) - results.paths.insert(printStorePath(i)); - return; - } + /* Either delete all garbage paths, or just the specified + paths (for gcDeleteSpecific). */ + std::visit(overloaded{ + [&](GCDelete del){ + if (del.pathsToDelete.has_value()) { + for (auto & i : del.pathsToDelete->paths) { + deleteReferrersClosure(i); + if (!del.pathsToDelete->skipAlive && !dead.count(i)) + throw Error( + "Cannot delete path '%1%' since it is still alive. " + "To find out why, use: " + "nix-store --query --roots", + printStorePath(i)); + } + } else if (options.maxFreed > 0) { + printInfo("deleting garbage..."); + findOrDeleteRoots(); + } - if (options.action == GCOptions::gcReturnDead) { - for (auto & i : dead) - results.paths.insert(printStorePath(i)); - return; - } + /* Unlink all files in /nix/store/.links that have a link count of 1, + which indicates that there are no other links and so they can be + safely deleted. FIXME: race condition with optimisePath(): we + might see a link count of 1 just before optimisePath() increases + the link count. */ - /* Unlink all files in /nix/store/.links that have a link count of 1, - which indicates that there are no other links and so they can be - safely deleted. FIXME: race condition with optimisePath(): we - might see a link count of 1 just before optimisePath() increases - the link count. */ - if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { - printInfo("deleting unused links..."); + printInfo("deleting unused links..."); - AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError("opening directory '%1%'", linksDir); + AutoCloseDir dir(opendir(linksDir.c_str())); + if (!dir) throw SysError("opening directory '%1%'", linksDir); - int64_t actualSize = 0, unsharedSize = 0; + int64_t actualSize = 0, unsharedSize = 0; - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { - checkInterrupt(); - std::string name = dirent->d_name; - if (name == "." || name == "..") continue; - Path path = linksDir + "/" + name; + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = linksDir + "/" + name; - auto st = lstat(path); + auto st = lstat(path); - if (st.st_nlink != 1) { - actualSize += st.st_size; - unsharedSize += (st.st_nlink - 1) * st.st_size; - continue; - } + if (st.st_nlink != 1) { + actualSize += st.st_size; + unsharedSize += (st.st_nlink - 1) * st.st_size; + continue; + } - printMsg(lvlTalkative, "deleting unused link '%1%'", path); + printMsg(lvlTalkative, "deleting unused link '%1%'", path); - if (unlink(path.c_str()) == -1) - throw SysError("deleting '%1%'", path); + if (unlink(path.c_str()) == -1) + throw SysError("deleting '%1%'", path); - /* Do not accound for deleted file here. Rely on deletePath() - accounting. */ - } + /* Do not accound for deleted file here. Rely on deletePath() + accounting. */ + } - struct stat st; - if (stat(linksDir.c_str(), &st) == -1) - throw SysError("statting '%1%'", linksDir); - int64_t overhead = st.st_blocks * 512ULL; + struct stat st; + if (stat(linksDir.c_str(), &st) == -1) + throw SysError("statting '%1%'", linksDir); + int64_t overhead = st.st_blocks * 512ULL; - printInfo("note: currently hard linking saves %.2f MiB", - ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); - } + printInfo("note: currently hard linking saves %.2f MiB", + ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); + }, + [&](GCReturn ret){ + printInfo("determining live/dead paths..."); + findOrDeleteRoots(); + if (ret == GCReturn::Live) { + for (auto & i : alive) + results.paths.insert(printStorePath(i)); + return; + } + + if (ret == GCReturn::Dead) { + for (auto & i : dead) + results.paths.insert(printStorePath(i)); + return; + } + }}, options.action); /* While we're at it, vacuum the database. */ //if (options.action == GCOptions::gcDeleteDead) vacuumDB(); @@ -945,8 +959,7 @@ void LocalStore::autoGC(bool sync) promise.set_value(); }); - GCOptions options; - options.maxFreed = settings.maxFree - avail; + GCOptions options{.action = GCAction{GCDelete{}}, .maxFreed = settings.maxFree - avail}; printInfo("running auto-GC to free %d bytes", options.maxFreed); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1e2104e1faa5..5acdda5684ba 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -865,15 +865,45 @@ Roots RemoteStore::findRoots(bool censor) void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) { auto conn(getConnection()); + int action; + StorePathSet pathsToDelete; + bool ignoreLiveness {false}; + bool recursive {false}; + std::visit(overloaded{ + [&](GCReturn ret){ + switch (ret) { + case (GCReturn::Live): + action = 0; + break; + case (GCReturn::Dead): + action = 1; + break; + } + }, + [&](GCDelete del) { + ignoreLiveness = del.ignoreLiveness; + if (!del.pathsToDelete.has_value()) { + action = 2; + } else { + action = 3; + pathsToDelete = del.pathsToDelete->paths; + recursive = del.pathsToDelete->skipAlive; + }; + } + }, options.action); conn->to - << WorkerProto::Op::CollectGarbage << options.action; - WorkerProto::write(*this, *conn, options.pathsToDelete); - conn->to << options.ignoreLiveness + << WorkerProto::Op::CollectGarbage << action; + WorkerProto::write(*this, conn->to, pathsToDelete); + conn->to << ignoreLiveness << options.maxFreed /* removed options */ << 0 << 0 << 0; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 31) { + conn->to << recursive; + } + conn.processStderr(); results.paths = readStrings(conn->from); diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 70af53b286ba..f18358b44b49 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -89,7 +89,7 @@ static int main_nix_collect_garbage(int argc, char * * argv) if (!dryRun) { auto store = openStore(); auto & gcStore = require(*store); - options.action = GCOptions::gcDeleteDead; + options.action = GCDelete {}; GCResults results; PrintFreed freed(true, results); gcStore.collectGarbage(options, results); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index caa0248f1aa9..893ac2e0a9e5 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -589,15 +589,15 @@ static void opGC(Strings opFlags, Strings opArgs) { bool printRoots = false; GCOptions options; - options.action = GCOptions::gcDeleteDead; + options.action = GCAction{GCDelete{}}; GCResults results; /* Do what? */ for (auto i = opFlags.begin(); i != opFlags.end(); ++i) if (*i == "--print-roots") printRoots = true; - else if (*i == "--print-live") options.action = GCOptions::gcReturnLive; - else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead; + else if (*i == "--print-live") options.action = GCAction{GCReturn::Live}; + else if (*i == "--print-dead") options.action = GCAction{GCReturn::Dead}; else if (*i == "--max-freed") options.maxFreed = std::max(getIntArg(*i, i, opFlags.end(), true), (int64_t) 0); else throw UsageError("bad sub-operation '%1%' in GC", *i); @@ -618,10 +618,10 @@ static void opGC(Strings opFlags, Strings opArgs) } else { - PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); + PrintFreed freed(std::holds_alternative(options.action), results); gcStore.collectGarbage(options, results); - if (options.action != GCOptions::gcDeleteDead) + if (!std::holds_alternative(options.action)) for (auto & i : results.paths) cout << i << std::endl; } @@ -633,18 +633,19 @@ static void opGC(Strings opFlags, Strings opArgs) roots). */ static void opDelete(Strings opFlags, Strings opArgs) { - GCOptions options; - options.action = GCOptions::gcDeleteSpecific; + GCDelete del = GCDelete{.pathsToDelete = GCPathsToDelete{}}; for (auto & i : opFlags) - if (i == "--ignore-liveness") options.ignoreLiveness = true; + if (i == "--ignore-liveness") del.ignoreLiveness = true; else throw UsageError("unknown flag '%1%'", i); for (auto & i : opArgs) - options.pathsToDelete.insert(store->followLinksToStorePath(i)); + del.pathsToDelete->paths.insert(store->followLinksToStorePath(i)); + auto & gcStore = require(*store); + GCOptions options{GCAction{del}}; GCResults results; PrintFreed freed(true, results); gcStore.collectGarbage(options, results); diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index 922cc699fc2a..07d4b10a4332 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -9,14 +9,20 @@ using namespace nix; struct CmdStoreDelete : StorePathsCommand { - GCOptions options { .action = GCOptions::gcDeleteDead }; + GCDelete deleteOpts{.pathsToDelete = GCPathsToDelete{}}; CmdStoreDelete() { addFlag({ .longName = "ignore-liveness", - .description = "Do not check whether the paths are reachable from a root.", - .handler = {&options.ignoreLiveness, true} + .description = "Delete all provided paths, even if they are alive (reachable from a garbage collection root).", + .handler = {&deleteOpts.ignoreLiveness, true} + }); + addFlag({ + .longName = "skip-alive", + .shortName = 's', + .description = "Skip paths that are still alive (referenced by a garbage collection root), instead of exiting with a non-zero exit code.", + .handler = {&deleteOpts.pathsToDelete->skipAlive, true} }); } @@ -37,8 +43,10 @@ struct CmdStoreDelete : StorePathsCommand auto & gcStore = require(*store); for (auto & path : storePaths) - options.pathsToDelete.insert(path); + deleteOpts.pathsToDelete->paths.insert(path); + + GCOptions options {GCAction{deleteOpts}}; GCResults results; PrintFreed freed(true, results); gcStore.collectGarbage(options, results); diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index 8b9b5d1642a8..aa7ef18723d6 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -37,9 +37,12 @@ struct CmdStoreGC : StoreCommand, MixDryRun { auto & gcStore = require(*store); - options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; + if (dryRun) + options.action = GCReturn::Dead; + else + options.action = GCDelete{}; GCResults results; - PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); + PrintFreed freed(!dryRun, results); gcStore.collectGarbage(options, results); } };