Skip to content

Commit

Permalink
Refactor GCOptions and GCAction, add recursive flag
Browse files Browse the repository at this point in the history
  • Loading branch information
@balsoft
balsoft committed Jul 27, 2023
1 parent 0d0f7b8 commit d30ea9e
Show file tree
Hide file tree
Showing 8 changed files with 213 additions and 147 deletions.
26 changes: 22 additions & 4 deletions src/libstore/daemon.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -712,18 +712,36 @@ static void performOp(TunnelLogger * logger, ref<Store> store,

case WorkerProto::Op::CollectGarbage: {
GCOptions options;
options.action = (GCOptions::GCAction) readInt(from);
options.pathsToDelete = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
from >> options.ignoreLiveness >> options.maxFreed;
int action;
bool ignoreLiveness;
StorePathSet pathsToDelete;
bool skipAlive {false};
action = readInt(from);
pathsToDelete = WorkerProto::Serialise<StorePathSet>::read(*store, from);
from >> ignoreLiveness >> options.maxFreed;
// obsolete fields
readInt(from);
readInt(from);
readInt(from);
if (GET_PROTOCOL_MINOR(clientVersion) >= 31) {
from >> recursive;
};

switch (action) {
case 0:
options.action = GCReturn::Live; break;
case 1:
options.action = GCReturn::Dead; break;
case 2:
options.action = GCDelete { .pathsToDelete = std::nullopt, .ignoreLiveness = ignoreLiveness }; break;
case 3:
options.action = GCDelete { .pathsToDelete = GCPathsToDelete { .paths = pathsToDelete, .skipAlive = skipAlive }, .ignoreLiveness = ignoreLiveness, }; break;
};

GCResults results;

logger->startWork();
if (options.ignoreLiveness)
if (ignoreLiveness)
throw Error("you are not allowed to ignore liveness");
auto & gcStore = require<GcStore>(*store);
gcStore.collectGarbage(options, results);
Expand Down
81 changes: 37 additions & 44 deletions src/libstore/gc-store.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,51 +9,45 @@ namespace nix {

typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots;

/**
* Return either live (reachable) or dead (unreachable) paths
*/
enum class GCReturn { Live, Dead };

/**
* Set of paths to delete, and whether to skip paths which are alive
*/
struct GCPathsToDelete {
StorePathSet paths;
bool skipAlive;
};

struct GCOptions
{
/**
* Garbage collector operation:
*
* - `gcReturnLive`: return the set of paths reachable from
* (i.e. in the closure of) the roots.
*
* - `gcReturnDead`: return the set of paths not reachable from
* the roots.
*
* - `gcDeleteDead`: actually delete the latter set.
*
* - `gcDeleteSpecific`: delete the paths listed in
* `pathsToDelete`, insofar as they are not reachable.
*/
typedef enum {
gcReturnLive,
gcReturnDead,
gcDeleteDead,
gcDeleteSpecific,
} GCAction;

GCAction action{gcDeleteDead};

/**
* If `ignoreLiveness` is set, then reachability from the roots is
* ignored (dangerous!). However, the paths must still be
* unreferenced *within* the store (i.e., there can be no other
* store paths that depend on them).
*/
bool ignoreLiveness{false};

/**
* For `gcDeleteSpecific`, the paths to delete.
*/
StorePathSet pathsToDelete;

/**
* Stop after at least `maxFreed` bytes have been freed.
*/
uint64_t maxFreed{std::numeric_limits<uint64_t>::max()};
/**
Delete either a given set of paths, or all dead paths
*/
struct GCDelete {
/* Delete this set, or all dead paths if it is std::nullopt */
std::optional<GCPathsToDelete> pathsToDelete;
/* If `ignoreLiveness' is set, then reachability from the roots is
ignored (dangerous!). However, the paths must still be
unreferenced *within* the store (i.e., there can be no other
store paths that depend on them). */
bool ignoreLiveness{false};
};

/**
* Garbage collection action: either return paths, or delete them
*/
using GCAction = std::variant<GCReturn, GCDelete>;

/**
* Options for the garbage collector
*/
struct GCOptions {
GCAction action;
/* Stop after at least `maxFreed' bytes have been freed. */
uint64_t maxFreed{std::numeric_limits<uint64_t>::max()};
};

struct GCResults
{
Expand All @@ -64,8 +58,7 @@ struct GCResults
PathSet paths;

/**
* For `gcReturnDead`, `gcDeleteDead` and `gcDeleteSpecific`, the
* number of bytes that would be or was freed.
* For `GCDelete', the number of bytes that would be or was freed.
*/
uint64_t bytesFreed = 0;
};
Expand Down
173 changes: 93 additions & 80 deletions src/libstore/gc.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,8 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
{
UncheckedRoots unchecked;

std::string pids = std::to_string(getpid());

auto procDir = AutoCloseDir{opendir("/proc")};
if (procDir) {
struct dirent * ent;
Expand All @@ -375,7 +377,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)");
while (errno = 0, ent = readdir(procDir.get())) {
checkInterrupt();
if (std::regex_match(ent->d_name, digitsRegex)) {
if (std::regex_match(ent->d_name, digitsRegex) && strcmp(pids.c_str(), ent->d_name)) {
try {
readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked);
readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked);
Expand Down Expand Up @@ -469,7 +471,7 @@ struct GCLimitReached { };

void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
{
bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
bool shouldDelete = std::holds_alternative<GCDelete>(options.action);
bool gcKeepOutputs = settings.gcKeepOutputs;
bool gcKeepDerivations = settings.gcKeepDerivations;

Expand All @@ -494,10 +496,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
consequences if `keep-outputs' or `keep-derivations' are true
(the garbage collector will recurse into deleting the outputs
or derivers, respectively). So disable them. */
if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) {
gcKeepOutputs = false;
gcKeepDerivations = false;
}
std::visit(overloaded{[&](GCDelete del){
if (del.pathsToDelete.has_value() && del.ignoreLiveness) {
gcKeepOutputs = false;
gcKeepDerivations = false;
}
}, [](GCReturn arg){}}, options.action);

if (shouldDelete)
deletePath(reservedPath);
Expand Down Expand Up @@ -617,8 +621,16 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
permanent roots cannot increase now. */
printInfo("finding garbage collector roots...");
Roots rootMap;
if (!options.ignoreLiveness)
findRootsNoTemp(rootMap, true);

std::visit(overloaded{
[&](GCDelete del) {
if (!del.ignoreLiveness)
findRootsNoTemp(rootMap, true);
},
[&](GCReturn r){
findRootsNoTemp(rootMap, true);
}},
options.action);

for (auto & i : rootMap) roots.insert(i.first);

Expand Down Expand Up @@ -721,9 +733,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
return markAlive();
}

if (options.action == GCOptions::gcDeleteSpecific
&& !options.pathsToDelete.count(*path))
return;
if (const GCDelete * del = std::get_if<GCDelete>(&options.action))
if (del->pathsToDelete.has_value() && !del->pathsToDelete->paths.count(*path))
return;

{
auto hashPart = std::string(path->hashPart());
Expand Down Expand Up @@ -781,27 +793,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
if (auto p = getEnv("_NIX_TEST_GC_SYNC"))
readFile(*p);

/* Either delete all garbage paths, or just the specified
paths (for gcDeleteSpecific). */
if (options.action == GCOptions::gcDeleteSpecific || options.action == GCOptions::gcDeleteDead) {

for (auto & i : options.pathsToDelete) {
deleteReferrersClosure(i);
if (options.action == GCOptions::gcDeleteSpecific && !dead.count(i))
throw Error(
"Cannot delete path '%1%' since it is still alive. "
"To find out why, use: "
"nix-store --query --roots",
printStorePath(i));
}

} else if (options.maxFreed > 0) {

if (shouldDelete)
printInfo("deleting garbage...");
else
printInfo("determining live/dead paths...");

auto findOrDeleteRoots = [&](){
try {
AutoCloseDir dir(opendir(realStoreDir.get().c_str()));
if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
Expand All @@ -825,65 +817,87 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
}
} catch (GCLimitReached & e) {
}
}
};

if (options.action == GCOptions::gcReturnLive) {
for (auto & i : alive)
results.paths.insert(printStorePath(i));
return;
}
/* Either delete all garbage paths, or just the specified
paths (for gcDeleteSpecific). */
std::visit(overloaded{
[&](GCDelete del){
if (del.pathsToDelete.has_value()) {
for (auto & i : del.pathsToDelete->paths) {
deleteReferrersClosure(i);
if (!del.pathsToDelete->skipAlive && !dead.count(i))
throw Error(
"Cannot delete path '%1%' since it is still alive. "
"To find out why, use: "
"nix-store --query --roots",
printStorePath(i));
}
} else if (options.maxFreed > 0) {
printInfo("deleting garbage...");
findOrDeleteRoots();
}

if (options.action == GCOptions::gcReturnDead) {
for (auto & i : dead)
results.paths.insert(printStorePath(i));
return;
}
/* Unlink all files in /nix/store/.links that have a link count of 1,
which indicates that there are no other links and so they can be
safely deleted. FIXME: race condition with optimisePath(): we
might see a link count of 1 just before optimisePath() increases
the link count. */

/* Unlink all files in /nix/store/.links that have a link count of 1,
which indicates that there are no other links and so they can be
safely deleted. FIXME: race condition with optimisePath(): we
might see a link count of 1 just before optimisePath() increases
the link count. */
if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
printInfo("deleting unused links...");
printInfo("deleting unused links...");

AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError("opening directory '%1%'", linksDir);
AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError("opening directory '%1%'", linksDir);

int64_t actualSize = 0, unsharedSize = 0;
int64_t actualSize = 0, unsharedSize = 0;

struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) {
checkInterrupt();
std::string name = dirent->d_name;
if (name == "." || name == "..") continue;
Path path = linksDir + "/" + name;
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) {
checkInterrupt();
std::string name = dirent->d_name;
if (name == "." || name == "..") continue;
Path path = linksDir + "/" + name;

auto st = lstat(path);
auto st = lstat(path);

if (st.st_nlink != 1) {
actualSize += st.st_size;
unsharedSize += (st.st_nlink - 1) * st.st_size;
continue;
}
if (st.st_nlink != 1) {
actualSize += st.st_size;
unsharedSize += (st.st_nlink - 1) * st.st_size;
continue;
}

printMsg(lvlTalkative, "deleting unused link '%1%'", path);
printMsg(lvlTalkative, "deleting unused link '%1%'", path);

if (unlink(path.c_str()) == -1)
throw SysError("deleting '%1%'", path);
if (unlink(path.c_str()) == -1)
throw SysError("deleting '%1%'", path);

/* Do not accound for deleted file here. Rely on deletePath()
accounting. */
}
/* Do not accound for deleted file here. Rely on deletePath()
accounting. */
}

struct stat st;
if (stat(linksDir.c_str(), &st) == -1)
throw SysError("statting '%1%'", linksDir);
int64_t overhead = st.st_blocks * 512ULL;
struct stat st;
if (stat(linksDir.c_str(), &st) == -1)
throw SysError("statting '%1%'", linksDir);
int64_t overhead = st.st_blocks * 512ULL;

printInfo("note: currently hard linking saves %.2f MiB",
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
}
printInfo("note: currently hard linking saves %.2f MiB",
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
},
[&](GCReturn ret){
printInfo("determining live/dead paths...");
findOrDeleteRoots();
if (ret == GCReturn::Live) {
for (auto & i : alive)
results.paths.insert(printStorePath(i));
return;
}

if (ret == GCReturn::Dead) {
for (auto & i : dead)
results.paths.insert(printStorePath(i));
return;
}
}}, options.action);

/* While we're at it, vacuum the database. */
//if (options.action == GCOptions::gcDeleteDead) vacuumDB();
Expand Down Expand Up @@ -945,8 +959,7 @@ void LocalStore::autoGC(bool sync)
promise.set_value();
});

GCOptions options;
options.maxFreed = settings.maxFree - avail;
GCOptions options{.action = GCAction{GCDelete{}}, .maxFreed = settings.maxFree - avail};

printInfo("running auto-GC to free %d bytes", options.maxFreed);

Expand Down
Loading

0 comments on commit d30ea9e

Please sign in to comment.