diff --git a/nix-trace/README.md b/nix-trace/README.md index f8f2981..9f10760 100644 --- a/nix-trace/README.md +++ b/nix-trace/README.md @@ -29,6 +29,9 @@ open() != -1 && read error: `f` FILENAME `\0` `e` `\0` opendir() == NULL: `d` FILENAME `\0` `-` `\0` opendir() != NULL: `d` FILENAME `\0` b3sum(directory listing) `\0` + +mkdir() == 0 `t` FILENAME `\0` `+` `\0` +unlinkat() == NULL `t` FILENAME `\0` `-` `\0` ``` Directory listing: diff --git a/nix-trace/test.sh b/nix-trace/test.sh index 8c3cc7e..fc6df83 100755 --- a/nix-trace/test.sh +++ b/nix-trace/test.sh @@ -2,14 +2,13 @@ run() { rm -f test-tmp/log - DYLD_INSERT_LIBRARIES=$PWD/build/trace-nix.so LD_PRELOAD=$PWD/build/trace-nix.so TRACE_NIX=test-tmp/log \ - nix-shell --run : -p -- "$@" 2>/dev/null -} - -run_without_p() { - rm -f test-tmp/log - DYLD_INSERT_LIBRARIES=$PWD/build/trace-nix.so LD_PRELOAD=$PWD/build/trace-nix.so TRACE_NIX=test-tmp/log \ - nix-shell --run : -- "$@" 2>/dev/null + env \ + DYLD_INSERT_LIBRARIES="$PWD"/build/trace-nix.so \ + LD_PRELOAD="$PWD"/build/trace-nix.so \ + TRACE_NIX=test-tmp/log \ + nix-shell --run : "$@" 2>/dev/null & + NIX_PID=$! + wait } result=0 @@ -23,6 +22,8 @@ dir_b3sum() { } check() { + local grep_opts= + [ "$1" != "--first" ] || { shift; grep_opts=-m1; } local name="$1" key="$2" val="$3" if ! grep -qzFx -- "$key" test-tmp/log; then @@ -31,7 +32,11 @@ check() { result=1 fi - local actual_val="$(grep -zFx -A1 -- "$key" test-tmp/log | tail -zn1 | tr -d '\0')" + local actual_val=$( + grep $grep_opts -zFx -A1 -- "$key" test-tmp/log | + tail -zn1 | + tr -d '\0' + ) if [ "$val" != "$actual_val" ]; then printf "\33[31mFail: %s: expected '%s', got '%s'\33[m\n" \ "$name" "$val" "$actual_val" @@ -48,71 +53,92 @@ echo '"foo"' > test-tmp/test.nix : > test-tmp/empty ln -s empty test-tmp/link +mkdir -p test-tmp/repo/data test-tmp/tmpdir +echo '{}' > test-tmp/repo/data/default.nix +tar -C test-tmp/repo -cf test-tmp/repo.tar . + x="" for i in {1..64};do x=x$x mkdir -p test-tmp/many-dirs/$x done -run 'with import {}; bash' +export XDG_CACHE_HOME="$PWD/test-tmp/xdg-cache" +export TMPDIR="$PWD/test-tmp/tmpdir" + + +run -p 'with import {}; bash' check import-channel \ "s/nix/var/nix/profiles/per-user/root/channels/unstable" \ "l$(readlink /nix/var/nix/profiles/per-user/root/channels/unstable)" -run 'with import {}; bash' +run -p 'with import {}; bash' check import-channel-ne \ "s/nix/var/nix/profiles/per-user/root/channels/nonexistentChannel" '-' -run 'import ./test-tmp/test.nix' +run -p 'import ./test-tmp/test.nix' check import-relative-nix \ "s$PWD/test-tmp/test.nix" "+" -run 'import ./test-tmp' +run -p 'import ./test-tmp' check import-relative-nix-dir \ "s$PWD/test-tmp" "d" -run 'import ./nonexistent.nix' +run -p 'import ./nonexistent.nix' check import-relative-nix-ne \ "s$PWD/nonexistent.nix" "-" -run 'builtins.readFile ./test-tmp/test.nix' +run -p 'builtins.readFile ./test-tmp/test.nix' check builtins.readFile \ "f$PWD/test-tmp/test.nix" \ "$(b3sum ./test-tmp/test.nix | head -c 32)" -run 'builtins.readFile "/nonexistent/readFile"' +run -p 'builtins.readFile "/nonexistent/readFile"' check builtins.readFile-ne \ "f/nonexistent/readFile" "-" -run 'builtins.readFile ./test-tmp' +run -p 'builtins.readFile ./test-tmp' check builtins.readFile-dir \ "f$PWD/test-tmp" "e" -run 'builtins.readFile ./test-tmp/empty' +run -p 'builtins.readFile ./test-tmp/empty' check builtins.readFile-empty \ "f$PWD/test-tmp/empty" \ "$(b3sum ./test-tmp/empty | head -c 32)" -run 'builtins.readDir ./test-tmp' +run -p 'builtins.readDir ./test-tmp' check builtins.readDir \ "d$PWD/test-tmp" "$(dir_b3sum ./test-tmp)" -run 'builtins.readDir "/nonexistent/readDir"' +run -p 'builtins.readDir "/nonexistent/readDir"' check builtins.readDir-ne \ "d/nonexistent/readDir" "-" -run 'builtins.readDir ./test-tmp/many-dirs' +run -p 'builtins.readDir ./test-tmp/many-dirs' check builtins.readDir-many-dirs \ "d$PWD/test-tmp/many-dirs" "$(dir_b3sum ./test-tmp/many-dirs)" -run_without_p +run check implicit:shell.nix \ "s$PWD/shell.nix" "-" check implicit:default.nix \ "s$PWD/default.nix" "-" + +run -p "fetchTarball file://$PWD/test-tmp/repo.tar?1" +check --first fetchTarball:create \ + "t$PWD/test-tmp/tmpdir/nix-$NIX_PID-1" "+" +check fetchTarball:delete \ + "t$PWD/test-tmp/tmpdir/nix-$NIX_PID-1" "-" + +run -I "q=file://$PWD/test-tmp/repo.tar?2" -p "import " +check --first tarball-I:create \ + "t$PWD/test-tmp/tmpdir/nix-$NIX_PID-1" "+" +check tarball-I:delete \ + "t$PWD/test-tmp/tmpdir/nix-$NIX_PID-1" "-" + exit $result diff --git a/nix-trace/trace-nix.c b/nix-trace/trace-nix.c index b57a40a..670affa 100644 --- a/nix-trace/trace-nix.c +++ b/nix-trace/trace-nix.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,9 @@ static FILE *log_f = NULL; static const char *pwd = NULL; +static char tmp_prefix[PATH_MAX]; // "$TMPDIR/nix-$$-" +static size_t tmp_prefix_dirname_len = 0; // Length of "$TMPDIR" +static size_t tmp_prefix_basename_len = 0; // Length of "nix-$$-" #define FATAL() \ do { \ @@ -88,6 +92,27 @@ static void __attribute__((constructor)) init() { INIT_MUTEX(print_mutex); INIT_MUTEX(buf_mutex); + + // References: + // https://github.com/NixOS/nix/blob/2.15.1/src/libutil/filesystem.cc#L18 + // https://github.com/NixOS/nix/blob/2.15.1/src/libutil/util.hh#L337-L338 + const char *tmpdir = getenv("TMPDIR"); + if (tmpdir == NULL) + tmpdir = "/tmp"; + const char *tmpdirend = tmpdir + strlen(tmpdir); + while (tmpdirend > tmpdir && tmpdirend[-1] == '/') + tmpdirend--; + int len = snprintf(tmp_prefix, sizeof(tmp_prefix), + "%.*s/nix-%" PRIu64 "-", + (int)(tmpdirend-tmpdir), + tmpdir, + (uint64_t)getpid()); + tmp_prefix_dirname_len = tmpdirend - tmpdir; + tmp_prefix_basename_len = len - tmp_prefix_dirname_len - 1; + if (len < 0 || len >= sizeof(tmp_prefix)) { + fprintf(stderr, "trace-nix: TMPDIR too long\n"); + tmp_prefix[0] = '\0'; + } } #ifdef __APPLE__ @@ -158,6 +183,62 @@ WRAPPER(DIR *, opendir, (const char *path)) { return dirp; } +WRAPPER(int, mkdir, (const char *path, mode_t mode)) { + int result = REAL(mkdir)(path, mode); + if (result == 0 && *tmp_prefix && memcmp(path, tmp_prefix, + tmp_prefix_dirname_len + 1 + tmp_prefix_basename_len) == 0) + print_log('t', path, "+"); + return result; +} + +WRAPPER(int, unlinkat, (int dirfd, const char *path, int flags)) { + int result = REAL(unlinkat)(dirfd, path, flags); + if (result != 0 || *tmp_prefix == '\0' || flags != AT_REMOVEDIR) + return result; + size_t path_len = strlen(path); + if (path_len > 45) // 45 == len(f"nix-{2**64}-{2**64}") + return result; + // Check that the path starts with 'nix-$$-' and do not contain slash. + if (memcmp(path, tmp_prefix + tmp_prefix_dirname_len + 1, tmp_prefix_basename_len) != 0 || + strchr(path + tmp_prefix_dirname_len + 1 + tmp_prefix_basename_len, '/')) + return result; + +#ifdef __linux__ + char file_path[tmp_prefix_dirname_len + 1 + path_len + 1]; + char fd_path[35]; // 35 == len(f"/proc/self/fd/{2**64}") + 1 + snprintf(fd_path, sizeof fd_path, "/proc/self/fd/%d", dirfd); + ssize_t len = readlink(fd_path, file_path, tmp_prefix_dirname_len); + if (len == -1 && errno == ENAMETOOLONG || len != tmp_prefix_dirname_len) + return result; + if (len == -1) { + // TODO: also readlink tmp_prefix to canonicalize it + fprintf(stderr, "trace-nix: readlink(%s): %s\n", fd_path, strerror(errno)); + return result; + } + file_path[len] = '\0'; +#elif defined(__APPLE__) + char file_path[PATH_MAX]; // Should be MAXPATHLEN? + if (fcntl(dirfd, F_GETPATH, file_path) == -1) { + fprintf(stderr, "trace-nix: fcntl(%d, F_GETPATH): %s\n", dirfd, strerror(errno)); + return result; + } +#else +#warning "Not implemented for this platform" + return result; +#endif + if (memcmp(file_path, tmp_prefix, tmp_prefix_dirname_len) != 0 || + file_path[tmp_prefix_dirname_len] != 0) + return result; + if (tmp_prefix_dirname_len + 1 + path_len + 1 > sizeof file_path) { + fprintf(stderr, "trace-nix: path too long: %s/%s\n", file_path, path); + return result; + } + file_path[tmp_prefix_dirname_len] = '/'; + memcpy(file_path + tmp_prefix_dirname_len + 1, path, path_len + 1); + print_log('t', file_path, "-"); + return result; +} + //////////////////////////////////////////////////////////////////////////////// static int enable(const char *path) { diff --git a/src/main.rs b/src/main.rs index bc9ef16..5029a26 100644 --- a/src/main.rs +++ b/src/main.rs @@ -279,7 +279,7 @@ fn run_nix_shell(inp: &NixShellInput) -> NixShellOutput { trace_file .read_to_end(&mut trace_data) .expect("Can't read trace file"); - let trace = Trace::load(trace_data); + let trace = Trace::load_raw(trace_data); if trace.check_for_changes() { eprintln!("cached-nix-shell: some files are already updated, cache won't be reused"); } @@ -572,7 +572,7 @@ fn check_cache(hash: &str) -> Option> { return None; } - let trace = read(trace_fname).unwrap().pipe(Trace::load); + let trace = read(trace_fname).unwrap().pipe(Trace::load_sorted); if trace.check_for_changes() { return None; } diff --git a/src/trace.rs b/src/trace.rs index d804945..47519c8 100644 --- a/src/trace.rs +++ b/src/trace.rs @@ -1,29 +1,100 @@ use itertools::Itertools; +use std::collections::btree_map::Entry; use std::collections::BTreeMap; use std::ffi::{OsStr, OsString}; use std::fs::{read, read_dir, read_link, symlink_metadata}; use std::io::ErrorKind; use std::os::unix::ffi::OsStrExt; +use ufcs::Pipe; -/// Output of trace-nix.so, sorted and deduplicated. -pub struct Trace { - items: BTreeMap, Vec>, -} +pub struct Trace(Vec<(Vec, Vec)>); impl Trace { - pub fn load(vec: Vec) -> Trace { - let items = vec + /// Load trace from a vector of bytes, as returned by `trace-nix.so`. + /// Sort and deduplicate entries. Remove entries inside temporary + /// directories created by fetchTarball. + pub fn load_raw(vec: Vec) -> Trace { + let mut result = BTreeMap::, Vec>::new(); + let mut inbetween = BTreeMap::, Vec<(Vec, Vec)>>::new(); + + // In between `tTMPDIR\0+\0` and `tTMPDIR\0-\0`, ignore all entries + // starting with `TMPDIR/`, including `TMPDIR` itself, if `TMPDIR` is + // no longer exists. This corresponds to the temporary directory created + // by fetchTarball. + + 'outer: for (key, value) in vec .split(|&b| b == 0) .filter(|&fname| !fname.is_empty()) // last entry has trailing NUL .map(Vec::from) .tuples::<(_, _)>() - .collect::, Vec>>(); - Trace { items } + { + let fname = OsStr::from_bytes(&key[1..]); + + if key.first() == Some(&b't') { + match value.as_slice() { + b"+" => { + // Handle mkdir + if symlink_metadata(fname).is_ok() { + // Unlikely: temporary directory still exists + continue; + } + match inbetween.entry(key[1..].to_vec()) { + Entry::Vacant(entry) => { + // Likely. + entry.insert(Vec::new()); + } + Entry::Occupied(mut entry) => { + // Unlikely: the directory with the same + // name was created twice. + for (k, v) in entry.get_mut().drain(..) { + result.insert(k, v); + } + *entry.get_mut() = Vec::new(); + } + } + } + b"-" => { + // Handle unlinkat + inbetween.remove(&key[1..]); + } + _ => panic!("Unexpected"), + } + } else { + for (k, v) in inbetween.iter_mut() { + if k == &key[1..] + || key[1..].starts_with(k) + && key.get(k.len() + 1) == Some(&b'/') + { + v.push((key.clone(), value.clone())); + continue 'outer; + } + } + result.insert(key, value); + } + } + + for (_, v) in inbetween { + for (k, v) in v { + result.insert(k, v); + } + } + + Trace(result.into_iter().collect()) + } + + /// Load trace from a vector of bytes, as stored in the cache. + pub fn load_sorted(vec: Vec) -> Trace { + vec.split(|&b| b == 0) + .filter(|&fname| !fname.is_empty()) // last entry has trailing NUL + .map(Vec::from) + .tuples::<(_, _)>() + .collect::>() + .pipe(Trace) } pub fn serialize(&self) -> Vec { let mut result = Vec::::new(); - for (a, b) in &self.items { + for (a, b) in &self.0 { result.push(0); result.extend(a); result.push(0); @@ -34,7 +105,7 @@ impl Trace { /// Return true if trace doesn't match (i.e. some file is changed) pub fn check_for_changes(&self) -> bool { - for (k, v) in &self.items { + for (k, v) in &self.0 { if check_item_updated(k, v) { return true; } diff --git a/tests/t19-tarball-unpack.sh b/tests/t19-tarball-unpack.sh new file mode 100755 index 0000000..2f42fd8 --- /dev/null +++ b/tests/t19-tarball-unpack.sh @@ -0,0 +1,21 @@ +#!/bin/sh +. ./lib.sh +# Check that tarball unpacking wont interfere with the cache. + +mkdir -p tmp/repo/data +echo '(import {}).hello' > tmp/repo/data/default.nix +tar -C tmp/repo -cf tmp/repo.tar data + +R=$$_$(date +%s) + +run cached-nix-shell -I q="file://$PWD/tmp/repo.tar?a$R" -p 'import ' --run : +check_slow + +run cached-nix-shell -I q="file://$PWD/tmp/repo.tar?a$R" -p 'import ' --run : +check_fast + +run cached-nix-shell -p "fetchTarball file://$PWD/tmp/repo.tar?b$R" --run : +check_slow + +run cached-nix-shell -p "fetchTarball file://$PWD/tmp/repo.tar?b$R" --run : +check_fast