From 5351f1bd36647617d26193f31ea58ef11d78280c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 15:25:43 +0200 Subject: [PATCH] Drop remaining uses of external "tar" Also, fetchGit now runs in O(1) memory since we pipe the output of 'git archive' directly into unpackTarball() (rather than first reading it all into memory). --- src/libexpr/primops/fetchGit.cc | 11 +++++++---- src/libstore/builtins/unpack-channel.cc | 12 +----------- src/libstore/download.cc | 12 ++++++++---- src/libutil/serialise.hh | 1 - src/libutil/tarfile.cc | 22 +++++++++++++++++++++- src/libutil/tarfile.hh | 5 ++++- src/nix-prefetch-url/nix-prefetch-url.cc | 4 ++-- 7 files changed, 43 insertions(+), 24 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 6229fef8d02e..1117443d9704 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -4,6 +4,7 @@ #include "store-api.hh" #include "pathlocks.hh" #include "hash.hh" +#include "tarfile.hh" #include @@ -166,14 +167,16 @@ GitInfo exportGit(ref store, const std::string & uri, if (e.errNo != ENOENT) throw; } - // FIXME: should pipe this, or find some better way to extract a - // revision. - auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev }); + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runProgram("tar", true, { "x", "-C", tmpDir }, tar); + unpackTarfile(*source, tmpDir); gitInfo.storePath = store->addToStore(name, tmpDir); diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 5fc68cd666ce..d18e3ddafbcc 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -1,5 +1,4 @@ #include "builtins.hh" -#include "compression.hh" #include "tarfile.hh" namespace nix { @@ -18,16 +17,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) createDirs(out); - auto source = sinkToSource([&](Sink & sink) { - auto decompressor = - hasSuffix(src, ".bz2") ? makeDecompressionSink("bzip2", sink) : - hasSuffix(src, ".xz") ? makeDecompressionSink("xz", sink) : - makeDecompressionSink("none", sink); - readFile(src, *decompressor); - decompressor->finish(); - }); - - unpackTarfile(*source, out); + unpackTarfile(src, out); auto entries = readDirectory(out); if (entries.size() != 1) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index cdf56e09d69a..16e5b24adb6a 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -8,6 +8,7 @@ #include "compression.hh" #include "pathlocks.hh" #include "finally.hh" +#include "tarfile.hh" #ifdef ENABLE_S3 #include @@ -898,12 +899,15 @@ CachedDownloadResult Downloader::downloadCached( unpackedStorePath = ""; } if (unpackedStorePath.empty()) { - printInfo(format("unpacking '%1%'...") % url); + printInfo("unpacking '%s'...", url); Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); - unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); + unpackTarfile(store->toRealPath(storePath), tmpDir, baseNameOf(url)); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); } replaceSymlink(unpackedStorePath, unpackedLink); storePath = unpackedStorePath; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a344a5ac7520..02e52c6383c6 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -77,7 +77,6 @@ struct BufferedSource : Source size_t read(unsigned char * data, size_t len) override; - bool hasData(); protected: diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index ae6d512bd03e..f7d3ad4172f2 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -1,4 +1,5 @@ #include "rust-ffi.hh" +#include "compression.hh" extern "C" { rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); @@ -6,9 +7,28 @@ extern "C" { namespace nix { -void unpackTarfile(Source & source, Path destDir) +void unpackTarfile(Source & source, const Path & destDir) { unpack_tarfile(source, destDir).use()->unwrap(); } +void unpackTarfile(const Path & tarFile, const Path & destDir, + std::optional baseName) +{ + if (!baseName) baseName = baseNameOf(tarFile); + + auto source = sinkToSource([&](Sink & sink) { + // FIXME: look at first few bytes to determine compression type. + auto decompressor = + // FIXME: add .gz support + hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) : + hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) : + makeDecompressionSink("none", sink); + readFile(tarFile, *decompressor); + decompressor->finish(); + }); + + unpackTarfile(*source, destDir); +} + } diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index c3e95fb0c92a..ce0911e2a326 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -2,6 +2,9 @@ namespace nix { -void unpackTarfile(Source & source, Path destDir); +void unpackTarfile(Source & source, const Path & destDir); + +void unpackTarfile(const Path & tarFile, const Path & destDir, + std::optional baseName = {}); } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index f54706a8a011..78c883833773 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -9,6 +9,7 @@ #include "legacy.hh" #include "finally.hh" #include "progress-bar.hh" +#include "tarfile.hh" #include @@ -192,8 +193,7 @@ static int _main(int argc, char * * argv) if (hasSuffix(baseNameOf(uri), ".zip")) runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}); else - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", tmpFile, "-C", unpacked}); + unpackTarfile(tmpFile, unpacked, baseNameOf(uri)); /* If the archive unpacks to a single file/directory, then use that as the top-level. */