Decompress tar files using libarchive

Makefile.config.in

@@ -18,6 +18,7 @@ SODIUM_LIBS = @SODIUM_LIBS@
 LIBLZMA_LIBS = @LIBLZMA_LIBS@
 SQLITE3_LIBS = @SQLITE3_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
+LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
 EDITLINE_LIBS = @EDITLINE_LIBS@
 bash = @bash@
 bindir = @bindir@

configure.ac

@@ -178,6 +178,8 @@ AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true],
   [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
 AC_CHECK_HEADERS([bzlib.h], [true],
   [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2.  See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])])
+# Checks for libarchive
+PKG_CHECK_MODULES([LIBARCHIVE], [libarchive >= 3.1.2], [CXXFLAGS="$LIBARCHIVE_CFLAGS $CXXFLAGS"])
 
 # Look for SQLite, a required dependency.
 PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"])

nix-rust/Cargo.toml

@@ -9,5 +9,4 @@ name = "nixrust"
 crate-type = ["cdylib"]
 
 [dependencies]
-tar = "0.4"
 libc = "0.2"

nix-rust/src/lib.rs

@@ -1,6 +1,5 @@
 mod error;
 mod foreign;
-mod tarfile;
 
 pub use error::Error;
 
@@ -23,10 +22,3 @@ impl<T> CBox<T> {
     }
 }
 
-#[no_mangle]
-pub extern "C" fn unpack_tarfile(
-    source: foreign::Source,
-    dest_dir: &str,
-) -> CBox<Result<(), error::CppException>> {
-    CBox::new(tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into()))
-}

release-common.nix

@@ -49,6 +49,7 @@ rec {
     [ curl
       bzip2 xz brotli editline
       openssl pkgconfig sqlite boehmgc
+      libarchive
       boost
       nlohmann_json
       rustc cargo

release.nix

@@ -394,10 +394,10 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" "bison" "flex" ]
+            [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libarchive-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" "bison" "flex" ]
             ++ extraPackages; };
       # At most 2047MB can be simulated in qemu-system-i386
-      memSize = 2047;
+    memSize = 2047;
       meta.schedulingPriority = 50;
       postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck";
       #enableParallelBuilding = true;
@@ -417,14 +417,14 @@ let
       src = jobs.tarball;
       diskImage = (diskImageFun vmTools.diskImageFuns)
         { extraPackages =
-            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" "libsodium-dev" "libboost-all-dev" ]
+            [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libarchive-dev" "libssl-dev" "liblzma-dev" "libseccomp-dev" "libsodium-dev" "libboost-all-dev" ]
             ++ extraPackages; };
       memSize = 2047;
       meta.schedulingPriority = 50;
       postInstall = "make installcheck";
       configureFlags = "--sysconfdir=/etc";
       debRequires =
-        [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" "libseccomp2" ]
+        [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libarchive" "libssl1.0.0" "liblzma5" "libseccomp2" ]
         ++ extraDebPackages;
       debMaintainer = "Eelco Dolstra <[email protected]>";
       doInstallCheck = true;

src/libstore/download.cc

@@ -907,7 +907,7 @@ CachedDownloadResult Downloader::downloadCached(
             printInfo("unpacking '%s'...", url);
             Path tmpDir = createTempDir();
             AutoDelete autoDelete(tmpDir, true);
-            unpackTarfile(store->toRealPath(storePath), tmpDir, baseNameOf(url));
+            unpackTarfile(store->toRealPath(storePath), tmpDir);
             auto members = readDirectory(tmpDir);
             if (members.size() != 1)
                 throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);

src/libutil/local.mk

@@ -6,6 +6,6 @@ libutil_DIR := $(d)
 
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
-libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context
+libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
 
 libutil_LIBS = libnixrust

src/libutil/tarfile.cc

@@ -1,36 +1,132 @@
-#include "rust-ffi.hh"
-#include "compression.hh"
+#include <archive.h>
+#include <archive_entry.h>
 
-extern "C" {
-    rust::Result<std::tuple<>> *
-    unpack_tarfile(rust::Source source, rust::StringSlice dest_dir);
-}
+#include "serialise.hh"
 
 namespace nix {
 
+struct TarArchive {
+    struct archive *archive;
+    Source *source;
+    std::vector<unsigned char> buffer;
+
+    void check(int err, const char *reason = "Failed to extract archive (%s)") {
+        if (err == ARCHIVE_EOF)
+            throw EndOfFile("reached end of archive");
+        else if (err != ARCHIVE_OK)
+            throw Error(reason, archive_error_string(this->archive));
+    }
+
+    TarArchive(Source& source) : buffer(4096) {
+        this->archive = archive_read_new();
+        this->source = &source;
+
+        archive_read_support_filter_all(archive);
+        archive_read_support_format_all(archive);
+        check(archive_read_open(archive, (void *)this, TarArchive::callback_open, TarArchive::callback_read, TarArchive::callback_close), "Failed to open archive (%s)");
+    }
+
+    TarArchive(const Path &path) {
+        this->archive = archive_read_new();
+
+        archive_read_support_filter_all(archive);
+        archive_read_support_format_all(archive);
+        check(archive_read_open_filename(archive, path.c_str(), 16384), "Failed to open archive (%s)");
+    }
+
+    // disable copy constructor
+    TarArchive(const TarArchive&) = delete;
+
+    void close() {
+        check(archive_read_close(archive), "Failed to close archive (%s)");
+    }
+
+    ~TarArchive() {
+        if (this->archive) archive_read_free(this->archive);
+    }
+
+private:
+    static int callback_open(struct archive *, void *self) {
+        return ARCHIVE_OK;
+    }
+
+    static ssize_t callback_read(struct archive *archive, void *_self, const void **buffer) {
+        TarArchive *self = (TarArchive *)_self;
+        *buffer = self->buffer.data();
+
+        try {
+            return self->source->read(self->buffer.data(), 4096);
+        } catch (EndOfFile &) {
+            return 0;
+        } catch (std::exception &err) {
+            archive_set_error(archive, EIO, "Source threw exception: %s", err.what());
+
+            return -1;
+        }
+    }
+
+    static int callback_close(struct archive *, void *self) {
+        return ARCHIVE_OK;
+    }
+};
+
+struct PushD {
+    char * oldDir;
+
+    PushD(const std::string &newDir)  {
+        oldDir = getcwd(0, 0);
+        if (!oldDir) throw SysError("getcwd");
+        int r = chdir(newDir.c_str());
+        if (r != 0) throw SysError("changing directory to tar output path");
+    }
+
+    ~PushD() {
+        int r = chdir(oldDir);
+        free(oldDir);
+        if (r != 0)
+            std::cerr << "warning: failed to change directory back after tar extraction";
+        /* can't throw out of a destructor */
+    }
+};
+
+static void extract_archive(TarArchive &archive, const Path & destDir) {
+    // need to chdir back *after* archive closing
+    PushD newDir(destDir);
+    struct archive_entry *entry;
+    int flags = ARCHIVE_EXTRACT_FFLAGS
+        | ARCHIVE_EXTRACT_PERM
+        | ARCHIVE_EXTRACT_SECURE_SYMLINKS
+        | ARCHIVE_EXTRACT_SECURE_NODOTDOT
+        | ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS;
+
+    for(;;) {
+        int r = archive_read_next_header(archive.archive, &entry);
+        if (r == ARCHIVE_EOF) break;
+        else if (r == ARCHIVE_WARN)
+            std::cerr << "warning: " << archive_error_string(archive.archive) << std::endl;
+        else
+            archive.check(r);
+
+        archive.check(archive_read_extract(archive.archive, entry, flags));
+    }
+
+    archive.close();
+}
+
 void unpackTarfile(Source & source, const Path & destDir)
 {
-    rust::Source source2(source);
-    rust::CBox(unpack_tarfile(source2, destDir))->unwrap();
+    auto archive = TarArchive(source);
+
+    createDirs(destDir);
+    extract_archive(archive, destDir);
 }
 
-void unpackTarfile(const Path & tarFile, const Path & destDir,
-    std::optional<std::string> baseName)
+void unpackTarfile(const Path & tarFile, const Path & destDir)
 {
-    if (!baseName) baseName = baseNameOf(tarFile);
-
-    auto source = sinkToSource([&](Sink & sink) {
-        // FIXME: look at first few bytes to determine compression type.
-        auto decompressor =
-            // FIXME: add .gz support
-            hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) :
-            hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) :
-            makeDecompressionSink("none", sink);
-        readFile(tarFile, *decompressor);
-        decompressor->finish();
-    });
-
-    unpackTarfile(*source, destDir);
+    auto archive = TarArchive(tarFile);
+
+    createDirs(destDir);
+    extract_archive(archive, destDir);
 }
 
 }

src/libutil/tarfile.hh

@@ -4,7 +4,6 @@ namespace nix {
 
 void unpackTarfile(Source & source, const Path & destDir);
 
-void unpackTarfile(const Path & tarFile, const Path & destDir,
-    std::optional<std::string> baseName = {});
+void unpackTarfile(const Path & tarFile, const Path & destDir);
 
 }