From 11da5b2816e11b081d8ff38db4330addd2014f7e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Mar 2019 14:12:20 +0100 Subject: [PATCH 01/23] Add some Rust code --- .gitignore | 2 ++ Makefile | 1 + nix-rust/Cargo.lock | 59 +++++++++++++++++++++++++++++++++++++++++++ nix-rust/Cargo.toml | 12 +++++++++ nix-rust/local.mk | 7 ++++++ nix-rust/src/lib.rs | 48 +++++++++++++++++++++++++++++++++++ release-common.nix | 1 + shell.nix | 2 +- src/nix/local.mk | 4 +-- src/nix/test.cc | 61 +++++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 nix-rust/Cargo.lock create mode 100644 nix-rust/Cargo.toml create mode 100644 nix-rust/local.mk create mode 100644 nix-rust/src/lib.rs create mode 100644 src/nix/test.cc diff --git a/.gitignore b/.gitignore index fd62dfb3856..f58e1f90fc8 100644 --- a/.gitignore +++ b/.gitignore @@ -119,3 +119,5 @@ GPATH GRTAGS GSYMS GTAGS + +nix-rust/target diff --git a/Makefile b/Makefile index 866c0961eb6..469070533b5 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ makefiles = \ mk/precompiled-headers.mk \ local.mk \ + nix-rust/local.mk \ src/libutil/local.mk \ src/libstore/local.mk \ src/libmain/local.mk \ diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock new file mode 100644 index 00000000000..cfaedc3ed4c --- /dev/null +++ b/nix-rust/Cargo.lock @@ -0,0 +1,59 @@ +[[package]] +name = "cfg-if" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "filetime" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "libc" +version = "0.2.50" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "nix-rust" +version = "0.1.0" +dependencies = [ + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "redox_syscall" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "tar" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", + "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11d43355396e872eefb45ce6342e4374ed7bc2b3a502d1b28e36d6e23c05d1f4" +"checksum filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a2df5c1a8c4be27e7707789dc42ae65976e60b394afd293d1419ab915833e646" +"checksum libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)" = "aab692d7759f5cd8c859e169db98ae5b52c924add2af5fbbca11d12fefb567c1" +"checksum redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)" = "423e376fffca3dfa06c9e9790a9ccd282fafb3cc6e6397d01dbf64f9bacc6b85" +"checksum tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)" = "c2167ff53da2a661702b3299f71a91b61b1dffef36b4b2884b1f9c67254c0133" +"checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml new file mode 100644 index 00000000000..2d2bf67523d --- /dev/null +++ b/nix-rust/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "nix-rust" +version = "0.1.0" +authors = ["Eelco Dolstra "] + +[lib] +name = "nixrust" +crate-type = ["staticlib"] + +[dependencies] +tar = "0.4" +libc = "0.2" diff --git a/nix-rust/local.mk b/nix-rust/local.mk new file mode 100644 index 00000000000..c69e3b9ea2c --- /dev/null +++ b/nix-rust/local.mk @@ -0,0 +1,7 @@ +libnixrust_PATH := $(d)/target/release/libnixrust.a +libnixrust_INSTALL_PATH := $(libnixrust_PATH) +libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl +libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) + +$(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml + $(trace-gen) cd nix-rust && RUSTC_BOOTSTRAP=1 cargo build --release && touch target/release/libnixrust.a diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs new file mode 100644 index 00000000000..abcb8905534 --- /dev/null +++ b/nix-rust/src/lib.rs @@ -0,0 +1,48 @@ +extern crate libc; +extern crate tar; + +use std::fs; +use std::io; +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; +use tar::Archive; + +#[no_mangle] +pub extern "C" fn unpack_tarfile(data: &[u8], dest_dir: &str) -> bool { + // FIXME: handle errors. + + let dest_dir = Path::new(dest_dir); + + let mut tar = Archive::new(data); + + for file in tar.entries().unwrap() { + let mut file = file.unwrap(); + + let dest_file = dest_dir.join(file.header().path().unwrap()); + + fs::create_dir_all(dest_file.parent().unwrap()).unwrap(); + + match file.header().entry_type() { + tar::EntryType::Directory => { + fs::create_dir(dest_file).unwrap(); + } + tar::EntryType::Regular => { + let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 { + 0o666 + } else { + 0o777 + }; + let mut f = fs::OpenOptions::new() + .create(true) + .write(true) + .mode(mode) + .open(dest_file) + .unwrap(); + io::copy(&mut file, &mut f).unwrap(); + } + t => panic!("Unsupported tar entry type '{:?}'.", t), + } + } + + true +} diff --git a/release-common.nix b/release-common.nix index 63f39f00548..dd5f939d969 100644 --- a/release-common.nix +++ b/release-common.nix @@ -51,6 +51,7 @@ rec { openssl pkgconfig sqlite boehmgc boost nlohmann_json + rustc cargo # Tests git diff --git a/shell.nix b/shell.nix index 9c504f024e9..21ed4712133 100644 --- a/shell.nix +++ b/shell.nix @@ -7,7 +7,7 @@ with import ./release-common.nix { inherit pkgs; }; (if useClang then clangStdenv else stdenv).mkDerivation { name = "nix"; - buildInputs = buildDeps ++ tarballDeps ++ perlDeps; + buildInputs = buildDeps ++ tarballDeps ++ perlDeps ++ [ pkgs.rustfmt ]; inherit configureFlags; diff --git a/src/nix/local.mk b/src/nix/local.mk index c09efd1fc89..a145cf9b1d1 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -15,9 +15,9 @@ nix_SOURCES := \ $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ -nix_LIBS = libexpr libmain libstore libutil +nix_LIBS = libexpr libmain libstore libutil libnixrust -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system -Lnix-rust/target/release $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ diff --git a/src/nix/test.cc b/src/nix/test.cc new file mode 100644 index 00000000000..70e0a903de4 --- /dev/null +++ b/src/nix/test.cc @@ -0,0 +1,61 @@ +#include "command.hh" +#include "store-api.hh" +#include "common-args.hh" + +using namespace nix; + +namespace rust { + +// Depending on the internal representation of Rust slices is slightly +// evil... +template struct Slice +{ + const T * ptr; + size_t size; + + Slice(const T * ptr, size_t size) : ptr(ptr), size(size) + { + assert(ptr); + } +}; + +struct StringSlice : Slice +{ + StringSlice(const std::string & s): Slice(s.data(), s.size()) { } +}; + +} + +extern "C" { + bool unpack_tarfile(rust::Slice data, rust::StringSlice dest_dir); +} + +struct CmdTest : StoreCommand +{ + CmdTest() + { + } + + std::string name() override + { + return "test"; + } + + std::string description() override + { + return "bla bla"; + } + + void run(ref store) override + { + auto data = readFile("./nix-2.2.tar"); + + std::string destDir = "./dest"; + + deletePath(destDir); + + unpack_tarfile({(uint8_t*) data.data(), data.size()}, destDir); + } +}; + +static RegisterCommand r(make_ref()); From e60f6bd4ce831ced94fafeb527c429b6f88159ac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Mar 2019 20:45:56 +0100 Subject: [PATCH 02/23] Enable Rust code to call C++ Source objects --- nix-rust/src/lib.rs | 19 +++++++++++++++++-- src/nix/test.cc | 34 ++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index abcb8905534..799d52e313c 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -7,13 +7,28 @@ use std::os::unix::fs::OpenOptionsExt; use std::path::Path; use tar::Archive; +/// A wrapper around Nix's Source class that provides the Read trait. +#[repr(C)] +pub struct Source { + fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize, + this: *mut libc::c_void, +} + +impl std::io::Read for Source { + fn read(&mut self, buf: &mut [u8]) -> std::result::Result { + let n = (self.fun)(self.this, buf); + assert!(n <= buf.len()); + Ok(n) + } +} + #[no_mangle] -pub extern "C" fn unpack_tarfile(data: &[u8], dest_dir: &str) -> bool { +pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool { // FIXME: handle errors. let dest_dir = Path::new(dest_dir); - let mut tar = Archive::new(data); + let mut tar = Archive::new(source); for file in tar.entries().unwrap() { let mut file = file.unwrap(); diff --git a/src/nix/test.cc b/src/nix/test.cc index 70e0a903de4..7f1f0d47aab 100644 --- a/src/nix/test.cc +++ b/src/nix/test.cc @@ -1,6 +1,7 @@ #include "command.hh" #include "store-api.hh" #include "common-args.hh" +#include "compression.hh" using namespace nix; @@ -10,10 +11,10 @@ namespace rust { // evil... template struct Slice { - const T * ptr; + T * ptr; size_t size; - Slice(const T * ptr, size_t size) : ptr(ptr), size(size) + Slice(T * ptr, size_t size) : ptr(ptr), size(size) { assert(ptr); } @@ -21,13 +22,30 @@ template struct Slice struct StringSlice : Slice { - StringSlice(const std::string & s): Slice(s.data(), s.size()) { } + StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {} +}; + +struct Source +{ + size_t (*fun)(void * source_this, rust::Slice data); + nix::Source * _this; + + Source(nix::Source & _this) + : fun(sourceWrapper), _this(&_this) + {} + + // FIXME: how to propagate exceptions? + static size_t sourceWrapper(void * _this, rust::Slice data) + { + auto n = ((nix::Source *) _this)->read(data.ptr, data.size); + return n; + } }; } extern "C" { - bool unpack_tarfile(rust::Slice data, rust::StringSlice dest_dir); + bool unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } struct CmdTest : StoreCommand @@ -48,13 +66,17 @@ struct CmdTest : StoreCommand void run(ref store) override { - auto data = readFile("./nix-2.2.tar"); + auto source = sinkToSource([&](Sink & sink) { + auto decompressor = makeDecompressionSink("bzip2", sink); + readFile("./nix-2.2.tar.bz2", *decompressor); + decompressor->finish(); + }); std::string destDir = "./dest"; deletePath(destDir); - unpack_tarfile({(uint8_t*) data.data(), data.size()}, destDir); + unpack_tarfile(*source, destDir); } }; From 045708db4343174f30f3647776971c852f72a9e8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Mar 2019 23:40:35 +0100 Subject: [PATCH 03/23] Make a builtin builder This was the last function using a shell script, so this allows us to get rid of tar, coreutils, bash etc. --- corepkgs/config.nix.in | 10 +----- corepkgs/unpack-channel.nix | 35 +++------------------ nix-rust/src/lib.rs | 5 ++- src/libstore/build.cc | 2 ++ src/libstore/builtins.hh | 1 + src/libstore/builtins/unpack-channel.cc | 39 +++++++++++++++++++++++ src/libstore/local.mk | 2 +- src/{nix/test.cc => libstore/rust.hh} | 41 +------------------------ src/nix/local.mk | 4 +-- 9 files changed, 55 insertions(+), 84 deletions(-) create mode 100644 src/libstore/builtins/unpack-channel.cc rename src/{nix/test.cc => libstore/rust.hh} (53%) diff --git a/corepkgs/config.nix.in b/corepkgs/config.nix.in index 32ce6b399f2..4ea182d8f58 100644 --- a/corepkgs/config.nix.in +++ b/corepkgs/config.nix.in @@ -1,3 +1,4 @@ +# FIXME: remove this file? let fromEnv = var: def: let val = builtins.getEnv var; in @@ -17,13 +18,4 @@ in rec { nixLocalstateDir = "@localstatedir@"; nixSysconfDir = "@sysconfdir@"; nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@"; - - # If Nix is installed in the Nix store, then automatically add it as - # a dependency to the core packages. This ensures that they work - # properly in a chroot. - chrootDeps = - if dirOf nixPrefix == builtins.storeDir then - [ (builtins.storePath nixPrefix) ] - else - [ ]; } diff --git a/corepkgs/unpack-channel.nix b/corepkgs/unpack-channel.nix index d39a2063781..10515bc8b91 100644 --- a/corepkgs/unpack-channel.nix +++ b/corepkgs/unpack-channel.nix @@ -1,39 +1,12 @@ -with import ; - -let - - builder = builtins.toFile "unpack-channel.sh" - '' - mkdir $out - cd $out - xzpat="\.xz\$" - gzpat="\.gz\$" - if [[ "$src" =~ $xzpat ]]; then - ${xz} -d < $src | ${tar} xf - ${tarFlags} - elif [[ "$src" =~ $gzpat ]]; then - ${gzip} -d < $src | ${tar} xf - ${tarFlags} - else - ${bzip2} -d < $src | ${tar} xf - ${tarFlags} - fi - if [ * != $channelName ]; then - mv * $out/$channelName - fi - ''; - -in - { name, channelName, src }: derivation { - system = builtins.currentSystem; - builder = shell; - args = [ "-e" builder ]; - inherit name channelName src; + builder = "builtin:unpack-channel"; + + system = "builtin"; - PATH = "${nixBinDir}:${coreutils}"; + inherit name channelName src; # No point in doing this remotely. preferLocalBuild = true; - - inherit chrootDeps; } diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index 799d52e313c..ac6dee543a2 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -33,7 +33,7 @@ pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool { for file in tar.entries().unwrap() { let mut file = file.unwrap(); - let dest_file = dest_dir.join(file.header().path().unwrap()); + let dest_file = dest_dir.join(file.path().unwrap()); fs::create_dir_all(dest_file.parent().unwrap()).unwrap(); @@ -55,6 +55,9 @@ pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool { .unwrap(); io::copy(&mut file, &mut f).unwrap(); } + tar::EntryType::Symlink => { + std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file).unwrap(); + } t => panic!("Unsupported tar entry type '{:?}'.", t), } } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 51a9fa35b2b..efbb7fc9f06 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3128,6 +3128,8 @@ void DerivationGoal::runChild() builtinFetchurl(drv2, netrcData); else if (drv->builder == "builtin:buildenv") builtinBuildenv(drv2); + else if (drv->builder == "builtin:unpack-channel") + builtinUnpackChannel(drv2); else throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8)); _exit(0); diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 0d2da873ece..87d6ce66533 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -7,5 +7,6 @@ namespace nix { // TODO: make pluggable. void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); void builtinBuildenv(const BasicDerivation & drv); +void builtinUnpackChannel(const BasicDerivation & drv); } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc new file mode 100644 index 00000000000..88202ec6b7f --- /dev/null +++ b/src/libstore/builtins/unpack-channel.cc @@ -0,0 +1,39 @@ +#include "rust.hh" +#include "builtins.hh" +#include "compression.hh" + +namespace nix { + +void builtinUnpackChannel(const BasicDerivation & drv) +{ + auto getAttr = [&](const string & name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + return i->second; + }; + + Path out = getAttr("out"); + auto channelName = getAttr("channelName"); + auto src = getAttr("src"); + + createDirs(out); + + auto source = sinkToSource([&](Sink & sink) { + auto decompressor = + hasSuffix(src, ".bz2") ? makeDecompressionSink("bzip2", sink) : + hasSuffix(src, ".xz") ? makeDecompressionSink("xz", sink) : + makeDecompressionSink("none", sink); + readFile(src, *decompressor); + decompressor->finish(); + }); + + unpack_tarfile(*source, out); + + auto entries = readDirectory(out); + if (entries.size() != 1) + throw Error("channel tarball '%s' contains more than one file", src); + if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1) + throw SysError("renaming channel directory"); +} + +} diff --git a/src/libstore/local.mk b/src/libstore/local.mk index d690fea28c2..d3254554dd3 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -6,7 +6,7 @@ libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) -libstore_LIBS = libutil +libstore_LIBS = libutil libnixrust libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread ifneq ($(OS), FreeBSD) diff --git a/src/nix/test.cc b/src/libstore/rust.hh similarity index 53% rename from src/nix/test.cc rename to src/libstore/rust.hh index 7f1f0d47aab..7e6c2f54d8a 100644 --- a/src/nix/test.cc +++ b/src/libstore/rust.hh @@ -1,9 +1,4 @@ -#include "command.hh" -#include "store-api.hh" -#include "common-args.hh" -#include "compression.hh" - -using namespace nix; +#include "serialise.hh" namespace rust { @@ -47,37 +42,3 @@ struct Source extern "C" { bool unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } - -struct CmdTest : StoreCommand -{ - CmdTest() - { - } - - std::string name() override - { - return "test"; - } - - std::string description() override - { - return "bla bla"; - } - - void run(ref store) override - { - auto source = sinkToSource([&](Sink & sink) { - auto decompressor = makeDecompressionSink("bzip2", sink); - readFile("./nix-2.2.tar.bz2", *decompressor); - decompressor->finish(); - }); - - std::string destDir = "./dest"; - - deletePath(destDir); - - unpack_tarfile(*source, destDir); - } -}; - -static RegisterCommand r(make_ref()); diff --git a/src/nix/local.mk b/src/nix/local.mk index a145cf9b1d1..c09efd1fc89 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -15,9 +15,9 @@ nix_SOURCES := \ $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ -nix_LIBS = libexpr libmain libstore libutil libnixrust +nix_LIBS = libexpr libmain libstore libutil -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system -Lnix-rust/target/release +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ From 6a9c815734de3549a5116796830bd4ebfd006025 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Mar 2019 23:49:14 +0100 Subject: [PATCH 04/23] Remove most of This is no longer needed. --- .gitignore | 1 + corepkgs/config.nix.in | 8 -------- tests/{config.nix => config.nix.in} | 6 ++---- tests/local.mk | 2 +- 4 files changed, 4 insertions(+), 13 deletions(-) rename tests/{config.nix => config.nix.in} (86%) diff --git a/.gitignore b/.gitignore index f58e1f90fc8..f64031e1220 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,7 @@ perl/Makefile.config /tests/restricted-innocent /tests/shell /tests/shell.drv +/tests/config.nix # /tests/lang/ /tests/lang/*.out diff --git a/corepkgs/config.nix.in b/corepkgs/config.nix.in index 4ea182d8f58..cb994594420 100644 --- a/corepkgs/config.nix.in +++ b/corepkgs/config.nix.in @@ -4,14 +4,6 @@ let let val = builtins.getEnv var; in if val != "" then val else def; in rec { - shell = "@bash@"; - coreutils = "@coreutils@"; - bzip2 = "@bzip2@"; - gzip = "@gzip@"; - xz = "@xz@"; - tar = "@tar@"; - tarFlags = "@tarFlags@"; - tr = "@tr@"; nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@"; nixPrefix = "@prefix@"; nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@"; diff --git a/tests/config.nix b/tests/config.nix.in similarity index 86% rename from tests/config.nix rename to tests/config.nix.in index 6ba91065b83..51aed539cb5 100644 --- a/tests/config.nix +++ b/tests/config.nix.in @@ -1,9 +1,7 @@ -with import ; - rec { - inherit shell; + shell = "@bash@"; - path = coreutils; + path = "@coreutils@"; system = builtins.currentSystem; diff --git a/tests/local.mk b/tests/local.mk index 187f96ea206..f1d215eeca9 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -39,4 +39,4 @@ tests-environment = NIX_REMOTE= $(bash) -e clean-files += $(d)/common.sh -installcheck: $(d)/common.sh $(d)/plugins/libplugintest.$(SO_EXT) +installcheck: $(d)/common.sh $(d)/config.nix $(d)/plugins/libplugintest.$(SO_EXT) From 87b7b25e13206bcfe200f45df822787612f159f7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 28 Mar 2019 00:02:10 +0100 Subject: [PATCH 05/23] Clean up the configure script --- configure.ac | 20 +++----------------- perl/lib/Nix/Config.pm.in | 4 ---- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/configure.ac b/configure.ac index 26f16b8460a..8e38f2b8e5f 100644 --- a/configure.ac +++ b/configure.ac @@ -117,26 +117,15 @@ fi ]) NEED_PROG(bash, bash) -NEED_PROG(patch, patch) AC_PATH_PROG(xmllint, xmllint, false) AC_PATH_PROG(xsltproc, xsltproc, false) AC_PATH_PROG(flex, flex, false) AC_PATH_PROG(bison, bison, false) -NEED_PROG(sed, sed) -NEED_PROG(tar, tar) -NEED_PROG(bzip2, bzip2) -NEED_PROG(gzip, gzip) -NEED_PROG(xz, xz) AC_PATH_PROG(dot, dot) AC_PATH_PROG(lsof, lsof, lsof) -NEED_PROG(cat, cat) -NEED_PROG(tr, tr) -AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH], - [path of cat, mkdir, etc.]), - coreutils=$withval, coreutils=$(dirname $cat)) -AC_SUBST(coreutils) +AC_SUBST(coreutils, [$(dirname $(type -p cat))]) AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], @@ -167,7 +156,8 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then LIBS="-latomic $LIBS" fi -# Look for OpenSSL, a required dependency. +# Look for OpenSSL, a required dependency. FIXME: this is only (maybe) +# used by S3BinaryCacheStore. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) @@ -177,11 +167,9 @@ AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true], AC_CHECK_HEADERS([bzlib.h], [true], [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) - # Look for SQLite, a required dependency. PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"]) - # Look for libcurl, a required dependency. PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) @@ -204,13 +192,11 @@ PKG_CHECK_MODULES([SODIUM], [libsodium], have_sodium=1], [have_sodium=]) AC_SUBST(HAVE_SODIUM, [$have_sodium]) - # Look for liblzma, a required dependency. PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt], [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])]) - # Look for libbrotli{enc,dec}. PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in index 67a20c3f41d..bc1749e601e 100644 --- a/perl/lib/Nix/Config.pm.in +++ b/perl/lib/Nix/Config.pm.in @@ -11,10 +11,6 @@ $logDir = $ENV{"NIX_LOG_DIR"} || "@nixlocalstatedir@/log/nix"; $confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix"; $storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@"; -$bzip2 = "@bzip2@"; -$xz = "@xz@"; -$curl = "@curl@"; - $useBindings = 1; %config = (); From 0dbb249b36f1d058e63982d26746fde48911cce4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 4 Jul 2019 17:06:38 +0200 Subject: [PATCH 06/23] Update Rust dependencies --- nix-rust/Cargo.lock | 61 +++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock index cfaedc3ed4c..ea7ac72db16 100644 --- a/nix-rust/Cargo.lock +++ b/nix-rust/Cargo.lock @@ -1,59 +1,82 @@ [[package]] name = "cfg-if" -version = "0.1.7" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "filetime" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "libc" -version = "0.2.50" +version = "0.2.58" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "nix-rust" version = "0.1.0" dependencies = [ - "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", - "tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "redox_syscall" -version = "0.1.51" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "tar" -version = "0.4.22" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)", + "filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "winapi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "xattr" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", ] [metadata] -"checksum cfg-if 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11d43355396e872eefb45ce6342e4374ed7bc2b3a502d1b28e36d6e23c05d1f4" -"checksum filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a2df5c1a8c4be27e7707789dc42ae65976e60b394afd293d1419ab915833e646" -"checksum libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)" = "aab692d7759f5cd8c859e169db98ae5b52c924add2af5fbbca11d12fefb567c1" -"checksum redox_syscall 0.1.51 (registry+https://github.com/rust-lang/crates.io-index)" = "423e376fffca3dfa06c9e9790a9ccd282fafb3cc6e6397d01dbf64f9bacc6b85" -"checksum tar 0.4.22 (registry+https://github.com/rust-lang/crates.io-index)" = "c2167ff53da2a661702b3299f71a91b61b1dffef36b4b2884b1f9c67254c0133" +"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33" +"checksum filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "450537dc346f0c4d738dda31e790da1da5d4bd12145aad4da0d03d713cb3794f" +"checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319" +"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +"checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3" +"checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" "checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" From d722e2175ef826cd60f05608fb7d58aa82261549 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Jul 2019 01:01:18 +0200 Subject: [PATCH 07/23] Include cargo dependencies in the Nix tarball --- nix-rust/local.mk | 4 +++- release.nix | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index c69e3b9ea2c..ce88fa51b33 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -4,4 +4,6 @@ libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) $(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml - $(trace-gen) cd nix-rust && RUSTC_BOOTSTRAP=1 cargo build --release && touch target/release/libnixrust.a + $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release -Z offline && touch target/release/libnixrust.a + +dist-files += $(d)/vendor diff --git a/release.nix b/release.nix index 1d7c2e2d628..64aa35e1134 100644 --- a/release.nix +++ b/release.nix @@ -10,6 +10,43 @@ let jobs = rec { + # Create a "vendor" directory that contains the crates listed in + # Cargo.lock, and include it in the Nix tarball. This allows Nix + # to be built without network access. + vendoredCrates = + let + lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock); + + files = map (pkg: import { + url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download"; + sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)"; + }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package); + + in pkgs.runCommand "cargo-vendor-dir" {} + '' + mkdir -p $out/vendor + + cat > $out/vendor/config < "$dir/.cargo-checksum.json" + + mv "$dir" $out/vendor/ + rmdir $out/vendor/tmp + '') files)} + ''; + tarball = with pkgs; @@ -38,6 +75,8 @@ let distPhase = '' + cp -prd ${vendoredCrates}/vendor/ nix-rust/vendor/ + runHook preDist make dist mkdir -p $out/tarballs From afb021893beb598213d4225de5edd746baa81d7a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Jul 2019 01:19:30 +0200 Subject: [PATCH 08/23] Reduce the size of the vendor directory by removing some winapi cruft --- release.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 64aa35e1134..9cf4c74f231 100644 --- a/release.nix +++ b/release.nix @@ -42,8 +42,15 @@ let # Add just enough metadata to keep Cargo happy. printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json" + # Clean up some cruft from the winapi crates. FIXME: find + # a way to remove winapi* from our dependencies. + if [[ $dir =~ /winapi ]]; then + find $dir -name "*.a" -print0 | xargs -0 rm -f -- + fi + mv "$dir" $out/vendor/ - rmdir $out/vendor/tmp + + rm -rf $out/vendor/tmp '') files)} ''; From 343ebcc048c69e103db3f87a9339388bd5dd93cb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Jul 2019 01:50:35 +0200 Subject: [PATCH 09/23] Only pass '-Z offline' to cargo if we have a vendor directory --- nix-rust/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index ce88fa51b33..a5e017cef79 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -4,6 +4,6 @@ libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) $(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml - $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release -Z offline && touch target/release/libnixrust.a + $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo -Z offline; fi) && touch target/release/libnixrust.a dist-files += $(d)/vendor From 8110b4ebb29174ecd4b22510da0285abf604b8a7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Sep 2019 21:55:32 +0200 Subject: [PATCH 10/23] Rust cleanup --- nix-rust/Cargo.toml | 1 + nix-rust/src/error.rs | 5 +++ nix-rust/src/foreign.rs | 14 +++++++++ nix-rust/src/lib.rs | 68 ++++------------------------------------- nix-rust/src/tarfile.rs | 47 ++++++++++++++++++++++++++++ src/libstore/rust.hh | 2 +- 6 files changed, 74 insertions(+), 63 deletions(-) create mode 100644 nix-rust/src/error.rs create mode 100644 nix-rust/src/foreign.rs create mode 100644 nix-rust/src/tarfile.rs diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml index 2d2bf67523d..8b56dc89cd0 100644 --- a/nix-rust/Cargo.toml +++ b/nix-rust/Cargo.toml @@ -2,6 +2,7 @@ name = "nix-rust" version = "0.1.0" authors = ["Eelco Dolstra "] +edition = "2018" [lib] name = "nixrust" diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs new file mode 100644 index 00000000000..28d0abdef99 --- /dev/null +++ b/nix-rust/src/error.rs @@ -0,0 +1,5 @@ +#[derive(Debug)] +pub enum Error { + Misc(String), + Foreign(libc::c_void), // == std::exception_ptr +} diff --git a/nix-rust/src/foreign.rs b/nix-rust/src/foreign.rs new file mode 100644 index 00000000000..7bce7753c05 --- /dev/null +++ b/nix-rust/src/foreign.rs @@ -0,0 +1,14 @@ +/// A wrapper around Nix's Source class that provides the Read trait. +#[repr(C)] +pub struct Source { + fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize, + this: *mut libc::c_void, +} + +impl std::io::Read for Source { + fn read(&mut self, buf: &mut [u8]) -> std::result::Result { + let n = (self.fun)(self.this, buf); + assert!(n <= buf.len()); + Ok(n) + } +} diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index ac6dee543a2..192ca29e4bb 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -1,66 +1,10 @@ -extern crate libc; -extern crate tar; +mod error; +mod foreign; +mod tarfile; -use std::fs; -use std::io; -use std::os::unix::fs::OpenOptionsExt; -use std::path::Path; -use tar::Archive; - -/// A wrapper around Nix's Source class that provides the Read trait. -#[repr(C)] -pub struct Source { - fun: extern "C" fn(this: *mut libc::c_void, data: &mut [u8]) -> usize, - this: *mut libc::c_void, -} - -impl std::io::Read for Source { - fn read(&mut self, buf: &mut [u8]) -> std::result::Result { - let n = (self.fun)(self.this, buf); - assert!(n <= buf.len()); - Ok(n) - } -} +pub use error::Error; #[no_mangle] -pub extern "C" fn unpack_tarfile(source: Source, dest_dir: &str) -> bool { - // FIXME: handle errors. - - let dest_dir = Path::new(dest_dir); - - let mut tar = Archive::new(source); - - for file in tar.entries().unwrap() { - let mut file = file.unwrap(); - - let dest_file = dest_dir.join(file.path().unwrap()); - - fs::create_dir_all(dest_file.parent().unwrap()).unwrap(); - - match file.header().entry_type() { - tar::EntryType::Directory => { - fs::create_dir(dest_file).unwrap(); - } - tar::EntryType::Regular => { - let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 { - 0o666 - } else { - 0o777 - }; - let mut f = fs::OpenOptions::new() - .create(true) - .write(true) - .mode(mode) - .open(dest_file) - .unwrap(); - io::copy(&mut file, &mut f).unwrap(); - } - tar::EntryType::Symlink => { - std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file).unwrap(); - } - t => panic!("Unsupported tar entry type '{:?}'.", t), - } - } - - true +pub extern "C" fn unpack_tarfile(source: foreign::Source, dest_dir: &str) { + tarfile::unpack_tarfile(source, dest_dir).unwrap(); } diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs new file mode 100644 index 00000000000..696118e4d29 --- /dev/null +++ b/nix-rust/src/tarfile.rs @@ -0,0 +1,47 @@ +use crate::{foreign::Source, Error}; +use std::fs; +use std::io; +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; +use tar::Archive; + +pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { + let dest_dir = Path::new(dest_dir); + + let mut tar = Archive::new(source); + + for file in tar.entries().unwrap() { + let mut file = file.unwrap(); + + let dest_file = dest_dir.join(file.path().unwrap()); + + fs::create_dir_all(dest_file.parent().unwrap()).unwrap(); + + match file.header().entry_type() { + tar::EntryType::Directory => { + fs::create_dir(dest_file).unwrap(); + } + tar::EntryType::Regular => { + let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 { + 0o666 + } else { + 0o777 + }; + let mut f = fs::OpenOptions::new() + .create(true) + .write(true) + .mode(mode) + .open(dest_file) + .unwrap(); + io::copy(&mut file, &mut f).unwrap(); + } + tar::EntryType::Symlink => { + std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file) + .unwrap(); + } + t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))), + } + } + + Ok(()) +} diff --git a/src/libstore/rust.hh b/src/libstore/rust.hh index 7e6c2f54d8a..4366e4723f0 100644 --- a/src/libstore/rust.hh +++ b/src/libstore/rust.hh @@ -40,5 +40,5 @@ struct Source } extern "C" { - bool unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); + void unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } From f738cd4d976f4f72159bbcbfa7b451c33f0ea74a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 01:15:20 +0200 Subject: [PATCH 11/23] More Rust FFI adventures We can now convert Rust Errors to C++ exceptions. At the Rust->C++ FFI boundary, Result will cause Error to be converted to and thrown as a C++ exception. --- nix-rust/src/error.rs | 27 ++++++++++- nix-rust/src/lib.rs | 27 ++++++++++- nix-rust/src/tarfile.rs | 20 ++++---- src/libstore/builtins/unpack-channel.cc | 2 +- src/libstore/rust.cc | 12 +++++ src/libstore/rust.hh | 61 ++++++++++++++++++++++++- 6 files changed, 132 insertions(+), 17 deletions(-) create mode 100644 src/libstore/rust.cc diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs index 28d0abdef99..a2003be6f70 100644 --- a/nix-rust/src/error.rs +++ b/nix-rust/src/error.rs @@ -1,5 +1,30 @@ #[derive(Debug)] pub enum Error { + IOError(std::io::Error), Misc(String), - Foreign(libc::c_void), // == std::exception_ptr + Foreign(CppException), +} + +impl From for Error { + fn from(err: std::io::Error) -> Self { + Error::IOError(err) + } +} + +impl From for CppException { + fn from(err: Error) -> Self { + match err { + Error::Foreign(ex) => ex, + Error::Misc(s) => unsafe { make_error(&s) }, + Error::IOError(err) => unsafe { make_error(&err.to_string()) }, + } + } +} + +#[repr(C)] +#[derive(Debug)] +pub struct CppException(*const libc::c_void); // == std::exception_ptr* + +extern "C" { + fn make_error(s: &str) -> CppException; } diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index 192ca29e4bb..b6b0d746dae 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -4,7 +4,30 @@ mod tarfile; pub use error::Error; +pub struct CBox { + ptr: *mut libc::c_void, + phantom: std::marker::PhantomData, +} + +impl CBox { + fn new(t: T) -> Self { + unsafe { + let size = std::mem::size_of::(); + let ptr = libc::malloc(size); + eprintln!("PTR = {:?}, SIZE = {}", ptr, size); + *(ptr as *mut T) = t; // FIXME: probably UB + Self { + ptr, + phantom: std::marker::PhantomData, + } + } + } +} + #[no_mangle] -pub extern "C" fn unpack_tarfile(source: foreign::Source, dest_dir: &str) { - tarfile::unpack_tarfile(source, dest_dir).unwrap(); +pub extern "C" fn unpack_tarfile( + source: foreign::Source, + dest_dir: &str, +) -> CBox> { + CBox::new(tarfile::unpack_tarfile(source, dest_dir).map_err(|err| err.into())) } diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs index 696118e4d29..797aa5064e4 100644 --- a/nix-rust/src/tarfile.rs +++ b/nix-rust/src/tarfile.rs @@ -10,19 +10,19 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { let mut tar = Archive::new(source); - for file in tar.entries().unwrap() { - let mut file = file.unwrap(); + for file in tar.entries()? { + let mut file = file?; - let dest_file = dest_dir.join(file.path().unwrap()); + let dest_file = dest_dir.join(file.path()?); - fs::create_dir_all(dest_file.parent().unwrap()).unwrap(); + fs::create_dir_all(dest_file.parent().unwrap())?; match file.header().entry_type() { tar::EntryType::Directory => { - fs::create_dir(dest_file).unwrap(); + fs::create_dir(dest_file)?; } tar::EntryType::Regular => { - let mode = if file.header().mode().unwrap() & libc::S_IXUSR == 0 { + let mode = if file.header().mode()? & libc::S_IXUSR == 0 { 0o666 } else { 0o777 @@ -31,13 +31,11 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { .create(true) .write(true) .mode(mode) - .open(dest_file) - .unwrap(); - io::copy(&mut file, &mut f).unwrap(); + .open(dest_file)?; + io::copy(&mut file, &mut f)?; } tar::EntryType::Symlink => { - std::os::unix::fs::symlink(file.header().link_name().unwrap().unwrap(), dest_file) - .unwrap(); + std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?; } t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))), } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 88202ec6b7f..2da26d98e64 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -27,7 +27,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) decompressor->finish(); }); - unpack_tarfile(*source, out); + unpack_tarfile(*source, out).use()->unwrap(); auto entries = readDirectory(out); if (entries.size() != 1) diff --git a/src/libstore/rust.cc b/src/libstore/rust.cc new file mode 100644 index 00000000000..a616d83a633 --- /dev/null +++ b/src/libstore/rust.cc @@ -0,0 +1,12 @@ +#include "logging.hh" +#include "rust.hh" + +namespace nix { + +extern "C" std::exception_ptr * make_error(rust::StringSlice s) +{ + // FIXME: leak + return new std::exception_ptr(std::make_exception_ptr(Error(std::string(s.ptr, s.size)))); +} + +} diff --git a/src/libstore/rust.hh b/src/libstore/rust.hh index 4366e4723f0..4c7720a4427 100644 --- a/src/libstore/rust.hh +++ b/src/libstore/rust.hh @@ -4,7 +4,8 @@ namespace rust { // Depending on the internal representation of Rust slices is slightly // evil... -template struct Slice +template +struct Slice { T * ptr; size_t size; @@ -37,8 +38,64 @@ struct Source } }; +/* C++ representation of Rust's Result. */ +template +struct Result +{ + unsigned int tag; + + union { + T data; + std::exception_ptr * exc; + }; + + /* Rethrow the wrapped exception or return the wrapped value. */ + T unwrap() + { + if (tag == 0) + return data; + else if (tag == 1) + std::rethrow_exception(*exc); + else + abort(); + } +}; + +template +struct CBox +{ + T * ptr; + + T * operator ->() + { + return ptr; + } + + CBox(T * ptr) : ptr(ptr) { } + CBox(const CBox &) = delete; + CBox(CBox &&) = delete; + + ~CBox() + { + free(ptr); + } +}; + +// Grrr, this is only needed because 'extern "C"' functions don't +// support non-POD return types (and CBox has a destructor so it's not +// POD). +template +struct CBox2 +{ + T * ptr; + CBox use() + { + return CBox(ptr); + } +}; + } extern "C" { - void unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); + rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } From b7fba16613aed5c2b2a0b4f98e3e0d32b0cddd40 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 12:43:07 +0200 Subject: [PATCH 12/23] Move code around --- src/libstore/rust.hh | 97 +------------------ src/{libstore/rust.cc => libutil/rust-ffi.cc} | 0 src/libutil/rust-ffi.hh | 97 +++++++++++++++++++ 3 files changed, 98 insertions(+), 96 deletions(-) rename src/{libstore/rust.cc => libutil/rust-ffi.cc} (100%) create mode 100644 src/libutil/rust-ffi.hh diff --git a/src/libstore/rust.hh b/src/libstore/rust.hh index 4c7720a4427..dccc98b4bdd 100644 --- a/src/libstore/rust.hh +++ b/src/libstore/rust.hh @@ -1,100 +1,5 @@ #include "serialise.hh" - -namespace rust { - -// Depending on the internal representation of Rust slices is slightly -// evil... -template -struct Slice -{ - T * ptr; - size_t size; - - Slice(T * ptr, size_t size) : ptr(ptr), size(size) - { - assert(ptr); - } -}; - -struct StringSlice : Slice -{ - StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {} -}; - -struct Source -{ - size_t (*fun)(void * source_this, rust::Slice data); - nix::Source * _this; - - Source(nix::Source & _this) - : fun(sourceWrapper), _this(&_this) - {} - - // FIXME: how to propagate exceptions? - static size_t sourceWrapper(void * _this, rust::Slice data) - { - auto n = ((nix::Source *) _this)->read(data.ptr, data.size); - return n; - } -}; - -/* C++ representation of Rust's Result. */ -template -struct Result -{ - unsigned int tag; - - union { - T data; - std::exception_ptr * exc; - }; - - /* Rethrow the wrapped exception or return the wrapped value. */ - T unwrap() - { - if (tag == 0) - return data; - else if (tag == 1) - std::rethrow_exception(*exc); - else - abort(); - } -}; - -template -struct CBox -{ - T * ptr; - - T * operator ->() - { - return ptr; - } - - CBox(T * ptr) : ptr(ptr) { } - CBox(const CBox &) = delete; - CBox(CBox &&) = delete; - - ~CBox() - { - free(ptr); - } -}; - -// Grrr, this is only needed because 'extern "C"' functions don't -// support non-POD return types (and CBox has a destructor so it's not -// POD). -template -struct CBox2 -{ - T * ptr; - CBox use() - { - return CBox(ptr); - } -}; - -} +#include "rust-ffi.hh" extern "C" { rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); diff --git a/src/libstore/rust.cc b/src/libutil/rust-ffi.cc similarity index 100% rename from src/libstore/rust.cc rename to src/libutil/rust-ffi.cc diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh new file mode 100644 index 00000000000..a488b96d69f --- /dev/null +++ b/src/libutil/rust-ffi.hh @@ -0,0 +1,97 @@ +#include "serialise.hh" + +namespace rust { + +// Depending on the internal representation of Rust slices is slightly +// evil... +template +struct Slice +{ + T * ptr; + size_t size; + + Slice(T * ptr, size_t size) : ptr(ptr), size(size) + { + assert(ptr); + } +}; + +struct StringSlice : Slice +{ + StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {} +}; + +struct Source +{ + size_t (*fun)(void * source_this, rust::Slice data); + nix::Source * _this; + + Source(nix::Source & _this) + : fun(sourceWrapper), _this(&_this) + {} + + // FIXME: how to propagate exceptions? + static size_t sourceWrapper(void * _this, rust::Slice data) + { + auto n = ((nix::Source *) _this)->read(data.ptr, data.size); + return n; + } +}; + +/* C++ representation of Rust's Result. */ +template +struct Result +{ + unsigned int tag; + + union { + T data; + std::exception_ptr * exc; + }; + + /* Rethrow the wrapped exception or return the wrapped value. */ + T unwrap() + { + if (tag == 0) + return data; + else if (tag == 1) + std::rethrow_exception(*exc); + else + abort(); + } +}; + +template +struct CBox +{ + T * ptr; + + T * operator ->() + { + return ptr; + } + + CBox(T * ptr) : ptr(ptr) { } + CBox(const CBox &) = delete; + CBox(CBox &&) = delete; + + ~CBox() + { + free(ptr); + } +}; + +// Grrr, this is only needed because 'extern "C"' functions don't +// support non-POD return types (and CBox has a destructor so it's not +// POD). +template +struct CBox2 +{ + T * ptr; + CBox use() + { + return CBox(ptr); + } +}; + +} From d14b1c261cd3dfb4be2da943d901a394c3f23205 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 12:44:31 +0200 Subject: [PATCH 13/23] Shut up some rust warnings --- nix-rust/src/error.rs | 1 + nix-rust/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nix-rust/src/error.rs b/nix-rust/src/error.rs index a2003be6f70..519007ea0aa 100644 --- a/nix-rust/src/error.rs +++ b/nix-rust/src/error.rs @@ -26,5 +26,6 @@ impl From for CppException { pub struct CppException(*const libc::c_void); // == std::exception_ptr* extern "C" { + #[allow(improper_ctypes)] // YOLO fn make_error(s: &str) -> CppException; } diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index b6b0d746dae..48952d8b9c9 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -5,7 +5,7 @@ mod tarfile; pub use error::Error; pub struct CBox { - ptr: *mut libc::c_void, + pub ptr: *mut libc::c_void, phantom: std::marker::PhantomData, } From d33dd6e6c09d87a59989057ad622a6265ddec2e0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 13:10:46 +0200 Subject: [PATCH 14/23] Move code around --- nix-rust/src/lib.rs | 1 - src/libstore/builtins/unpack-channel.cc | 4 ++-- src/libstore/local.mk | 2 +- src/libutil/local.mk | 2 ++ src/libutil/rust-ffi.cc | 2 +- src/{libstore/rust.hh => libutil/tarfile.cc} | 10 +++++++++- src/libutil/tarfile.hh | 7 +++++++ 7 files changed, 22 insertions(+), 6 deletions(-) rename src/{libstore/rust.hh => libutil/tarfile.cc} (53%) create mode 100644 src/libutil/tarfile.hh diff --git a/nix-rust/src/lib.rs b/nix-rust/src/lib.rs index 48952d8b9c9..1b88ac8af88 100644 --- a/nix-rust/src/lib.rs +++ b/nix-rust/src/lib.rs @@ -14,7 +14,6 @@ impl CBox { unsafe { let size = std::mem::size_of::(); let ptr = libc::malloc(size); - eprintln!("PTR = {:?}, SIZE = {}", ptr, size); *(ptr as *mut T) = t; // FIXME: probably UB Self { ptr, diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 2da26d98e64..5fc68cd666c 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -1,6 +1,6 @@ -#include "rust.hh" #include "builtins.hh" #include "compression.hh" +#include "tarfile.hh" namespace nix { @@ -27,7 +27,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) decompressor->finish(); }); - unpack_tarfile(*source, out).use()->unwrap(); + unpackTarfile(*source, out); auto entries = readDirectory(out); if (entries.size() != 1) diff --git a/src/libstore/local.mk b/src/libstore/local.mk index d3254554dd3..d690fea28c2 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -6,7 +6,7 @@ libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) -libstore_LIBS = libutil libnixrust +libstore_LIBS = libutil libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread ifneq ($(OS), FreeBSD) diff --git a/src/libutil/local.mk b/src/libutil/local.mk index e41a67d1f9e..35c1f6c13dc 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -7,3 +7,5 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context + +libutil_LIBS = libnixrust diff --git a/src/libutil/rust-ffi.cc b/src/libutil/rust-ffi.cc index a616d83a633..931d29542ee 100644 --- a/src/libutil/rust-ffi.cc +++ b/src/libutil/rust-ffi.cc @@ -1,5 +1,5 @@ #include "logging.hh" -#include "rust.hh" +#include "rust-ffi.hh" namespace nix { diff --git a/src/libstore/rust.hh b/src/libutil/tarfile.cc similarity index 53% rename from src/libstore/rust.hh rename to src/libutil/tarfile.cc index dccc98b4bdd..ae6d512bd03 100644 --- a/src/libstore/rust.hh +++ b/src/libutil/tarfile.cc @@ -1,6 +1,14 @@ -#include "serialise.hh" #include "rust-ffi.hh" extern "C" { rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } + +namespace nix { + +void unpackTarfile(Source & source, Path destDir) +{ + unpack_tarfile(source, destDir).use()->unwrap(); +} + +} diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh new file mode 100644 index 00000000000..c3e95fb0c92 --- /dev/null +++ b/src/libutil/tarfile.hh @@ -0,0 +1,7 @@ +#include "serialise.hh" + +namespace nix { + +void unpackTarfile(Source & source, Path destDir); + +} From f2bd8470926686361602e545d63a69d4bfc22f90 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 15:03:17 +0200 Subject: [PATCH 15/23] Ignore tar header entries In particular, these are emitted by 'git archive' (in fetchGit). --- nix-rust/src/tarfile.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs index 797aa5064e4..9e32a3f0082 100644 --- a/nix-rust/src/tarfile.rs +++ b/nix-rust/src/tarfile.rs @@ -37,6 +37,7 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { tar::EntryType::Symlink => { std::os::unix::fs::symlink(file.header().link_name()?.unwrap(), dest_file)?; } + tar::EntryType::XGlobalHeader | tar::EntryType::XHeader => {} t => return Err(Error::Misc(format!("unsupported tar entry type '{:?}'", t))), } } From 8918bae09828133259acb36d6aef60ffbfad252c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 11 Sep 2019 15:25:43 +0200 Subject: [PATCH 16/23] Drop remaining uses of external "tar" Also, fetchGit now runs in O(1) memory since we pipe the output of 'git archive' directly into unpackTarball() (rather than first reading it all into memory). --- src/libexpr/primops/fetchGit.cc | 11 +++++++---- src/libstore/builtins/unpack-channel.cc | 12 +----------- src/libstore/download.cc | 12 ++++++++---- src/libutil/serialise.hh | 1 - src/libutil/tarfile.cc | 22 +++++++++++++++++++++- src/libutil/tarfile.hh | 5 ++++- src/nix-prefetch-url/nix-prefetch-url.cc | 4 ++-- 7 files changed, 43 insertions(+), 24 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 7ef3b382398..9d0c642911d 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -4,6 +4,7 @@ #include "store-api.hh" #include "pathlocks.hh" #include "hash.hh" +#include "tarfile.hh" #include @@ -164,14 +165,16 @@ GitInfo exportGit(ref store, const std::string & uri, if (e.errNo != ENOENT) throw; } - // FIXME: should pipe this, or find some better way to extract a - // revision. - auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev }); + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runProgram("tar", true, { "x", "-C", tmpDir }, tar); + unpackTarfile(*source, tmpDir); gitInfo.storePath = store->addToStore(name, tmpDir); diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 5fc68cd666c..d18e3ddafbc 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -1,5 +1,4 @@ #include "builtins.hh" -#include "compression.hh" #include "tarfile.hh" namespace nix { @@ -18,16 +17,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) createDirs(out); - auto source = sinkToSource([&](Sink & sink) { - auto decompressor = - hasSuffix(src, ".bz2") ? makeDecompressionSink("bzip2", sink) : - hasSuffix(src, ".xz") ? makeDecompressionSink("xz", sink) : - makeDecompressionSink("none", sink); - readFile(src, *decompressor); - decompressor->finish(); - }); - - unpackTarfile(*source, out); + unpackTarfile(src, out); auto entries = readDirectory(out); if (entries.size() != 1) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index e80663dfffe..61e88c5c1ef 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -8,6 +8,7 @@ #include "compression.hh" #include "pathlocks.hh" #include "finally.hh" +#include "tarfile.hh" #ifdef ENABLE_S3 #include @@ -903,12 +904,15 @@ CachedDownloadResult Downloader::downloadCached( unpackedStorePath = ""; } if (unpackedStorePath.empty()) { - printInfo(format("unpacking '%1%'...") % url); + printInfo("unpacking '%s'...", url); Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); - unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); + unpackTarfile(store->toRealPath(storePath), tmpDir, baseNameOf(url)); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); } replaceSymlink(unpackedStorePath, unpackedLink); storePath = unpackedStorePath; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 128e287f31c..5780c93a630 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -77,7 +77,6 @@ struct BufferedSource : Source size_t read(unsigned char * data, size_t len) override; - bool hasData(); protected: diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index ae6d512bd03..f7d3ad4172f 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -1,4 +1,5 @@ #include "rust-ffi.hh" +#include "compression.hh" extern "C" { rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); @@ -6,9 +7,28 @@ extern "C" { namespace nix { -void unpackTarfile(Source & source, Path destDir) +void unpackTarfile(Source & source, const Path & destDir) { unpack_tarfile(source, destDir).use()->unwrap(); } +void unpackTarfile(const Path & tarFile, const Path & destDir, + std::optional baseName) +{ + if (!baseName) baseName = baseNameOf(tarFile); + + auto source = sinkToSource([&](Sink & sink) { + // FIXME: look at first few bytes to determine compression type. + auto decompressor = + // FIXME: add .gz support + hasSuffix(*baseName, ".bz2") ? makeDecompressionSink("bzip2", sink) : + hasSuffix(*baseName, ".xz") ? makeDecompressionSink("xz", sink) : + makeDecompressionSink("none", sink); + readFile(tarFile, *decompressor); + decompressor->finish(); + }); + + unpackTarfile(*source, destDir); +} + } diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index c3e95fb0c92..ce0911e2a32 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -2,6 +2,9 @@ namespace nix { -void unpackTarfile(Source & source, Path destDir); +void unpackTarfile(Source & source, const Path & destDir); + +void unpackTarfile(const Path & tarFile, const Path & destDir, + std::optional baseName = {}); } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index f54706a8a01..78c88383377 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -9,6 +9,7 @@ #include "legacy.hh" #include "finally.hh" #include "progress-bar.hh" +#include "tarfile.hh" #include @@ -192,8 +193,7 @@ static int _main(int argc, char * * argv) if (hasSuffix(baseNameOf(uri), ".zip")) runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}); else - // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", tmpFile, "-C", unpacked}); + unpackTarfile(tmpFile, unpacked, baseNameOf(uri)); /* If the archive unpacks to a single file/directory, then use that as the top-level. */ From 88f8063917a4261abfabfe94bc33c63f7405d1d5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Nov 2019 22:45:15 +0100 Subject: [PATCH 17/23] -Z offline -> --offline --- nix-rust/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index a5e017cef79..fda1c3dee88 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -4,6 +4,6 @@ libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) $(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml - $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo -Z offline; fi) && touch target/release/libnixrust.a + $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo --offline; fi) && touch target/release/libnixrust.a dist-files += $(d)/vendor From e6c1d1b474b1f14753b27f35b2aad4051ffd9e8d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Nov 2019 22:46:36 +0100 Subject: [PATCH 18/23] Update Cargo.lock --- nix-rust/Cargo.lock | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/nix-rust/Cargo.lock b/nix-rust/Cargo.lock index ea7ac72db16..0112ed471db 100644 --- a/nix-rust/Cargo.lock +++ b/nix-rust/Cargo.lock @@ -1,29 +1,31 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. [[package]] name = "cfg-if" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "filetime" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "libc" -version = "0.2.58" +version = "0.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "nix-rust" version = "0.1.0" dependencies = [ - "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -37,15 +39,15 @@ name = "tar" version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", "xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "winapi" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -67,16 +69,16 @@ name = "xattr" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", ] [metadata] -"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33" -"checksum filetime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "450537dc346f0c4d738dda31e790da1da5d4bd12145aad4da0d03d713cb3794f" -"checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319" +"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum filetime 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1ff6d4dab0aa0c8e6346d46052e93b13a16cf847b54ed357087c35011048cc7d" +"checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8" "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" "checksum tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)" = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3" -"checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" +"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" "checksum xattr 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" From dbc4f9d478814f3ce4ee23531502247d02c85911 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Nov 2019 00:17:24 +0100 Subject: [PATCH 19/23] Fix macOS build https://hydra.nixos.org/build/107466992 --- nix-rust/src/tarfile.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix-rust/src/tarfile.rs b/nix-rust/src/tarfile.rs index 9e32a3f0082..379d9098f1c 100644 --- a/nix-rust/src/tarfile.rs +++ b/nix-rust/src/tarfile.rs @@ -22,7 +22,7 @@ pub fn unpack_tarfile(source: Source, dest_dir: &str) -> Result<(), Error> { fs::create_dir(dest_file)?; } tar::EntryType::Regular => { - let mode = if file.header().mode()? & libc::S_IXUSR == 0 { + let mode = if file.header().mode()? & (libc::S_IXUSR as u32) == 0 { 0o666 } else { 0o777 From 949dc848940b68ffd1327a278df4472e98a455dc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Nov 2019 14:17:15 +0100 Subject: [PATCH 20/23] Fix segfault on i686-linux https://hydra.nixos.org/build/107467517 Seems that on i686-linux, gcc and rustc disagree on how to return 1-word structs: gcc has the caller pass a pointer to the result, while rustc has the callee return the result in a register. Work around this by using a bare pointer. --- src/libutil/rust-ffi.hh | 13 ------------- src/libutil/tarfile.cc | 6 ++++-- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh index a488b96d69f..663758bfcb6 100644 --- a/src/libutil/rust-ffi.hh +++ b/src/libutil/rust-ffi.hh @@ -81,17 +81,4 @@ struct CBox } }; -// Grrr, this is only needed because 'extern "C"' functions don't -// support non-POD return types (and CBox has a destructor so it's not -// POD). -template -struct CBox2 -{ - T * ptr; - CBox use() - { - return CBox(ptr); - } -}; - } diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index f7d3ad4172f..2cc7793fde8 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -2,14 +2,16 @@ #include "compression.hh" extern "C" { - rust::CBox2>> unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); + rust::Result> * + unpack_tarfile(rust::Source source, rust::StringSlice dest_dir); } namespace nix { void unpackTarfile(Source & source, const Path & destDir) { - unpack_tarfile(source, destDir).use()->unwrap(); + rust::Source source2(source); + rust::CBox(unpack_tarfile(source2, destDir))->unwrap(); } void unpackTarfile(const Path & tarFile, const Path & destDir, From f553a8bdea96afe18b97dee6eba91c0bbc545b79 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Nov 2019 14:18:57 +0100 Subject: [PATCH 21/23] When OPTIMIZE=0, build rust code in debug mode --- nix-rust/local.mk | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index fda1c3dee88..c147f987407 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -1,9 +1,19 @@ -libnixrust_PATH := $(d)/target/release/libnixrust.a +ifeq ($(OPTIMIZE), 1) + RUST_MODE = --release + RUST_DIR = release +else + RUST_MODE = + RUST_DIR = debug +endif + +libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.a libnixrust_INSTALL_PATH := $(libnixrust_PATH) -libnixrust_LDFLAGS_USE := -L$(d)/target/release -lnixrust -ldl +libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) -$(d)/target/release/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml - $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo build --release $$(if [[ -d vendor ]]; then echo --offline; fi) && touch target/release/libnixrust.a +$(d)/target/$(RUST_DIR)/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml + $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \ + cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \ + && touch target/$(RUST_DIR)/libnixrust.a dist-files += $(d)/vendor From 895ce1bb6cc77324c377baa08eb98666768e5c6d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Nov 2019 17:33:59 +0100 Subject: [PATCH 22/23] make clean: Delete nix-rust/target --- nix-rust/local.mk | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nix-rust/local.mk b/nix-rust/local.mk index c147f987407..ed1e640c913 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -17,3 +17,8 @@ $(d)/target/$(RUST_DIR)/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml && touch target/$(RUST_DIR)/libnixrust.a dist-files += $(d)/vendor + +clean: clean-rust + +clean-rust: + $(suppress) rm -rfv nix-rust/target From 39954a958623431acb8642372f881cbdb7bb789d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 29 Nov 2019 18:27:40 +0100 Subject: [PATCH 23/23] Make libnixrust a dynamic library This is a hack to fix the build on macOS, which was failing because libnixrust.a contains compiler builtins that clash with libclang_rt.osx.a. There's probably a better solution... https://hydra.nixos.org/build/107473280 --- nix-rust/Cargo.toml | 2 +- nix-rust/local.mk | 26 ++++++++++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/nix-rust/Cargo.toml b/nix-rust/Cargo.toml index 8b56dc89cd0..c4f4ceb0949 100644 --- a/nix-rust/Cargo.toml +++ b/nix-rust/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [lib] name = "nixrust" -crate-type = ["staticlib"] +crate-type = ["cdylib"] [dependencies] tar = "0.4" diff --git a/nix-rust/local.mk b/nix-rust/local.mk index ed1e640c913..7645d5394ef 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -6,15 +6,29 @@ else RUST_DIR = debug endif -libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.a -libnixrust_INSTALL_PATH := $(libnixrust_PATH) +libnixrust_PATH := $(d)/target/$(RUST_DIR)/libnixrust.$(SO_EXT) +libnixrust_INSTALL_PATH := $(libdir)/libnixrust.$(SO_EXT) libnixrust_LDFLAGS_USE := -L$(d)/target/$(RUST_DIR) -lnixrust -ldl -libnixrust_LDFLAGS_USE_INSTALLED := $(libnixrust_LDFLAGS_USE) +libnixrust_LDFLAGS_USE_INSTALLED := -L$(libdir) -lnixrust -ldl -$(d)/target/$(RUST_DIR)/libnixrust.a: $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml +ifeq ($(OS), Darwin) +libnixrust_BUILD_FLAGS = NIX_LDFLAGS="-undefined dynamic_lookup" +else +libnixrust_LDFLAGS_USE += -Wl,-rpath,$(abspath $(d)/target/$(RUST_DIR)) +libnixrust_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$(libdir) +endif + +$(libnixrust_PATH): $(wildcard $(d)/src/*.rs) $(d)/Cargo.toml $(trace-gen) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) \ - cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \ - && touch target/$(RUST_DIR)/libnixrust.a + $(libnixrust_BUILD_FLAGS) \ + cargo build $(RUST_MODE) $$(if [[ -d vendor ]]; then echo --offline; fi) \ + && touch target/$(RUST_DIR)/libnixrust.$(SO_EXT) + +$(libnixrust_INSTALL_PATH): $(libnixrust_PATH) + $(target-gen) cp $^ $@ +ifeq ($(OS), Darwin) + install_name_tool -id $@ $@ +endif dist-files += $(d)/vendor