Merge pull request #1 from obsidiansystems/ipfs-git-method

Add git file ingestion method

src/libexpr/primops.cc

@@ -725,8 +725,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
         drv.outputs.insert_or_assign("out", DerivationOutput {
             std::move(outPath),
-            (ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "")
-                + printHashType(h.type),
+            ingestionMethodPrefix(ingestionMethod) + printHashType(h.type),
             h.to_string(Base16, false),
         });
     }

src/libstore/binary-cache-store.cc

@@ -336,13 +336,19 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
        small files. */
     StringSink sink;
     Hash h;
-    if (method == FileIngestionMethod::Recursive) {
+    switch (method) {
+    case FileIngestionMethod::Recursive:
         dumpPath(srcPath, sink, filter);
         h = hashString(hashAlgo, *sink.s);
-    } else {
+        break;
+    case FileIngestionMethod::Flat: {
         auto s = readFile(srcPath);
         dumpString(s, sink);
         h = hashString(hashAlgo, s);
+        break;
+    }
+    case FileIngestionMethod::Git:
+        throw Error("cannot add to binary cache store using the git file ingestion method");
     }
 
     ValidPathInfo info(makeFixedOutputPath(method, h, name));

src/libstore/derivations.cc

@@ -9,13 +9,16 @@
 namespace nix {
 
 
-void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const
+void DerivationOutput::parseHashInfo(FileIngestionMethod & method, Hash & hash) const
 {
-    recursive = FileIngestionMethod::Flat;
+    method = FileIngestionMethod::Flat;
     string algo = hashAlgo;
 
     if (string(algo, 0, 2) == "r:") {
-        recursive = FileIngestionMethod::Recursive;
+        method = FileIngestionMethod::Recursive;
+        algo = string(algo, 2);
+    } else if (string(algo, 0, 4) == "git:") {
+        method = FileIngestionMethod::Git;
         algo = string(algo, 2);
     }
 

src/libstore/derivations.hh

@@ -22,7 +22,7 @@ struct DerivationOutput
         , hashAlgo(std::move(hashAlgo))
         , hash(std::move(hash))
     { }
-    void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const;
+    void parseHashInfo(FileIngestionMethod & method, Hash & hash) const;
 };
 
 typedef std::map<string, DerivationOutput> DerivationOutputs;

src/libstore/ipfs.cc

@@ -0,0 +1,61 @@
+#include "binary-cache-store.hh"
+
+namespace nix {
+
+class IpfsStore : public BinaryCacheStore
+{
+    Path daemonUri;
+
+public:
+
+    IpfsStore(
+        const Params & params, const Path & _daemonUri)
+        : BinaryCacheStore(params)
+        , daemonUri(_daemonUri)
+    {
+        if (daemonUri.back() == '/')
+            daemonUri.pop_back();
+    }
+
+    std::string getUri() override
+    {
+        return daemonUri;
+    }
+
+    void init() override
+    {
+    }
+
+    bool fileExists(const std::string & path) override
+    {
+        return false;
+    }
+
+    void upsertFile(const std::string & path,
+        const std::string & data,
+        const std::string & mimeType) override
+    {
+    }
+
+    void getFile(const std::string & path, Sink & sink) override
+    {
+    }
+
+    void getFile(const std::string & path, Callback<std::shared_ptr<std::string>> callback) noexcept override
+    {
+    }
+
+};
+
+static RegisterStoreImplementation regStore([](
+    const std::string & uri, const Store::Params & params)
+    -> std::shared_ptr<Store>
+{
+    if (std::string(uri, 0, 12) != "ipfs+http://")
+        return 0;
+    auto store = std::make_shared<IpfsStore>(params, std::string(uri, 5));
+    store->init();
+    return store;
+});
+
+}

src/libstore/local-store.cc

@@ -1,5 +1,6 @@
 #include "local-store.hh"
 #include "globals.hh"
+#include "git.hh"
 #include "archive.hh"
 #include "pathlocks.hh"
 #include "worker-protocol.hh"
@@ -1067,11 +1068,21 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
 
             autoGC();
 
-            if (method == FileIngestionMethod::Recursive) {
+            switch (method) {
+            case FileIngestionMethod::Flat:
+                writeFile(realPath, dump);
+                break;
+            case FileIngestionMethod::Recursive: {
                 StringSource source(dump);
                 restorePath(realPath, source);
-            } else
-                writeFile(realPath, dump);
+                break;
+            }
+            case FileIngestionMethod::Git: {
+                StringSource source(dump);
+                restoreGit(realPath, source);
+                break;
+            }
+            }
 
             canonicalisePathMetaData(realPath, -1);
 

src/libstore/path.hh

@@ -79,6 +79,18 @@ enum struct FileIngestionMethod : uint8_t {
     Git,
 };
 
+inline std::string ingestionMethodPrefix(FileIngestionMethod method) {
+    switch (method) {
+    case FileIngestionMethod::Flat:
+        return "";
+    case FileIngestionMethod::Recursive:
+        return "r:";
+    case FileIngestionMethod::Git:
+        return "git:";
+    }
+    throw;
+}
+
 struct StorePathWithOutputs
 {
     StorePath path;

src/libstore/remote-store.cc

@@ -488,6 +488,8 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath,
 {
     if (repair) throw Error("repairing is not supported when building through the Nix daemon");
 
+    if (method == FileIngestionMethod::Git) throw Error("cannot remotely add to store using the git file ingestion method");
+
     auto conn(getConnection());
 
     Path srcPath(absPath(_srcPath));

src/libstore/store-api.cc

@@ -118,8 +118,8 @@ string storePathToHash(const Path & path)
        for paths copied by addToStore() or produced by fixed-output
        derivations:
          the string "fixed:out:<rec><algo>:<hash>:", where
-           <rec> = "r:" for recursive (path) hashes, or "" for flat
-             (file) hashes
+           <rec> = "r:" for recursive (path) hashes, "git:" for git
+             paths, or "" for flat (file) hashes
            <algo> = "md5", "sha1" or "sha256"
            <hash> = base-16 representation of the path or flat hash of
              the contents of the path (or expected contents of the
@@ -172,20 +172,20 @@ static std::string makeType(
 
 
 StorePath Store::makeFixedOutputPath(
-    FileIngestionMethod recursive,
+    FileIngestionMethod method,
     const Hash & hash,
     std::string_view name,
     const StorePathSet & references,
     bool hasSelfReference) const
 {
-    if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) {
+    if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) {
         return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name);
     } else {
         assert(references.empty());
         return makeStorePath("output:out",
             hashString(htSHA256,
                 "fixed:out:"
-                + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "")
+                + ingestionMethodPrefix(method)
                 + hash.to_string(Base16) + ":"),
             name);
     }
@@ -787,15 +787,19 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
     }
 
     else if (hasPrefix(ca, "fixed:")) {
-        FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 };
-        Hash hash(std::string(ca, recursive == FileIngestionMethod::Recursive ? 8 : 6));
+        FileIngestionMethod method = FileIngestionMethod::Flat;
+        if (ca.compare(6, 2, "r:") == 0)
+            method = FileIngestionMethod::Recursive;
+        else if (ca.compare(6, 4, "git:") == 0)
+            method = FileIngestionMethod::Git;
+        Hash hash(std::string(ca, 6 + ingestionMethodPrefix(method).length()));
         auto refs = cloneStorePathSet(references);
         bool hasSelfReference = false;
         if (refs.count(path)) {
             hasSelfReference = true;
             refs.erase(path);
         }
-        if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path)
+        if (store.makeFixedOutputPath(method, hash, path.name(), refs, hasSelfReference) == path)
             return true;
         else
             warn();
@@ -832,11 +836,9 @@ Strings ValidPathInfo::shortRefs() const
 }
 
 
-std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash)
+std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
 {
-    return "fixed:"
-        + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "")
-        + hash.to_string();
+    return "fixed:" + ingestionMethodPrefix(method) + hash.to_string();
 }
 
 

src/libutil/archive.cc

@@ -294,61 +294,6 @@ void parseDump(ParseSink & sink, Source & source)
 }
 
 
-struct RestoreSink : ParseSink
-{
-    Path dstPath;
-    AutoCloseFD fd;
-
-    void createDirectory(const Path & path)
-    {
-        Path p = dstPath + path;
-        if (mkdir(p.c_str(), 0777) == -1)
-            throw SysError(format("creating directory '%1%'") % p);
-    };
-
-    void createRegularFile(const Path & path)
-    {
-        Path p = dstPath + path;
-        fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
-        if (!fd) throw SysError(format("creating file '%1%'") % p);
-    }
-
-    void isExecutable()
-    {
-        struct stat st;
-        if (fstat(fd.get(), &st) == -1)
-            throw SysError("fstat");
-        if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1)
-            throw SysError("fchmod");
-    }
-
-    void preallocateContents(unsigned long long len)
-    {
-#if HAVE_POSIX_FALLOCATE
-        if (len) {
-            errno = posix_fallocate(fd.get(), 0, len);
-            /* Note that EINVAL may indicate that the underlying
-               filesystem doesn't support preallocation (e.g. on
-               OpenSolaris).  Since preallocation is just an
-               optimisation, ignore it. */
-            if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS)
-                throw SysError(format("preallocating file of %1% bytes") % len);
-        }
-#endif
-    }
-
-    void receiveContents(unsigned char * data, unsigned int len)
-    {
-        writeFull(fd.get(), data, len);
-    }
-
-    void createSymlink(const Path & path, const string & target)
-    {
-        Path p = dstPath + path;
-        nix::createSymlink(target, p);
-    }
-};
-
 
 void restorePath(const Path & path, Source & source)
 {

src/libutil/archive.hh

@@ -2,7 +2,7 @@
 
 #include "types.hh"
 #include "serialise.hh"
-
+#include "fs-sink.hh"
 
 namespace nix {
 
@@ -50,19 +50,6 @@ void dumpPath(const Path & path, Sink & sink,
 
 void dumpString(const std::string & s, Sink & sink);
 
-/* FIXME: fix this API, it sucks. */
-struct ParseSink
-{
-    virtual void createDirectory(const Path & path) { };
-
-    virtual void createRegularFile(const Path & path) { };
-    virtual void isExecutable() { };
-    virtual void preallocateContents(unsigned long long size) { };
-    virtual void receiveContents(unsigned char * data, unsigned int len) { };
-
-    virtual void createSymlink(const Path & path, const string & target) { };
-};
-
 struct TeeSink : ParseSink
 {
     TeeSource source;