fetchTree: Return a path instead of a store path

rebase of NixOS#10252

src/libcmd/common-eval-args.cc

@@ -32,8 +32,8 @@ EvalSettings evalSettings {
                 // FIXME `parseFlakeRef` should take a `std::string_view`.
                 auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
                 debug("fetching flake search path element '%s''", rest);
-                auto storePath = flakeRef.resolve(store).fetchTree(store).first;
-                return store->toRealPath(storePath);
+                auto storePath = flakeRef.resolve(store).lazyFetch(store).first;
+                return storePath;
             },
         },
     },
@@ -174,15 +174,15 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
             state.store,
             state.fetchSettings,
             EvalSettings::resolvePseudoUrl(s));
-        auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy);
-        return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
+        state.registerAccessor(accessor);
+        return SourcePath(accessor);
     }
 
     else if (hasPrefix(s, "flake:")) {
         experimentalFeatureSettings.require(Xp::Flakes);
         auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
-        auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
-        return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
+        auto accessor = flakeRef.resolve(state.store).lazyFetch(state.store).first;
+        return SourcePath(accessor);
     }
 
     else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {

src/libexpr/eval-settings.hh

@@ -7,6 +7,7 @@
 namespace nix {
 
 class Store;
+struct SourcePath;
 
 struct EvalSettings : Config
 {
@@ -22,7 +23,7 @@ struct EvalSettings : Config
      * @todo Return (`std::optional` of) `SourceAccssor` or something
      * more structured instead of mere `std::string`?
      */
-    using LookupPathHook = std::optional<std::string>(ref<Store> store, std::string_view);
+    using LookupPathHook = std::optional<SourcePath>(ref<Store> store, std::string_view);
 
     /**
      * Map from "scheme" to a `LookupPathHook`.

src/libexpr/eval.cc

@@ -859,6 +859,11 @@ void Value::mkPath(const SourcePath & path)
     mkPath(&*path.accessor, makeImmutableString(path.path.abs()));
 }
 
+void EvalState::registerAccessor(const ref<SourceAccessor> accessor)
+{
+    sourceAccessors.push_back(accessor);
+}
+
 
 inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 {
@@ -1973,10 +1978,21 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * const * lists, co
     v.mkList(list);
 }
 
+Value * resolveOutPath(EvalState & state, Value * v, const PosIdx pos)
+{
+    state.forceValue(*v, pos);
+    if (v->type() != nAttrs)
+        return v;
+    auto found = v->attrs()->find(state.sOutPath);
+    if (found != v->attrs()->end())
+        return resolveOutPath(state, found->value, pos);
+    return v;
+}
 
 void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
 {
     NixStringContext context;
+    std::shared_ptr<SourceAccessor> accessor;
     std::vector<BackedStringView> s;
     size_t sSize = 0;
     NixInt n{0};
@@ -2010,15 +2026,19 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     Value * vTmpP = values.data();
 
     for (auto & [i_pos, i] : *es) {
-        Value & vTmp = *vTmpP++;
-        i->eval(state, env, vTmp);
+        Value & vTmp0 = *vTmpP++;
+        i->eval(state, env, vTmp0);
+        Value & vTmp = *resolveOutPath(state, &vTmp0, i_pos);
 
         /* If the first element is a path, then the result will also
            be a path, we don't copy anything (yet - that's done later,
            since paths are copied when they are used in a derivation),
            and none of the strings are allowed to have contexts. */
         if (first) {
             firstType = vTmp.type();
+            if (firstType == nPath) {
+                accessor = vTmp.path().accessor;
+            }
         }
 
         if (firstType == nInt) {
@@ -2065,7 +2085,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     else if (firstType == nPath) {
         if (!context.empty())
             state.error<EvalError>("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow();
-        v.mkPath(state.rootPath(CanonPath(canonPath(str()))));
+        v.mkPath({ref(accessor), CanonPath(str())});
     } else
         v.mkStringMove(c_str(), context);
 }
@@ -3053,12 +3073,14 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
         if (!suffixOpt) continue;
         auto suffix = *suffixOpt;
 
-        auto rOpt = resolveLookupPathPath(i.path);
+        auto rOpt = resolveLookupPathPath(
+                i.path
+        , true);
         if (!rOpt) continue;
         auto r = *rOpt;
 
-        Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
-        if (pathExists(res)) return rootPath(CanonPath(canonPath(res)));
+        auto res = r / suffix;
+        if (res.pathExists()) return res;
     }
 
     if (hasPrefix(path, "nix/"))
@@ -3073,26 +3095,28 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
 }
 
 
-std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl)
+std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl)
 {
     auto & value = value0.s;
     auto i = lookupPathResolved.find(value);
     if (i != lookupPathResolved.end()) return i->second;
 
-    auto finish = [&](std::string res) {
+    auto finish = [&](SourcePath res) {
         debug("resolved search path element '%s' to '%s'", value, res);
         lookupPathResolved.emplace(value, res);
         return res;
     };
 
+    std::optional<SourcePath> res;
+
     if (EvalSettings::isPseudoUrl(value)) {
         try {
             auto accessor = fetchers::downloadTarball(
                 store,
                 fetchSettings,
                 EvalSettings::resolvePseudoUrl(value));
             auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
-            return finish(store->toRealPath(storePath));
+            res.emplace(rootPath(CanonPath(store->toRealPath(storePath))));
         } catch (Error & e) {
             logWarning({
                 .msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
@@ -3110,23 +3134,23 @@ std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Pa
         }
     }
 
-    {
-        auto path = absPath(value);
+    else {
+        auto path = rootPath(value);
 
         /* Allow access to paths in the search path. */
         if (initAccessControl) {
-            allowPath(path);
-            if (store->isInStore(path)) {
+            allowPath(path.path.abs());
+            if (store->isInStore(path.path.abs())) {
                 try {
                     StorePathSet closure;
-                    store->computeFSClosure(store->toStorePath(path).first, closure);
+                    store->computeFSClosure(store->toStorePath(path.path.abs()).first, closure);
                     for (auto & p : closure)
                         allowPath(p);
                 } catch (InvalidPath &) { }
             }
         }
 
-        if (pathExists(path))
+        if (path.pathExists())
             return finish(std::move(path));
         else {
             logWarning({

src/libexpr/eval.hh

@@ -249,6 +249,9 @@ public:
 
     const SourcePath callFlakeInternal;
 
+    /* A collection of InputAccessors, just to keep them alive. */
+    std::list<ref<SourceAccessor>> sourceAccessors;
+
     /**
      * Store used to materialise .drv files.
      */
@@ -339,7 +342,7 @@ private:
 
     LookupPath lookupPath;
 
-    std::map<std::string, std::optional<std::string>> lookupPathResolved;
+    std::map<std::string, std::optional<SourcePath>> lookupPathResolved;
 
     /**
      * Cache used by prim_match().
@@ -381,6 +384,8 @@ public:
      */
     SourcePath rootPath(PathView path);
 
+    void registerAccessor(ref<SourceAccessor> accessor);
+
     /**
      * Allow access to a path.
      */
@@ -446,7 +451,7 @@ public:
      *
      * If it is not found, return `std::nullopt`
      */
-    std::optional<std::string> resolveLookupPathPath(
+    std::optional<SourcePath> resolveLookupPathPath(
         const LookupPath::Path & elem,
         bool initAccessControl = false);
 

src/libexpr/nixexpr.cc

@@ -45,7 +45,7 @@ void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
 
 void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    str << s;
+    str << path;
 }
 
 void ExprVar::show(const SymbolTable & symbols, std::ostream & str) const

src/libexpr/nixexpr.hh

@@ -131,12 +131,12 @@ struct ExprString : Expr
 
 struct ExprPath : Expr
 {
-    ref<SourceAccessor> accessor;
-    std::string s;
+    const SourcePath path;
     Value v;
-    ExprPath(ref<SourceAccessor> accessor, std::string s) : accessor(accessor), s(std::move(s))
+    ExprPath(SourcePath && path)
+        : path(path)
     {
-        v.mkPath(&*accessor, this->s.c_str());
+        v.mkPath(&*path.accessor, strdup(path.path.abs().c_str()));
     }
     Value * maybeThunk(EvalState & state, Env & env) override;
     COMMON_METHODS

src/libexpr/parser.y

@@ -136,6 +136,10 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
   std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs;
   std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts;
   std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts;
+  struct {
+      nix::Expr * e;
+      bool appendSlash;
+  } pathStart;
 }
 
 %type <e> start expr expr_function expr_if expr_op
@@ -149,7 +153,8 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
 %type <inheritAttrs> attrs
 %type <string_parts> string_parts_interpolated
 %type <ind_string_parts> ind_string_parts
-%type <e> path_start string_parts string_attr
+%type <pathStart> path_start
+%type <e> string_parts string_attr
 %type <id> attr
 %token <id> ID
 %token <str> STR IND_STR
@@ -295,9 +300,11 @@ expr_simple
       $$ = state->stripIndentation(CUR_POS, std::move(*$2));
       delete $2;
   }
-  | path_start PATH_END
+  | path_start PATH_END { $$ = $1.e; }
   | path_start string_parts_interpolated PATH_END {
-      $2->insert($2->begin(), {state->at(@1), $1});
+      if ($1.appendSlash)
+          $2->insert($2->begin(), {noPos, new ExprString("/")});
+      $2->insert($2->begin(), {state->at(@1), $1.e});
       $$ = new ExprConcatStrings(CUR_POS, false, $2);
   }
   | SPATH {
@@ -350,11 +357,17 @@ string_parts_interpolated
 
 path_start
   : PATH {
-    Path path(absPath({$1.p, $1.l}, state->basePath.path.abs()));
-    /* add back in the trailing '/' to the first segment */
-    if ($1.p[$1.l-1] == '/' && $1.l > 1)
-      path += "/";
-    $$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path));
+      std::string_view path({$1.p, $1.l});
+      $$ = {
+          .e = new ExprPath(
+              /* Absolute paths are always interpreted relative to the
+                 root filesystem accessor, rather than the accessor of the
+                 current Nix expression. */
+              hasPrefix(path, "/")
+              ? SourcePath{state->rootFS, CanonPath(path)}
+              : SourcePath{state->basePath.accessor, CanonPath(path, state->basePath.path)}),
+          .appendSlash = hasSuffix(path, "/")
+      };
   }
   | HPATH {
     if (state->settings.pureEval) {
@@ -363,8 +376,8 @@ path_start
             std::string_view($1.p, $1.l)
         );
     }
-    Path path(getHome() + std::string($1.p + 1, $1.l - 1));
-    $$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path));
+    CanonPath path(getHome() + std::string($1.p + 1, $1.l - 1));
+    $$ = {.e = new ExprPath(SourcePath{state->rootFS, std::move(path)}), .appendSlash = true};
   }
   ;
 

src/libexpr/primops.cc

@@ -178,6 +178,9 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
     auto path = realisePath(state, pos, vPath, std::nullopt);
     auto path2 = path.path.abs();
 
+// FIXME: corruption?
+// TODO(tomberek): re-enable this code?
+#if 0
     // FIXME
     auto isValidDerivationInStore = [&]() -> std::optional<StorePath> {
         if (!state.store->isStorePath(path2))
@@ -218,7 +221,9 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
         state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh");
     }
 
-    else {
+    else
+#endif
+    {
         if (!vScope)
             state.evalFile(path, v);
         else {