Refactor the parser somewhat

src/libexpr/eval.cc

@@ -20,6 +20,9 @@
 #include "gc-small-vector.hh"
 #include "url.hh"
 #include "fetch-to-store.hh"
+#include "tarball.hh"
+#include "flake/flakeref.hh"
+#include "parser-tab.hh"
 
 #include <algorithm>
 #include <chrono>
@@ -416,6 +419,16 @@ EvalState::EvalState(
     , sPath(symbols.create("path"))
     , sPrefix(symbols.create("prefix"))
     , sOutputSpecified(symbols.create("outputSpecified"))
+    , exprSymbols{
+        .sub = symbols.create("__sub"),
+        .lessThan = symbols.create("__lessThan"),
+        .mul = symbols.create("__mul"),
+        .div = symbols.create("__div"),
+        .or_ = symbols.create("or"),
+        .findFile = symbols.create("__findFile"),
+        .nixPath = symbols.create("__nixPath"),
+        .body = symbols.create("body"),
+    }
     , repair(NoRepair)
     , emptyBindings(0)
     , rootFS(
@@ -2636,6 +2649,183 @@ void EvalState::printStatistics()
 }
 
 
+SourcePath resolveExprPath(SourcePath path)
+{
+    unsigned int followCount = 0, maxFollow = 1024;
+
+    /* If `path' is a symlink, follow it.  This is so that relative
+       path references work. */
+    while (!path.path.isRoot()) {
+        // Basic cycle/depth limit to avoid infinite loops.
+        if (++followCount >= maxFollow)
+            throw Error("too many symbolic links encountered while traversing the path '%s'", path);
+        auto p = path.parent().resolveSymlinks() + path.baseName();
+        if (p.lstat().type != InputAccessor::tSymlink) break;
+        path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))};
+    }
+
+    /* If `path' refers to a directory, append `/default.nix'. */
+    if (path.resolveSymlinks().lstat().type == InputAccessor::tDirectory)
+        return path + "default.nix";
+
+    return path;
+}
+
+
+Expr * EvalState::parseExprFromFile(const SourcePath & path)
+{
+    return parseExprFromFile(path, staticBaseEnv);
+}
+
+
+Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
+{
+    auto buffer = path.resolveSymlinks().readFile();
+    // readFile hopefully have left some extra space for terminators
+    buffer.append("\0\0", 2);
+    return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
+}
+
+
+Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
+{
+    auto s = make_ref<std::string>(std::move(s_));
+    s->append("\0\0", 2);
+    return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv);
+}
+
+
+Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath)
+{
+    return parseExprFromString(std::move(s), basePath, staticBaseEnv);
+}
+
+
+Expr * EvalState::parseStdin()
+{
+    //Activity act(*logger, lvlTalkative, "parsing standard input");
+    auto buffer = drainFD(0);
+    // drainFD should have left some extra space for terminators
+    buffer.append("\0\0", 2);
+    auto s = make_ref<std::string>(std::move(buffer));
+    return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv);
+}
+
+
+SourcePath EvalState::findFile(const std::string_view path)
+{
+    return findFile(searchPath, path);
+}
+
+
+SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos)
+{
+    for (auto & i : searchPath.elements) {
+        auto suffixOpt = i.prefix.suffixIfPotentialMatch(path);
+
+        if (!suffixOpt) continue;
+        auto suffix = *suffixOpt;
+
+        auto rOpt = resolveSearchPathPath(i.path);
+        if (!rOpt) continue;
+        auto r = *rOpt;
+
+        Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
+        if (pathExists(res)) return rootPath(CanonPath(canonPath(res)));
+    }
+
+    if (hasPrefix(path, "nix/"))
+        return {corepkgsFS, CanonPath(path.substr(3))};
+
+    debugThrow(ThrownError({
+        .msg = hintfmt(evalSettings.pureEval
+            ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
+            : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
+            path),
+        .errPos = positions[pos]
+    }), 0, 0);
+}
+
+
+std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Path & value0, bool initAccessControl)
+{
+    auto & value = value0.s;
+    auto i = searchPathResolved.find(value);
+    if (i != searchPathResolved.end()) return i->second;
+
+    std::optional<std::string> res;
+
+    if (EvalSettings::isPseudoUrl(value)) {
+        try {
+            auto storePath = fetchers::downloadTarball(
+                store, EvalSettings::resolvePseudoUrl(value), "source", false).storePath;
+            res = { store->toRealPath(storePath) };
+        } catch (FileTransferError & e) {
+            logWarning({
+                .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
+            });
+        }
+    }
+
+    else if (hasPrefix(value, "flake:")) {
+        experimentalFeatureSettings.require(Xp::Flakes);
+        auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false);
+        debug("fetching flake search path element '%s''", value);
+        auto storePath = flakeRef.resolve(store).fetchTree(store).first;
+        res = { store->toRealPath(storePath) };
+    }
+
+    else {
+        auto path = absPath(value);
+
+        /* Allow access to paths in the search path. */
+        if (initAccessControl) {
+            allowPath(path);
+            if (store->isInStore(path)) {
+                try {
+                    StorePathSet closure;
+                    store->computeFSClosure(store->toStorePath(path).first, closure);
+                    for (auto & p : closure)
+                        allowPath(p);
+                } catch (InvalidPath &) { }
+            }
+        }
+
+        if (pathExists(path))
+            res = { path };
+        else {
+            logWarning({
+                .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value)
+            });
+            res = std::nullopt;
+        }
+    }
+
+    if (res)
+        debug("resolved search path element '%s' to '%s'", value, *res);
+    else
+        debug("failed to resolve search path element '%s'", value);
+
+    searchPathResolved.emplace(value, res);
+    return res;
+}
+
+
+Expr * EvalState::parse(
+    char * text,
+    size_t length,
+    Pos::Origin origin,
+    const SourcePath & basePath,
+    std::shared_ptr<StaticEnv> & staticEnv)
+{
+    auto result = parseExprFromBuf(text, length, origin, basePath, symbols, positions, rootFS, exprSymbols);
+
+    result->bindVars(*this, staticEnv);
+
+    return result;
+}
+
+
 std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
 {
     throw TypeError({

src/libexpr/eval.hh

@@ -207,6 +207,8 @@ public:
         sPrefix,
         sOutputSpecified;
 
+    const Expr::AstSymbols exprSymbols;
+
     /**
      * If set, force copying files to the Nix store even if they
      * already exist there.

src/libexpr/lexer.l

@@ -29,12 +29,7 @@ using namespace nix;
 
 namespace nix {
 
-static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data)
-{
-    return data->state.positions.add(data->origin, loc.first_line, loc.first_column);
-}
-
-#define CUR_POS makeCurPos(*yylloc, data)
+#define CUR_POS state->at(*yylloc)
 
 static void initLoc(YYLTYPE * loc)
 {
@@ -153,7 +148,7 @@ or          { return OR_KW; }
               } catch (const boost::bad_lexical_cast &) {
                   throw ParseError({
                       .msg = hintfmt("invalid integer '%1%'", yytext),
-                      .errPos = data->state.positions[CUR_POS],
+                      .errPos = state->positions[CUR_POS],
                   });
               }
               return INT_LIT;
@@ -163,7 +158,7 @@ or          { return OR_KW; }
               if (errno != 0)
                   throw ParseError({
                       .msg = hintfmt("invalid float '%1%'", yytext),
-                      .errPos = data->state.positions[CUR_POS],
+                      .errPos = state->positions[CUR_POS],
                   });
               return FLOAT_LIT;
             }
@@ -186,7 +181,7 @@ or          { return OR_KW; }
                 /* It is impossible to match strings ending with '$' with one
                    regex because trailing contexts are only valid at the end
                    of a rule. (A sane but undocumented limitation.) */
-                yylval->str = unescapeStr(data->symbols, yytext, yyleng);
+                yylval->str = unescapeStr(state->symbols, yytext, yyleng);
                 return STR;
               }
 <STRING>\$\{  { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
@@ -214,7 +209,7 @@ or          { return OR_KW; }
                    return IND_STR;
                  }
 <IND_STRING>\'\'\\{ANY} {
-                   yylval->str = unescapeStr(data->symbols, yytext + 2, yyleng - 2);
+                   yylval->str = unescapeStr(state->symbols, yytext + 2, yyleng - 2);
                    return IND_STR;
                  }
 <IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
@@ -292,7 +287,7 @@ or          { return OR_KW; }
 <INPATH_SLASH><<EOF>> {
   throw ParseError({
       .msg = hintfmt("path has a trailing slash"),
-      .errPos = data->state.positions[CUR_POS],
+      .errPos = state->positions[CUR_POS],
   });
 }
 

src/libexpr/nixexpr.cc

@@ -9,6 +9,8 @@
 
 namespace nix {
 
+unsigned long Expr::nrExprs = 0;
+
 ExprBlackHole eBlackHole;
 
 // FIXME: remove, because *symbols* are abstract and do not have a single

src/libexpr/nixexpr.hh

@@ -140,6 +140,11 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath)
 
 struct Expr
 {
+    struct AstSymbols {
+        Symbol sub, lessThan, mul, div, or_, findFile, nixPath, body;
+    };
+
+
     static unsigned long nrExprs;
     Expr() {
         nrExprs++;