Fix segfault on infinite recursion in some cases

This fixes a segfault on infinite function call recursion (rather than
infinite thunk recursion) by tracking the function call depth in
`EvalState`.

Additionally, to avoid printing extremely long stack traces, stack
frames are now deduplicated, with a `(19997 duplicate traces omitted)`
message. This should only really be triggered in infinite recursion
scenarios.

Before:

    $ nix-instantiate --eval --expr '(x: x x) (x: x x)'
    Segmentation fault: 11

After:

    $ nix-instantiate --eval --expr '(x: x x) (x: x x)'
    error: stack depth exhausted

           at «string»:1:14:

                1| (x: x x) (x: x x)
                 |              ^

    $ nix-instantiate --eval --expr '(x: x x) (x: x x)' --show-trace
    error:
           … from call site

             at «string»:1:1:

                1| (x: x x) (x: x x)
                 | ^

           … while calling anonymous lambda

             at «string»:1:2:

                1| (x: x x) (x: x x)
                 |  ^

           … from call site

             at «string»:1:5:

                1| (x: x x) (x: x x)
                 |     ^

           … while calling anonymous lambda

             at «string»:1:11:

                1| (x: x x) (x: x x)
                 |           ^

           … from call site

             at «string»:1:14:

                1| (x: x x) (x: x x)
                 |              ^

           (19997 duplicate traces omitted)

           error: stack depth exhausted

           at «string»:1:14:

                1| (x: x x) (x: x x)
                 |              ^

src/libexpr/eval.cc

@@ -1508,6 +1508,10 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v)
 
 void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos)
 {
+    if (depth > 10000)
+        error("stack depth exhausted").atPos(pos).template debugThrow<EvalError>();
+    depth++;
+
     auto trace = evalSettings.traceFunctionCalls
         ? std::make_unique<FunctionCallTrace>(positions[pos])
         : nullptr;
@@ -1732,6 +1736,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
     }
 
     vRes = vCur;
+    depth--;
 }
 
 

src/libexpr/eval.hh

@@ -623,6 +623,8 @@ private:
         const SourcePath & basePath,
         std::shared_ptr<StaticEnv> & staticEnv);
 
+    unsigned int depth = 0;
+
 public:
 
     /**

src/libutil/error.cc

@@ -50,6 +50,29 @@ std::ostream & operator <<(std::ostream & str, const AbstractPos & pos)
     return str;
 }
 
+inline bool operator<(const AbstractPos& lhs, const AbstractPos& rhs)
+{
+    return lhs.line < rhs.line
+        || lhs.column < rhs.column;
+}
+inline bool operator> (const AbstractPos& lhs, const AbstractPos& rhs) { return rhs < lhs; }
+inline bool operator<=(const AbstractPos& lhs, const AbstractPos& rhs) { return !(lhs > rhs); }
+inline bool operator>=(const AbstractPos& lhs, const AbstractPos& rhs) { return !(lhs < rhs); }
+
+inline bool operator<(const Trace& lhs, const Trace& rhs)
+{
+    if (!lhs.pos && rhs.pos) {
+        return true;
+    }
+
+    return (lhs.pos && rhs.pos && *lhs.pos < *rhs.pos)
+        || lhs.hint.str() < rhs.hint.str()
+        || lhs.frame < rhs.frame;
+}
+inline bool operator> (const Trace& lhs, const Trace& rhs) { return rhs < lhs; }
+inline bool operator<=(const Trace& lhs, const Trace& rhs) { return !(lhs > rhs); }
+inline bool operator>=(const Trace& lhs, const Trace& rhs) { return !(lhs < rhs); }
+
 std::optional<LinesOfCode> AbstractPos::getCodeLines() const
 {
     if (line == 0)
@@ -334,7 +357,15 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
 
     bool frameOnly = false;
     if (!einfo.traces.empty()) {
+        std::set<Trace> tracesSeen;
+        size_t skippedTraces = 0;
         size_t count = 0;
+        const auto printSkippedTracesMaybe = [&] {
+            if (skippedTraces > 0) {
+                oss << "\n" << ANSI_WARNING "(" << skippedTraces << " duplicate traces omitted)" ANSI_NORMAL << "\n";
+            }
+        };
+
         for (const auto & trace : einfo.traces) {
             if (trace.hint.str().empty()) continue;
             if (frameOnly && !trace.frame) continue;
@@ -344,6 +375,16 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
                 break;
             }
 
+            if (tracesSeen.count(trace)) {
+                skippedTraces++;
+                continue;
+            }
+
+            printSkippedTracesMaybe();
+
+            tracesSeen.insert(trace);
+            skippedTraces = 0;
+
             count++;
             frameOnly = trace.frame;
 
@@ -352,6 +393,8 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
             if (printPosMaybe(oss, ellipsisIndent, trace.pos))
                 count++;
         }
+
+        printSkippedTracesMaybe();
         oss << "\n" << prefix;
     }
 

tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp

@@ -0,0 +1,50 @@
+error:
+       … from call site
+
+         at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:1:
+
+            1| (x: x x) (x: x x)
+             | ^
+            2|
+
+       … while calling anonymous lambda
+
+         at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:2:
+
+            1| (x: x x) (x: x x)
+             |  ^
+            2|
+
+       … from call site
+
+         at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:5:
+
+            1| (x: x x) (x: x x)
+             |     ^
+            2|
+
+       … while calling anonymous lambda
+
+         at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:11:
+
+            1| (x: x x) (x: x x)
+             |           ^
+            2|
+
+       … from call site
+
+         at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:14:
+
+            1| (x: x x) (x: x x)
+             |              ^
+            2|
+
+       (19997 duplicate traces omitted)
+
+       error: stack depth exhausted
+
+       at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:14:
+
+            1| (x: x x) (x: x x)
+             |              ^
+            2|

tests/functional/lang/eval-fail-infinite-recursion-lambda.nix

@@ -0,0 +1 @@
+(x: x x) (x: x x)