Detect search explosions

This patch detects some search explosions due to double extensions in
our search algorithm which can happen in some pathological positions,
and takes measure to ensure progress even in these pathological situations.

While a small number of double extensions can be useful during search
(for example to resolve a tactical sequence), a sustained regime of
double extensions leads to search explosion and a non-finishing search.
See the discussion in official-stockfish/Stockfish#3544
and the issue official-stockfish/Stockfish#3532 .

The implemented algorithm is as follows:

a) at each node during search, store the current depth in the stack.
   Double extensions are by definition levels of the stack where the
   depth at ply N is strictly higher than depth at ply N-1.

b) during search, calculate for each thread a running average of the
   number of double extensions in the last 4096 visited nodes.

c) if one thread has more than 2% of double extensions for a sustained
   period of time (6 millions consecutive nodes, or about 4 seconds on
   my iMac), we decide that this thread is in an explosion state and
   we calm down this thread by preventing it to do any double extension
   for the next 6 millions nodes.

-----------

Example where the patch solves a search explosion:

```
./stockfish
ucinewgame
position fen 8/Pk6/8/1p6/8/P1K5/8/6B1 w - - 37 130
go infinite
```

This algorithm does not affect search in normal, non-pathological positions.
We verified that usual bench is unchanged up to depth 20 at least, for instance,
and that the node numbers are unchanged for a search of the starting position
at depth 32.

-------------

Bench: 5575265

src/search.cpp

@@ -88,6 +88,30 @@ namespace {
     return VALUE_DRAW + Value(2 * (thisThread->nodes & 1) - 1);
   }
 
+  // Check if the current thread is in a search explosion
+  ExplosionState search_explosion(Thread* thisThread) {
+
+    uint64_t now = thisThread->nodes;
+    bool explosive =    thisThread->doubleExtensionAverage[WHITE].is_greater(2, 100)
+                     || thisThread->doubleExtensionAverage[BLACK].is_greater(2, 100);
+
+    if (explosive)
+       thisThread->lastExplosiveTime = now;
+    else
+       thisThread->lastNormalTime = now;
+
+    if (   explosive
+        && thisThread->state == EXPLOSION_NONE
+        && now - thisThread->lastNormalTime > 6000000)
+        thisThread->state = MUST_CALM_DOWN;
+
+    if (   thisThread->state == MUST_CALM_DOWN
+        && now - thisThread->lastExplosiveTime > 6000000)
+        thisThread->state = EXPLOSION_NONE;
+
+    return thisThread->state;
+  }
+
   // Skill structure is used to implement strength limit
   struct Skill {
     explicit Skill(int l) : level(l) {}
@@ -308,8 +332,13 @@ void Thread::search() {
 
   multiPV = std::min(multiPV, rootMoves.size());
 
-  ttHitAverage.set(50, 100); // initialize the running average at 50%
+  ttHitAverage.set(50, 100);                  // initialize the running average at 50%
+  doubleExtensionAverage[WHITE].set(0, 100);  // initialize the running average at 0%
+  doubleExtensionAverage[BLACK].set(0, 100);  // initialize the running average at 0%
 
+  lastExplosiveTime = nodes;
+  lastNormalTime    = nodes;
+  state = EXPLOSION_NONE;
   trend = SCORE_ZERO;
 
   int searchAgainCounter = 0;
@@ -516,6 +545,14 @@ namespace {
   template <NodeType nodeType>
   Value search(Position& pos, Stack* ss, Value alpha, Value beta, Depth depth, bool cutNode) {
 
+    Thread* thisThread = pos.this_thread();
+
+    // Step 0. Limit search explosion
+    if (   ss->ply > 10
+        && search_explosion(thisThread) == MUST_CALM_DOWN
+        && depth > (ss-1)->depth)
+       depth = (ss-1)->depth;
+
     constexpr bool PvNode = nodeType != NonPV;
     constexpr bool rootNode = nodeType == Root;
     const Depth maxNextDepth = rootNode ? depth : depth + 1;
@@ -552,12 +589,11 @@ namespace {
     Value bestValue, value, ttValue, eval, maxValue, probCutBeta;
     bool givesCheck, improving, didLMR, priorCapture;
     bool captureOrPromotion, doFullDepthSearch, moveCountPruning,
-         ttCapture, singularQuietLMR;
+         ttCapture, singularQuietLMR, noLMRExtension;
     Piece movedPiece;
     int moveCount, captureCount, quietCount;
 
     // Step 1. Initialize node
-    Thread* thisThread = pos.this_thread();
     ss->inCheck        = pos.checkers();
     priorCapture       = pos.captured_piece();
     Color us           = pos.side_to_move();
@@ -600,8 +636,12 @@ namespace {
     (ss+1)->excludedMove = bestMove = MOVE_NONE;
     (ss+2)->killers[0]   = (ss+2)->killers[1] = MOVE_NONE;
     ss->doubleExtensions = (ss-1)->doubleExtensions;
+    ss->depth            = depth;
     Square prevSq        = to_sq((ss-1)->currentMove);
 
+    // Update the running average statistics for double extensions
+    thisThread->doubleExtensionAverage[us].update(ss->depth > (ss-1)->depth);
+
     // Initialize statScore to zero for the grandchildren of the current position.
     // So statScore is shared between all grandchildren and only the first grandchild
     // starts with statScore = 0. Later grandchildren start with the last calculated
@@ -945,8 +985,7 @@ namespace {
                                       ss->ply);
 
     value = bestValue;
-    singularQuietLMR = moveCountPruning = false;
-    bool doubleExtension = false;
+    singularQuietLMR = moveCountPruning = noLMRExtension = false;
 
     // Indicate PvNodes that will probably fail low if the node was searched
     // at a depth equal or greater than the current depth, and the result of this search was a fail low.
@@ -1066,13 +1105,13 @@ namespace {
               extension = 1;
               singularQuietLMR = !ttCapture;
 
-              // Avoid search explosion by limiting the number of double extensions to at most 3
+              // Avoid search explosion by limiting the number of double extensions
               if (   !PvNode
                   && value < singularBeta - 93
                   && ss->doubleExtensions < 3)
               {
                   extension = 2;
-                  doubleExtension = true;
+                  noLMRExtension = true;
               }
           }
 
@@ -1184,8 +1223,16 @@ namespace {
 
           // In general we want to cap the LMR depth search at newDepth. But if
           // reductions are really negative and movecount is low, we allow this move
-          // to be searched deeper than the first move in specific cases.
-          Depth d = std::clamp(newDepth - r, 1, newDepth + (r < -1 && (moveCount <= 5 || (depth > 6 && PvNode)) && !doubleExtension));
+          // to be searched deeper than the first move in specific cases (note that
+          // this may lead to hidden double extensions if newDepth got it own extension
+          // before).
+          int deeper =   r >= -1               ? 0
+                       : noLMRExtension        ? 0
+                       : moveCount <= 5        ? 1
+                       : (depth > 6 && PvNode) ? 1
+                       :                         0;
+
+          Depth d = std::clamp(newDepth - r, 1, newDepth + deeper);
 
           value = -search<NonPV>(pos, ss+1, -(alpha+1), -alpha, d, true);
 

src/search.h

@@ -47,6 +47,7 @@ struct Stack {
   Move excludedMove;
   Move killers[2];
   Value staticEval;
+  Depth depth;
   int statScore;
   int moveCount;
   bool inCheck;

src/thread.h

@@ -61,9 +61,13 @@ class Thread {
   Material::Table materialTable;
   size_t pvIdx, pvLast;
   RunningAverage ttHitAverage;
+  RunningAverage doubleExtensionAverage[COLOR_NB];
+  uint64_t lastExplosiveTime;
+  uint64_t lastNormalTime;
+  std::atomic<uint64_t> nodes, tbHits, bestMoveChanges;
   int selDepth, nmpMinPly;
   Color nmpColor;
-  std::atomic<uint64_t> nodes, tbHits, bestMoveChanges;
+  ExplosionState state;
 
   Position rootPos;
   StateInfo rootState;

src/types.h

@@ -173,6 +173,11 @@ enum Bound {
   BOUND_EXACT = BOUND_UPPER | BOUND_LOWER
 };
 
+enum ExplosionState {
+  EXPLOSION_NONE,
+  MUST_CALM_DOWN
+};
+
 enum Value : int {
   VALUE_ZERO      = 0,
   VALUE_DRAW      = 0,