Improve excluded move logic

PR consists of 2 improvements on nodes with excludeMove:

1. Remove xoring the posKey with make_key(excludedMove)

   Since we never call tte->save anymore with excludedMove,
   the unique left purpose of the xoring was to avoid a TT hit.
   Nevertheless on a normal bench run this produced ~25 false positives
   (key collisions)
   To avoid that we now forbid early TT cutoff's with excludeMove
   Maybe these accesses to TT with xored key caused useless misses
   in the CPU caches (L1, L2 ...)
   Now doing the probe with the same key as the enclosing search does,
   should hit the CPU cache.

2. Don't probe Tablebases with excludedMove.

   This can't be tested on fishtest, but it's obvious that
   tablebases don't deliver any information about suboptimal moves.

Side note:
   Very surprisingly it looks like we cannot use static eval's from
   TT since they slightly differ over time due to changing optimism.
   Attempts to use static eval's from TT did loose about 13 ELO.
   This is something about to investigate.

LTC: https://tests.stockfishchess.org/tests/view/63dc0f8de9d4cdfbe672d0c6
LLR: 2.94 (-2.94,2.94) <0.50,2.50>
Total: 44736 W: 12046 L: 11733 D: 20957
Ptnml(0-2): 12, 4212, 13617, 4505, 22

An analogue of this passed STC & LTC
see PR #4374 (thanks Dubslow for reviewing!)

closes https://github.com/official-stockfish/Stockfish/pull/4380

Bench: 4758694

diff --git a/src/search.cpp b/src/search.cpp
index f6bc0aa9a8098a7250037749b3c81f231b277307..30a08cb77290532d8feafab0535c12ae513b564d 100644 (file)
--- a/src/search.cpp
+++ b/src/search.cpp
@@ -615,22 +615,24 @@ namespace {
     if (!rootNode)
         (ss+2)->statScore = 0;
 
-    // Step 4. Transposition table lookup. We don't want the score of a partial
-    // search to overwrite a previous full search TT value, so we use a different
-    // position key in case of an excluded move.
+    // Step 4. Transposition table lookup.
     excludedMove = ss->excludedMove;
-    posKey = excludedMove == MOVE_NONE ? pos.key() : pos.key() ^ make_key(excludedMove);
+    posKey = pos.key();
     tte = TT.probe(posKey, ss->ttHit);
     ttValue = ss->ttHit ? value_from_tt(tte->value(), ss->ply, pos.rule50_count()) : VALUE_NONE;
     ttMove =  rootNode ? thisThread->rootMoves[thisThread->pvIdx].pv[0]
             : ss->ttHit    ? tte->move() : MOVE_NONE;
     ttCapture = ttMove && pos.capture(ttMove);
+
+    // At this point, if excluded, skip straight to step 6, static eval. However,
+    // to save indentation, we list the condition in all code between here and there.
     if (!excludedMove)
         ss->ttPv = PvNode || (ss->ttHit && tte->is_pv());
 
     // At non-PV nodes we check for an early TT cutoff
     if (  !PvNode
         && ss->ttHit
+        && !excludedMove
         && tte->depth() > depth - (tte->bound() == BOUND_EXACT)
         && ttValue != VALUE_NONE // Possible in case of TT access race
         && (tte->bound() & (ttValue >= beta ? BOUND_LOWER : BOUND_UPPER)))
@@ -664,7 +666,7 @@ namespace {
     }
 
     // Step 5. Tablebases probe
-    if (!rootNode && TB::Cardinality)
+    if (!rootNode && !excludedMove && TB::Cardinality)
     {
         int piecesCount = pos.count<ALL_PIECES>();
 
@@ -727,6 +729,12 @@ namespace {
         complexity = 0;
         goto moves_loop;
     }
+    else if (excludedMove) {
+        // excludeMove implies that we had a ttHit on the containing non-excluded search with ss->staticEval filled from TT
+        // However static evals from the TT aren't good enough (-13 elo), presumably due to changing optimism context
+        // Recalculate value with current optimism (without updating thread avgComplexity)
+        ss->staticEval = eval = evaluate(pos, &complexity);
+    }
     else if (ss->ttHit)
     {
         // Never assume anything about values stored in TT
@@ -735,6 +743,7 @@ namespace {
             ss->staticEval = eval = evaluate(pos, &complexity);
         else // Fall back to (semi)classical complexity for TT hits, the NNUE complexity is lost
             complexity = abs(ss->staticEval - pos.psq_eg_stm());
+        thisThread->complexityAverage.update(complexity);
 
         // ttValue can be used as a better position evaluation (~7 Elo)
         if (    ttValue != VALUE_NONE
@@ -744,14 +753,12 @@ namespace {
     else
     {
         ss->staticEval = eval = evaluate(pos, &complexity);
+        thisThread->complexityAverage.update(complexity);
 
         // Save static evaluation into transposition table
-        if (!excludedMove)
-            tte->save(posKey, VALUE_NONE, ss->ttPv, BOUND_NONE, DEPTH_NONE, MOVE_NONE, eval);
+        tte->save(posKey, VALUE_NONE, ss->ttPv, BOUND_NONE, DEPTH_NONE, MOVE_NONE, eval);
     }
 
-    thisThread->complexityAverage.update(complexity);
-
     // Use static evaluation difference to improve quiet move ordering (~4 Elo)
     if (is_ok((ss-1)->currentMove) && !(ss-1)->inCheck && !priorCapture)
     {
@@ -1061,6 +1068,7 @@ moves_loop: // When in check, search starts here
               Depth singularDepth = (depth - 1) / 2;
 
               ss->excludedMove = move;
+              // the search with excludedMove will update ss->staticEval
               value = search<NonPV>(pos, ss, singularBeta - 1, singularBeta, singularDepth, cutNode);
               ss->excludedMove = MOVE_NONE;