X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=tests%2Finstrumented.sh;h=bfb50e94c196710680ec999f0a079e334c5f686c;hb=HEAD;hp=137d0e4af2c8ea0a93c3c6975d2118b804363f10;hpb=c959871a59d106aa0294916890c31c475aa195ea;p=stockfish

diff --git a/tests/instrumented.sh b/tests/instrumented.sh
index 137d0e4a..5fc6ca9a 100755
--- a/tests/instrumented.sh
+++ b/tests/instrumented.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# check for errors under valgrind or sanitizers.
+# check for errors under Valgrind or sanitizers.
 
 error()
 {
@@ -13,41 +13,37 @@ case $1 in
   --valgrind)
     echo "valgrind testing started"
     prefix=''
-    exeprefix='valgrind --error-exitcode=42'
-    postfix='1>/dev/null'
+    exeprefix='valgrind --error-exitcode=42 --errors-for-leak-kinds=all --leak-check=full'
+    postfix=''
     threads="1"
   ;;
   --valgrind-thread)
     echo "valgrind-thread testing started"
     prefix=''
-    exeprefix='valgrind --error-exitcode=42'
-    postfix='1>/dev/null'
+    exeprefix='valgrind --fair-sched=try --error-exitcode=42'
+    postfix=''
     threads="2"
   ;;
   --sanitizer-undefined)
     echo "sanitizer-undefined testing started"
     prefix='!'
     exeprefix=''
-    postfix='2>&1 | grep "runtime error:"'
+    postfix='2>&1 | grep -A50 "runtime error:"'
     threads="1"
   ;;
   --sanitizer-thread)
     echo "sanitizer-thread testing started"
     prefix='!'
     exeprefix=''
-    postfix='2>&1 | grep "WARNING: ThreadSanitizer:"'
+    postfix='2>&1 | grep -A50 "WARNING: ThreadSanitizer:"'
     threads="2"
 
 cat << EOF > tsan.supp
-race:TTEntry::move
-race:TTEntry::depth
-race:TTEntry::bound
-race:TTEntry::save
-race:TTEntry::value
-race:TTEntry::eval
+race:Stockfish::TTEntry::read
+race:Stockfish::TTEntry::save
 
-race:TranspositionTable::probe
-race:TranspositionTable::hashfull
+race:Stockfish::TranspositionTable::probe
+race:Stockfish::TranspositionTable::hashfull
 
 EOF
 
@@ -63,13 +59,32 @@ EOF
   ;;
 esac
 
+cat << EOF > bench_tmp.epd
+Rn6/1rbq1bk1/2p2n1p/2Bp1p2/3Pp1pP/1N2P1P1/2Q1NPB1/6K1 w - - 2 26
+rnbqkb1r/ppp1pp2/5n1p/3p2p1/P2PP3/5P2/1PP3PP/RNBQKBNR w KQkq - 0 3
+3qnrk1/4bp1p/1p2p1pP/p2bN3/1P1P1B2/P2BQ3/5PP1/4R1K1 w - - 9 28
+r4rk1/1b2ppbp/pq4pn/2pp1PB1/1p2P3/1P1P1NN1/1PP3PP/R2Q1RK1 w - - 0 13
+EOF
+
 # simple command line testing
 for args in "eval" \
             "go nodes 1000" \
             "go depth 10" \
+            "go perft 4" \
             "go movetime 1000" \
             "go wtime 8000 btime 8000 winc 500 binc 500" \
-            "bench 128 $threads 10 default depth"
+            "go wtime 1000 btime 1000 winc 0 binc 0" \
+            "go wtime 1000 btime 1000 winc 0 binc 0" \
+            "go wtime 1000 btime 1000 winc 0 binc 0 movestogo 5" \
+            "go movetime 200" \
+            "go nodes 20000 searchmoves e2e4 d2d4" \
+            "bench 128 $threads 8 default depth" \
+            "bench 128 $threads 3 bench_tmp.epd depth" \
+            "export_net verify.nnue" \
+            "d" \
+            "compiler" \
+            "license" \
+            "uci"
 do
 
    echo "$prefix $exeprefix ./stockfish $args $postfix"
@@ -77,14 +92,25 @@ do
 
 done
 
+# verify the generated net equals the base net
+network=`./stockfish uci | grep 'option name EvalFile type string default' | awk '{print $NF}'`
+echo "Comparing $network to the written verify.nnue"
+diff $network verify.nnue
+
 # more general testing, following an uci protocol exchange
 cat << EOF > game.exp
- set timeout 10
+ set timeout 240
+ # to correctly catch eof we need the following line
+ # expect_before timeout { exit 2 } eof { exit 3 }
+ expect_before timeout { exit 2 }
+
  spawn $exeprefix ./stockfish
+ expect "Stockfish"
 
  send "uci\n"
  expect "uciok"
 
+ # send "setoption name Debug Log File value debug.log\n"
  send "setoption name Threads value $threads\n"
 
  send "ucinewgame\n"
@@ -92,18 +118,126 @@ cat << EOF > game.exp
  send "go nodes 1000\n"
  expect "bestmove"
 
+ send "ucinewgame\n"
  send "position startpos moves e2e4 e7e6\n"
  send "go nodes 1000\n"
  expect "bestmove"
 
+ send "ucinewgame\n"
  send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n"
- send "go depth 30\n"
+ send "go depth 10\n"
  expect "bestmove"
 
+ send "ucinewgame\n"
+ send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n"
+ send "flip\n"
+ send "go depth 10\n"
+ expect "bestmove"
+
+ send "ucinewgame\n"
+ send "position startpos\n"
+ send "go depth 5\n"
+ expect -re {info depth \d+ seldepth \d+ multipv \d+ score cp \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect "bestmove"
+
+ send "ucinewgame\n"
+ send "setoption name UCI_ShowWDL value true\n"
+ send "position startpos\n"
+ send "go depth 9\n"
+ expect -re {info depth 1 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 2 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 3 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 4 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 5 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 6 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 7 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 8 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 9 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect "bestmove"
+
+ send "setoption name Clear Hash\n"
+
+ send "ucinewgame\n"
+ send "position fen 5K2/8/2qk4/2nPp3/3r4/6B1/B7/3R4 w - e6\n"
+ send "go depth 18\n"
+ expect "score mate 1"
+ expect "pv d5e6"
+ expect "bestmove d5e6"
+
+ send "ucinewgame\n"
+ send "position fen 2brrb2/8/p7/Q7/1p1kpPp1/1P1pN1K1/3P4/8 b - -\n"
+ send "go depth 18\n"
+ expect "score mate -1"
+ expect "bestmove"
+
+ send "ucinewgame\n"
+ send "position fen 7K/P1p1p1p1/2P1P1Pk/6pP/3p2P1/1P6/3P4/8 w - - 0 1\n"
+ send "go nodes 500000\n"
+ expect "bestmove"
+
+ send "ucinewgame\n"
+ send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
+ send "go depth 18 searchmoves c6d7\n"
+ expect "score mate 2 * pv c6d7 * f7f5"
+ expect "bestmove c6d7"
+
+ send "ucinewgame\n"
+ send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
+ send "go mate 2 searchmoves c6d7\n"
+ expect "score mate 2 * pv c6d7"
+ expect "bestmove c6d7"
+
+ send "ucinewgame\n"
+ send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
+ send "go nodes 500000 searchmoves c6d7\n"
+ expect "score mate 2 * pv c6d7 * f7f5"
+ expect "bestmove c6d7"
+
+ send "ucinewgame\n"
+ send "position fen 1NR2B2/5p2/5p2/1p1kpp2/1P2rp2/2P1pB2/2P1P1K1/8 b - - \n"
+ send "go depth 27\n"
+ expect "score mate -2"
+ expect "pv d5e6 c8d8"
+ expect "bestmove d5e6"
+
+ send "ucinewgame\n"
+ send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7 f2f1q\n"
+ send "go depth 18\n"
+ expect "score mate 1 * pv f7f5"
+ expect "bestmove f7f5"
+ 
+ send "ucinewgame\n"
+ send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
+ send "go depth 18 searchmoves c6d7\n"
+ expect "score mate 2 * pv c6d7 * f7f5"
+ expect "bestmove c6d7"
+ 
+ send "ucinewgame\n"
+ send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7\n"
+ send "go depth 18 searchmoves e3e2\n"
+ expect "score mate -1 * pv e3e2 f7f5"
+ expect "bestmove e3e2"
+
+ send "setoption name EvalFile value verify.nnue\n"
+ send "position startpos\n"
+ send "go depth 5\n"
+ expect "bestmove"
+
+ send "setoption name MultiPV value 4\n"
+ send "position startpos\n"
+ send "go depth 5\n"
+ expect "bestmove"
+
+ send "setoption name Skill Level value 10\n"
+ send "position startpos\n"
+ send "go depth 5\n"
+ expect "bestmove"
+ send "setoption name Skill Level value 20\n"
+
  send "quit\n"
  expect eof
 
- # return error code of the spawned program, useful for valgrind
+ # return error code of the spawned program, useful for Valgrind
  lassign [wait] pid spawnid os_error_flag value
  exit \$value
 EOF
@@ -116,15 +250,35 @@ fi
 
 cat << EOF > syzygy.exp
  set timeout 240
+ # to correctly catch eof we need the following line
+ # expect_before timeout { exit 2 } eof { exit 3 }
+ expect_before timeout { exit 2 }
  spawn $exeprefix ./stockfish
+ expect "Stockfish"
  send "uci\n"
  send "setoption name SyzygyPath value ../tests/syzygy/\n"
- expect "info string Found 35 tablebases" {} timeout {exit 1}
- send "bench 128 1 10 default depth\n"
+ expect "info string Found 35 WDL and 35 DTZ tablebase files (up to 4-man)."
+ send "bench 128 1 8 default depth\n"
+ expect "Nodes searched  :"
+ send "ucinewgame\n"
+ send "position fen 4k3/PP6/8/8/8/8/8/4K3 w - - 0 1\n"
+ send "go depth 5\n"
+ expect -re {score cp 20000|score mate}
+ expect "bestmove"
+ send "ucinewgame\n"
+ send "position fen 8/1P6/2B5/8/4K3/8/6k1/8 w - - 0 1\n"
+ send "go depth 5\n"
+ expect -re {score cp 20000|score mate}
+ expect "bestmove"
+ send "ucinewgame\n"
+ send "position fen 8/1P6/2B5/8/4K3/8/6k1/8 b - - 0 1\n"
+ send "go depth 5\n"
+ expect -re {score cp -20000|score mate}
+ expect "bestmove"
  send "quit\n"
  expect eof
 
- # return error code of the spawned program, useful for valgrind
+ # return error code of the spawned program, useful for Valgrind
  lassign [wait] pid spawnid os_error_flag value
  exit \$value
 EOF
@@ -132,6 +286,9 @@ EOF
 for exp in game.exp syzygy.exp
 do
 
+  echo "======== $exp =============="
+  cat $exp
+  echo "============================"
   echo "$prefix expect $exp $postfix"
   eval "$prefix expect $exp $postfix"
 
@@ -139,6 +296,6 @@ do
 
 done
 
-rm -f tsan.supp
+rm -f tsan.supp bench_tmp.epd
 
 echo "instrumented testing OK"