Merge remote-tracking branch 'upstream/master'

[stockfish] / tests / instrumented.sh

diff --git a/tests/instrumented.sh b/tests/instrumented.sh
index ae6d5c4b905f2e79464a1a36c45ddc72a323a184..5fc6ca9a9745bc5b6951a0d65e30a43fe6e06083 100755 (executable)
--- a/tests/instrumented.sh
+++ b/tests/instrumented.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
 #!/bin/bash

-# check for errors under valgrind or sanitizers.
+# check for errors under Valgrind or sanitizers.

 
 error()
 {
 
 error()
 {


@@ -13,15 +13,15 @@ case $1 in
   --valgrind)
     echo "valgrind testing started"
     prefix=''
   --valgrind)
     echo "valgrind testing started"
     prefix=''

-    exeprefix='valgrind --error-exitcode=42'
-    postfix='1>/dev/null'
+    exeprefix='valgrind --error-exitcode=42 --errors-for-leak-kinds=all --leak-check=full'
+    postfix=''

     threads="1"
   ;;
   --valgrind-thread)
     echo "valgrind-thread testing started"
     prefix=''
     threads="1"
   ;;
   --valgrind-thread)
     echo "valgrind-thread testing started"
     prefix=''

-    exeprefix='valgrind --error-exitcode=42'
-    postfix='1>/dev/null'
+    exeprefix='valgrind --fair-sched=try --error-exitcode=42'
+    postfix=''

     threads="2"
   ;;
   --sanitizer-undefined)
     threads="2"
   ;;
   --sanitizer-undefined)


@@ -39,16 +39,11 @@ case $1 in
     threads="2"
 
 cat << EOF > tsan.supp
     threads="2"
 
 cat << EOF > tsan.supp

-race:TTEntry::move
-race:TTEntry::depth
-race:TTEntry::bound
-race:TTEntry::save
-race:TTEntry::value
-race:TTEntry::eval
-race:TTEntry::is_pv
+race:Stockfish::TTEntry::read
+race:Stockfish::TTEntry::save

 
 

-race:TranspositionTable::probe
-race:TranspositionTable::hashfull
+race:Stockfish::TranspositionTable::probe
+race:Stockfish::TranspositionTable::hashfull

 
 EOF
 
 
 EOF
 


@@ -64,13 +59,32 @@ EOF
   ;;
 esac
 
   ;;
 esac
 

+cat << EOF > bench_tmp.epd
+Rn6/1rbq1bk1/2p2n1p/2Bp1p2/3Pp1pP/1N2P1P1/2Q1NPB1/6K1 w - - 2 26
+rnbqkb1r/ppp1pp2/5n1p/3p2p1/P2PP3/5P2/1PP3PP/RNBQKBNR w KQkq - 0 3
+3qnrk1/4bp1p/1p2p1pP/p2bN3/1P1P1B2/P2BQ3/5PP1/4R1K1 w - - 9 28
+r4rk1/1b2ppbp/pq4pn/2pp1PB1/1p2P3/1P1P1NN1/1PP3PP/R2Q1RK1 w - - 0 13
+EOF
+

 # simple command line testing
 for args in "eval" \
             "go nodes 1000" \
             "go depth 10" \
 # simple command line testing
 for args in "eval" \
             "go nodes 1000" \
             "go depth 10" \

+            "go perft 4" \

             "go movetime 1000" \
             "go wtime 8000 btime 8000 winc 500 binc 500" \
             "go movetime 1000" \
             "go wtime 8000 btime 8000 winc 500 binc 500" \

-            "bench 128 $threads 10 default depth"
+            "go wtime 1000 btime 1000 winc 0 binc 0" \
+            "go wtime 1000 btime 1000 winc 0 binc 0" \
+            "go wtime 1000 btime 1000 winc 0 binc 0 movestogo 5" \
+            "go movetime 200" \
+            "go nodes 20000 searchmoves e2e4 d2d4" \
+            "bench 128 $threads 8 default depth" \
+            "bench 128 $threads 3 bench_tmp.epd depth" \
+            "export_net verify.nnue" \
+            "d" \
+            "compiler" \
+            "license" \
+            "uci"

 do
 
    echo "$prefix $exeprefix ./stockfish $args $postfix"
 do
 
    echo "$prefix $exeprefix ./stockfish $args $postfix"


@@ -78,14 +92,25 @@ do
 
 done
 
 
 done
 

+# verify the generated net equals the base net
+network=`./stockfish uci | grep 'option name EvalFile type string default' | awk '{print $NF}'`
+echo "Comparing $network to the written verify.nnue"
+diff $network verify.nnue
+

 # more general testing, following an uci protocol exchange
 cat << EOF > game.exp
 # more general testing, following an uci protocol exchange
 cat << EOF > game.exp

- set timeout 10
+ set timeout 240
+ # to correctly catch eof we need the following line
+ # expect_before timeout { exit 2 } eof { exit 3 }
+ expect_before timeout { exit 2 }
+

  spawn $exeprefix ./stockfish
  spawn $exeprefix ./stockfish

+ expect "Stockfish"

 
  send "uci\n"
  expect "uciok"
 
 
  send "uci\n"
  expect "uciok"
 

+ # send "setoption name Debug Log File value debug.log\n"

  send "setoption name Threads value $threads\n"
 
  send "ucinewgame\n"
  send "setoption name Threads value $threads\n"
 
  send "ucinewgame\n"


@@ -93,18 +118,126 @@ cat << EOF > game.exp
  send "go nodes 1000\n"
  expect "bestmove"
 
  send "go nodes 1000\n"
  expect "bestmove"
 

+ send "ucinewgame\n"

  send "position startpos moves e2e4 e7e6\n"
  send "go nodes 1000\n"
  expect "bestmove"
 
  send "position startpos moves e2e4 e7e6\n"
  send "go nodes 1000\n"
  expect "bestmove"
 

+ send "ucinewgame\n"

  send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n"
  send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n"

- send "go depth 30\n"
+ send "go depth 10\n"

  expect "bestmove"
 
  expect "bestmove"
 

+ send "ucinewgame\n"
+ send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n"
+ send "flip\n"
+ send "go depth 10\n"
+ expect "bestmove"
+
+ send "ucinewgame\n"
+ send "position startpos\n"
+ send "go depth 5\n"
+ expect -re {info depth \d+ seldepth \d+ multipv \d+ score cp \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect "bestmove"
+
+ send "ucinewgame\n"
+ send "setoption name UCI_ShowWDL value true\n"
+ send "position startpos\n"
+ send "go depth 9\n"
+ expect -re {info depth 1 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 2 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 3 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 4 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 5 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 6 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 7 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 8 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect -re {info depth 9 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
+ expect "bestmove"
+
+ send "setoption name Clear Hash\n"
+
+ send "ucinewgame\n"
+ send "position fen 5K2/8/2qk4/2nPp3/3r4/6B1/B7/3R4 w - e6\n"
+ send "go depth 18\n"
+ expect "score mate 1"
+ expect "pv d5e6"
+ expect "bestmove d5e6"
+
+ send "ucinewgame\n"
+ send "position fen 2brrb2/8/p7/Q7/1p1kpPp1/1P1pN1K1/3P4/8 b - -\n"
+ send "go depth 18\n"
+ expect "score mate -1"
+ expect "bestmove"
+
+ send "ucinewgame\n"
+ send "position fen 7K/P1p1p1p1/2P1P1Pk/6pP/3p2P1/1P6/3P4/8 w - - 0 1\n"
+ send "go nodes 500000\n"
+ expect "bestmove"
+
+ send "ucinewgame\n"
+ send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
+ send "go depth 18 searchmoves c6d7\n"
+ expect "score mate 2 * pv c6d7 * f7f5"
+ expect "bestmove c6d7"
+
+ send "ucinewgame\n"
+ send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
+ send "go mate 2 searchmoves c6d7\n"
+ expect "score mate 2 * pv c6d7"
+ expect "bestmove c6d7"
+
+ send "ucinewgame\n"
+ send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
+ send "go nodes 500000 searchmoves c6d7\n"
+ expect "score mate 2 * pv c6d7 * f7f5"
+ expect "bestmove c6d7"
+
+ send "ucinewgame\n"
+ send "position fen 1NR2B2/5p2/5p2/1p1kpp2/1P2rp2/2P1pB2/2P1P1K1/8 b - - \n"
+ send "go depth 27\n"
+ expect "score mate -2"
+ expect "pv d5e6 c8d8"
+ expect "bestmove d5e6"
+
+ send "ucinewgame\n"
+ send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7 f2f1q\n"
+ send "go depth 18\n"
+ expect "score mate 1 * pv f7f5"
+ expect "bestmove f7f5"
+ 
+ send "ucinewgame\n"
+ send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
+ send "go depth 18 searchmoves c6d7\n"
+ expect "score mate 2 * pv c6d7 * f7f5"
+ expect "bestmove c6d7"
+ 
+ send "ucinewgame\n"
+ send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7\n"
+ send "go depth 18 searchmoves e3e2\n"
+ expect "score mate -1 * pv e3e2 f7f5"
+ expect "bestmove e3e2"
+
+ send "setoption name EvalFile value verify.nnue\n"
+ send "position startpos\n"
+ send "go depth 5\n"
+ expect "bestmove"
+
+ send "setoption name MultiPV value 4\n"
+ send "position startpos\n"
+ send "go depth 5\n"
+ expect "bestmove"
+
+ send "setoption name Skill Level value 10\n"
+ send "position startpos\n"
+ send "go depth 5\n"
+ expect "bestmove"
+ send "setoption name Skill Level value 20\n"
+

  send "quit\n"
  expect eof
 
  send "quit\n"
  expect eof
 

- # return error code of the spawned program, useful for valgrind
+ # return error code of the spawned program, useful for Valgrind

  lassign [wait] pid spawnid os_error_flag value
  exit \$value
 EOF
  lassign [wait] pid spawnid os_error_flag value
  exit \$value
 EOF


@@ -117,15 +250,35 @@ fi
 
 cat << EOF > syzygy.exp
  set timeout 240
 
 cat << EOF > syzygy.exp
  set timeout 240

+ # to correctly catch eof we need the following line
+ # expect_before timeout { exit 2 } eof { exit 3 }
+ expect_before timeout { exit 2 }

  spawn $exeprefix ./stockfish
  spawn $exeprefix ./stockfish

+ expect "Stockfish"

  send "uci\n"
  send "setoption name SyzygyPath value ../tests/syzygy/\n"
  send "uci\n"
  send "setoption name SyzygyPath value ../tests/syzygy/\n"

- expect "info string Found 35 tablebases" {} timeout {exit 1}
- send "bench 128 1 10 default depth\n"
+ expect "info string Found 35 WDL and 35 DTZ tablebase files (up to 4-man)."
+ send "bench 128 1 8 default depth\n"
+ expect "Nodes searched  :"
+ send "ucinewgame\n"
+ send "position fen 4k3/PP6/8/8/8/8/8/4K3 w - - 0 1\n"
+ send "go depth 5\n"
+ expect -re {score cp 20000|score mate}
+ expect "bestmove"
+ send "ucinewgame\n"
+ send "position fen 8/1P6/2B5/8/4K3/8/6k1/8 w - - 0 1\n"
+ send "go depth 5\n"
+ expect -re {score cp 20000|score mate}
+ expect "bestmove"
+ send "ucinewgame\n"
+ send "position fen 8/1P6/2B5/8/4K3/8/6k1/8 b - - 0 1\n"
+ send "go depth 5\n"
+ expect -re {score cp -20000|score mate}
+ expect "bestmove"

  send "quit\n"
  expect eof
 
  send "quit\n"
  expect eof
 

- # return error code of the spawned program, useful for valgrind
+ # return error code of the spawned program, useful for Valgrind

  lassign [wait] pid spawnid os_error_flag value
  exit \$value
 EOF
  lassign [wait] pid spawnid os_error_flag value
  exit \$value
 EOF


@@ -133,6 +286,9 @@ EOF
 for exp in game.exp syzygy.exp
 do
 
 for exp in game.exp syzygy.exp
 do
 

+  echo "======== $exp =============="
+  cat $exp
+  echo "============================"

   echo "$prefix expect $exp $postfix"
   eval "$prefix expect $exp $postfix"
 
   echo "$prefix expect $exp $postfix"
   eval "$prefix expect $exp $postfix"
 


@@ -140,6 +296,6 @@ do
 
 done
 
 
 done
 

-rm -f tsan.supp
+rm -f tsan.supp bench_tmp.epd

 
 echo "instrumented testing OK"
 
 echo "instrumented testing OK"