2 # check for errors under valgrind or sanitizers.
6 echo "instrumented testing failed on line $1"
9 trap 'error ${LINENO}' ERR
11 # define suitable post and prefixes for testing options
14 echo "valgrind testing started"
16 exeprefix='valgrind --error-exitcode=42 --errors-for-leak-kinds=all --leak-check=full'
21 echo "valgrind-thread testing started"
23 exeprefix='valgrind --fair-sched=try --error-exitcode=42'
27 --sanitizer-undefined)
28 echo "sanitizer-undefined testing started"
31 postfix='2>&1 | grep -A50 "runtime error:"'
35 echo "sanitizer-thread testing started"
38 postfix='2>&1 | grep -A50 "WARNING: ThreadSanitizer:"'
41 cat << EOF > tsan.supp
42 race:Stockfish::TTEntry::move
43 race:Stockfish::TTEntry::depth
44 race:Stockfish::TTEntry::bound
45 race:Stockfish::TTEntry::save
46 race:Stockfish::TTEntry::value
47 race:Stockfish::TTEntry::eval
48 race:Stockfish::TTEntry::is_pv
50 race:Stockfish::TranspositionTable::probe
51 race:Stockfish::TranspositionTable::hashfull
55 export TSAN_OPTIONS="suppressions=./tsan.supp"
59 echo "unknown testing started"
67 cat << EOF > bench_tmp.epd
68 Rn6/1rbq1bk1/2p2n1p/2Bp1p2/3Pp1pP/1N2P1P1/2Q1NPB1/6K1 w - - 2 26
69 rnbqkb1r/ppp1pp2/5n1p/3p2p1/P2PP3/5P2/1PP3PP/RNBQKBNR w KQkq - 0 3
70 3qnrk1/4bp1p/1p2p1pP/p2bN3/1P1P1B2/P2BQ3/5PP1/4R1K1 w - - 9 28
71 r4rk1/1b2ppbp/pq4pn/2pp1PB1/1p2P3/1P1P1NN1/1PP3PP/R2Q1RK1 w - - 0 13
74 # simple command line testing
80 "go wtime 8000 btime 8000 winc 500 binc 500" \
81 "go wtime 1000 btime 1000 winc 0 binc 0" \
82 "go wtime 1000 btime 1000 winc 0 binc 0" \
83 "go wtime 1000 btime 1000 winc 0 binc 0 movestogo 5" \
85 "go nodes 20000 searchmoves e2e4 d2d4" \
86 "bench 128 $threads 8 default depth" \
87 "bench 128 $threads 3 bench_tmp.epd depth" \
88 "export_net verify.nnue" \
95 echo "$prefix $exeprefix ./stockfish $args $postfix"
96 eval "$prefix $exeprefix ./stockfish $args $postfix"
100 # verify the generated net equals the base net
101 network=`./stockfish uci | grep 'option name EvalFile type string default' | awk '{print $NF}'`
102 echo "Comparing $network to the written verify.nnue"
103 diff $network verify.nnue
105 # more general testing, following an uci protocol exchange
106 cat << EOF > game.exp
108 spawn $exeprefix ./stockfish
113 # send "setoption name Debug Log File value debug.log\n"
114 send "setoption name Threads value $threads\n"
117 send "position startpos\n"
118 send "go nodes 1000\n"
121 send "position startpos moves e2e4 e7e6\n"
122 send "go nodes 1000\n"
125 send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n"
129 send "setoption name UCI_ShowWDL value true\n"
130 send "position startpos\n"
135 send "setoption name Skill Level value 10\n"
136 send "position startpos\n"
140 send "setoption name Clear Hash\n"
142 send "setoption name EvalFile value verify.nnue\n"
143 send "position startpos\n"
147 send "setoption name MultiPV value 4\n"
148 send "position startpos\n"
154 # return error code of the spawned program, useful for valgrind
155 lassign [wait] pid spawnid os_error_flag value
159 #download TB as needed
160 if [ ! -d ../tests/syzygy ]; then
161 curl -sL https://api.github.com/repos/niklasf/python-chess/tarball/9b9aa13f9f36d08aadfabff872882f4ab1494e95 | tar -xzf -
162 mv niklasf-python-chess-9b9aa13 ../tests/syzygy
165 cat << EOF > syzygy.exp
167 spawn $exeprefix ./stockfish
169 send "setoption name SyzygyPath value ../tests/syzygy/\n"
170 expect "info string Found 35 tablebases" {} timeout {exit 1}
171 send "bench 128 1 8 default depth\n"
173 send "position fen 4k3/PP6/8/8/8/8/8/4K3 w - - 0 1\n"
176 send "position fen 8/1P6/2B5/8/4K3/8/6k1/8 w - - 0 1\n"
182 # return error code of the spawned program, useful for valgrind
183 lassign [wait] pid spawnid os_error_flag value
187 for exp in game.exp syzygy.exp
190 echo "$prefix expect $exp $postfix"
191 eval "$prefix expect $exp $postfix"
197 rm -f tsan.supp bench_tmp.epd
199 echo "instrumented testing OK"