From ea6220f3813e5b76b444a02905eaf2c556bdb368 Mon Sep 17 00:00:00 2001 From: Guy Vreuls Date: Fri, 7 Aug 2020 17:07:46 +0200 Subject: [PATCH] This commit enables a mixed bench, to improve CI and allow for PGO (profile-build) of the NNUE part of the code. Joint work gvreuls / vondele * Download the default NNUE net in AppVeyor * Download net in travis CI `make net` * Adjust tests to cover more archs, speedup instrumented testing * Introduce 'mixed' bench as default, with further options: classical, NNUE, mixed. mixed (default) and NNUE require the default net to be present, which can be obtained with ``` make net ``` Further examples (first is equivalent to `./stockfish bench`): ``` ./stockfish bench 16 1 13 default depth mixed ./stockfish bench 16 1 13 default depth classical ./stockfish bench 16 1 13 default depth NNUE ``` The net is now downloaded automatically if needed for `profile-build` (usual `build` works fine without net present) PGO gives a nice speedup on fishtest: passed STC: LLR: 2.93 (-2.94,2.94) {-0.50,1.50} Total: 3360 W: 469 L: 343 D: 2548 Ptnml(0-2): 20, 246, 1030, 356, 28 https://tests.stockfishchess.org/tests/view/5f31b5499081672066537569 passed LTC: LLR: 2.97 (-2.94,2.94) {0.25,1.75} Total: 8824 W: 609 L: 502 D: 7713 Ptnml(0-2): 8, 430, 3438, 519, 17 https://tests.stockfishchess.org/tests/view/5f31c87b908167206653757c closes https://github.com/official-stockfish/Stockfish/pull/2931 fixes https://github.com/official-stockfish/Stockfish/issues/2907 requires fishtest updates before commit Bench: 4290577 --- .travis.yml | 27 +++++++++++++++++++++------ appveyor.yml | 14 ++++++++++++++ src/Makefile | 2 +- src/benchmark.cpp | 13 +++++++++++-- tests/instrumented.sh | 8 ++++---- 5 files changed, 51 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index d563a1e1..0dd38047 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,9 @@ before_script: - cd src script: + # Download net + - make net + # Obtain bench reference from git log - git log HEAD | grep "\b[Bb]ench[ :]\+[0-9]\{7\}" | head -n 1 | sed "s/[^0-9]*\([0-9]*\).*/\1/g" > git_sig - export benchref=$(cat git_sig) @@ -55,14 +58,26 @@ script: # # Verify bench number against various builds - export CXXFLAGS="-Werror -D_GLIBCXX_DEBUG" - - make clean && make -j2 ARCH=x86-64 optimize=no debug=yes build && ../tests/signature.sh $benchref + - make clean && make -j2 ARCH=x86-64-modern optimize=no debug=yes build && ../tests/signature.sh $benchref + - export CXXFLAGS="-Werror" + - make clean && make -j2 ARCH=x86-64-modern build && ../tests/signature.sh $benchref + - make clean && make -j2 ARCH=x86-64-ssse3 build && ../tests/signature.sh $benchref + - make clean && make -j2 ARCH=x86-64-sse3-popcnt build && ../tests/signature.sh $benchref + - make clean && make -j2 ARCH=x86-64 build && ../tests/signature.sh $benchref - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32 optimize=no debug=yes build && ../tests/signature.sh $benchref; fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32 build && ../tests/signature.sh $benchref; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-32-old build && ../tests/signature.sh $benchref; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" && "$COMP" == "gcc" ]]; then make clean && make -j2 ARCH=x86-64-modern profile-build && ../tests/signature.sh $benchref; fi + + # compile only for some more advanced architectures (might not run in travis) + - make clean && make -j2 ARCH=x86-64-avx2 build + - make clean && make -j2 ARCH=x86-64-bmi2 build + # needs gcc 10 to compile + - if [[ "$COMPILER" != "g++-8" ]]; then make clean && make -j2 ARCH=x86-64-avx512 build; fi # # Check perft and reproducible search - - export CXXFLAGS="-Werror" - - make clean && make -j2 ARCH=x86-64 build + - make clean && make -j2 ARCH=x86-64-modern build - ../tests/perft.sh - ../tests/reprosearch.sh @@ -70,11 +85,11 @@ script: # Valgrind # - export CXXFLAGS="-O1 -fno-inline" - - if [ -x "$(command -v valgrind )" ]; then make clean && make -j2 ARCH=x86-64 debug=yes optimize=no build > /dev/null && ../tests/instrumented.sh --valgrind; fi + - if [ -x "$(command -v valgrind )" ]; then make clean && make -j2 ARCH=x86-64-modern debug=yes optimize=no build > /dev/null && ../tests/instrumented.sh --valgrind; fi - if [ -x "$(command -v valgrind )" ]; then ../tests/instrumented.sh --valgrind-thread; fi # # Sanitizer # - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-64 sanitize=undefined optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-undefined; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-64 sanitize=thread optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-thread; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-64-modern sanitize=undefined optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-undefined; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make clean && make -j2 ARCH=x86-64-modern sanitize=thread optimize=no debug=yes build > /dev/null && ../tests/instrumented.sh --sanitizer-thread; fi diff --git a/appveyor.yml b/appveyor.yml index d356ba2f..a3732a23 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -61,6 +61,20 @@ before_build: build_script: - cmake --build . --config %CONFIGURATION% -- /verbosity:minimal + - ps: | + # Download default NNUE net from fishtest + $nnuenet = Get-Content -Path src\ucioption.cpp | Select-String -CaseSensitive -Pattern "Option" | Select-String -CaseSensitive -Pattern "nn-[a-z0-9]{12}.nnue" + $dummy = $nnuenet -match "(?nn-[a-z0-9]{12}.nnue)" + $nnuenet = $Matches.nnuenet + Write-Host "Default net:" $nnuenet + $nnuedownloadurl = "https://tests.stockfishchess.org/api/nn/$nnuenet" + $nnuefilepath = "src\${env:CONFIGURATION}\$nnuenet" + if (Test-Path -Path $nnuefilepath) { + Write-Host "Already available." + } else { + Write-Host "Downloading $nnuedownloadurl to $nnuefilepath" + Invoke-WebRequest -Uri $nnuedownloadurl -OutFile $nnuefilepath + } before_test: - cd src/%CONFIGURATION% diff --git a/src/Makefile b/src/Makefile index e34fbf61..c00b60b5 100644 --- a/src/Makefile +++ b/src/Makefile @@ -569,7 +569,7 @@ help: build: config-sanity $(MAKE) ARCH=$(ARCH) COMP=$(COMP) all -profile-build: config-sanity objclean profileclean +profile-build: config-sanity objclean profileclean net @echo "" @echo "Step 1/4. Building instrumented executable ..." $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_make) diff --git a/src/benchmark.cpp b/src/benchmark.cpp index 6041d642..806e9840 100644 --- a/src/benchmark.cpp +++ b/src/benchmark.cpp @@ -95,8 +95,9 @@ const vector Defaults = { /// setup_bench() builds a list of UCI commands to be run by bench. There /// are five parameters: TT size in MB, number of search threads that /// should be used, the limit value spent for each position, a file name -/// where to look for positions in FEN format and the type of the limit: -/// depth, perft, nodes and movetime (in millisecs). +/// where to look for positions in FEN format, the type of the limit: +/// depth, perft, nodes and movetime (in millisecs), and evaluation type +/// mixed (default), classical, NNUE. /// /// bench -> search default positions up to depth 13 /// bench 64 1 15 -> search default positions up to depth 15 (TT = 64MB) @@ -115,6 +116,7 @@ vector setup_bench(const Position& current, istream& is) { string limit = (is >> token) ? token : "13"; string fenFile = (is >> token) ? token : "default"; string limitType = (is >> token) ? token : "depth"; + string evalType = (is >> token) ? token : "mixed"; go = limitType == "eval" ? "eval" : "go " + limitType + " " + limit; @@ -146,13 +148,20 @@ vector setup_bench(const Position& current, istream& is) { list.emplace_back("setoption name Hash value " + ttSize); list.emplace_back("ucinewgame"); + size_t posCounter = 0; + for (const string& fen : fens) if (fen.find("setoption") != string::npos) list.emplace_back(fen); else { + if (evalType == "classical" || (evalType == "mixed" && posCounter % 2 == 0)) + list.emplace_back("setoption name Use NNUE value false"); + else if (evalType == "NNUE" || (evalType == "mixed" && posCounter % 2 != 0)) + list.emplace_back("setoption name Use NNUE value true"); list.emplace_back("position fen " + fen); list.emplace_back(go); + ++posCounter; } return list; diff --git a/tests/instrumented.sh b/tests/instrumented.sh index ae6d5c4b..03ded74a 100755 --- a/tests/instrumented.sh +++ b/tests/instrumented.sh @@ -70,7 +70,7 @@ for args in "eval" \ "go depth 10" \ "go movetime 1000" \ "go wtime 8000 btime 8000 winc 500 binc 500" \ - "bench 128 $threads 10 default depth" + "bench 128 $threads 8 default depth" do echo "$prefix $exeprefix ./stockfish $args $postfix" @@ -80,7 +80,7 @@ done # more general testing, following an uci protocol exchange cat << EOF > game.exp - set timeout 10 + set timeout 240 spawn $exeprefix ./stockfish send "uci\n" @@ -98,7 +98,7 @@ cat << EOF > game.exp expect "bestmove" send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n" - send "go depth 30\n" + send "go depth 20\n" expect "bestmove" send "quit\n" @@ -121,7 +121,7 @@ cat << EOF > syzygy.exp send "uci\n" send "setoption name SyzygyPath value ../tests/syzygy/\n" expect "info string Found 35 tablebases" {} timeout {exit 1} - send "bench 128 1 10 default depth\n" + send "bench 128 1 8 default depth\n" send "quit\n" expect eof -- 2.39.2