From: Marco Costalba Date: Mon, 14 Aug 2017 17:41:04 +0000 (-0700) Subject: Rewrite benchmark X-Git-Url: https://git.sesse.net/?p=stockfish;a=commitdiff_plain;h=444d99b6d24ba823c8c1ed7a94f3c15b0a536024 Rewrite benchmark First step in improving bench to handle arbitrary UCI commands so to test many more code paths. This first patch just set the new code structure. No functional change. --- diff --git a/src/benchmark.cpp b/src/benchmark.cpp index f61b5da3..c1c74f7b 100644 --- a/src/benchmark.cpp +++ b/src/benchmark.cpp @@ -23,10 +23,7 @@ #include #include -#include "misc.h" #include "position.h" -#include "search.h" -#include "thread.h" #include "uci.h" using namespace std; @@ -88,19 +85,17 @@ const vector Defaults = { } // namespace -/// benchmark() runs a simple benchmark by letting Stockfish analyze a set -/// of positions for a given limit each. There are five parameters: the -/// transposition table size, the number of search threads that should +/// setup_bench() builds a list of UCI commands to be run by bench. There +/// are five parameters: TT size, number of search threads that should /// be used, the limit value spent for each position (optional, default is /// depth 13), an optional file name where to look for positions in FEN /// format (defaults are the positions defined above) and the type of the /// limit value: depth (default), time in millisecs or number of nodes. -void benchmark(const Position& current, istream& is) { +std::vector setup_bench(const Position& current , istream& is) { - string token; - vector fens; - Search::LimitsType limits; + vector fens, list; + string go, token; // Assign default values to missing arguments string ttSize = (is >> token) ? token : "16"; @@ -109,22 +104,10 @@ void benchmark(const Position& current, istream& is) { string fenFile = (is >> token) ? token : "default"; string limitType = (is >> token) ? token : "depth"; - Search::clear(); // Wait for search finished - Options["Threads"] = threads; - Options["Hash"] = ttSize; - - if (limitType == "time") - limits.movetime = stoi(limit); // movetime is in millisecs - - else if (limitType == "nodes") - limits.nodes = stoll(limit); - - else if (limitType == "mate") - limits.mate = stoi(limit); - - else - limits.depth = stoi(limit); + // Build 'go' string (movetime is in millisecs) + go = "go " + limitType + " " + limit; + // Get test positions fens if (fenFile == "default") fens = Defaults; @@ -139,7 +122,7 @@ void benchmark(const Position& current, istream& is) { if (!file.is_open()) { cerr << "Unable to open file " << fenFile << endl; - return; + exit(EXIT_FAILURE); } while (getline(file, fen)) @@ -149,35 +132,16 @@ void benchmark(const Position& current, istream& is) { file.close(); } - uint64_t nodes = 0; - TimePoint elapsed = now(); - Position pos; + // Build UCI command list + list.emplace_back("ucinewgame"); + list.emplace_back("setoption name Threads value " + threads); + list.emplace_back("setoption name Hash value " + ttSize); - for (size_t i = 0; i < fens.size(); ++i) + for (const string& fen : fens) { - StateListPtr states(new std::deque(1)); - pos.set(fens[i], Options["UCI_Chess960"], &states->back(), Threads.main()); - - cerr << "\nPosition: " << i + 1 << '/' << fens.size() << endl; - - if (limitType == "perft") - nodes += Search::perft(pos, limits.depth * ONE_PLY); - - else - { - limits.startTime = now(); - Threads.start_thinking(pos, states, limits); - Threads.main()->wait_for_search_finished(); - nodes += Threads.nodes_searched(); - } + list.emplace_back("position fen " + fen); + list.emplace_back(go); } - elapsed = now() - elapsed + 1; // Ensure positivity to avoid a 'divide by zero' - - dbg_print(); // Just before exiting - - cerr << "\n===========================" - << "\nTotal time (ms) : " << elapsed - << "\nNodes searched : " << nodes - << "\nNodes/second : " << 1000 * nodes / elapsed << endl; + return list; } diff --git a/src/uci.cpp b/src/uci.cpp index 4604c52b..cf54497d 100644 --- a/src/uci.cpp +++ b/src/uci.cpp @@ -35,7 +35,7 @@ using namespace std; -extern void benchmark(const Position& pos, istream& is); +extern std::vector setup_bench(const Position&, istream&); namespace { @@ -134,6 +134,48 @@ namespace { Threads.start_thinking(pos, states, limits, ponderMode); } + + // bench() is called when engine receives the "bench" command. Firstly + // a list of UCI commands is setup according to bench parameters, then + // it is run one by one printing summaries at the end. + + void bench(Position& pos, istream& args, StateListPtr& states) { + + string token; + uint64_t num, nodes = 0, cnt = 1; + + vector list = setup_bench(pos, args); + num = count_if (list.begin(), list.end(), [](string s) { return s.find("go ") == 0; }); + + TimePoint elapsed = now(); + + for (const auto& cmd : list) + { + istringstream is(cmd); + is >> skipws >> token; + + if (token == "go") + { + cerr << "\nPosition: " << cnt++ << '/' << num << endl; + go(pos, is, states); + Threads.main()->wait_for_search_finished(); + nodes += Threads.nodes_searched(); + } + else if (token == "setoption") setoption(is); + else if (token == "position") position(pos, is, states); + else if (token == "ucinewgame") Search::clear(); + } + + elapsed = now() - elapsed + 1; // Ensure positivity to avoid a 'divide by zero' + + dbg_print(); // Just before exiting + + cerr << "\n===========================" + << "\nTotal time (ms) : " << elapsed + << "\nNodes searched : " << nodes + << "\nNodes/second : " << 1000 * nodes / elapsed << endl; + } + } // namespace @@ -190,7 +232,7 @@ void UCI::loop(int argc, char* argv[]) { // Additional custom non-UCI commands, mainly for debugging else if (token == "flip") pos.flip(); - else if (token == "bench") benchmark(pos, is); + else if (token == "bench") bench(pos, is, states); else if (token == "d") sync_cout << pos << sync_endl; else if (token == "eval") sync_cout << Eval::trace(pos) << sync_endl; else if (token == "perft") @@ -202,7 +244,7 @@ void UCI::loop(int argc, char* argv[]) { ss << Options["Hash"] << " " << Options["Threads"] << " " << depth << " current perft"; - benchmark(pos, ss); + // TODO benchmark(pos, ss); } else sync_cout << "Unknown command: " << cmd << sync_endl;