From: Steinar H. Gunderson Date: Tue, 1 Dec 2020 22:03:04 +0000 (+0100) Subject: Merge remote-tracking branch 'upstream/master' into HEAD X-Git-Url: https://git.sesse.net/?p=stockfish;a=commitdiff_plain;h=aa75388ec136a8cf83b09da2328c5fefd5a010bd;hp=-c Merge remote-tracking branch 'upstream/master' into HEAD --- aa75388ec136a8cf83b09da2328c5fefd5a010bd diff --combined src/Makefile index 48180660,87203547..e3466ea1 --- a/src/Makefile +++ b/src/Makefile @@@ -39,12 -39,9 +39,12 @@@ PGOBENCH = ./$(EXE) benc SRCS = benchmark.cpp bitbase.cpp bitboard.cpp endgame.cpp evaluate.cpp main.cpp \ material.cpp misc.cpp movegen.cpp movepick.cpp pawns.cpp position.cpp psqt.cpp \ search.cpp thread.cpp timeman.cpp tt.cpp uci.cpp ucioption.cpp tune.cpp syzygy/tbprobe.cpp \ - nnue/evaluate_nnue.cpp nnue/features/half_kp.cpp + nnue/evaluate_nnue.cpp nnue/features/half_kp.cpp \ + hashprobe.grpc.pb.cc hashprobe.pb.cc +CLISRCS = client.cpp hashprobe.grpc.pb.cc hashprobe.pb.cc uci.cpp OBJS = $(notdir $(SRCS:.cpp=.o)) +CLIOBJS = $(notdir $(CLISRCS:.cpp=.o)) VPATH = syzygy:nnue:nnue/features @@@ -310,7 -307,7 +310,7 @@@ endi ifeq ($(COMP),gcc) comp=gcc CXX=g++ - CXXFLAGS += -pedantic -Wextra -Wshadow + CXXFLAGS += -pedantic -Wextra ifeq ($(arch),$(filter $(arch),armv7 armv8)) ifeq ($(OS),Android) @@@ -384,19 -381,6 +384,6 @@@ ifeq ($(COMP),clang endif endif - ifeq ($(comp),icc) - profile_make = icc-profile-make - profile_use = icc-profile-use - else - ifeq ($(comp),clang) - profile_make = clang-profile-make - profile_use = clang-profile-use - else - profile_make = gcc-profile-make - profile_use = gcc-profile-use - endif - endif - ifeq ($(KERNEL),Darwin) CXXFLAGS += -arch $(arch) -mmacosx-version-min=10.14 LDFLAGS += -arch $(arch) -mmacosx-version-min=10.14 @@@ -408,20 -392,30 +395,30 @@@ endi # Currently we don't know how to make PGO builds with the NDK yet. ifeq ($(COMP),ndk) CXXFLAGS += -stdlib=libc++ -fPIE + comp=clang ifeq ($(arch),armv7) - comp=armv7a-linux-androideabi16-clang CXX=armv7a-linux-androideabi16-clang++ CXXFLAGS += -mthumb -march=armv7-a -mfloat-abi=softfp -mfpu=neon STRIP=arm-linux-androideabi-strip endif ifeq ($(arch),armv8) - comp=aarch64-linux-android21-clang CXX=aarch64-linux-android21-clang++ STRIP=aarch64-linux-android-strip endif LDFLAGS += -static-libstdc++ -pie -lm -latomic endif + ifeq ($(comp),icc) + profile_make = icc-profile-make + profile_use = icc-profile-use + else ifeq ($(comp),clang) + profile_make = clang-profile-make + profile_use = clang-profile-use + else + profile_make = gcc-profile-make + profile_use = gcc-profile-use + endif + ### Travis CI script uses COMPILER to overwrite CXX ifdef COMPILER COMPCXX=$(COMPILER) @@@ -472,7 -466,7 +469,7 @@@ endi ### 3.3 Optimization ifeq ($(optimize),yes) - CXXFLAGS += -O3 + CXXFLAGS += -O3 -g ifeq ($(comp),gcc) ifeq ($(OS), Android) @@@ -593,10 -587,7 +590,7 @@@ endi ### needs access to the optimization flags. ifeq ($(optimize),yes) ifeq ($(debug), no) - ifeq ($(COMP),ndk) - CXXFLAGS += -flto=thin - LDFLAGS += $(CXXFLAGS) - else ifeq ($(comp),clang) + ifeq ($(comp),clang) CXXFLAGS += -flto=thin ifneq ($(findstring MINGW,$(KERNEL)),) CXXFLAGS += -fuse-ld=lld @@@ -720,7 -711,7 +714,7 @@@ endi config-sanity icc-profile-use icc-profile-make gcc-profile-use gcc-profile-make \ clang-profile-use clang-profile-make - build: config-sanity net + build: net config-sanity $(MAKE) ARCH=$(ARCH) COMP=$(COMP) all profile-build: net config-sanity objclean profileclean @@@ -791,9 -782,9 +785,9 @@@ default ### Section 5. Private Targets ### ========================================================================== -all: $(EXE) .depend +all: $(EXE) client .depend - config-sanity: + config-sanity: net @echo "" @echo "Config:" @echo "debug: '$(debug)'" @@@ -887,32 -878,6 +881,32 @@@ icc-profile-use EXTRACXXFLAGS='-prof_use -prof_dir ./profdir' \ all +### GRPC + +PROTOS_PATH = . +PROTOC = protoc +GRPC_CPP_PLUGIN = grpc_cpp_plugin +GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)` + +%.grpc.pb.h %.grpc.pb.cc: %.proto + $(PROTOC) -I $(PROTOS_PATH) --grpc_out=. --plugin=protoc-gen-grpc=$(GRPC_CPP_PLUGIN_PATH) $< + +# oh my +%.cpp: %.cc + cp $< $@ + +%.pb.h %.pb.cc: %.proto + $(PROTOC) -I $(PROTOS_PATH) --cpp_out=. $< + +#LDFLAGS += -Wl,-Bstatic -Wl,-\( -lprotobuf -lgrpc++_unsecure -lgrpc_unsecure -lgrpc -lz -Wl,-\) -Wl,-Bdynamic -ldl +LDFLAGS += /usr/lib/x86_64-linux-gnu/libprotobuf.a /usr/lib/x86_64-linux-gnu/libgrpc++_unsecure.a /usr/lib/x86_64-linux-gnu/libgrpc_unsecure.a /usr/lib/x86_64-linux-gnu/libgrpc.a /usr/lib/x86_64-linux-gnu/libcares.a -ldl -lz +#LDFLAGS += /usr/lib/x86_64-linux-gnu/libprotobuf.a /usr/lib/libgrpc++_unsecure.a /usr/lib/libgrpc_unsecure.a /usr/lib/libgrpc.a /usr/lib/x86_64-linux-gnu/libcares.a -ldl -lz + +client: $(CLIOBJS) + $(CXX) -o $@ $(CLIOBJS) $(LDFLAGS) + +# Other stuff + .depend: -@$(CXX) $(DEPENDFLAGS) -MM $(SRCS) > $@ 2> /dev/null diff --combined src/main.cpp index a62c3a85,e6dff918..e8548ac1 --- a/src/main.cpp +++ b/src/main.cpp @@@ -16,10 -16,7 +16,10 @@@ along with this program. If not, see . */ +#include #include +#include +#include #include "bitboard.h" #include "endgame.h" @@@ -30,197 -27,6 +30,197 @@@ #include "uci.h" #include "syzygy/tbprobe.h" +#include +#include +#include +#include "hashprobe.h" +#include "hashprobe.grpc.pb.h" +#include "tt.h" + +using grpc::Server; +using grpc::ServerBuilder; +using grpc::ServerContext; +using grpc::Status; +using grpc::StatusCode; +using namespace hashprobe; + +Status HashProbeImpl::Probe(ServerContext* context, + const HashProbeRequest* request, + HashProbeResponse *response) { + Position pos; + StateInfo st; + pos.set(request->fen(), /*isChess960=*/false, &st, Threads.main()); + if (!pos.pos_is_ok()) { + return Status(StatusCode::INVALID_ARGUMENT, "Invalid FEN"); + } + + bool invert = (pos.side_to_move() == BLACK); + StateListPtr setup_states = StateListPtr(new std::deque(1)); + + ProbeMove(&pos, setup_states.get(), invert, response->mutable_root()); + + MoveList moves(pos); + for (const ExtMove* em = moves.begin(); em != moves.end(); ++em) { + HashProbeLine *line = response->add_line(); + FillMove(&pos, em->move, line->mutable_move()); + setup_states->push_back(StateInfo()); + pos.do_move(em->move, setup_states->back()); + ProbeMove(&pos, setup_states.get(), !invert, line); + pos.undo_move(em->move); + } + + return Status::OK; +} + +void HashProbeImpl::FillMove(Position *pos, Move move, HashProbeMove* decoded) { + if (!is_ok(move)) return; + + Square from = from_sq(move); + Square to = to_sq(move); + + if (type_of(move) == CASTLING) { + to = make_square(to > from ? FILE_G : FILE_C, rank_of(from)); + } + + Piece moved_piece = pos->moved_piece(move); + std::string pretty; + if (type_of(move) == CASTLING) { + if (to > from) { + pretty = "O-O"; + } else { + pretty = "O-O-O"; + } + } else if (type_of(moved_piece) == PAWN) { + if (type_of(move) == ENPASSANT || pos->piece_on(to) != NO_PIECE) { + // Capture. + pretty = char('a' + file_of(from)); + pretty += "x"; + } + pretty += UCI::square(to); + if (type_of(move) == PROMOTION) { + pretty += "="; + pretty += " PNBRQK"[promotion_type(move)]; + } + } else { + pretty = " PNBRQK"[type_of(moved_piece)]; + Bitboard attackers = pos->attackers_to(to) & pos->pieces(color_of(moved_piece), type_of(moved_piece)); + if (more_than_one(attackers)) { + // Remove all illegal moves to disambiguate. + Bitboard att_copy = attackers; + while (att_copy) { + Square s = pop_lsb(&att_copy); + Move m = make_move(s, to); + if (!pos->pseudo_legal(m) || !pos->legal(m)) { + attackers &= ~SquareBB[s]; + } + } + } + if (more_than_one(attackers)) { + // Disambiguate by file if possible. + Bitboard attackers_this_file = attackers & file_bb(file_of(from)); + if (attackers != attackers_this_file) { + pretty += char('a' + file_of(from)); + attackers = attackers_this_file; + } + if (more_than_one(attackers)) { + // Still ambiguous, so need to disambiguate by rank. + pretty += char('1' + rank_of(from)); + } + } + + if (type_of(move) == ENPASSANT || pos->piece_on(to) != NO_PIECE) { + pretty += "x"; + } + + pretty += UCI::square(to); + } + + if (pos->gives_check(move)) { + // Check if mate. + StateInfo si; + pos->do_move(move, si, true); + if (MoveList(*pos).size() > 0) { + pretty += "+"; + } else { + pretty += "#"; + } + pos->undo_move(move); + } + + decoded->set_pretty(pretty); +} + +void HashProbeImpl::ProbeMove(Position* pos, std::deque* setup_states, bool invert, HashProbeLine* response) { + bool found; + TTEntry *entry = TT.probe(pos->key(), found); + response->set_found(found); + if (found) { + Value value = entry->value(); + Value eval = entry->eval(); + Bound bound = entry->bound(); + + if (invert) { + value = -value; + eval = -eval; + if (bound == BOUND_UPPER) { + bound = BOUND_LOWER; + } else if (bound == BOUND_LOWER) { + bound = BOUND_UPPER; + } + } + + response->set_depth(entry->depth()); + FillValue(eval, response->mutable_eval()); + if (entry->depth() > DEPTH_NONE) { + FillValue(value, response->mutable_value()); + } + response->set_bound(HashProbeLine::ValueBound(bound)); + + // Follow the PV until we hit an illegal move. + std::stack pv; + std::set seen; + while (found && is_ok(entry->move()) && + pos->pseudo_legal(entry->move()) && + pos->legal(entry->move())) { + FillMove(pos, entry->move(), response->add_pv()); + if (seen.count(pos->key())) break; + pv.push(entry->move()); + seen.insert(pos->key()); + setup_states->push_back(StateInfo()); + pos->do_move(entry->move(), setup_states->back()); + entry = TT.probe(pos->key(), found); + } + + // Unroll the PV back again, so the Position object remains unchanged. + while (!pv.empty()) { + pos->undo_move(pv.top()); + pv.pop(); + } + } +} + +void HashProbeImpl::FillValue(Value value, HashProbeScore* score) { + if (abs(value) < VALUE_MATE - MAX_PLY) { + score->set_score_type(HashProbeScore::SCORE_CP); + score->set_score_cp(value * 100 / PawnValueEg); + } else { + score->set_score_type(HashProbeScore::SCORE_MATE); + score->set_score_mate((value > 0 ? VALUE_MATE - value + 1 : -VALUE_MATE - value) / 2); + } +} + +HashProbeThread::HashProbeThread(const std::string &server_address) { + builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); + builder.RegisterService(&service); + server = std::move(builder.BuildAndStart()); + std::cout << "Server listening on " << server_address << std::endl; + std::thread([this]{ server->Wait(); }).detach(); +} + +void HashProbeThread::Shutdown() { + server->Shutdown(); +} + namespace PSQT { void init(); } @@@ -239,7 -45,7 +239,7 @@@ int main(int argc, char* argv[]) Endgames::init(); Threads.set(size_t(Options["Threads"])); Search::clear(); // After threads are up - Eval::init_NNUE(); + Eval::NNUE::init(); UCI::loop(argc, argv); diff --combined src/misc.cpp index 6a32e7cb,f2bce6b0..832a9ac1 --- a/src/misc.cpp +++ b/src/misc.cpp @@@ -150,7 -150,6 +150,7 @@@ const string engine_info(bool to_uci) { date >> month >> day >> year; ss << setw(2) << day << setw(2) << (1 + months.find(month) / 4) << year.substr(2); + ss << "-asn"; } ss << (to_uci ? "\nid author ": " by ") @@@ -358,27 -357,11 +358,11 @@@ void std_aligned_free(void* ptr) #endif } - /// aligned_ttmem_alloc() will return suitably aligned memory, if possible using large pages. - /// The returned pointer is the aligned one, while the mem argument is the one that needs - /// to be passed to free. With c++17 some of this functionality could be simplified. + /// aligned_large_pages_alloc() will return suitably aligned memory, if possible using large pages. - #if defined(__linux__) && !defined(__ANDROID__) + #if defined(_WIN32) - void* aligned_ttmem_alloc(size_t allocSize, void*& mem) { - - constexpr size_t alignment = 2 * 1024 * 1024; // assumed 2MB page sizes - size_t size = ((allocSize + alignment - 1) / alignment) * alignment; // multiple of alignment - if (posix_memalign(&mem, alignment, size)) - mem = nullptr; - #if defined(MADV_HUGEPAGE) - madvise(mem, allocSize, MADV_HUGEPAGE); - #endif - return mem; - } - - #elif defined(_WIN64) - - static void* aligned_ttmem_alloc_large_pages(size_t allocSize) { + static void* aligned_large_pages_alloc_win(size_t allocSize) { HANDLE hProcessToken { }; LUID luid { }; @@@ -423,23 -406,10 +407,10 @@@ return mem; } - void* aligned_ttmem_alloc(size_t allocSize, void*& mem) { - - static bool firstCall = true; + void* aligned_large_pages_alloc(size_t allocSize) { // Try to allocate large pages - mem = aligned_ttmem_alloc_large_pages(allocSize); - - // Suppress info strings on the first call. The first call occurs before 'uci' - // is received and in that case this output confuses some GUIs. - if (!firstCall) - { - if (mem) - sync_cout << "info string Hash table allocation: Windows large pages used." << sync_endl; - else - sync_cout << "info string Hash table allocation: Windows large pages not used." << sync_endl; - } - firstCall = false; + void* mem = aligned_large_pages_alloc_win(allocSize); // Fall back to regular, page aligned, allocation if necessary if (!mem) @@@ -450,23 -420,31 +421,31 @@@ #else - void* aligned_ttmem_alloc(size_t allocSize, void*& mem) { + void* aligned_large_pages_alloc(size_t allocSize) { - constexpr size_t alignment = 64; // assumed cache line size - size_t size = allocSize + alignment - 1; // allocate some extra space - mem = malloc(size); - void* ret = reinterpret_cast((uintptr_t(mem) + alignment - 1) & ~uintptr_t(alignment - 1)); - return ret; + #if defined(__linux__) + constexpr size_t alignment = 2 * 1024 * 1024; // assumed 2MB page size + #else + constexpr size_t alignment = 4096; // assumed small page size + #endif + + // round up to multiples of alignment + size_t size = ((allocSize + alignment - 1) / alignment) * alignment; + void *mem = std_aligned_alloc(alignment, size); + #if defined(MADV_HUGEPAGE) + madvise(mem, size, MADV_HUGEPAGE); + #endif + return mem; } #endif - /// aligned_ttmem_free() will free the previously allocated ttmem + /// aligned_large_pages_free() will free the previously allocated ttmem - #if defined(_WIN64) + #if defined(_WIN32) - void aligned_ttmem_free(void* mem) { + void aligned_large_pages_free(void* mem) { if (mem && !VirtualFree(mem, 0, MEM_RELEASE)) { @@@ -479,8 -457,8 +458,8 @@@ #else - void aligned_ttmem_free(void *mem) { - free(mem); + void aligned_large_pages_free(void *mem) { + std_aligned_free(mem); } #endif @@@ -605,11 -583,10 +584,10 @@@ namespace CommandLine string argv0; // path+name of the executable binary, as given by argv[0] string binaryDirectory; // path of the executable directory string workingDirectory; // path of the working directory - string pathSeparator; // Separator for our current OS void init(int argc, char* argv[]) { (void)argc; - string separator; + string pathSeparator; // extract the path+name of the executable binary argv0 = argv[0]; diff --combined src/position.cpp index 2658c71a,07ce0a7c..13010c1a --- a/src/position.cpp +++ b/src/position.cpp @@@ -77,6 -77,8 +77,8 @@@ std::ostream& operator<<(std::ostream& && !pos.can_castle(ANY_CASTLING)) { StateInfo st; + ASSERT_ALIGNED(&st, Eval::NNUE::kCacheLineSize); + Position p; p.set(pos.fen(), pos.is_chess960(), &st, pos.this_thread()); Tablebases::ProbeState s1, s2; @@@ -195,7 -197,6 +197,6 @@@ Position& Position::set(const string& f std::memset(this, 0, sizeof(Position)); std::memset(si, 0, sizeof(StateInfo)); - std::fill_n(&pieceList[0][0], sizeof(pieceList) / sizeof(Square), SQ_NONE); st = si; ss >> std::noskipws; @@@ -279,7 -280,11 +280,9 @@@ chess960 = isChess960; thisThread = th; set_state(st); + st->accumulator.state[WHITE] = Eval::NNUE::INIT; + st->accumulator.state[BLACK] = Eval::NNUE::INIT; - assert(pos_is_ok()); - return *this; } @@@ -701,7 -706,8 +704,8 @@@ void Position::do_move(Move m, StateInf ++st->pliesFromNull; // Used by NNUE - st->accumulator.computed_accumulation = false; + st->accumulator.state[WHITE] = Eval::NNUE::EMPTY; + st->accumulator.state[BLACK] = Eval::NNUE::EMPTY; auto& dp = st->dirtyPiece; dp.dirty_num = 1; @@@ -994,16 -1000,16 +998,16 @@@ void Position::do_null_move(StateInfo& assert(!checkers()); assert(&newSt != st); - if (Eval::useNNUE) - { - std::memcpy(&newSt, st, sizeof(StateInfo)); - } - else - std::memcpy(&newSt, st, offsetof(StateInfo, accumulator)); + std::memcpy(&newSt, st, offsetof(StateInfo, accumulator)); newSt.previous = st; st = &newSt; + st->dirtyPiece.dirty_num = 0; + st->dirtyPiece.piece[0] = NO_PIECE; // Avoid checks in UpdateAccumulator() + st->accumulator.state[WHITE] = Eval::NNUE::EMPTY; + st->accumulator.state[BLACK] = Eval::NNUE::EMPTY; + if (st->epSquare != SQ_NONE) { st->key ^= Zobrist::enpassant[file_of(st->epSquare)]; @@@ -1313,21 -1319,17 +1317,17 @@@ bool Position::pos_is_ok() const assert(0 && "pos_is_ok: Bitboards"); StateInfo si = *st; + ASSERT_ALIGNED(&si, Eval::NNUE::kCacheLineSize); + set_state(&si); if (std::memcmp(&si, st, sizeof(StateInfo))) assert(0 && "pos_is_ok: State"); for (Piece pc : Pieces) - { if ( pieceCount[pc] != popcount(pieces(color_of(pc), type_of(pc))) || pieceCount[pc] != std::count(board, board + SQUARE_NB, pc)) assert(0 && "pos_is_ok: Pieces"); - for (int i = 0; i < pieceCount[pc]; ++i) - if (board[pieceList[pc][i]] != pc || index[pieceList[pc][i]] != i) - assert(0 && "pos_is_ok: Index"); - } - for (Color c : { WHITE, BLACK }) for (CastlingRights cr : {c & KING_SIDE, c & QUEEN_SIDE}) { diff --combined src/ucioption.cpp index df6283cb,bb0b8311..71239159 --- a/src/ucioption.cpp +++ b/src/ucioption.cpp @@@ -27,13 -27,11 +27,13 @@@ #include "thread.h" #include "tt.h" #include "uci.h" +#include "hashprobe.h" #include "syzygy/tbprobe.h" using std::string; UCI::OptionsMap Options; // Global object +std::unique_ptr hash_probe_thread; namespace UCI { @@@ -43,15 -41,8 +43,15 @@@ void on_hash_size(const Option& o) { TT void on_logger(const Option& o) { start_logger(o); } void on_threads(const Option& o) { Threads.set(size_t(o)); } void on_tb_path(const Option& o) { Tablebases::init(o); } - void on_use_NNUE(const Option& ) { Eval::init_NNUE(); } - void on_eval_file(const Option& ) { Eval::init_NNUE(); } + void on_use_NNUE(const Option& ) { Eval::NNUE::init(); } + void on_eval_file(const Option& ) { Eval::NNUE::init(); } +void on_rpc_server_address(const Option& o) { + if (hash_probe_thread) { + hash_probe_thread->Shutdown(); + } + std::string addr = o; + hash_probe_thread.reset(new HashProbeThread(addr)); +} /// Our case insensitive less() function as required by UCI protocol bool CaseInsensitiveLess::operator() (const string& s1, const string& s2) const { @@@ -90,7 -81,6 +90,7 @@@ void init(OptionsMap& o) o["SyzygyProbeLimit"] << Option(7, 0, 7); o["Use NNUE"] << Option(true, on_use_NNUE); o["EvalFile"] << Option(EvalFileDefaultName, on_eval_file); + o["RPCServerAddress"] << Option("", on_rpc_server_address); }