From: syzygy1 <3028851+syzygy1@users.noreply.github.com>
Date: Sat, 15 Aug 2020 14:50:39 +0000 (+0200)
Subject: Assume network file is in little-endian byte order
X-Git-Url: https://git.sesse.net/?p=stockfish;a=commitdiff_plain;h=72dc7a5c54554a8c7c4bf68aa7de2d4de05f3294

Assume network file is in little-endian byte order

This patch fixes the byte order when reading 16- and 32-bit values from the network file on a big-endian machine.

Bytes are ordered in read_le() using unsigned arithmetic, which doesn't need tricks to determine the endianness of the machine. Unfortunately the compiler doesn't seem to be able to optimise the ordering operation, but reading in the weights is not a time-critical operation and the extra time it takes should not be noticeable.

Big endian systems are still untested with NNUE.

fixes #3007

closes https://github.com/official-stockfish/Stockfish/pull/3009

No functional change.
---

diff --git a/src/nnue/evaluate_nnue.cpp b/src/nnue/evaluate_nnue.cpp
index a6ece8e2..3aa85943 100644
--- a/src/nnue/evaluate_nnue.cpp
+++ b/src/nnue/evaluate_nnue.cpp
@@ -77,7 +77,7 @@ namespace Eval::NNUE {
   bool ReadParameters(std::istream& stream, const AlignedPtr<T>& pointer) {
 
     std::uint32_t header;
-    stream.read(reinterpret_cast<char*>(&header), sizeof(header));
+    header = read_le<std::uint32_t>(stream);
     if (!stream || header != T::GetHashValue()) return false;
     return pointer->ReadParameters(stream);
   }
@@ -96,9 +96,9 @@ namespace Eval::NNUE {
     std::uint32_t* hash_value, std::string* architecture) {
 
     std::uint32_t version, size;
-    stream.read(reinterpret_cast<char*>(&version), sizeof(version));
-    stream.read(reinterpret_cast<char*>(hash_value), sizeof(*hash_value));
-    stream.read(reinterpret_cast<char*>(&size), sizeof(size));
+    version = read_le<std::uint32_t>(stream);
+    *hash_value = read_le<std::uint32_t>(stream);
+    size = read_le<std::uint32_t>(stream);
     if (!stream || version != kVersion) return false;
     architecture->resize(size);
     stream.read(&(*architecture)[0], size);
diff --git a/src/nnue/layers/affine_transform.h b/src/nnue/layers/affine_transform.h
index 322e3240..bac258e8 100644
--- a/src/nnue/layers/affine_transform.h
+++ b/src/nnue/layers/affine_transform.h
@@ -62,11 +62,10 @@ namespace Eval::NNUE::Layers {
    // Read network parameters
     bool ReadParameters(std::istream& stream) {
       if (!previous_layer_.ReadParameters(stream)) return false;
-      stream.read(reinterpret_cast<char*>(biases_),
-                  kOutputDimensions * sizeof(BiasType));
-      stream.read(reinterpret_cast<char*>(weights_),
-                  kOutputDimensions * kPaddedInputDimensions *
-                  sizeof(WeightType));
+      for (std::size_t i = 0; i < kOutputDimensions; ++i)
+        biases_[i] = read_le<BiasType>(stream);
+      for (std::size_t i = 0; i < kOutputDimensions * kPaddedInputDimensions; ++i)
+        weights_[i] = read_le<WeightType>(stream);
       return !stream.fail();
     }
 
diff --git a/src/nnue/nnue_common.h b/src/nnue/nnue_common.h
index eab7d258..61f18aee 100644
--- a/src/nnue/nnue_common.h
+++ b/src/nnue/nnue_common.h
@@ -21,6 +21,9 @@
 #ifndef NNUE_COMMON_H_INCLUDED
 #define NNUE_COMMON_H_INCLUDED
 
+#include <cstring>
+#include <iostream>
+
 #if defined(USE_AVX2)
 #include <immintrin.h>
 
@@ -101,6 +104,22 @@ namespace Eval::NNUE {
     return (n + base - 1) / base * base;
   }
 
+  // Read a signed or unsigned integer from  a stream in little-endian order
+  template <typename IntType>
+  inline IntType read_le(std::istream& stream) {
+    // Read the relevant bytes from the stream in little-endian order
+    std::uint8_t u[sizeof(IntType)];
+    stream.read(reinterpret_cast<char*>(u), sizeof(IntType));
+    // Use unsigned arithmetic to convert to machine order
+    typename std::make_unsigned<IntType>::type v = 0;
+    for (std::size_t i = 0; i < sizeof(IntType); ++i)
+      v = (v << 8) | u[sizeof(IntType) - i - 1];
+    // Copy the machine-ordered bytes into a potentially signed value
+    IntType w;
+    std::memcpy(&w, &v, sizeof(IntType));
+    return w;
+  }
+
 }  // namespace Eval::NNUE
 
 #endif // #ifndef NNUE_COMMON_H_INCLUDED
diff --git a/src/nnue/nnue_feature_transformer.h b/src/nnue/nnue_feature_transformer.h
index 40f2603d..4db9be9f 100644
--- a/src/nnue/nnue_feature_transformer.h
+++ b/src/nnue/nnue_feature_transformer.h
@@ -55,10 +55,10 @@ namespace Eval::NNUE {
 
     // Read network parameters
     bool ReadParameters(std::istream& stream) {
-      stream.read(reinterpret_cast<char*>(biases_),
-                  kHalfDimensions * sizeof(BiasType));
-      stream.read(reinterpret_cast<char*>(weights_),
-                  kHalfDimensions * kInputDimensions * sizeof(WeightType));
+      for (std::size_t i = 0; i < kHalfDimensions; ++i)
+        biases_[i] = read_le<BiasType>(stream);
+      for (std::size_t i = 0; i < kHalfDimensions * kInputDimensions; ++i)
+        weights_[i] = read_le<WeightType>(stream);
       return !stream.fail();
     }