template <IndexType InputDimensions, IndexType PaddedInputDimensions, IndexType OutputDimensions>
static void affine_transform_non_ssse3(std::int32_t* output, const std::int8_t* weights, const std::int32_t* biases, const std::uint8_t* input)
{
+# if defined(USE_SSE2) || defined(USE_MMX) || defined(USE_NEON_DOTPROD) || defined(USE_NEON)
# if defined(USE_SSE2)
// At least a multiple of 16, with SSE2.
constexpr IndexType NumChunks = ceil_to_multiple<IndexType>(InputDimensions, 16) / 16;
}
output[i] = sum[0] + sum[1] + sum[2] + sum[3];
-# else
- std::int32_t sum = biases[i];
- for (IndexType j = 0; j < InputDimensions; ++j) {
- sum += weights[offset + j] * input[j];
- }
- output[i] = sum;
# endif
}
# if defined(USE_MMX)
_mm_empty();
# endif
+
+# else
+ std::memcpy(output, biases, sizeof(std::int32_t) * OutputDimensions);
+
+ // Traverse weights in transpose order to take advantage of input sparsity
+ for (IndexType i = 0; i < InputDimensions; ++i)
+ if (input[i]) {
+ const std::int8_t* w = &weights[i];
+ const int in = input[i];
+ for (IndexType j = 0; j < OutputDimensions; ++j)
+ output[j] += w[j * PaddedInputDimensions] * in;
+ }
+# endif
}
#endif
for (IndexType i = Start; i < InputDimensions; ++i) {
output[i] = static_cast<OutputType>(
- std::max(0, std::min(127, input[i] >> WeightScaleBits)));
+ std::clamp(input[i] >> WeightScaleBits, 0, 127));
}
}
};
for (IndexType i = Start; i < InputDimensions; ++i) {
output[i] = static_cast<OutputType>(
- // really should be /127 but we need to make it fast
- // needs to be accounted for in the trainer
- std::min(127ll, (((long long)input[i] * input[i]) >> (2 * WeightScaleBits)) / 128));
+ // Really should be /127 but we need to make it fast so we right shift
+ // by an extra 7 bits instead. Needs to be accounted for in the trainer.
+ std::min(127ll, ((long long)input[i] * input[i]) >> (2 * WeightScaleBits + 7)));
}
}
};
for (IndexType j = 0; j < HalfDimensions / 2; ++j) {
BiasType sum0 = accumulation[static_cast<int>(perspectives[p])][j + 0];
BiasType sum1 = accumulation[static_cast<int>(perspectives[p])][j + HalfDimensions / 2];
- sum0 = std::max<int>(0, std::min<int>(127, sum0));
- sum1 = std::max<int>(0, std::min<int>(127, sum1));
- output[offset + j] = static_cast<OutputType>(sum0 * sum1 / 128);
+ sum0 = std::clamp<BiasType>(sum0, 0, 127);
+ sum1 = std::clamp<BiasType>(sum1, 0, 127);
+ output[offset + j] = static_cast<OutputType>(unsigned(sum0 * sum1) / 128);
}
#endif