perf(hex8): prepare guarded private four-column paired tangent packet
This commit is contained in:
KOKO\Mimi
@@ -0,0 +1,183 @@
|
||||
#include "hex8_pair_simd.h"
|
||||
|
||||
#include "hex8_pair_arithmetic.h"
|
||||
|
||||
#if defined(FESA_HEX8_PAIR_AVX2) && defined(_MSC_VER) && defined(_M_X64)
|
||||
#include <immintrin.h>
|
||||
|
||||
namespace fesa::hex8_internal {
|
||||
namespace {
|
||||
struct VectorPair { __m256d high, low; };
|
||||
|
||||
__m256d Select(__m256d mask, __m256d selected, __m256d other) {
|
||||
return _mm256_blendv_pd(other, selected, mask);
|
||||
}
|
||||
__m256d Finite(__m256d value) {
|
||||
const auto exponent = _mm256_set1_epi64x(0x7ff0000000000000LL);
|
||||
const auto nonfinite = _mm256_cmpeq_epi64(
|
||||
_mm256_and_si256(_mm256_castpd_si256(value), exponent), exponent);
|
||||
return _mm256_castsi256_pd(_mm256_xor_si256(
|
||||
nonfinite, _mm256_set1_epi64x(-1)));
|
||||
}
|
||||
__m256d Zero(__m256d value) {
|
||||
return _mm256_castsi256_pd(_mm256_cmpeq_epi64(
|
||||
_mm256_and_si256(_mm256_castpd_si256(value),
|
||||
_mm256_set1_epi64x(0x7fffffffffffffffLL)),
|
||||
_mm256_setzero_si256()));
|
||||
}
|
||||
VectorPair InvalidOutside(VectorPair value, __m256d valid) {
|
||||
return {Select(valid, value.high, _mm256_set1_pd(
|
||||
std::numeric_limits<double>::quiet_NaN())),
|
||||
Select(valid, value.low, _mm256_setzero_pd())};
|
||||
}
|
||||
VectorPair Load(const PairPacket& value) {
|
||||
return {_mm256_setr_pd(value[0].high, value[1].high,
|
||||
value[2].high, value[3].high),
|
||||
_mm256_setr_pd(value[0].low, value[1].low,
|
||||
value[2].low, value[3].low)};
|
||||
}
|
||||
PairPacket Store(VectorPair value) {
|
||||
double high[4], low[4];
|
||||
_mm256_storeu_pd(high, value.high);
|
||||
_mm256_storeu_pd(low, value.low);
|
||||
return {{{high[0], low[0]}, {high[1], low[1]},
|
||||
{high[2], low[2]}, {high[3], low[3]}}};
|
||||
}
|
||||
VectorPair TwoSum(__m256d left, __m256d right) {
|
||||
const auto sum = _mm256_add_pd(left, right);
|
||||
const auto bridge = _mm256_sub_pd(sum, left);
|
||||
const auto restored = _mm256_sub_pd(sum, bridge);
|
||||
const auto error = _mm256_add_pd(_mm256_sub_pd(left, restored),
|
||||
_mm256_sub_pd(right, bridge));
|
||||
const auto valid = _mm256_and_pd(_mm256_and_pd(Finite(sum), Finite(bridge)),
|
||||
_mm256_and_pd(Finite(restored), Finite(error)));
|
||||
const auto exact_zero = _mm256_and_pd(Zero(sum), Zero(error));
|
||||
return InvalidOutside({Select(exact_zero, _mm256_setzero_pd(), sum),
|
||||
Select(exact_zero, _mm256_setzero_pd(), error)}, valid);
|
||||
}
|
||||
VectorPair ScalarAdd(VectorPair left, VectorPair right, bool pair_shortcuts) {
|
||||
const auto a = Store(left), b = Store(right);
|
||||
PairPacket result;
|
||||
for (std::size_t lane = 0; lane < result.size(); ++lane)
|
||||
result[lane] = pair_shortcuts ? AddPair(a[lane], b[lane])
|
||||
: math_internal::Add(a[lane], b[lane]);
|
||||
return Load(result);
|
||||
}
|
||||
VectorPair ScalarMultiply(const PairPacket& left, const PairPacket& right) {
|
||||
PairPacket result;
|
||||
for (std::size_t lane = 0; lane < result.size(); ++lane)
|
||||
result[lane] = MultiplyPair(left[lane], right[lane]);
|
||||
return Load(result);
|
||||
}
|
||||
VectorPair Add(VectorPair left, VectorPair right, bool pair_shortcuts) {
|
||||
const auto a = Store(left), b = Store(right);
|
||||
int left_zero = 0, right_zero = 0;
|
||||
for (std::size_t lane = 0; lane < a.size(); ++lane) {
|
||||
if (!math_internal::IsNormalized(a[lane]) ||
|
||||
!math_internal::IsNormalized(b[lane]))
|
||||
return ScalarAdd(left, right, pair_shortcuts);
|
||||
if (a[lane].high == 0 && a[lane].low == 0) left_zero |= 1 << lane;
|
||||
if (b[lane].high == 0 && b[lane].low == 0) right_zero |= 1 << lane;
|
||||
}
|
||||
if (pair_shortcuts) {
|
||||
if (left_zero == 15)
|
||||
return {right.high, Select(Zero(right.low), _mm256_setzero_pd(), right.low)};
|
||||
if (right_zero == 15)
|
||||
return {left.high, Select(Zero(left.low), _mm256_setzero_pd(), left.low)};
|
||||
if (left_zero != 0 || right_zero != 0)
|
||||
return ScalarAdd(left, right, true);
|
||||
}
|
||||
const auto head = TwoSum(left.high, right.high);
|
||||
const auto tail = TwoSum(left.low, right.low);
|
||||
const auto middle = TwoSum(head.low, tail.high);
|
||||
const auto joined = TwoSum(head.high, middle.high);
|
||||
const auto remainder = _mm256_add_pd(
|
||||
_mm256_add_pd(joined.low, middle.low), tail.low);
|
||||
const auto valid = _mm256_and_pd(Finite(remainder), _mm256_and_pd(
|
||||
_mm256_and_pd(Finite(head.high), Finite(head.low)), _mm256_and_pd(
|
||||
_mm256_and_pd(Finite(tail.high), Finite(tail.low)), _mm256_and_pd(
|
||||
_mm256_and_pd(Finite(middle.high), Finite(middle.low)),
|
||||
_mm256_and_pd(Finite(joined.high), Finite(joined.low))))));
|
||||
// Original Add returns before its last TwoSum when a prior finite gate fails.
|
||||
return InvalidOutside(TwoSum(
|
||||
Select(valid, joined.high, _mm256_setzero_pd()),
|
||||
Select(valid, remainder, _mm256_setzero_pd())), valid);
|
||||
}
|
||||
VectorPair Product(__m256d left, __m256d right) {
|
||||
double a[4], b[4];
|
||||
_mm256_storeu_pd(a, left);
|
||||
_mm256_storeu_pd(b, right);
|
||||
bool special = false;
|
||||
for (std::size_t lane = 0; lane < 4; ++lane)
|
||||
if (!math_internal::IsFiniteBinary64(a[lane]) ||
|
||||
!math_internal::IsFiniteBinary64(b[lane]) ||
|
||||
a[lane] == 0 || b[lane] == 0 ||
|
||||
math_internal::LowestBitExponent(a[lane]) +
|
||||
math_internal::LowestBitExponent(b[lane]) < -1074)
|
||||
special = true;
|
||||
if (special) {
|
||||
PairPacket result;
|
||||
for (std::size_t lane = 0; lane < 4; ++lane)
|
||||
result[lane] = math_internal::TwoProduct(a[lane], b[lane]);
|
||||
return Load(result);
|
||||
}
|
||||
const auto product = _mm256_mul_pd(left, right);
|
||||
const auto product_valid = Finite(product);
|
||||
const auto one = _mm256_set1_pd(1.0);
|
||||
// Invalid products never reach the residual FMA in the original helper.
|
||||
const auto error = _mm256_fmsub_pd(Select(product_valid, left, one),
|
||||
Select(product_valid, right, one), Select(product_valid, product, one));
|
||||
const auto valid = _mm256_and_pd(product_valid, Finite(error));
|
||||
return InvalidOutside(TwoSum(
|
||||
Select(valid, product, _mm256_setzero_pd()),
|
||||
Select(valid, error, _mm256_setzero_pd())), valid);
|
||||
}
|
||||
VectorPair Multiply(VectorPair left, VectorPair right) {
|
||||
const auto a = Store(left), b = Store(right);
|
||||
int branch = -1;
|
||||
for (std::size_t lane = 0; lane < a.size(); ++lane) {
|
||||
const bool normalized = math_internal::IsNormalized(a[lane]) &&
|
||||
math_internal::IsNormalized(b[lane]);
|
||||
if (!normalized) return ScalarMultiply(a, b);
|
||||
const int shape = (a[lane].low != 0 ? 1 : 0) |
|
||||
(b[lane].low != 0 ? 2 : 0);
|
||||
if (branch != -1 && shape != branch) return ScalarMultiply(a, b);
|
||||
branch = shape;
|
||||
}
|
||||
auto product = Product(left.high, right.high);
|
||||
if (branch == 3) {
|
||||
product = Add(product, Product(left.high, right.low), false);
|
||||
product = Add(product, Product(left.low, right.high), false);
|
||||
return Add(product, Product(left.low, right.low), false);
|
||||
}
|
||||
const VectorPair zero{_mm256_setzero_pd(), _mm256_setzero_pd()};
|
||||
if ((branch & 2) != 0)
|
||||
product = Add(product, Product(left.high, right.low), true);
|
||||
else if ((branch & 1) != 0)
|
||||
product = Add(product, zero, true);
|
||||
if ((branch & 1) != 0)
|
||||
product = Add(product, Product(left.low, right.high), true);
|
||||
return Add(product, zero, true);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void EvaluateMaterialPacketAvx2(const MaterialPacketInput& input,
|
||||
PairPacket* output) {
|
||||
const auto first = Multiply(Load(input.deviatoric_left),
|
||||
Load(input.deviatoric_right));
|
||||
const auto second = Multiply(Load(input.shear_left), Load(input.shear_right));
|
||||
const auto third = Multiply(Load(input.tensor_left), Load(input.gradient_dot));
|
||||
const auto material = Add(Add(first, second, true), third, true);
|
||||
const PairPacket weight{{input.weight, input.weight, input.weight, input.weight}};
|
||||
*output = Store(Add(Load(input.accumulated),
|
||||
Multiply(material, Load(weight)), true));
|
||||
}
|
||||
} // namespace fesa::hex8_internal
|
||||
#else
|
||||
namespace fesa::hex8_internal {
|
||||
void EvaluateMaterialPacketAvx2(const MaterialPacketInput& input,
|
||||
PairPacket* output) {
|
||||
EvaluateMaterialPacketScalar(input, output);
|
||||
}
|
||||
} // namespace fesa::hex8_internal
|
||||
#endif
|
||||
Reference in New Issue
Block a user