baram2584/FESADev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(hex8): isolate AVX2 packet code from baseline COMDATs

Browse Source
This commit is contained in:
KOKO\Mimi
2026-10-14 21:10:27 +09:00
parent c0e23503e8
commit 446d3fa059
5 changed files with 344 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fesa/elements/hex8_pair_simd_avx2.cpp
+94 -36
View File
@@ -1,17 +1,61 @@
#include "hex8_pair_simd.h"
#include "hex8_pair_arithmetic.h"
#if defined(FESA_HEX8_PAIR_AVX2) && defined(_MSC_VER) && defined(_M_X64)
#include <cstring>
#include <immintrin.h>
#include <intrin.h>
#include <limits>
namespace fesa::hex8_internal {
namespace {
struct VectorPair { __m256d high, low; };
struct VectorPair {
__m256d high;
__m256d low;
};
// TU-local predicates mirror the checked scalar premises without publishing
// AVX2-compiled common math or HighLow COMDAT definitions to baseline callers.
/// @brief Reads binary64 fields without FP classification or comparison.
std::uint64_t ScalarBits(double value) {
static_assert(sizeof(double) == sizeof(std::uint64_t) &&
std::numeric_limits<double>::is_iec559 &&
std::numeric_limits<double>::digits == 53);
std::uint64_t bits = 0;
std::memcpy(&bits, &value, sizeof(bits));
return bits;
}
/// @brief Bitwise finite gate before any scalar FP comparison.
bool FiniteScalar(double value) {
return (ScalarBits(value) & 0x7ff0000000000000ULL) !=
0x7ff0000000000000ULL;
}
/// @brief Preserves finite, canonical-zero and RN64 normalization gates.
bool NormalizedScalar(PacketPair value) {
if (!FiniteScalar(value.high) || !FiniteScalar(value.low)) return false;
if (value.high == 0 && value.low == 0)
return (ScalarBits(value.high) >> 63U) == 0 &&
(ScalarBits(value.low) >> 63U) == 0;
return value.high + value.low == value.high;
}
/// @brief Exact least-bit exponent; caller has checked finite/nonzero first.
int LowestBitExponentScalar(double value) {
const auto bits = ScalarBits(value);
const auto encoded_exponent = (bits >> 52U) & 0x7ffU;
auto significand = bits & 0xfffffffffffffULL;
const int exponent = encoded_exponent == 0U
? -1074
: static_cast<int>(encoded_exponent) - 1075;
if (encoded_exponent != 0U) significand |= 1ULL << 52U;
unsigned long trailing = 0;
_BitScanForward64(&trailing, significand);
return exponent + static_cast<int>(trailing);
}
/// @brief Selects each lane without evaluating additional FP arithmetic.
__m256d Select(__m256d mask, __m256d selected, __m256d other) {
return _mm256_blendv_pd(other, selected, mask);
}
/// @brief Produces independent finite masks without signaling NaN compares.
__m256d Finite(__m256d value) {
const auto exponent = _mm256_set1_epi64x(0x7ff0000000000000LL);
const auto nonfinite = _mm256_cmpeq_epi64(
@@ -19,30 +63,36 @@ __m256d Finite(__m256d value) {
return _mm256_castsi256_pd(_mm256_xor_si256(
nonfinite, _mm256_set1_epi64x(-1)));
}
/// @brief Recognizes both signed zero encodings using integer fields.
__m256d Zero(__m256d value) {
return _mm256_castsi256_pd(_mm256_cmpeq_epi64(
_mm256_and_si256(_mm256_castpd_si256(value),
_mm256_set1_epi64x(0x7fffffffffffffffLL)),
_mm256_setzero_si256()));
}
/// @brief Preserves valid lanes and encodes invalid lanes as quiet NaN/+0.
VectorPair InvalidOutside(VectorPair value, __m256d valid) {
return {Select(valid, value.high, _mm256_set1_pd(
std::numeric_limits<double>::quiet_NaN())),
return {Select(valid, value.high, _mm256_castsi256_pd(
_mm256_set1_epi64x(0x7ff8000000000000LL))),
Select(valid, value.low, _mm256_setzero_pd())};
}
/// @brief Loads raw high/low fields in lane order 0..3, with no normalization.
VectorPair Load(const PairPacket& value) {
return {_mm256_setr_pd(value[0].high, value[1].high,
value[2].high, value[3].high),
_mm256_setr_pd(value[0].low, value[1].low,
value[2].low, value[3].low)};
return {_mm256_setr_pd(value.values[0].high, value.values[1].high,
value.values[2].high, value.values[3].high),
_mm256_setr_pd(value.values[0].low, value.values[1].low,
value.values[2].low, value.values[3].low)};
}
/// @brief Stores raw high/low fields without rounding or lane reduction.
PairPacket Store(VectorPair value) {
double high[4], low[4];
double high[4];
double low[4];
_mm256_storeu_pd(high, value.high);
_mm256_storeu_pd(low, value.low);
return {{{high[0], low[0]}, {high[1], low[1]},
{high[2], low[2]}, {high[3], low[3]}}};
}
/// @brief Retains the original per-lane subtraction/addition association and gates.
VectorPair TwoSum(__m256d left, __m256d right) {
const auto sum = _mm256_add_pd(left, right);
const auto bridge = _mm256_sub_pd(sum, left);
@@ -55,29 +105,32 @@ VectorPair TwoSum(__m256d left, __m256d right) {
return InvalidOutside({Select(exact_zero, _mm256_setzero_pd(), sum),
Select(exact_zero, _mm256_setzero_pd(), error)}, valid);
}
/// @brief Routes exceptional packets to baseline checked Add in lane order.
VectorPair ScalarAdd(VectorPair left, VectorPair right, bool pair_shortcuts) {
const auto a = Store(left), b = Store(right);
const auto a = Store(left);
const auto b = Store(right);
PairPacket result;
for (std::size_t lane = 0; lane < result.size(); ++lane)
result[lane] = pair_shortcuts ? AddPair(a[lane], b[lane])
: math_internal::Add(a[lane], b[lane]);
ScalarPacketAdd(a, b, pair_shortcuts, &result);
return Load(result);
}
/// @brief Routes exceptional packets to baseline checked MultiplyPair.
VectorPair ScalarMultiply(const PairPacket& left, const PairPacket& right) {
PairPacket result;
for (std::size_t lane = 0; lane < result.size(); ++lane)
result[lane] = MultiplyPair(left[lane], right[lane]);
ScalarPacketMultiply(left, right, &result);
return Load(result);
}
/// @brief Retains normalized/zero branches and the checked per-lane Add graph.
VectorPair Add(VectorPair left, VectorPair right, bool pair_shortcuts) {
const auto a = Store(left), b = Store(right);
int left_zero = 0, right_zero = 0;
for (std::size_t lane = 0; lane < a.size(); ++lane) {
if (!math_internal::IsNormalized(a[lane]) ||
!math_internal::IsNormalized(b[lane]))
const auto a = Store(left);
const auto b = Store(right);
int left_zero = 0;
int right_zero = 0;
for (std::size_t lane = 0; lane < 4; ++lane) {
if (!NormalizedScalar(a.values[lane]) ||
!NormalizedScalar(b.values[lane]))
return ScalarAdd(left, right, pair_shortcuts);
if (a[lane].high == 0 && a[lane].low == 0) left_zero |= 1 << lane;
if (b[lane].high == 0 && b[lane].low == 0) right_zero |= 1 << lane;
if (a.values[lane].high == 0 && a.values[lane].low == 0) left_zero |= 1 << lane;
if (b.values[lane].high == 0 && b.values[lane].low == 0) right_zero |= 1 << lane;
}
if (pair_shortcuts) {
if (left_zero == 15)
@@ -103,22 +156,24 @@ VectorPair Add(VectorPair left, VectorPair right, bool pair_shortcuts) {
Select(valid, joined.high, _mm256_setzero_pd()),
Select(valid, remainder, _mm256_setzero_pd())), valid);
}
/// @brief Checks finite/zero/tail premises before products; neutralizes rejected
/// lanes before the explicit residual FMA and final TwoSum.
VectorPair Product(__m256d left, __m256d right) {
double a[4], b[4];
double a[4];
double b[4];
_mm256_storeu_pd(a, left);
_mm256_storeu_pd(b, right);
bool special = false;
for (std::size_t lane = 0; lane < 4; ++lane)
if (!math_internal::IsFiniteBinary64(a[lane]) ||
!math_internal::IsFiniteBinary64(b[lane]) ||
if (!FiniteScalar(a[lane]) ||
!FiniteScalar(b[lane]) ||
a[lane] == 0 || b[lane] == 0 ||
math_internal::LowestBitExponent(a[lane]) +
math_internal::LowestBitExponent(b[lane]) < -1074)
LowestBitExponentScalar(a[lane]) +
LowestBitExponentScalar(b[lane]) < -1074)
special = true;
if (special) {
PairPacket result;
for (std::size_t lane = 0; lane < 4; ++lane)
result[lane] = math_internal::TwoProduct(a[lane], b[lane]);
ScalarPacketTwoProduct(a, b, &result);
return Load(result);
}
const auto product = _mm256_mul_pd(left, right);
@@ -132,15 +187,18 @@ VectorPair Product(__m256d left, __m256d right) {
Select(valid, product, _mm256_setzero_pd()),
Select(valid, error, _mm256_setzero_pd())), valid);
}
/// @brief Uses uniform low-zero branches with original four-product association;
/// mixed or invalid packets preserve the baseline checked path.
VectorPair Multiply(VectorPair left, VectorPair right) {
const auto a = Store(left), b = Store(right);
const auto a = Store(left);
const auto b = Store(right);
int branch = -1;
for (std::size_t lane = 0; lane < a.size(); ++lane) {
const bool normalized = math_internal::IsNormalized(a[lane]) &&
math_internal::IsNormalized(b[lane]);
for (std::size_t lane = 0; lane < 4; ++lane) {
const bool normalized = NormalizedScalar(a.values[lane]) &&
NormalizedScalar(b.values[lane]);
if (!normalized) return ScalarMultiply(a, b);
const int shape = (a[lane].low != 0 ? 1 : 0) |
(b[lane].low != 0 ? 2 : 0);
const int shape = (a.values[lane].low != 0 ? 1 : 0) |
(b.values[lane].low != 0 ? 2 : 0);
if (branch != -1 && shape != branch) return ScalarMultiply(a, b);
branch = shape;
}