baram2584/FESADev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge reviewed vector pair guards and exact tail screen

Browse Source
This commit is contained in:
KOKO\Mimi
2026-10-14 22:02:20 +09:00
parent bd1cc3bf30 33e6aa0563
commit b8d1dff670
3 changed files with 286 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fesa/elements/hex8_pair_simd_avx2.cpp
+54 -33
View File
@@ -29,14 +29,6 @@ bool FiniteScalar(double value) {
return (ScalarBits(value) & 0x7ff0000000000000ULL) !=
0x7ff0000000000000ULL;
}
/// @brief Preserves finite, canonical-zero and RN64 normalization gates.
bool NormalizedScalar(PacketPair value) {
if (!FiniteScalar(value.high) || !FiniteScalar(value.low)) return false;
if (value.high == 0 && value.low == 0)
return (ScalarBits(value.high) >> 63U) == 0 &&
(ScalarBits(value.low) >> 63U) == 0;
return value.high + value.low == value.high;
}
/// @brief Exact least-bit exponent; caller has checked finite/nonzero first.
int LowestBitExponentScalar(double value) {
const auto bits = ScalarBits(value);
@@ -51,6 +43,17 @@ int LowestBitExponentScalar(double value) {
return exponent + static_cast<int>(trailing);
}
/// @brief Proves the exact tail bound using integer exponents only.
/// @pre Both operands are finite and nonzero; false requires the exact scans.
bool TailBoundGuaranteed(double left, double right) {
const auto left_exponent = (ScalarBits(left) >> 52U) & 0x7ffU;
const auto right_exponent = (ScalarBits(right) >> 52U) & 0x7ffU;
const auto left_adjusted = left_exponent == 0 ? 1U : left_exponent;
const auto right_adjusted = right_exponent == 0 ? 1U : right_exponent;
// LBE >= adjusted exponent - 1075, including nonzero subnormals.
return left_adjusted + right_adjusted >= 1076U;
}
/// @brief Selects each lane without evaluating additional FP arithmetic.
__m256d Select(__m256d mask, __m256d selected, __m256d other) {
return _mm256_blendv_pd(other, selected, mask);
@@ -70,6 +73,31 @@ __m256d Zero(__m256d value) {
_mm256_set1_epi64x(0x7fffffffffffffffLL)),
_mm256_setzero_si256()));
}
/// @brief Preserves finite/canonical-zero/RN64 gates for active lanes only.
/// @note Both sum operands and equality RHS are neutral before FP evaluation.
__m256d Normalized(VectorPair value, __m256d active) {
const auto finite_active = _mm256_and_pd(active,
_mm256_and_pd(Finite(value.high), Finite(value.low)));
const auto both_zero = _mm256_and_pd(Zero(value.high), Zero(value.low));
const auto canonical_zero = _mm256_castsi256_pd(_mm256_and_si256(
_mm256_cmpeq_epi64(_mm256_castpd_si256(value.high),
_mm256_setzero_si256()),
_mm256_cmpeq_epi64(_mm256_castpd_si256(value.low),
_mm256_setzero_si256())));
const auto sum_active = _mm256_andnot_pd(both_zero, finite_active);
const auto high = Select(sum_active, value.high, _mm256_setzero_pd());
const auto low = Select(sum_active, value.low, _mm256_setzero_pd());
const auto equality = _mm256_cmp_pd(_mm256_add_pd(high, low), high, _CMP_EQ_OQ);
return _mm256_and_pd(finite_active,
Select(both_zero, canonical_zero, equality));
}
/// @brief Retains left-to-right normalized short-circuit for every lane.
bool NormalizedInputs(VectorPair left, VectorPair right) {
const auto all = _mm256_castsi256_pd(_mm256_set1_epi64x(-1));
const auto left_valid = Normalized(left, all);
return _mm256_movemask_pd(Normalized(right, left_valid)) == 15;
}
/// @brief Preserves valid lanes and encodes invalid lanes as quiet NaN/+0.
VectorPair InvalidOutside(VectorPair value, __m256d valid) {
return {Select(valid, value.high, _mm256_castsi256_pd(
@@ -121,17 +149,12 @@ VectorPair ScalarMultiply(const PairPacket& left, const PairPacket& right) {
}
/// @brief Retains normalized/zero branches and the checked per-lane Add graph.
VectorPair Add(VectorPair left, VectorPair right, bool pair_shortcuts) {
const auto a = Store(left);
const auto b = Store(right);
int left_zero = 0;
int right_zero = 0;
for (std::size_t lane = 0; lane < 4; ++lane) {
if (!NormalizedScalar(a.values[lane]) ||
!NormalizedScalar(b.values[lane]))
return ScalarAdd(left, right, pair_shortcuts);
if (a.values[lane].high == 0 && a.values[lane].low == 0) left_zero |= 1 << lane;
if (b.values[lane].high == 0 && b.values[lane].low == 0) right_zero |= 1 << lane;
}
if (!NormalizedInputs(left, right))
return ScalarAdd(left, right, pair_shortcuts);
const int left_zero = _mm256_movemask_pd(
_mm256_and_pd(Zero(left.high), Zero(left.low)));
const int right_zero = _mm256_movemask_pd(
_mm256_and_pd(Zero(right.high), Zero(right.low)));
if (pair_shortcuts) {
if (left_zero == 15)
return {right.high, Select(Zero(right.low), _mm256_setzero_pd(), right.low)};
@@ -168,8 +191,9 @@ VectorPair Product(__m256d left, __m256d right) {
if (!FiniteScalar(a[lane]) ||
!FiniteScalar(b[lane]) ||
a[lane] == 0 || b[lane] == 0 ||
LowestBitExponentScalar(a[lane]) +
LowestBitExponentScalar(b[lane]) < -1074)
(!TailBoundGuaranteed(a[lane], b[lane]) &&
LowestBitExponentScalar(a[lane]) +
LowestBitExponentScalar(b[lane]) < -1074))
special = true;
if (special) {
PairPacket result;
@@ -190,18 +214,15 @@ VectorPair Product(__m256d left, __m256d right) {
/// @brief Uses uniform low-zero branches with original four-product association;
/// mixed or invalid packets preserve the baseline checked path.
VectorPair Multiply(VectorPair left, VectorPair right) {
const auto a = Store(left);
const auto b = Store(right);
int branch = -1;
for (std::size_t lane = 0; lane < 4; ++lane) {
const bool normalized = NormalizedScalar(a.values[lane]) &&
NormalizedScalar(b.values[lane]);
if (!normalized) return ScalarMultiply(a, b);
const int shape = (a.values[lane].low != 0 ? 1 : 0) |
(b.values[lane].low != 0 ? 2 : 0);
if (branch != -1 && shape != branch) return ScalarMultiply(a, b);
branch = shape;
}
if (!NormalizedInputs(left, right))
return ScalarMultiply(Store(left), Store(right));
const int left_low_zero = _mm256_movemask_pd(Zero(left.low));
const int right_low_zero = _mm256_movemask_pd(Zero(right.low));
if ((left_low_zero != 0 && left_low_zero != 15) ||
(right_low_zero != 0 && right_low_zero != 15))
return ScalarMultiply(Store(left), Store(right));
const int branch = (left_low_zero == 0 ? 1 : 0) |
(right_low_zero == 0 ? 2 : 0);
auto product = Product(left.high, right.high);
if (branch == 3) {
product = Add(product, Product(left.high, right.low), false);