baram2584/FESADev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(hex8): isolate AVX2 packet code from baseline COMDATs

Browse Source
This commit is contained in:
KOKO\Mimi
2026-10-14 21:10:27 +09:00
parent c0e23503e8
commit 446d3fa059
5 changed files with 344 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fesa/elements/hex8_high_low.cpp
+14 -10
View File
@@ -271,25 +271,29 @@ Result<Hex8HighLowResponse> Hex8Element::EvaluateHighLow(
if (use_packets)
for (std::size_t i = 0; i < 3; ++i)
for (std::size_t first = 0; first < 24; first += 4) {
hex8_internal::MaterialPacketInput input;
input.weight = HighLow{point.weight};
hex8_internal::MaterialPacketInput input{};
const auto raw_pair = [](HighLow pair) {
return hex8_internal::PacketPair{pair.high, pair.low};
};
input.weight = raw_pair(HighLow{point.weight});
const auto row = 3U * a + i;
for (std::size_t lane = 0; lane < 4; ++lane) {
const auto column = first + lane;
const auto b = column / 3U;
const auto k = column % 3U;
input.deviatoric_left[lane] = deviatoric_fg[a][i];
input.deviatoric_right[lane] = fg[b][k];
input.shear_left[lane] = shear_fg[b][i];
input.shear_right[lane] = fg[a][k];
input.tensor_left[lane] = shear_fft[i][k];
input.gradient_dot[lane] = dots[b];
input.accumulated[lane] = material[row][column];
input.deviatoric_left.values[lane] = raw_pair(deviatoric_fg[a][i]);
input.deviatoric_right.values[lane] = raw_pair(fg[b][k]);
input.shear_left.values[lane] = raw_pair(shear_fg[b][i]);
input.shear_right.values[lane] = raw_pair(fg[a][k]);
input.tensor_left.values[lane] = raw_pair(shear_fft[i][k]);
input.gradient_dot.values[lane] = raw_pair(dots[b]);
input.accumulated.values[lane] = raw_pair(material[row][column]);
}
hex8_internal::PairPacket output;
packet_evaluator(input, &output);
for (std::size_t lane = 0; lane < 4; ++lane)
material[row][first + lane] = output[lane];
material[row][first + lane] =
{output.values[lane].high, output.values[lane].low};
}
}
}
src/fesa/elements/hex8_pair_simd.h
+40 -8
View File
@@ -1,28 +1,60 @@
#ifndef FESA_SRC_FESA_ELEMENTS_HEX8_PAIR_SIMD_H_
#define FESA_SRC_FESA_ELEMENTS_HEX8_PAIR_SIMD_H_
#include <array>
#include "../math/high_low_arithmetic.h"
#include <cstdint>
namespace fesa::hex8_internal {
using PairPacket = std::array<math_internal::HighLow, 4>;
/// @brief Trivial raw carrier; neither component is normalized or rounded here.
struct PacketPair {
double high;
double low;
};
/// @brief Four independent lanes with no executable constructor or operator.
struct PairPacket {
PacketPair values[4];
};
struct MaterialPacketInput {
PairPacket deviatoric_left, deviatoric_right;
PairPacket shear_left, shear_right;
PairPacket tensor_left, gradient_dot;
PairPacket deviatoric_left;
PairPacket deviatoric_right;
PairPacket shear_left;
PairPacket shear_right;
PairPacket tensor_left;
PairPacket gradient_dot;
PairPacket accumulated;
math_internal::HighLow weight;
PacketPair weight;
};
using MaterialPacketEvaluator = void (*)(const MaterialPacketInput&, PairPacket*);
/// @brief Evaluates each lane's original checked material graph in order 0..3.
/// @note Invalid arithmetic remains a raw invalid pair for the caller's gates.
void EvaluateMaterialPacketScalar(const MaterialPacketInput& input,
PairPacket* output);
/// @brief Evaluates independent material lanes with the checked vector graph.
/// @pre The selector has admitted the CPU/OS ISA and current masked FP state.
/// @note No lane reduction or model failure is introduced; special packets bridge
/// to original scalar arithmetic, preserving components and operation order.
void EvaluateMaterialPacketAvx2(const MaterialPacketInput& input,
PairPacket* output);
/// @brief Baseline-ISA checked addition bridge in lane order 0..3.
/// @note pair_shortcuts chooses the original local AddPair or general Add graph;
/// invalid pairs are preserved, with no FP-state mutation or normalization here.
void ScalarPacketAdd(const PairPacket& left, const PairPacket& right,
bool pair_shortcuts, PairPacket* output);
/// @brief Baseline-ISA original checked MultiplyPair bridge in lane order 0..3.
/// @note Raw invalid outputs remain invalid; no component is rounded away.
void ScalarPacketMultiply(const PairPacket& left, const PairPacket& right,
PairPacket* output);
/// @brief Baseline-ISA original checked TwoProduct bridge in lane order 0..3.
/// @note Keeps finite/zero/underflow/FMA checks and the caller's FP state.
void ScalarPacketTwoProduct(const double left[4], const double right[4],
PairPacket* output);
/// @brief Selects a private backend from CPU/OS and this thread's FP state.
/// @note Never mutates FP state or caches a thread's environment decision.
MaterialPacketEvaluator SelectMaterialPacketEvaluator();
/// @brief Pure instruction/OS-state predicate; does not execute ISA probes.
bool SupportsMaterialPacketInstructions(std::uint32_t leaf1_ecx,
std::uint64_t xcr0,
std::uint32_t leaf7_ebx);
} // namespace fesa::hex8_internal
#endif // FESA_SRC_FESA_ELEMENTS_HEX8_PAIR_SIMD_H_
src/fesa/elements/hex8_pair_simd_avx2.cpp
+94 -36
View File
@@ -1,17 +1,61 @@
#include "hex8_pair_simd.h"
#include "hex8_pair_arithmetic.h"
#if defined(FESA_HEX8_PAIR_AVX2) && defined(_MSC_VER) && defined(_M_X64)
#include <cstring>
#include <immintrin.h>
#include <intrin.h>
#include <limits>
namespace fesa::hex8_internal {
namespace {
struct VectorPair { __m256d high, low; };
struct VectorPair {
__m256d high;
__m256d low;
};
// TU-local predicates mirror the checked scalar premises without publishing
// AVX2-compiled common math or HighLow COMDAT definitions to baseline callers.
/// @brief Reads binary64 fields without FP classification or comparison.
std::uint64_t ScalarBits(double value) {
static_assert(sizeof(double) == sizeof(std::uint64_t) &&
std::numeric_limits<double>::is_iec559 &&
std::numeric_limits<double>::digits == 53);
std::uint64_t bits = 0;
std::memcpy(&bits, &value, sizeof(bits));
return bits;
}
/// @brief Bitwise finite gate before any scalar FP comparison.
bool FiniteScalar(double value) {
return (ScalarBits(value) & 0x7ff0000000000000ULL) !=
0x7ff0000000000000ULL;
}
/// @brief Preserves finite, canonical-zero and RN64 normalization gates.
bool NormalizedScalar(PacketPair value) {
if (!FiniteScalar(value.high) || !FiniteScalar(value.low)) return false;
if (value.high == 0 && value.low == 0)
return (ScalarBits(value.high) >> 63U) == 0 &&
(ScalarBits(value.low) >> 63U) == 0;
return value.high + value.low == value.high;
}
/// @brief Exact least-bit exponent; caller has checked finite/nonzero first.
int LowestBitExponentScalar(double value) {
const auto bits = ScalarBits(value);
const auto encoded_exponent = (bits >> 52U) & 0x7ffU;
auto significand = bits & 0xfffffffffffffULL;
const int exponent = encoded_exponent == 0U
? -1074
: static_cast<int>(encoded_exponent) - 1075;
if (encoded_exponent != 0U) significand |= 1ULL << 52U;
unsigned long trailing = 0;
_BitScanForward64(&trailing, significand);
return exponent + static_cast<int>(trailing);
}
/// @brief Selects each lane without evaluating additional FP arithmetic.
__m256d Select(__m256d mask, __m256d selected, __m256d other) {
return _mm256_blendv_pd(other, selected, mask);
}
/// @brief Produces independent finite masks without signaling NaN compares.
__m256d Finite(__m256d value) {
const auto exponent = _mm256_set1_epi64x(0x7ff0000000000000LL);
const auto nonfinite = _mm256_cmpeq_epi64(
@@ -19,30 +63,36 @@ __m256d Finite(__m256d value) {
return _mm256_castsi256_pd(_mm256_xor_si256(
nonfinite, _mm256_set1_epi64x(-1)));
}
/// @brief Recognizes both signed zero encodings using integer fields.
__m256d Zero(__m256d value) {
return _mm256_castsi256_pd(_mm256_cmpeq_epi64(
_mm256_and_si256(_mm256_castpd_si256(value),
_mm256_set1_epi64x(0x7fffffffffffffffLL)),
_mm256_setzero_si256()));
}
/// @brief Preserves valid lanes and encodes invalid lanes as quiet NaN/+0.
VectorPair InvalidOutside(VectorPair value, __m256d valid) {
return {Select(valid, value.high, _mm256_set1_pd(
std::numeric_limits<double>::quiet_NaN())),
return {Select(valid, value.high, _mm256_castsi256_pd(
_mm256_set1_epi64x(0x7ff8000000000000LL))),
Select(valid, value.low, _mm256_setzero_pd())};
}
/// @brief Loads raw high/low fields in lane order 0..3, with no normalization.
VectorPair Load(const PairPacket& value) {
return {_mm256_setr_pd(value[0].high, value[1].high,
value[2].high, value[3].high),
_mm256_setr_pd(value[0].low, value[1].low,
value[2].low, value[3].low)};
return {_mm256_setr_pd(value.values[0].high, value.values[1].high,
value.values[2].high, value.values[3].high),
_mm256_setr_pd(value.values[0].low, value.values[1].low,
value.values[2].low, value.values[3].low)};
}
/// @brief Stores raw high/low fields without rounding or lane reduction.
PairPacket Store(VectorPair value) {
double high[4], low[4];
double high[4];
double low[4];
_mm256_storeu_pd(high, value.high);
_mm256_storeu_pd(low, value.low);
return {{{high[0], low[0]}, {high[1], low[1]},
{high[2], low[2]}, {high[3], low[3]}}};
}
/// @brief Retains the original per-lane subtraction/addition association and gates.
VectorPair TwoSum(__m256d left, __m256d right) {
const auto sum = _mm256_add_pd(left, right);
const auto bridge = _mm256_sub_pd(sum, left);
@@ -55,29 +105,32 @@ VectorPair TwoSum(__m256d left, __m256d right) {
return InvalidOutside({Select(exact_zero, _mm256_setzero_pd(), sum),
Select(exact_zero, _mm256_setzero_pd(), error)}, valid);
}
/// @brief Routes exceptional packets to baseline checked Add in lane order.
VectorPair ScalarAdd(VectorPair left, VectorPair right, bool pair_shortcuts) {
const auto a = Store(left), b = Store(right);
const auto a = Store(left);
const auto b = Store(right);
PairPacket result;
for (std::size_t lane = 0; lane < result.size(); ++lane)
result[lane] = pair_shortcuts ? AddPair(a[lane], b[lane])
: math_internal::Add(a[lane], b[lane]);
ScalarPacketAdd(a, b, pair_shortcuts, &result);
return Load(result);
}
/// @brief Routes exceptional packets to baseline checked MultiplyPair.
VectorPair ScalarMultiply(const PairPacket& left, const PairPacket& right) {
PairPacket result;
for (std::size_t lane = 0; lane < result.size(); ++lane)
result[lane] = MultiplyPair(left[lane], right[lane]);
ScalarPacketMultiply(left, right, &result);
return Load(result);
}
/// @brief Retains normalized/zero branches and the checked per-lane Add graph.
VectorPair Add(VectorPair left, VectorPair right, bool pair_shortcuts) {
const auto a = Store(left), b = Store(right);
int left_zero = 0, right_zero = 0;
for (std::size_t lane = 0; lane < a.size(); ++lane) {
if (!math_internal::IsNormalized(a[lane]) ||
!math_internal::IsNormalized(b[lane]))
const auto a = Store(left);
const auto b = Store(right);
int left_zero = 0;
int right_zero = 0;
for (std::size_t lane = 0; lane < 4; ++lane) {
if (!NormalizedScalar(a.values[lane]) ||
!NormalizedScalar(b.values[lane]))
return ScalarAdd(left, right, pair_shortcuts);
if (a[lane].high == 0 && a[lane].low == 0) left_zero |= 1 << lane;
if (b[lane].high == 0 && b[lane].low == 0) right_zero |= 1 << lane;
if (a.values[lane].high == 0 && a.values[lane].low == 0) left_zero |= 1 << lane;
if (b.values[lane].high == 0 && b.values[lane].low == 0) right_zero |= 1 << lane;
}
if (pair_shortcuts) {
if (left_zero == 15)
@@ -103,22 +156,24 @@ VectorPair Add(VectorPair left, VectorPair right, bool pair_shortcuts) {
Select(valid, joined.high, _mm256_setzero_pd()),
Select(valid, remainder, _mm256_setzero_pd())), valid);
}
/// @brief Checks finite/zero/tail premises before products; neutralizes rejected
/// lanes before the explicit residual FMA and final TwoSum.
VectorPair Product(__m256d left, __m256d right) {
double a[4], b[4];
double a[4];
double b[4];
_mm256_storeu_pd(a, left);
_mm256_storeu_pd(b, right);
bool special = false;
for (std::size_t lane = 0; lane < 4; ++lane)
if (!math_internal::IsFiniteBinary64(a[lane]) ||
!math_internal::IsFiniteBinary64(b[lane]) ||
if (!FiniteScalar(a[lane]) ||
!FiniteScalar(b[lane]) ||
a[lane] == 0 || b[lane] == 0 ||
math_internal::LowestBitExponent(a[lane]) +
math_internal::LowestBitExponent(b[lane]) < -1074)
LowestBitExponentScalar(a[lane]) +
LowestBitExponentScalar(b[lane]) < -1074)
special = true;
if (special) {
PairPacket result;
for (std::size_t lane = 0; lane < 4; ++lane)
result[lane] = math_internal::TwoProduct(a[lane], b[lane]);
ScalarPacketTwoProduct(a, b, &result);
return Load(result);
}
const auto product = _mm256_mul_pd(left, right);
@@ -132,15 +187,18 @@ VectorPair Product(__m256d left, __m256d right) {
Select(valid, product, _mm256_setzero_pd()),
Select(valid, error, _mm256_setzero_pd())), valid);
}
/// @brief Uses uniform low-zero branches with original four-product association;
/// mixed or invalid packets preserve the baseline checked path.
VectorPair Multiply(VectorPair left, VectorPair right) {
const auto a = Store(left), b = Store(right);
const auto a = Store(left);
const auto b = Store(right);
int branch = -1;
for (std::size_t lane = 0; lane < a.size(); ++lane) {
const bool normalized = math_internal::IsNormalized(a[lane]) &&
math_internal::IsNormalized(b[lane]);
for (std::size_t lane = 0; lane < 4; ++lane) {
const bool normalized = NormalizedScalar(a.values[lane]) &&
NormalizedScalar(b.values[lane]);
if (!normalized) return ScalarMultiply(a, b);
const int shape = (a[lane].low != 0 ? 1 : 0) |
(b[lane].low != 0 ? 2 : 0);
const int shape = (a.values[lane].low != 0 ? 1 : 0) |
(b.values[lane].low != 0 ? 2 : 0);
if (branch != -1 && shape != branch) return ScalarMultiply(a, b);
branch = shape;
}
src/fesa/elements/hex8_pair_simd_dispatch.cpp
+65 -11
View File
@@ -9,18 +9,70 @@
#endif
namespace fesa::hex8_internal {
#if defined(_MSC_VER)
#define FESA_HEX8_BASELINE_NOINLINE __declspec(noinline)
#else
#define FESA_HEX8_BASELINE_NOINLINE
#endif
FESA_HEX8_BASELINE_NOINLINE
void ScalarPacketAdd(const PairPacket& left, const PairPacket& right,
bool pair_shortcuts, PairPacket* output) {
for (std::size_t lane = 0; lane < 4; ++lane) {
const math_internal::HighLow a{left.values[lane].high, left.values[lane].low};
const math_internal::HighLow b{right.values[lane].high, right.values[lane].low};
const auto value = pair_shortcuts ? AddPair(a, b) : math_internal::Add(a, b);
output->values[lane] = {value.high, value.low};
}
}
FESA_HEX8_BASELINE_NOINLINE
void ScalarPacketMultiply(const PairPacket& left, const PairPacket& right,
PairPacket* output) {
for (std::size_t lane = 0; lane < 4; ++lane) {
const auto value = MultiplyPair(
{left.values[lane].high, left.values[lane].low},
{right.values[lane].high, right.values[lane].low});
output->values[lane] = {value.high, value.low};
}
}
FESA_HEX8_BASELINE_NOINLINE
void ScalarPacketTwoProduct(const double left[4], const double right[4],
PairPacket* output) {
for (std::size_t lane = 0; lane < 4; ++lane) {
const auto value = math_internal::TwoProduct(left[lane], right[lane]);
output->values[lane] = {value.high, value.low};
}
}
#undef FESA_HEX8_BASELINE_NOINLINE
bool SupportsMaterialPacketInstructions(std::uint32_t leaf1_ecx,
std::uint64_t xcr0,
std::uint32_t leaf7_ebx) {
constexpr std::uint32_t required_leaf1 =
(1U << 12) | (1U << 26) | (1U << 27) | (1U << 28);
// MSVC /arch:AVX2 permits BMI codegen; ANDN is present in this kernel.
constexpr std::uint32_t required_leaf7 =
(1U << 3) | (1U << 5) | (1U << 8);
return (leaf1_ecx & required_leaf1) == required_leaf1 &&
(xcr0 & 6U) == 6U &&
(leaf7_ebx & required_leaf7) == required_leaf7;
}
void EvaluateMaterialPacketScalar(const MaterialPacketInput& input,
PairPacket* output) {
for (std::size_t lane = 0; lane < output->size(); ++lane) {
const auto first = MultiplyPair(input.deviatoric_left[lane],
input.deviatoric_right[lane]);
const auto second = MultiplyPair(input.shear_left[lane],
input.shear_right[lane]);
const auto third = MultiplyPair(input.tensor_left[lane],
input.gradient_dot[lane]);
const auto value = [](PacketPair pair) {
return math_internal::HighLow{pair.high, pair.low};
};
for (std::size_t lane = 0; lane < 4; ++lane) {
const auto first = MultiplyPair(value(input.deviatoric_left.values[lane]),
value(input.deviatoric_right.values[lane]));
const auto second = MultiplyPair(value(input.shear_left.values[lane]),
value(input.shear_right.values[lane]));
const auto third = MultiplyPair(value(input.tensor_left.values[lane]),
value(input.gradient_dot.values[lane]));
const auto material = AddPair(AddPair(first, second), third);
(*output)[lane] = AddPair(input.accumulated[lane],
MultiplyPair(material, input.weight));
const auto accumulated = AddPair(value(input.accumulated.values[lane]),
MultiplyPair(material, value(input.weight)));
output->values[lane] = {accumulated.high, accumulated.low};
}
}
@@ -33,9 +85,11 @@ MaterialPacketEvaluator SelectMaterialPacketEvaluator() {
__cpuidex(info, 1, 0);
constexpr int required = (1 << 12) | (1 << 26) | (1 << 27) | (1 << 28);
if ((info[2] & required) != required) return false;
if ((_xgetbv(0) & 6U) != 6U) return false;
const auto leaf1 = static_cast<std::uint32_t>(info[2]);
const auto xcr0 = _xgetbv(0);
__cpuidex(info, 7, 0);
return (info[1] & (1 << 5)) != 0;
return SupportsMaterialPacketInstructions(
leaf1, xcr0, static_cast<std::uint32_t>(info[1]));
}();
if (!hardware_supported || !math_internal::ArithmeticEnvironmentSupported())
return EvaluateMaterialPacketScalar;