baram2584/FESADev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(hex8): isolate AVX2 packet code from baseline COMDATs

Browse Source
This commit is contained in:
KOKO\Mimi
2026-10-14 21:10:27 +09:00
parent c0e23503e8
commit 446d3fa059
5 changed files with 344 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/elements/hex8_high_low_test.cpp
+131 -64
View File
@@ -12,6 +12,7 @@
#include <limits>
#include <random>
#include <thread>
#include <type_traits>
#include <utility>
#include <vector>
@@ -85,27 +86,31 @@ TEST(Hex8HighLow, ScalarCrossProductShortcutMatchesOriginalBits) {
}
}
fesa::hex8_internal::PacketPair RawPacketPair(HighLow value) {
return {value.high, value.low};
}
fesa::hex8_internal::PairPacket FrozenMaterialPacket(
const fesa::hex8_internal::MaterialPacketInput& input) {
using Original = fesa::high_low_original::HighLow;
const auto original = [](HighLow value) {
const auto original = [](fesa::hex8_internal::PacketPair value) {
return Original{value.high, value.low};
};
fesa::hex8_internal::PairPacket result;
for (std::size_t lane = 0; lane < 4; ++lane) {
const auto first = fesa::high_low_original::Multiply(
original(input.deviatoric_left[lane]),
original(input.deviatoric_right[lane]));
original(input.deviatoric_left.values[lane]),
original(input.deviatoric_right.values[lane]));
const auto second = fesa::high_low_original::Multiply(
original(input.shear_left[lane]), original(input.shear_right[lane]));
original(input.shear_left.values[lane]), original(input.shear_right.values[lane]));
const auto third = fesa::high_low_original::Multiply(
original(input.tensor_left[lane]), original(input.gradient_dot[lane]));
original(input.tensor_left.values[lane]), original(input.gradient_dot.values[lane]));
const auto value = fesa::high_low_original::Add(
fesa::high_low_original::Add(first, second), third);
const auto accumulated = fesa::high_low_original::Add(
original(input.accumulated[lane]),
original(input.accumulated.values[lane]),
fesa::high_low_original::Multiply(value, original(input.weight)));
result[lane] = {accumulated.high, accumulated.low};
result.values[lane] = {accumulated.high, accumulated.low};
}
return result;
}
@@ -119,10 +124,10 @@ bool ExpectFrozenMaterialPacket(
evaluator(input, &selected);
for (std::size_t lane = 0; lane < 4; ++lane) {
SCOPED_TRACE(lane);
EXPECT_EQ(PairBits(scalar[lane].high), PairBits(expected[lane].high));
EXPECT_EQ(PairBits(scalar[lane].low), PairBits(expected[lane].low));
EXPECT_EQ(PairBits(selected[lane].high), PairBits(expected[lane].high));
EXPECT_EQ(PairBits(selected[lane].low), PairBits(expected[lane].low));
EXPECT_EQ(PairBits(scalar.values[lane].high), PairBits(expected.values[lane].high));
EXPECT_EQ(PairBits(scalar.values[lane].low), PairBits(expected.values[lane].low));
EXPECT_EQ(PairBits(selected.values[lane].high), PairBits(expected.values[lane].high));
EXPECT_EQ(PairBits(selected.values[lane].low), PairBits(expected.values[lane].low));
}
return evaluator != fesa::hex8_internal::EvaluateMaterialPacketScalar;
}
@@ -147,51 +152,55 @@ TEST(Hex8HighLow, FourColumnMaterialPacketMatchesFrozenScalarBits) {
{0, std::numeric_limits<double>::signaling_NaN()}};
for (std::size_t first = 0; first < boundary.size(); ++first)
for (std::size_t second = 0; second < boundary.size(); ++second) {
MaterialPacketInput input;
MaterialPacketInput input{};
input.weight = {0.125};
for (std::size_t lane = 0; lane < 4; ++lane) {
input.deviatoric_left[lane] = boundary[first];
input.deviatoric_right[lane] = boundary[second];
input.shear_left[lane] = {2, std::ldexp(1.0, -54)};
input.shear_right[lane] = {-0.5, std::ldexp(1.0, -56)};
input.tensor_left[lane] = {0.75};
input.gradient_dot[lane] = {-0.25};
input.accumulated[lane] = lane == 0 ? HighLow{} : HighLow{0.125};
input.deviatoric_left.values[lane] = RawPacketPair(boundary[first]);
input.deviatoric_right.values[lane] = RawPacketPair(boundary[second]);
input.shear_left.values[lane] = {2, std::ldexp(1.0, -54)};
input.shear_right.values[lane] = {-0.5, std::ldexp(1.0, -56)};
input.tensor_left.values[lane] = {0.75};
input.gradient_dot.values[lane] = {-0.25};
input.accumulated.values[lane] = lane == 0 ? fesa::hex8_internal::PacketPair{0, 0}
: fesa::hex8_internal::PacketPair{0.125, 0};
}
avx2_evaluator_checks += ExpectFrozenMaterialPacket(input) ? 1U : 0U;
++packet_checks;
}
for (int exponent = -1074; exponent <= 1023; ++exponent) {
MaterialPacketInput input;
MaterialPacketInput input{};
input.weight = {1};
for (std::size_t lane = 0; lane < 4; ++lane) {
input.deviatoric_left[lane] = {
input.deviatoric_left.values[lane] = {
std::ldexp(lane % 2U == 0 ? 1.0 : -1.0, exponent)};
input.deviatoric_right[lane] = {lane < 2 ? 0.5 : 2.0};
input.deviatoric_right.values[lane] = {lane < 2 ? 0.5 : 2.0};
}
avx2_evaluator_checks += ExpectFrozenMaterialPacket(input) ? 1U : 0U;
++packet_checks;
}
std::mt19937_64 random(0x4c0116U);
for (std::size_t sample = 0; sample < 2048; ++sample) {
MaterialPacketInput input;
MaterialPacketInput input{};
const auto pair = [&]() {
const double high = std::ldexp(
1.0 + static_cast<double>(random() % 256U) / 256.0,
static_cast<int>(random() % 201U) - 100);
const double signed_high = (random() & 1U) != 0 ? high : -high;
const double low = sample % 3U == 0 ? 0.0 : std::ldexp(high, -55);
return HighLow{signed_high, (random() & 1U) != 0 ? low : -low};
return fesa::hex8_internal::PacketPair{
signed_high, (random() & 1U) != 0 ? low : -low};
};
input.weight = sample % 2U == 0 ? HighLow{0.125} : pair();
input.weight = sample % 2U == 0
? fesa::hex8_internal::PacketPair{0.125, 0} : pair();
for (std::size_t lane = 0; lane < 4; ++lane) {
input.deviatoric_left[lane] = pair();
input.deviatoric_right[lane] = pair();
input.shear_left[lane] = pair();
input.shear_right[lane] = pair();
input.tensor_left[lane] = pair();
input.gradient_dot[lane] = pair();
input.accumulated[lane] = sample % 4U == 0 ? HighLow{} : pair();
input.deviatoric_left.values[lane] = pair();
input.deviatoric_right.values[lane] = pair();
input.shear_left.values[lane] = pair();
input.shear_right.values[lane] = pair();
input.tensor_left.values[lane] = pair();
input.gradient_dot.values[lane] = pair();
input.accumulated.values[lane] = sample % 4U == 0
? fesa::hex8_internal::PacketPair{0, 0} : pair();
}
avx2_evaluator_checks += ExpectFrozenMaterialPacket(input) ? 1U : 0U;
++packet_checks;
@@ -202,15 +211,15 @@ TEST(Hex8HighLow, FourColumnMaterialPacketMatchesFrozenScalarBits) {
}
TEST(Hex8HighLow, PacketSpecialLaneDoesNotChangeOtherColumns) {
fesa::hex8_internal::MaterialPacketInput input;
fesa::hex8_internal::MaterialPacketInput input{};
input.weight = {0.125};
for (std::size_t lane = 0; lane < 4; ++lane) {
input.deviatoric_left[lane] = {1, std::ldexp(1.0, -54)};
input.deviatoric_right[lane] = {2, std::ldexp(1.0, -55)};
input.shear_left[lane] = {0.5};
input.shear_right[lane] = {-0.25};
input.tensor_left[lane] = {0.75};
input.gradient_dot[lane] = {0.125};
input.deviatoric_left.values[lane] = {1, std::ldexp(1.0, -54)};
input.deviatoric_right.values[lane] = {2, std::ldexp(1.0, -55)};
input.shear_left.values[lane] = {0.5};
input.shear_right.values[lane] = {-0.25};
input.tensor_left.values[lane] = {0.75};
input.gradient_dot.values[lane] = {0.125};
}
const auto reference = FrozenMaterialPacket(input);
const std::vector<HighLow> special{{}, {-0.0, 0}, {1},
@@ -220,14 +229,14 @@ TEST(Hex8HighLow, PacketSpecialLaneDoesNotChangeOtherColumns) {
for (std::size_t lane = 0; lane < 4; ++lane)
for (const auto value : special) {
auto changed = input;
changed.deviatoric_left[lane] = value;
changed.deviatoric_left.values[lane] = RawPacketPair(value);
ExpectFrozenMaterialPacket(changed);
fesa::hex8_internal::PairPacket actual;
fesa::hex8_internal::SelectMaterialPacketEvaluator()(changed, &actual);
for (std::size_t other = 0; other < 4; ++other)
if (other != lane) {
EXPECT_EQ(PairBits(actual[other].high), PairBits(reference[other].high));
EXPECT_EQ(PairBits(actual[other].low), PairBits(reference[other].low));
EXPECT_EQ(PairBits(actual.values[other].high), PairBits(reference.values[other].high));
EXPECT_EQ(PairBits(actual.values[other].low), PairBits(reference.values[other].low));
}
}
}
@@ -237,7 +246,7 @@ TEST(Hex8HighLow, UniformDyadicPacketsExerciseEachLowZeroShape) {
for (int shape = 0; shape < 4; ++shape)
for (int low_weight = 0; low_weight < 2; ++low_weight)
for (int zero_accumulator = 0; zero_accumulator < 2; ++zero_accumulator) {
fesa::hex8_internal::MaterialPacketInput input;
fesa::hex8_internal::MaterialPacketInput input{};
input.weight = {0.5, low_weight != 0 ? std::ldexp(1.0, -57) : 0.0};
for (std::size_t lane = 0; lane < 4; ++lane) {
const bool left_low = (shape & 1) != 0;
@@ -247,14 +256,15 @@ TEST(Hex8HighLow, UniformDyadicPacketsExerciseEachLowZeroShape) {
// Thus each Multiply low-shape is uniform and its component products
// bypass zero/unsupported-tail fallback. No mixed-zero Add fallback
// is possible; the initial zero accumulator is uniform across lanes.
input.deviatoric_left[lane] = {1, left_low ? std::ldexp(1.0, -54) : 0};
input.deviatoric_right[lane] = {2, right_low ? std::ldexp(1.0, -54) : 0};
input.shear_left[lane] = {0.5, left_low ? std::ldexp(1.0, -56) : 0};
input.shear_right[lane] = {0.25, right_low ? std::ldexp(1.0, -57) : 0};
input.tensor_left[lane] = {0.75, left_low ? std::ldexp(1.0, -56) : 0};
input.gradient_dot[lane] = {0.125, right_low ? std::ldexp(1.0, -58) : 0};
input.accumulated[lane] = zero_accumulator != 0 ? HighLow{}
: HighLow{0.0625};
input.deviatoric_left.values[lane] = {1, left_low ? std::ldexp(1.0, -54) : 0};
input.deviatoric_right.values[lane] = {2, right_low ? std::ldexp(1.0, -54) : 0};
input.shear_left.values[lane] = {0.5, left_low ? std::ldexp(1.0, -56) : 0};
input.shear_right.values[lane] = {0.25, right_low ? std::ldexp(1.0, -57) : 0};
input.tensor_left.values[lane] = {0.75, left_low ? std::ldexp(1.0, -56) : 0};
input.gradient_dot.values[lane] = {0.125, right_low ? std::ldexp(1.0, -58) : 0};
input.accumulated.values[lane] = zero_accumulator != 0
? fesa::hex8_internal::PacketPair{0, 0}
: fesa::hex8_internal::PacketPair{0.0625, 0};
}
avx2_checks += ExpectFrozenMaterialPacket(input) ? 1U : 0U;
++checks;
@@ -270,17 +280,17 @@ TEST(Hex8HighLow, OverflowLaneIsNeutralizedBeforeResidualFma) {
std::fenv_t original;
ASSERT_EQ(std::fegetenv(&original), 0);
for (std::size_t bad = 0; bad < 4; ++bad) {
fesa::hex8_internal::MaterialPacketInput input;
fesa::hex8_internal::MaterialPacketInput input{};
input.weight = {0.5};
for (std::size_t lane = 0; lane < 4; ++lane) {
input.deviatoric_left[lane] = {
input.deviatoric_left.values[lane] = {
lane == bad ? std::numeric_limits<double>::max() : 1};
input.deviatoric_right[lane] = {2};
input.shear_left[lane] = {0.5};
input.shear_right[lane] = {0.25};
input.tensor_left[lane] = {0.75};
input.gradient_dot[lane] = {0.125};
input.accumulated[lane] = {0.0625};
input.deviatoric_right.values[lane] = {2};
input.shear_left.values[lane] = {0.5};
input.shear_right.values[lane] = {0.25};
input.tensor_left.values[lane] = {0.75};
input.gradient_dot.values[lane] = {0.125};
input.accumulated.values[lane] = {0.0625};
}
// All operands are finite/nonzero and max*2 has LBE sum 972, so Product
// reaches its vector multiply; only the post-product finite mask rejects it.
@@ -292,8 +302,8 @@ TEST(Hex8HighLow, OverflowLaneIsNeutralizedBeforeResidualFma) {
EXPECT_EQ(std::fesetenv(&original), 0);
EXPECT_FALSE(extra_invalid);
for (std::size_t lane = 0; lane < 4; ++lane) {
EXPECT_EQ(PairBits(output[lane].high), PairBits(expected[lane].high));
EXPECT_EQ(PairBits(output[lane].low), PairBits(expected[lane].low));
EXPECT_EQ(PairBits(output.values[lane].high), PairBits(expected.values[lane].high));
EXPECT_EQ(PairBits(output.values[lane].low), PairBits(expected.values[lane].low));
}
}
RecordProperty("avx2_product_overflow_positions", 4);
@@ -320,16 +330,73 @@ TEST(Hex8HighLow, PacketBackendSelectionRecoversAfterEnvironmentRestoration) {
EXPECT_EQ(fesa::hex8_internal::SelectMaterialPacketEvaluator(), initial);
}
TEST(Hex8HighLow, PacketScalarBridgesPreserveOriginalRawBits) {
using fesa::hex8_internal::PacketPair;
using fesa::hex8_internal::PairPacket;
static_assert(std::is_trivial_v<PacketPair> &&
std::is_standard_layout_v<PacketPair>);
static_assert(std::is_trivial_v<PairPacket> &&
std::is_standard_layout_v<PairPacket>);
static_assert(std::is_trivial_v<fesa::hex8_internal::MaterialPacketInput>);
const double tiny = std::numeric_limits<double>::denorm_min();
const std::vector<PacketPair> pairs{{0, 0}, {-0.0, 0}, {0, -0.0},
{1, -0.0}, {1, std::ldexp(1.0, -54)}, {tiny, 0}, {1, tiny},
{std::numeric_limits<double>::max(), 0}, {1, 1},
{std::numeric_limits<double>::infinity(), 0},
{std::numeric_limits<double>::signaling_NaN(), 0}};
for (const auto left : pairs)
for (const auto right : pairs) {
PairPacket a{{left, left, left, left}};
PairPacket b{{right, right, right, right}};
PairPacket result;
const fesa::high_low_original::HighLow x{left.high, left.low};
const fesa::high_low_original::HighLow y{right.high, right.low};
const auto expect = [&](fesa::high_low_original::HighLow expected) {
for (std::size_t lane = 0; lane < 4; ++lane) {
EXPECT_EQ(PairBits(result.values[lane].high), PairBits(expected.high));
EXPECT_EQ(PairBits(result.values[lane].low), PairBits(expected.low));
}
};
fesa::hex8_internal::ScalarPacketAdd(a, b, false, &result);
expect(fesa::high_low_original::Add(x, y));
fesa::hex8_internal::ScalarPacketAdd(a, b, true, &result);
expect(fesa::high_low_original::Add(x, y));
fesa::hex8_internal::ScalarPacketMultiply(a, b, &result);
expect(fesa::high_low_original::Multiply(x, y));
const double high_left[4]{left.high, left.high, left.high, left.high};
const double high_right[4]{right.high, right.high, right.high, right.high};
fesa::hex8_internal::ScalarPacketTwoProduct(high_left, high_right, &result);
expect(fesa::high_low_original::TwoProduct(left.high, right.high));
}
}
TEST(Hex8HighLow, PacketCapabilityRejectsMissingIntegerIsaAndOsState) {
constexpr std::uint32_t leaf1 =
(1U << 12) | (1U << 26) | (1U << 27) | (1U << 28);
constexpr std::uint32_t leaf7 = (1U << 3) | (1U << 5) | (1U << 8);
EXPECT_TRUE(fesa::hex8_internal::SupportsMaterialPacketInstructions(
leaf1, 6, leaf7));
for (const auto missing : {1U << 3, 1U << 5, 1U << 8})
EXPECT_FALSE(fesa::hex8_internal::SupportsMaterialPacketInstructions(
leaf1, 6, leaf7 & ~missing));
for (const auto missing : {1U << 12, 1U << 26, 1U << 27, 1U << 28})
EXPECT_FALSE(fesa::hex8_internal::SupportsMaterialPacketInstructions(
leaf1 & ~missing, 6, leaf7));
for (std::uint64_t state : {0ULL, 2ULL, 4ULL})
EXPECT_FALSE(fesa::hex8_internal::SupportsMaterialPacketInstructions(
leaf1, state, leaf7));
}
TEST(Hex8HighLow, PacketDoesNotClearPriorMaskedStickyFlags) {
std::fenv_t original;
ASSERT_EQ(std::fegetenv(&original), 0);
const auto evaluator = fesa::hex8_internal::SelectMaterialPacketEvaluator();
ASSERT_EQ(std::feraiseexcept(FE_INEXACT), 0);
fesa::hex8_internal::MaterialPacketInput input;
fesa::hex8_internal::MaterialPacketInput input{};
input.weight = {1};
for (std::size_t lane = 0; lane < 4; ++lane) {
input.deviatoric_left[lane] = {1, std::ldexp(1.0, -54)};
input.deviatoric_right[lane] = {2, std::ldexp(1.0, -54)};
input.deviatoric_left.values[lane] = {1, std::ldexp(1.0, -54)};
input.deviatoric_right.values[lane] = {2, std::ldexp(1.0, -54)};
}
fesa::hex8_internal::PairPacket output;
evaluator(input, &output);