perf(hex8): packetize ordered gradient and geometric contractions
This commit is contained in:
KOKO\Mimi
@@ -449,6 +449,231 @@ TEST(Hex8HighLow, PacketTailScreenRetainsExactBoundaryAndFallback) {
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(_M_X64) || defined(__SSE2__)
|
||||
fesa::hex8_internal::GeometryPacketOutput FrozenGeometryPacket(
|
||||
const fesa::hex8_internal::GeometryPacketInput& input) {
|
||||
const auto pair = [](fesa::hex8_internal::PacketPair value) {
|
||||
return fesa::high_low_original::HighLow{value.high, value.low};
|
||||
};
|
||||
fesa::hex8_internal::GeometryPacketOutput output{};
|
||||
for (std::size_t lane = 0; lane < 4; ++lane) {
|
||||
fesa::high_low_original::HighLow dot;
|
||||
for (std::size_t i = 0; i < 3; ++i)
|
||||
dot = fesa::high_low_original::Add(dot,
|
||||
fesa::high_low_original::Multiply(pair(input.gradient_a[i]),
|
||||
pair(input.gradient_b[i].values[lane])));
|
||||
fesa::high_low_original::HighLow geo;
|
||||
for (std::size_t j = 0; j < 3; ++j)
|
||||
for (std::size_t l = 0; l < 3; ++l)
|
||||
geo = fesa::high_low_original::Add(geo,
|
||||
fesa::high_low_original::Multiply(pair(input.gradient_stress[j][l]),
|
||||
pair(input.gradient_b[l].values[lane])));
|
||||
const auto weighted = fesa::high_low_original::Multiply(geo, pair(input.weight));
|
||||
output.gradient_dot.values[lane] = {dot.high, dot.low};
|
||||
output.weighted_geometric.values[lane] = {weighted.high, weighted.low};
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
fesa::hex8_internal::GeometryPacketInput GeometryDyadicFixture() {
|
||||
fesa::hex8_internal::GeometryPacketInput input{};
|
||||
input.weight = {0.125, 0};
|
||||
for (std::size_t j = 0; j < 3; ++j) {
|
||||
input.gradient_a[j] = {static_cast<double>(j + 1) * 0.25, 0};
|
||||
for (std::size_t l = 0; l < 3; ++l) {
|
||||
// Deliberately asymmetric first-products and distinct b/component mapping.
|
||||
input.gradient_stress[j][l] =
|
||||
{static_cast<double>(1 + 3 * j + l) * 0.125, 0};
|
||||
for (std::size_t lane = 0; lane < 4; ++lane)
|
||||
input.gradient_b[l].values[lane] =
|
||||
{static_cast<double>(1 + 4 * l + lane) * 0.0625, 0};
|
||||
}
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
// Original observation repeats geo*weight for three independent accumulators.
|
||||
// Selected observation reuses the weighted pair but retains each checked Add.
|
||||
void ExpectGeometryPacketBitsAndFlags(
|
||||
const fesa::hex8_internal::GeometryPacketInput& input, bool prior_status) {
|
||||
std::fenv_t original;
|
||||
ASSERT_EQ(std::fegetenv(&original), 0);
|
||||
const auto original_csr = _mm_getcsr();
|
||||
struct Observation {
|
||||
fesa::hex8_internal::GeometryPacketOutput output;
|
||||
fesa::hex8_internal::PairPacket accumulators[3];
|
||||
int flags;
|
||||
unsigned int status;
|
||||
};
|
||||
const auto current_pair = [](fesa::hex8_internal::PacketPair value) {
|
||||
return HighLow{value.high, value.low};
|
||||
};
|
||||
const auto observe = [&](bool original_order) {
|
||||
EXPECT_EQ(std::fesetenv(&original), 0);
|
||||
_mm_setcsr(original_csr & ~0x3fU);
|
||||
EXPECT_EQ(std::feclearexcept(FE_ALL_EXCEPT), 0);
|
||||
if (prior_status) {
|
||||
EXPECT_EQ(std::feraiseexcept(FE_INEXACT), 0);
|
||||
_mm_setcsr(_mm_getcsr() | 0x02U);
|
||||
}
|
||||
Observation value{};
|
||||
if (!original_order)
|
||||
fesa::hex8_internal::EvaluateGeometryPacketAvx2(input, &value.output);
|
||||
for (std::size_t lane = 0; lane < 4; ++lane) {
|
||||
HighLow geo;
|
||||
if (original_order) {
|
||||
HighLow dot;
|
||||
for (std::size_t i = 0; i < 3; ++i)
|
||||
dot = fesa::hex8_internal::AddPair(dot,
|
||||
fesa::hex8_internal::MultiplyPair(current_pair(input.gradient_a[i]),
|
||||
current_pair(input.gradient_b[i].values[lane])));
|
||||
for (std::size_t j = 0; j < 3; ++j)
|
||||
for (std::size_t l = 0; l < 3; ++l)
|
||||
geo = fesa::hex8_internal::AddPair(geo,
|
||||
fesa::hex8_internal::MultiplyPair(
|
||||
current_pair(input.gradient_stress[j][l]),
|
||||
current_pair(input.gradient_b[l].values[lane])));
|
||||
value.output.gradient_dot.values[lane] = {dot.high, dot.low};
|
||||
}
|
||||
const HighLow accumulators[3]{{},
|
||||
{static_cast<double>(lane + 1), std::ldexp(1.0, -54)}, {-1, -0.0}};
|
||||
for (std::size_t component = 0; component < 3; ++component) {
|
||||
const auto weighted = original_order
|
||||
? fesa::hex8_internal::MultiplyPair(geo, current_pair(input.weight))
|
||||
: current_pair(value.output.weighted_geometric.values[lane]);
|
||||
if (original_order)
|
||||
value.output.weighted_geometric.values[lane] =
|
||||
{weighted.high, weighted.low};
|
||||
const auto accumulated = fesa::hex8_internal::AddPair(
|
||||
accumulators[component], weighted);
|
||||
value.accumulators[component].values[lane] =
|
||||
{accumulated.high, accumulated.low};
|
||||
}
|
||||
}
|
||||
value.flags = std::fetestexcept(FE_ALL_EXCEPT);
|
||||
value.status = _mm_getcsr() & 0x3fU;
|
||||
return value;
|
||||
};
|
||||
const auto baseline = observe(true);
|
||||
const auto selected = observe(false);
|
||||
fesa::hex8_internal::GeometryPacketOutput scalar;
|
||||
fesa::hex8_internal::EvaluateGeometryPacketScalar(input, &scalar);
|
||||
const auto expected = FrozenGeometryPacket(input);
|
||||
EXPECT_EQ(std::fesetenv(&original), 0);
|
||||
_mm_setcsr(original_csr);
|
||||
EXPECT_EQ(selected.flags, baseline.flags);
|
||||
EXPECT_EQ(selected.status, baseline.status);
|
||||
const auto same_bits = [](const fesa::hex8_internal::PairPacket& a,
|
||||
const fesa::hex8_internal::PairPacket& b) {
|
||||
for (std::size_t lane = 0; lane < 4; ++lane) {
|
||||
EXPECT_EQ(PairBits(a.values[lane].high), PairBits(b.values[lane].high));
|
||||
EXPECT_EQ(PairBits(a.values[lane].low), PairBits(b.values[lane].low));
|
||||
}
|
||||
};
|
||||
same_bits(selected.output.gradient_dot, expected.gradient_dot);
|
||||
same_bits(selected.output.weighted_geometric, expected.weighted_geometric);
|
||||
same_bits(scalar.gradient_dot, expected.gradient_dot);
|
||||
same_bits(scalar.weighted_geometric, expected.weighted_geometric);
|
||||
for (std::size_t component = 0; component < 3; ++component)
|
||||
same_bits(selected.accumulators[component], baseline.accumulators[component]);
|
||||
}
|
||||
|
||||
TEST(Hex8HighLow, GeometryPacketPreservesAsymmetricLanesAndUniformShapes) {
|
||||
if (fesa::hex8_internal::SelectMaterialPacketEvaluator() ==
|
||||
fesa::hex8_internal::EvaluateMaterialPacketScalar)
|
||||
GTEST_SKIP() << "Geometry AVX2 masked environment is not selected.";
|
||||
RecordProperty("private_geometry_packet_backend", "avx2");
|
||||
static_assert(std::is_trivial_v<fesa::hex8_internal::GeometryPacketInput>);
|
||||
static_assert(std::is_trivial_v<fesa::hex8_internal::GeometryPacketOutput>);
|
||||
std::size_t checks = 0;
|
||||
for (int shape = 0; shape < 4; ++shape) {
|
||||
auto input = GeometryDyadicFixture();
|
||||
for (std::size_t j = 0; j < 3; ++j) {
|
||||
if ((shape & 1) != 0) {
|
||||
input.gradient_a[j].low = std::ldexp(input.gradient_a[j].high, -55);
|
||||
for (std::size_t l = 0; l < 3; ++l)
|
||||
input.gradient_stress[j][l].low =
|
||||
std::ldexp(input.gradient_stress[j][l].high, -55);
|
||||
}
|
||||
if ((shape & 2) != 0)
|
||||
for (std::size_t lane = 0; lane < 4; ++lane)
|
||||
input.gradient_b[j].values[lane].low =
|
||||
std::ldexp(input.gradient_b[j].values[lane].high, -55);
|
||||
}
|
||||
ExpectGeometryPacketBitsAndFlags(input, false);
|
||||
ExpectGeometryPacketBitsAndFlags(input, true);
|
||||
checks += 2;
|
||||
}
|
||||
RecordProperty("avx2_geometry_uniform_bit_flag_checks", static_cast<int>(checks));
|
||||
}
|
||||
|
||||
TEST(Hex8HighLow, GeometryPacketRetainsSpecialLanesAndExactTailBoundary) {
|
||||
if (fesa::hex8_internal::SelectMaterialPacketEvaluator() ==
|
||||
fesa::hex8_internal::EvaluateMaterialPacketScalar)
|
||||
GTEST_SKIP() << "Geometry AVX2 masked environment is not selected.";
|
||||
using fesa::hex8_internal::PacketPair;
|
||||
const double tiny = std::numeric_limits<double>::denorm_min();
|
||||
const double maximum = std::numeric_limits<double>::max();
|
||||
const std::vector<PacketPair> special{{0, 0}, {-0.0, 0}, {1, -0.0}, {1, 1},
|
||||
{tiny, 0}, {maximum, maximum},
|
||||
{std::numeric_limits<double>::signaling_NaN(), 0},
|
||||
{0, std::numeric_limits<double>::signaling_NaN()}};
|
||||
std::size_t checks = 0;
|
||||
for (const auto pair : special)
|
||||
for (std::size_t lane = 0; lane < 4; ++lane) {
|
||||
auto input = GeometryDyadicFixture();
|
||||
input.gradient_b[1].values[lane] = pair;
|
||||
ExpectGeometryPacketBitsAndFlags(input, false);
|
||||
ExpectGeometryPacketBitsAndFlags(input, true);
|
||||
checks += 2;
|
||||
}
|
||||
const auto odd = [](int exponent) {
|
||||
return std::nextafter(std::ldexp(1.0, exponent),
|
||||
std::numeric_limits<double>::infinity());
|
||||
};
|
||||
for (int exponent : {-470, -471})
|
||||
for (std::size_t lane = 0; lane < 4; ++lane) {
|
||||
auto input = GeometryDyadicFixture();
|
||||
input.gradient_stress[0][0] = {odd(-500), 0};
|
||||
input.gradient_b[0].values[lane] = {odd(exponent), 0};
|
||||
ExpectGeometryPacketBitsAndFlags(input, false);
|
||||
checks++;
|
||||
}
|
||||
RecordProperty("avx2_geometry_special_tail_checks", static_cast<int>(checks));
|
||||
}
|
||||
|
||||
TEST(Hex8HighLow, GeometryPacketPreservesOrderedOverflowAndPreparedInvalid) {
|
||||
if (fesa::hex8_internal::SelectMaterialPacketEvaluator() ==
|
||||
fesa::hex8_internal::EvaluateMaterialPacketScalar)
|
||||
GTEST_SKIP() << "Geometry AVX2 masked environment is not selected.";
|
||||
const double maximum = std::numeric_limits<double>::max();
|
||||
std::size_t checks = 0;
|
||||
for (bool overflow_order : {false, true}) {
|
||||
fesa::hex8_internal::GeometryPacketInput input{};
|
||||
input.weight = {0.5, 0};
|
||||
for (std::size_t l = 0; l < 3; ++l)
|
||||
for (std::size_t lane = 0; lane < 4; ++lane)
|
||||
input.gradient_b[l].values[lane] = {1, 0};
|
||||
input.gradient_stress[0][0] = {maximum, 0};
|
||||
input.gradient_stress[0][1] = {overflow_order ? maximum : -maximum, 0};
|
||||
input.gradient_stress[0][2] = {overflow_order ? -maximum : maximum, 0};
|
||||
ExpectGeometryPacketBitsAndFlags(input, false);
|
||||
ExpectGeometryPacketBitsAndFlags(input, true);
|
||||
checks += 2;
|
||||
}
|
||||
auto input = GeometryDyadicFixture();
|
||||
const auto invalid_first = fesa::high_low_original::Multiply(
|
||||
{std::ldexp(1.0, -600), 0}, {std::ldexp(1.0, -500), 0});
|
||||
input.gradient_stress[0][0] = {invalid_first.high, invalid_first.low};
|
||||
for (std::size_t lane = 0; lane < 4; ++lane)
|
||||
input.gradient_b[0].values[lane] = {std::ldexp(1.0, 600), 0};
|
||||
ExpectGeometryPacketBitsAndFlags(input, false);
|
||||
ExpectGeometryPacketBitsAndFlags(input, true);
|
||||
checks += 2;
|
||||
RecordProperty("avx2_geometry_ordered_invalid_checks", static_cast<int>(checks));
|
||||
}
|
||||
#endif
|
||||
|
||||
TEST(Hex8HighLow, PacketBackendSelectionRecoversAfterEnvironmentRestoration) {
|
||||
const auto initial = fesa::hex8_internal::SelectMaterialPacketEvaluator();
|
||||
const int rounding = std::fegetround();
|
||||
|
||||
Reference in New Issue
Block a user