baram2584/FESADev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge reviewed geometric packet optimization and FP fallback

Browse Source
This commit is contained in:
KOKO\Mimi
2026-10-14 23:34:29 +09:00
parent b11e12a146 f81a0b6739
commit 9936994e49
6 changed files with 478 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/fesa/elements/hex8_high_low.cpp
+31 -3
View File
@@ -246,11 +246,38 @@ Result<Hex8HighLowResponse> Hex8Element::EvaluateHighLow(
gradient_stress[a][j][l] = gradients[a][j] * s[l][j];
for (std::size_t a = 0; a < 8; ++a) {
std::array<HighLow, 8> dots{};
std::array<HighLow, 8> weighted_geo{};
if (use_packets)
for (std::size_t first_b = 0; first_b < 8; first_b += 4) {
hex8_internal::GeometryPacketInput input{};
const auto raw_pair = [](HighLow pair) {
return hex8_internal::PacketPair{pair.high, pair.low};
};
input.weight = raw_pair(HighLow{point.weight});
for (std::size_t j = 0; j < 3; ++j) {
input.gradient_a[j] = raw_pair(gradients[a][j]);
for (std::size_t l = 0; l < 3; ++l)
input.gradient_stress[j][l] = raw_pair(gradient_stress[a][j][l]);
for (std::size_t lane = 0; lane < 4; ++lane)
input.gradient_b[j].values[lane] =
raw_pair(gradients[first_b + lane][j]);
}
hex8_internal::GeometryPacketOutput output;
// Same selector contract as material; unsupported/linear stays below.
hex8_internal::EvaluateGeometryPacketAvx2(input, &output);
for (std::size_t lane = 0; lane < 4; ++lane) {
dots[first_b + lane] = {output.gradient_dot.values[lane].high,
output.gradient_dot.values[lane].low};
weighted_geo[first_b + lane] =
{output.weighted_geometric.values[lane].high,
output.weighted_geometric.values[lane].low};
}
}
for (std::size_t b = 0; b < 8; ++b) {
const auto dot = Dot(gradients[a], gradients[b]);
const auto dot = use_packets ? dots[b] : Dot(gradients[a], gradients[b]);
dots[b] = dot;
HighLow geo;
if (nonlinear)
if (nonlinear && !use_packets)
for (std::size_t j = 0; j < 3; ++j)
for (std::size_t l = 0; l < 3; ++l)
geo += gradient_stress[a][j][l] * gradients[b][l];
@@ -265,7 +292,8 @@ Result<Hex8HighLowResponse> Hex8Element::EvaluateHighLow(
material[row][column] += mat * HighLow{point.weight};
}
if (i == k)
geometric[row][column] += geo * HighLow{point.weight};
geometric[row][column] +=
use_packets ? weighted_geo[b] : geo * HighLow{point.weight};
}
}
if (use_packets)
src/fesa/elements/hex8_pair_simd.h
+24
View File
@@ -23,6 +23,30 @@ struct MaterialPacketInput {
PairPacket accumulated;
PacketPair weight;
};
/// @brief One owned trial point/a and four independent b gradients, raw only.
struct GeometryPacketInput {
PacketPair gradient_a[3];
PacketPair gradient_stress[3][3];
PairPacket gradient_b[3];
PacketPair weight;
};
/// @brief Independent raw contractions; accumulator entries remain caller-owned.
struct GeometryPacketOutput {
PairPacket gradient_dot;
PairPacket weighted_geometric;
};
using GeometryPacketEvaluator = void (*)(const GeometryPacketInput&,
GeometryPacketOutput*);
/// @brief Baseline checked zero-start Dot3/ordered j,l Geo9 and weight products.
/// @note Lane order is 0..3. Invalid pairs remain invalid for original gates.
void EvaluateGeometryPacketScalar(const GeometryPacketInput& input,
GeometryPacketOutput* output);
/// @brief Evaluates independent b lanes without horizontal reduction.
/// @pre The material selector has admitted CPU/OS/current masked FP state.
/// @note Keeps original zero-start, product association and j,l Add order.
void EvaluateGeometryPacketAvx2(const GeometryPacketInput& input,
GeometryPacketOutput* output);
using MaterialPacketEvaluator = void (*)(const MaterialPacketInput&, PairPacket*);
/// @brief Evaluates each lane's original checked material graph in order 0..3.
src/fesa/elements/hex8_pair_simd_avx2.cpp
+54 -14
View File
@@ -111,6 +111,10 @@ VectorPair Load(const PairPacket& value) {
_mm256_setr_pd(value.values[0].low, value.values[1].low,
value.values[2].low, value.values[3].low)};
}
/// @brief Replicates one raw first product without changing either component.
VectorPair Splat(PacketPair value) {
return {_mm256_set1_pd(value.high), _mm256_set1_pd(value.low)};
}
/// @brief Stores raw high/low fields without rounding or lane reduction.
PairPacket Store(VectorPair value) {
double high[4];
@@ -120,9 +124,14 @@ PairPacket Store(VectorPair value) {
return {{{high[0], low[0]}, {high[1], low[1]},
{high[2], low[2]}, {high[3], low[3]}}};
}
/// @brief Retains the original per-lane subtraction/addition association and gates.
VectorPair TwoSum(__m256d left, __m256d right) {
/// @brief Retains finite-sum error association; false requests original graph.
/// @note False leaves output unreadable and preserves the necessary sum flags.
/// Prefix flags still require optimized scalar-oracle/codegen validation.
bool TwoSum(__m256d left, __m256d right, VectorPair* output) {
const auto sum = _mm256_add_pd(left, right);
// Do not speculate Inf-Inf error arithmetic beyond a failed first sum.
// The caller retains original inputs for its existing baseline bridge.
if (_mm256_movemask_pd(Finite(sum)) != 15) return false;
const auto bridge = _mm256_sub_pd(sum, left);
const auto restored = _mm256_sub_pd(sum, bridge);
const auto error = _mm256_add_pd(_mm256_sub_pd(left, restored),
@@ -130,8 +139,9 @@ VectorPair TwoSum(__m256d left, __m256d right) {
const auto valid = _mm256_and_pd(_mm256_and_pd(Finite(sum), Finite(bridge)),
_mm256_and_pd(Finite(restored), Finite(error)));
const auto exact_zero = _mm256_and_pd(Zero(sum), Zero(error));
return InvalidOutside({Select(exact_zero, _mm256_setzero_pd(), sum),
Select(exact_zero, _mm256_setzero_pd(), error)}, valid);
*output = InvalidOutside({Select(exact_zero, _mm256_setzero_pd(), sum),
Select(exact_zero, _mm256_setzero_pd(), error)}, valid);
return true;
}
/// @brief Routes exceptional packets to baseline checked Add in lane order.
VectorPair ScalarAdd(VectorPair left, VectorPair right, bool pair_shortcuts) {
@@ -163,10 +173,15 @@ VectorPair Add(VectorPair left, VectorPair right, bool pair_shortcuts) {
if (left_zero != 0 || right_zero != 0)
return ScalarAdd(left, right, true);
}
const auto head = TwoSum(left.high, right.high);
const auto tail = TwoSum(left.low, right.low);
const auto middle = TwoSum(head.low, tail.high);
const auto joined = TwoSum(head.high, middle.high);
VectorPair head;
VectorPair tail;
VectorPair middle;
VectorPair joined;
if (!TwoSum(left.high, right.high, &head) ||
!TwoSum(left.low, right.low, &tail) ||
!TwoSum(head.low, tail.high, &middle) ||
!TwoSum(head.high, middle.high, &joined))
return ScalarAdd(left, right, pair_shortcuts);
const auto remainder = _mm256_add_pd(
_mm256_add_pd(joined.low, middle.low), tail.low);
const auto valid = _mm256_and_pd(Finite(remainder), _mm256_and_pd(
@@ -175,9 +190,11 @@ VectorPair Add(VectorPair left, VectorPair right, bool pair_shortcuts) {
_mm256_and_pd(Finite(middle.high), Finite(middle.low)),
_mm256_and_pd(Finite(joined.high), Finite(joined.low))))));
// Original Add returns before its last TwoSum when a prior finite gate fails.
return InvalidOutside(TwoSum(
Select(valid, joined.high, _mm256_setzero_pd()),
Select(valid, remainder, _mm256_setzero_pd())), valid);
VectorPair result;
if (!TwoSum(Select(valid, joined.high, _mm256_setzero_pd()),
Select(valid, remainder, _mm256_setzero_pd()), &result))
return ScalarAdd(left, right, pair_shortcuts);
return InvalidOutside(result, valid);
}
/// @brief Checks finite/zero/tail premises before products; neutralizes rejected
/// lanes before the explicit residual FMA and final TwoSum.
@@ -207,9 +224,14 @@ VectorPair Product(__m256d left, __m256d right) {
const auto error = _mm256_fmsub_pd(Select(product_valid, left, one),
Select(product_valid, right, one), Select(product_valid, product, one));
const auto valid = _mm256_and_pd(product_valid, Finite(error));
return InvalidOutside(TwoSum(
Select(valid, product, _mm256_setzero_pd()),
Select(valid, error, _mm256_setzero_pd())), valid);
VectorPair result;
if (!TwoSum(Select(valid, product, _mm256_setzero_pd()),
Select(valid, error, _mm256_setzero_pd()), &result)) {
PairPacket scalar;
ScalarPacketTwoProduct(a, b, &scalar);
return Load(scalar);
}
return InvalidOutside(result, valid);
}
/// @brief Uses uniform low-zero branches with original four-product association;
/// mixed or invalid packets preserve the baseline checked path.
@@ -251,6 +273,20 @@ void EvaluateMaterialPacketAvx2(const MaterialPacketInput& input,
*output = Store(Add(Load(input.accumulated),
Multiply(material, Load(weight)), true));
}
void EvaluateGeometryPacketAvx2(const GeometryPacketInput& input,
GeometryPacketOutput* output) {
VectorPair dot{_mm256_setzero_pd(), _mm256_setzero_pd()};
for (std::size_t i = 0; i < 3; ++i)
dot = Add(dot, Multiply(Splat(input.gradient_a[i]),
Load(input.gradient_b[i])), true);
VectorPair geo{_mm256_setzero_pd(), _mm256_setzero_pd()};
for (std::size_t j = 0; j < 3; ++j)
for (std::size_t l = 0; l < 3; ++l)
geo = Add(geo, Multiply(Splat(input.gradient_stress[j][l]),
Load(input.gradient_b[l])), true);
output->gradient_dot = Store(dot);
output->weighted_geometric = Store(Multiply(geo, Splat(input.weight)));
}
} // namespace fesa::hex8_internal
#else
namespace fesa::hex8_internal {
@@ -258,5 +294,9 @@ void EvaluateMaterialPacketAvx2(const MaterialPacketInput& input,
PairPacket* output) {
EvaluateMaterialPacketScalar(input, output);
}
void EvaluateGeometryPacketAvx2(const GeometryPacketInput& input,
GeometryPacketOutput* output) {
EvaluateGeometryPacketScalar(input, output);
}
} // namespace fesa::hex8_internal
#endif
src/fesa/elements/hex8_pair_simd_dispatch.cpp
+21
View File
@@ -42,6 +42,27 @@ void ScalarPacketTwoProduct(const double left[4], const double right[4],
output->values[lane] = {value.high, value.low};
}
}
FESA_HEX8_BASELINE_NOINLINE
void EvaluateGeometryPacketScalar(const GeometryPacketInput& input,
GeometryPacketOutput* output) {
const auto value = [](PacketPair pair) {
return math_internal::HighLow{pair.high, pair.low};
};
for (std::size_t lane = 0; lane < 4; ++lane) {
math_internal::HighLow dot;
for (std::size_t i = 0; i < 3; ++i)
dot = AddPair(dot, MultiplyPair(value(input.gradient_a[i]),
value(input.gradient_b[i].values[lane])));
math_internal::HighLow geo;
for (std::size_t j = 0; j < 3; ++j)
for (std::size_t l = 0; l < 3; ++l)
geo = AddPair(geo, MultiplyPair(value(input.gradient_stress[j][l]),
value(input.gradient_b[l].values[lane])));
const auto weighted = MultiplyPair(geo, value(input.weight));
output->gradient_dot.values[lane] = {dot.high, dot.low};
output->weighted_geometric.values[lane] = {weighted.high, weighted.low};
}
}
#undef FESA_HEX8_BASELINE_NOINLINE
bool SupportsMaterialPacketInstructions(std::uint32_t leaf1_ecx,