perf(hex8): packetize ordered gradient and geometric contractions

This commit is contained in:
KOKO\Mimi
2026-10-14 22:37:39 +09:00
parent 01590bbb0d
commit 96772651d8
5 changed files with 323 additions and 3 deletions
+31 -3
View File
@@ -246,11 +246,38 @@ Result<Hex8HighLowResponse> Hex8Element::EvaluateHighLow(
gradient_stress[a][j][l] = gradients[a][j] * s[l][j];
for (std::size_t a = 0; a < 8; ++a) {
std::array<HighLow, 8> dots{};
std::array<HighLow, 8> weighted_geo{};
if (use_packets)
for (std::size_t first_b = 0; first_b < 8; first_b += 4) {
hex8_internal::GeometryPacketInput input{};
const auto raw_pair = [](HighLow pair) {
return hex8_internal::PacketPair{pair.high, pair.low};
};
input.weight = raw_pair(HighLow{point.weight});
for (std::size_t j = 0; j < 3; ++j) {
input.gradient_a[j] = raw_pair(gradients[a][j]);
for (std::size_t l = 0; l < 3; ++l)
input.gradient_stress[j][l] = raw_pair(gradient_stress[a][j][l]);
for (std::size_t lane = 0; lane < 4; ++lane)
input.gradient_b[j].values[lane] =
raw_pair(gradients[first_b + lane][j]);
}
hex8_internal::GeometryPacketOutput output;
// Same selector contract as material; unsupported/linear stays below.
hex8_internal::EvaluateGeometryPacketAvx2(input, &output);
for (std::size_t lane = 0; lane < 4; ++lane) {
dots[first_b + lane] = {output.gradient_dot.values[lane].high,
output.gradient_dot.values[lane].low};
weighted_geo[first_b + lane] =
{output.weighted_geometric.values[lane].high,
output.weighted_geometric.values[lane].low};
}
}
for (std::size_t b = 0; b < 8; ++b) {
const auto dot = Dot(gradients[a], gradients[b]);
const auto dot = use_packets ? dots[b] : Dot(gradients[a], gradients[b]);
dots[b] = dot;
HighLow geo;
if (nonlinear)
if (nonlinear && !use_packets)
for (std::size_t j = 0; j < 3; ++j)
for (std::size_t l = 0; l < 3; ++l)
geo += gradient_stress[a][j][l] * gradients[b][l];
@@ -265,7 +292,8 @@ Result<Hex8HighLowResponse> Hex8Element::EvaluateHighLow(
material[row][column] += mat * HighLow{point.weight};
}
if (i == k)
geometric[row][column] += geo * HighLow{point.weight};
geometric[row][column] +=
use_packets ? weighted_geo[b] : geo * HighLow{point.weight};
}
}
if (use_packets)
+24
View File
@@ -23,6 +23,30 @@ struct MaterialPacketInput {
PairPacket accumulated;
PacketPair weight;
};
/// @brief One owned trial point/a and four independent b gradients, raw only.
struct GeometryPacketInput {
PacketPair gradient_a[3];
PacketPair gradient_stress[3][3];
PairPacket gradient_b[3];
PacketPair weight;
};
/// @brief Independent raw contractions; accumulator entries remain caller-owned.
struct GeometryPacketOutput {
PairPacket gradient_dot;
PairPacket weighted_geometric;
};
using GeometryPacketEvaluator = void (*)(const GeometryPacketInput&,
GeometryPacketOutput*);
/// @brief Baseline checked zero-start Dot3/ordered j,l Geo9 and weight products.
/// @note Lane order is 0..3. Invalid pairs remain invalid for original gates.
void EvaluateGeometryPacketScalar(const GeometryPacketInput& input,
GeometryPacketOutput* output);
/// @brief Evaluates independent b lanes without horizontal reduction.
/// @pre The material selector has admitted CPU/OS/current masked FP state.
/// @note Keeps original zero-start, product association and j,l Add order.
void EvaluateGeometryPacketAvx2(const GeometryPacketInput& input,
GeometryPacketOutput* output);
using MaterialPacketEvaluator = void (*)(const MaterialPacketInput&, PairPacket*);
/// @brief Evaluates each lane's original checked material graph in order 0..3.
+22
View File
@@ -111,6 +111,10 @@ VectorPair Load(const PairPacket& value) {
_mm256_setr_pd(value.values[0].low, value.values[1].low,
value.values[2].low, value.values[3].low)};
}
/// @brief Replicates one raw first product without changing either component.
VectorPair Splat(PacketPair value) {
return {_mm256_set1_pd(value.high), _mm256_set1_pd(value.low)};
}
/// @brief Stores raw high/low fields without rounding or lane reduction.
PairPacket Store(VectorPair value) {
double high[4];
@@ -251,6 +255,20 @@ void EvaluateMaterialPacketAvx2(const MaterialPacketInput& input,
*output = Store(Add(Load(input.accumulated),
Multiply(material, Load(weight)), true));
}
void EvaluateGeometryPacketAvx2(const GeometryPacketInput& input,
GeometryPacketOutput* output) {
VectorPair dot{_mm256_setzero_pd(), _mm256_setzero_pd()};
for (std::size_t i = 0; i < 3; ++i)
dot = Add(dot, Multiply(Splat(input.gradient_a[i]),
Load(input.gradient_b[i])), true);
VectorPair geo{_mm256_setzero_pd(), _mm256_setzero_pd()};
for (std::size_t j = 0; j < 3; ++j)
for (std::size_t l = 0; l < 3; ++l)
geo = Add(geo, Multiply(Splat(input.gradient_stress[j][l]),
Load(input.gradient_b[l])), true);
output->gradient_dot = Store(dot);
output->weighted_geometric = Store(Multiply(geo, Splat(input.weight)));
}
} // namespace fesa::hex8_internal
#else
namespace fesa::hex8_internal {
@@ -258,5 +276,9 @@ void EvaluateMaterialPacketAvx2(const MaterialPacketInput& input,
PairPacket* output) {
EvaluateMaterialPacketScalar(input, output);
}
void EvaluateGeometryPacketAvx2(const GeometryPacketInput& input,
GeometryPacketOutput* output) {
EvaluateGeometryPacketScalar(input, output);
}
} // namespace fesa::hex8_internal
#endif
@@ -42,6 +42,27 @@ void ScalarPacketTwoProduct(const double left[4], const double right[4],
output->values[lane] = {value.high, value.low};
}
}
FESA_HEX8_BASELINE_NOINLINE
void EvaluateGeometryPacketScalar(const GeometryPacketInput& input,
GeometryPacketOutput* output) {
const auto value = [](PacketPair pair) {
return math_internal::HighLow{pair.high, pair.low};
};
for (std::size_t lane = 0; lane < 4; ++lane) {
math_internal::HighLow dot;
for (std::size_t i = 0; i < 3; ++i)
dot = AddPair(dot, MultiplyPair(value(input.gradient_a[i]),
value(input.gradient_b[i].values[lane])));
math_internal::HighLow geo;
for (std::size_t j = 0; j < 3; ++j)
for (std::size_t l = 0; l < 3; ++l)
geo = AddPair(geo, MultiplyPair(value(input.gradient_stress[j][l]),
value(input.gradient_b[l].values[lane])));
const auto weighted = MultiplyPair(geo, value(input.weight));
output->gradient_dot.values[lane] = {dot.high, dot.low};
output->weighted_geometric.values[lane] = {weighted.high, weighted.low};
}
}
#undef FESA_HEX8_BASELINE_NOINLINE
bool SupportsMaterialPacketInstructions(std::uint32_t leaf1_ecx,