perf(hex8): packetize ordered gradient and geometric contractions
This commit is contained in:
@@ -246,11 +246,38 @@ Result<Hex8HighLowResponse> Hex8Element::EvaluateHighLow(
|
||||
gradient_stress[a][j][l] = gradients[a][j] * s[l][j];
|
||||
for (std::size_t a = 0; a < 8; ++a) {
|
||||
std::array<HighLow, 8> dots{};
|
||||
std::array<HighLow, 8> weighted_geo{};
|
||||
if (use_packets)
|
||||
for (std::size_t first_b = 0; first_b < 8; first_b += 4) {
|
||||
hex8_internal::GeometryPacketInput input{};
|
||||
const auto raw_pair = [](HighLow pair) {
|
||||
return hex8_internal::PacketPair{pair.high, pair.low};
|
||||
};
|
||||
input.weight = raw_pair(HighLow{point.weight});
|
||||
for (std::size_t j = 0; j < 3; ++j) {
|
||||
input.gradient_a[j] = raw_pair(gradients[a][j]);
|
||||
for (std::size_t l = 0; l < 3; ++l)
|
||||
input.gradient_stress[j][l] = raw_pair(gradient_stress[a][j][l]);
|
||||
for (std::size_t lane = 0; lane < 4; ++lane)
|
||||
input.gradient_b[j].values[lane] =
|
||||
raw_pair(gradients[first_b + lane][j]);
|
||||
}
|
||||
hex8_internal::GeometryPacketOutput output;
|
||||
// Same selector contract as material; unsupported/linear stays below.
|
||||
hex8_internal::EvaluateGeometryPacketAvx2(input, &output);
|
||||
for (std::size_t lane = 0; lane < 4; ++lane) {
|
||||
dots[first_b + lane] = {output.gradient_dot.values[lane].high,
|
||||
output.gradient_dot.values[lane].low};
|
||||
weighted_geo[first_b + lane] =
|
||||
{output.weighted_geometric.values[lane].high,
|
||||
output.weighted_geometric.values[lane].low};
|
||||
}
|
||||
}
|
||||
for (std::size_t b = 0; b < 8; ++b) {
|
||||
const auto dot = Dot(gradients[a], gradients[b]);
|
||||
const auto dot = use_packets ? dots[b] : Dot(gradients[a], gradients[b]);
|
||||
dots[b] = dot;
|
||||
HighLow geo;
|
||||
if (nonlinear)
|
||||
if (nonlinear && !use_packets)
|
||||
for (std::size_t j = 0; j < 3; ++j)
|
||||
for (std::size_t l = 0; l < 3; ++l)
|
||||
geo += gradient_stress[a][j][l] * gradients[b][l];
|
||||
@@ -265,7 +292,8 @@ Result<Hex8HighLowResponse> Hex8Element::EvaluateHighLow(
|
||||
material[row][column] += mat * HighLow{point.weight};
|
||||
}
|
||||
if (i == k)
|
||||
geometric[row][column] += geo * HighLow{point.weight};
|
||||
geometric[row][column] +=
|
||||
use_packets ? weighted_geo[b] : geo * HighLow{point.weight};
|
||||
}
|
||||
}
|
||||
if (use_packets)
|
||||
|
||||
@@ -23,6 +23,30 @@ struct MaterialPacketInput {
|
||||
PairPacket accumulated;
|
||||
PacketPair weight;
|
||||
};
|
||||
/// @brief One owned trial point/a and four independent b gradients, raw only.
|
||||
struct GeometryPacketInput {
|
||||
PacketPair gradient_a[3];
|
||||
PacketPair gradient_stress[3][3];
|
||||
PairPacket gradient_b[3];
|
||||
PacketPair weight;
|
||||
};
|
||||
/// @brief Independent raw contractions; accumulator entries remain caller-owned.
|
||||
struct GeometryPacketOutput {
|
||||
PairPacket gradient_dot;
|
||||
PairPacket weighted_geometric;
|
||||
};
|
||||
using GeometryPacketEvaluator = void (*)(const GeometryPacketInput&,
|
||||
GeometryPacketOutput*);
|
||||
/// @brief Baseline checked zero-start Dot3/ordered j,l Geo9 and weight products.
|
||||
/// @note Lane order is 0..3. Invalid pairs remain invalid for original gates.
|
||||
void EvaluateGeometryPacketScalar(const GeometryPacketInput& input,
|
||||
GeometryPacketOutput* output);
|
||||
/// @brief Evaluates independent b lanes without horizontal reduction.
|
||||
/// @pre The material selector has admitted CPU/OS/current masked FP state.
|
||||
/// @note Keeps original zero-start, product association and j,l Add order.
|
||||
void EvaluateGeometryPacketAvx2(const GeometryPacketInput& input,
|
||||
GeometryPacketOutput* output);
|
||||
|
||||
using MaterialPacketEvaluator = void (*)(const MaterialPacketInput&, PairPacket*);
|
||||
|
||||
/// @brief Evaluates each lane's original checked material graph in order 0..3.
|
||||
|
||||
@@ -111,6 +111,10 @@ VectorPair Load(const PairPacket& value) {
|
||||
_mm256_setr_pd(value.values[0].low, value.values[1].low,
|
||||
value.values[2].low, value.values[3].low)};
|
||||
}
|
||||
/// @brief Replicates one raw first product without changing either component.
|
||||
VectorPair Splat(PacketPair value) {
|
||||
return {_mm256_set1_pd(value.high), _mm256_set1_pd(value.low)};
|
||||
}
|
||||
/// @brief Stores raw high/low fields without rounding or lane reduction.
|
||||
PairPacket Store(VectorPair value) {
|
||||
double high[4];
|
||||
@@ -251,6 +255,20 @@ void EvaluateMaterialPacketAvx2(const MaterialPacketInput& input,
|
||||
*output = Store(Add(Load(input.accumulated),
|
||||
Multiply(material, Load(weight)), true));
|
||||
}
|
||||
void EvaluateGeometryPacketAvx2(const GeometryPacketInput& input,
|
||||
GeometryPacketOutput* output) {
|
||||
VectorPair dot{_mm256_setzero_pd(), _mm256_setzero_pd()};
|
||||
for (std::size_t i = 0; i < 3; ++i)
|
||||
dot = Add(dot, Multiply(Splat(input.gradient_a[i]),
|
||||
Load(input.gradient_b[i])), true);
|
||||
VectorPair geo{_mm256_setzero_pd(), _mm256_setzero_pd()};
|
||||
for (std::size_t j = 0; j < 3; ++j)
|
||||
for (std::size_t l = 0; l < 3; ++l)
|
||||
geo = Add(geo, Multiply(Splat(input.gradient_stress[j][l]),
|
||||
Load(input.gradient_b[l])), true);
|
||||
output->gradient_dot = Store(dot);
|
||||
output->weighted_geometric = Store(Multiply(geo, Splat(input.weight)));
|
||||
}
|
||||
} // namespace fesa::hex8_internal
|
||||
#else
|
||||
namespace fesa::hex8_internal {
|
||||
@@ -258,5 +276,9 @@ void EvaluateMaterialPacketAvx2(const MaterialPacketInput& input,
|
||||
PairPacket* output) {
|
||||
EvaluateMaterialPacketScalar(input, output);
|
||||
}
|
||||
void EvaluateGeometryPacketAvx2(const GeometryPacketInput& input,
|
||||
GeometryPacketOutput* output) {
|
||||
EvaluateGeometryPacketScalar(input, output);
|
||||
}
|
||||
} // namespace fesa::hex8_internal
|
||||
#endif
|
||||
|
||||
@@ -42,6 +42,27 @@ void ScalarPacketTwoProduct(const double left[4], const double right[4],
|
||||
output->values[lane] = {value.high, value.low};
|
||||
}
|
||||
}
|
||||
FESA_HEX8_BASELINE_NOINLINE
|
||||
void EvaluateGeometryPacketScalar(const GeometryPacketInput& input,
|
||||
GeometryPacketOutput* output) {
|
||||
const auto value = [](PacketPair pair) {
|
||||
return math_internal::HighLow{pair.high, pair.low};
|
||||
};
|
||||
for (std::size_t lane = 0; lane < 4; ++lane) {
|
||||
math_internal::HighLow dot;
|
||||
for (std::size_t i = 0; i < 3; ++i)
|
||||
dot = AddPair(dot, MultiplyPair(value(input.gradient_a[i]),
|
||||
value(input.gradient_b[i].values[lane])));
|
||||
math_internal::HighLow geo;
|
||||
for (std::size_t j = 0; j < 3; ++j)
|
||||
for (std::size_t l = 0; l < 3; ++l)
|
||||
geo = AddPair(geo, MultiplyPair(value(input.gradient_stress[j][l]),
|
||||
value(input.gradient_b[l].values[lane])));
|
||||
const auto weighted = MultiplyPair(geo, value(input.weight));
|
||||
output->gradient_dot.values[lane] = {dot.high, dot.low};
|
||||
output->weighted_geometric.values[lane] = {weighted.high, weighted.low};
|
||||
}
|
||||
}
|
||||
#undef FESA_HEX8_BASELINE_NOINLINE
|
||||
|
||||
bool SupportsMaterialPacketInstructions(std::uint32_t leaf1_ecx,
|
||||
|
||||
Reference in New Issue
Block a user