fix(hex8): isolate AVX2 packet code from baseline COMDATs

This commit is contained in:
KOKO\Mimi
2026-10-14 21:10:27 +09:00
parent c0e23503e8
commit 446d3fa059
5 changed files with 344 additions and 129 deletions
+65 -11
View File
@@ -9,18 +9,70 @@
#endif
namespace fesa::hex8_internal {
#if defined(_MSC_VER)
#define FESA_HEX8_BASELINE_NOINLINE __declspec(noinline)
#else
#define FESA_HEX8_BASELINE_NOINLINE
#endif
FESA_HEX8_BASELINE_NOINLINE
void ScalarPacketAdd(const PairPacket& left, const PairPacket& right,
bool pair_shortcuts, PairPacket* output) {
for (std::size_t lane = 0; lane < 4; ++lane) {
const math_internal::HighLow a{left.values[lane].high, left.values[lane].low};
const math_internal::HighLow b{right.values[lane].high, right.values[lane].low};
const auto value = pair_shortcuts ? AddPair(a, b) : math_internal::Add(a, b);
output->values[lane] = {value.high, value.low};
}
}
FESA_HEX8_BASELINE_NOINLINE
void ScalarPacketMultiply(const PairPacket& left, const PairPacket& right,
PairPacket* output) {
for (std::size_t lane = 0; lane < 4; ++lane) {
const auto value = MultiplyPair(
{left.values[lane].high, left.values[lane].low},
{right.values[lane].high, right.values[lane].low});
output->values[lane] = {value.high, value.low};
}
}
FESA_HEX8_BASELINE_NOINLINE
void ScalarPacketTwoProduct(const double left[4], const double right[4],
PairPacket* output) {
for (std::size_t lane = 0; lane < 4; ++lane) {
const auto value = math_internal::TwoProduct(left[lane], right[lane]);
output->values[lane] = {value.high, value.low};
}
}
#undef FESA_HEX8_BASELINE_NOINLINE
bool SupportsMaterialPacketInstructions(std::uint32_t leaf1_ecx,
std::uint64_t xcr0,
std::uint32_t leaf7_ebx) {
constexpr std::uint32_t required_leaf1 =
(1U << 12) | (1U << 26) | (1U << 27) | (1U << 28);
// MSVC /arch:AVX2 permits BMI codegen; ANDN is present in this kernel.
constexpr std::uint32_t required_leaf7 =
(1U << 3) | (1U << 5) | (1U << 8);
return (leaf1_ecx & required_leaf1) == required_leaf1 &&
(xcr0 & 6U) == 6U &&
(leaf7_ebx & required_leaf7) == required_leaf7;
}
void EvaluateMaterialPacketScalar(const MaterialPacketInput& input,
PairPacket* output) {
for (std::size_t lane = 0; lane < output->size(); ++lane) {
const auto first = MultiplyPair(input.deviatoric_left[lane],
input.deviatoric_right[lane]);
const auto second = MultiplyPair(input.shear_left[lane],
input.shear_right[lane]);
const auto third = MultiplyPair(input.tensor_left[lane],
input.gradient_dot[lane]);
const auto value = [](PacketPair pair) {
return math_internal::HighLow{pair.high, pair.low};
};
for (std::size_t lane = 0; lane < 4; ++lane) {
const auto first = MultiplyPair(value(input.deviatoric_left.values[lane]),
value(input.deviatoric_right.values[lane]));
const auto second = MultiplyPair(value(input.shear_left.values[lane]),
value(input.shear_right.values[lane]));
const auto third = MultiplyPair(value(input.tensor_left.values[lane]),
value(input.gradient_dot.values[lane]));
const auto material = AddPair(AddPair(first, second), third);
(*output)[lane] = AddPair(input.accumulated[lane],
MultiplyPair(material, input.weight));
const auto accumulated = AddPair(value(input.accumulated.values[lane]),
MultiplyPair(material, value(input.weight)));
output->values[lane] = {accumulated.high, accumulated.low};
}
}
@@ -33,9 +85,11 @@ MaterialPacketEvaluator SelectMaterialPacketEvaluator() {
__cpuidex(info, 1, 0);
constexpr int required = (1 << 12) | (1 << 26) | (1 << 27) | (1 << 28);
if ((info[2] & required) != required) return false;
if ((_xgetbv(0) & 6U) != 6U) return false;
const auto leaf1 = static_cast<std::uint32_t>(info[2]);
const auto xcr0 = _xgetbv(0);
__cpuidex(info, 7, 0);
return (info[1] & (1 << 5)) != 0;
return SupportsMaterialPacketInstructions(
leaf1, xcr0, static_cast<std::uint32_t>(info[1]));
}();
if (!hardware_supported || !math_internal::ArithmeticEnvironmentSupported())
return EvaluateMaterialPacketScalar;