fix(hex8): isolate AVX2 packet code from baseline COMDATs
This commit is contained in:
@@ -9,18 +9,70 @@
|
||||
#endif
|
||||
|
||||
namespace fesa::hex8_internal {
|
||||
#if defined(_MSC_VER)
|
||||
#define FESA_HEX8_BASELINE_NOINLINE __declspec(noinline)
|
||||
#else
|
||||
#define FESA_HEX8_BASELINE_NOINLINE
|
||||
#endif
|
||||
FESA_HEX8_BASELINE_NOINLINE
|
||||
void ScalarPacketAdd(const PairPacket& left, const PairPacket& right,
|
||||
bool pair_shortcuts, PairPacket* output) {
|
||||
for (std::size_t lane = 0; lane < 4; ++lane) {
|
||||
const math_internal::HighLow a{left.values[lane].high, left.values[lane].low};
|
||||
const math_internal::HighLow b{right.values[lane].high, right.values[lane].low};
|
||||
const auto value = pair_shortcuts ? AddPair(a, b) : math_internal::Add(a, b);
|
||||
output->values[lane] = {value.high, value.low};
|
||||
}
|
||||
}
|
||||
FESA_HEX8_BASELINE_NOINLINE
|
||||
void ScalarPacketMultiply(const PairPacket& left, const PairPacket& right,
|
||||
PairPacket* output) {
|
||||
for (std::size_t lane = 0; lane < 4; ++lane) {
|
||||
const auto value = MultiplyPair(
|
||||
{left.values[lane].high, left.values[lane].low},
|
||||
{right.values[lane].high, right.values[lane].low});
|
||||
output->values[lane] = {value.high, value.low};
|
||||
}
|
||||
}
|
||||
FESA_HEX8_BASELINE_NOINLINE
|
||||
void ScalarPacketTwoProduct(const double left[4], const double right[4],
|
||||
PairPacket* output) {
|
||||
for (std::size_t lane = 0; lane < 4; ++lane) {
|
||||
const auto value = math_internal::TwoProduct(left[lane], right[lane]);
|
||||
output->values[lane] = {value.high, value.low};
|
||||
}
|
||||
}
|
||||
#undef FESA_HEX8_BASELINE_NOINLINE
|
||||
|
||||
bool SupportsMaterialPacketInstructions(std::uint32_t leaf1_ecx,
|
||||
std::uint64_t xcr0,
|
||||
std::uint32_t leaf7_ebx) {
|
||||
constexpr std::uint32_t required_leaf1 =
|
||||
(1U << 12) | (1U << 26) | (1U << 27) | (1U << 28);
|
||||
// MSVC /arch:AVX2 permits BMI codegen; ANDN is present in this kernel.
|
||||
constexpr std::uint32_t required_leaf7 =
|
||||
(1U << 3) | (1U << 5) | (1U << 8);
|
||||
return (leaf1_ecx & required_leaf1) == required_leaf1 &&
|
||||
(xcr0 & 6U) == 6U &&
|
||||
(leaf7_ebx & required_leaf7) == required_leaf7;
|
||||
}
|
||||
|
||||
void EvaluateMaterialPacketScalar(const MaterialPacketInput& input,
|
||||
PairPacket* output) {
|
||||
for (std::size_t lane = 0; lane < output->size(); ++lane) {
|
||||
const auto first = MultiplyPair(input.deviatoric_left[lane],
|
||||
input.deviatoric_right[lane]);
|
||||
const auto second = MultiplyPair(input.shear_left[lane],
|
||||
input.shear_right[lane]);
|
||||
const auto third = MultiplyPair(input.tensor_left[lane],
|
||||
input.gradient_dot[lane]);
|
||||
const auto value = [](PacketPair pair) {
|
||||
return math_internal::HighLow{pair.high, pair.low};
|
||||
};
|
||||
for (std::size_t lane = 0; lane < 4; ++lane) {
|
||||
const auto first = MultiplyPair(value(input.deviatoric_left.values[lane]),
|
||||
value(input.deviatoric_right.values[lane]));
|
||||
const auto second = MultiplyPair(value(input.shear_left.values[lane]),
|
||||
value(input.shear_right.values[lane]));
|
||||
const auto third = MultiplyPair(value(input.tensor_left.values[lane]),
|
||||
value(input.gradient_dot.values[lane]));
|
||||
const auto material = AddPair(AddPair(first, second), third);
|
||||
(*output)[lane] = AddPair(input.accumulated[lane],
|
||||
MultiplyPair(material, input.weight));
|
||||
const auto accumulated = AddPair(value(input.accumulated.values[lane]),
|
||||
MultiplyPair(material, value(input.weight)));
|
||||
output->values[lane] = {accumulated.high, accumulated.low};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,9 +85,11 @@ MaterialPacketEvaluator SelectMaterialPacketEvaluator() {
|
||||
__cpuidex(info, 1, 0);
|
||||
constexpr int required = (1 << 12) | (1 << 26) | (1 << 27) | (1 << 28);
|
||||
if ((info[2] & required) != required) return false;
|
||||
if ((_xgetbv(0) & 6U) != 6U) return false;
|
||||
const auto leaf1 = static_cast<std::uint32_t>(info[2]);
|
||||
const auto xcr0 = _xgetbv(0);
|
||||
__cpuidex(info, 7, 0);
|
||||
return (info[1] & (1 << 5)) != 0;
|
||||
return SupportsMaterialPacketInstructions(
|
||||
leaf1, xcr0, static_cast<std::uint32_t>(info[1]));
|
||||
}();
|
||||
if (!hardware_supported || !math_internal::ArithmeticEnvironmentSupported())
|
||||
return EvaluateMaterialPacketScalar;
|
||||
|
||||
Reference in New Issue
Block a user