FESADev/src/fesa/elements/hex8_pair_simd_dispatch.cpp

#include "hex8_pair_simd.h"

#include "hex8_pair_arithmetic.h"

#if defined(FESA_HEX8_PAIR_AVX2) && defined(_MSC_VER) && defined(_M_X64)
#include <float.h>
#include <intrin.h>
#include <xmmintrin.h>
#endif

namespace fesa::hex8_internal {
#if defined(_MSC_VER)
#define FESA_HEX8_BASELINE_NOINLINE __declspec(noinline)
#else
#define FESA_HEX8_BASELINE_NOINLINE
#endif
FESA_HEX8_BASELINE_NOINLINE
void ScalarPacketAdd(const PairPacket& left, const PairPacket& right,
                     bool pair_shortcuts, PairPacket* output) {
  for (std::size_t lane = 0; lane < 4; ++lane) {
    const math_internal::HighLow a{left.values[lane].high, left.values[lane].low};
    const math_internal::HighLow b{right.values[lane].high, right.values[lane].low};
    const auto value = pair_shortcuts ? AddPair(a, b) : math_internal::Add(a, b);
    output->values[lane] = {value.high, value.low};
  }
}
FESA_HEX8_BASELINE_NOINLINE
void ScalarPacketMultiply(const PairPacket& left, const PairPacket& right,
                          PairPacket* output) {
  for (std::size_t lane = 0; lane < 4; ++lane) {
    const auto value = MultiplyPair(
        {left.values[lane].high, left.values[lane].low},
        {right.values[lane].high, right.values[lane].low});
    output->values[lane] = {value.high, value.low};
  }
}
FESA_HEX8_BASELINE_NOINLINE
void ScalarPacketTwoProduct(const double left[4], const double right[4],
                            PairPacket* output) {
  for (std::size_t lane = 0; lane < 4; ++lane) {
    const auto value = math_internal::TwoProduct(left[lane], right[lane]);
    output->values[lane] = {value.high, value.low};
  }
}
FESA_HEX8_BASELINE_NOINLINE
void EvaluateGeometryPacketScalar(const GeometryPacketInput& input,
                                  GeometryPacketOutput* output) {
  const auto value = [](PacketPair pair) {
    return math_internal::HighLow{pair.high, pair.low};
  };
  for (std::size_t lane = 0; lane < 4; ++lane) {
    math_internal::HighLow dot;
    for (std::size_t i = 0; i < 3; ++i)
      dot = AddPair(dot, MultiplyPair(value(input.gradient_a[i]),
                                      value(input.gradient_b[i].values[lane])));
    math_internal::HighLow geo;
    for (std::size_t j = 0; j < 3; ++j)
      for (std::size_t l = 0; l < 3; ++l)
        geo = AddPair(geo, MultiplyPair(value(input.gradient_stress[j][l]),
                                        value(input.gradient_b[l].values[lane])));
    const auto weighted = MultiplyPair(geo, value(input.weight));
    output->gradient_dot.values[lane] = {dot.high, dot.low};
    output->weighted_geometric.values[lane] = {weighted.high, weighted.low};
  }
}
#undef FESA_HEX8_BASELINE_NOINLINE

bool SupportsMaterialPacketInstructions(std::uint32_t leaf1_ecx,
                                        std::uint64_t xcr0,
                                        std::uint32_t leaf7_ebx) {
  constexpr std::uint32_t required_leaf1 =
      (1U << 12) | (1U << 26) | (1U << 27) | (1U << 28);
  // MSVC /arch:AVX2 permits BMI codegen; ANDN is present in this kernel.
  constexpr std::uint32_t required_leaf7 =
      (1U << 3) | (1U << 5) | (1U << 8);
  return (leaf1_ecx & required_leaf1) == required_leaf1 &&
         (xcr0 & 6U) == 6U &&
         (leaf7_ebx & required_leaf7) == required_leaf7;
}

void EvaluateMaterialPacketScalar(const MaterialPacketInput& input,
                                  PairPacket* output) {
  const auto value = [](PacketPair pair) {
    return math_internal::HighLow{pair.high, pair.low};
  };
  for (std::size_t lane = 0; lane < 4; ++lane) {
    const auto first = MultiplyPair(value(input.deviatoric_left.values[lane]),
                                    value(input.deviatoric_right.values[lane]));
    const auto second = MultiplyPair(value(input.shear_left.values[lane]),
                                     value(input.shear_right.values[lane]));
    const auto third = MultiplyPair(value(input.tensor_left.values[lane]),
                                    value(input.gradient_dot.values[lane]));
    const auto material = AddPair(AddPair(first, second), third);
    const auto accumulated = AddPair(value(input.accumulated.values[lane]),
                                     MultiplyPair(material, value(input.weight)));
    output->values[lane] = {accumulated.high, accumulated.low};
  }
}

MaterialPacketEvaluator SelectMaterialPacketEvaluator() {
#if defined(FESA_HEX8_PAIR_AVX2) && defined(_MSC_VER) && defined(_M_X64)
  static const bool hardware_supported = [] {
    int info[4]{};
    __cpuid(info, 0);
    if (info[0] < 7) return false;
    __cpuidex(info, 1, 0);
    constexpr int required = (1 << 12) | (1 << 26) | (1 << 27) | (1 << 28);
    if ((info[2] & required) != required) return false;
    const auto leaf1 = static_cast<std::uint32_t>(info[2]);
    const auto xcr0 = _xgetbv(0);
    __cpuidex(info, 7, 0);
    return SupportsMaterialPacketInstructions(
        leaf1, xcr0, static_cast<std::uint32_t>(info[1]));
  }();
  if (!hardware_supported || !math_internal::ArithmeticEnvironmentSupported())
    return EvaluateMaterialPacketScalar;
  if ((_mm_getcsr() & 0x1f80U) != 0x1f80U)
    return EvaluateMaterialPacketScalar;
  unsigned int control = 0;
  if (_controlfp_s(&control, 0, 0) != 0 ||
      (control & _MCW_EM) != _MCW_EM)
    return EvaluateMaterialPacketScalar;
  return EvaluateMaterialPacketAvx2;
#else
  return EvaluateMaterialPacketScalar;
#endif
}
}  // namespace fesa::hex8_internal