Merge reviewed geometric packet optimization and FP fallback

2026-10-14 23:34:29 +09:00
parent b11e12a146 f81a0b6739
commit 9936994e49
6 changed files with 478 additions and 17 deletions
@@ -449,6 +449,296 @@ TEST(Hex8HighLow, PacketTailScreenRetainsExactBoundaryAndFallback) {
 }
 #endif

+#if defined(_M_X64) || defined(__SSE2__)
+fesa::hex8_internal::GeometryPacketOutput FrozenGeometryPacket(
+    const fesa::hex8_internal::GeometryPacketInput& input) {
+  const auto pair = [](fesa::hex8_internal::PacketPair value) {
+    return fesa::high_low_original::HighLow{value.high, value.low};
+  };
+  fesa::hex8_internal::GeometryPacketOutput output{};
+  for (std::size_t lane = 0; lane < 4; ++lane) {
+    fesa::high_low_original::HighLow dot;
+    for (std::size_t i = 0; i < 3; ++i)
+      dot = fesa::high_low_original::Add(dot,
+          fesa::high_low_original::Multiply(pair(input.gradient_a[i]),
+              pair(input.gradient_b[i].values[lane])));
+    fesa::high_low_original::HighLow geo;
+    for (std::size_t j = 0; j < 3; ++j)
+      for (std::size_t l = 0; l < 3; ++l)
+        geo = fesa::high_low_original::Add(geo,
+            fesa::high_low_original::Multiply(pair(input.gradient_stress[j][l]),
+                pair(input.gradient_b[l].values[lane])));
+    const auto weighted = fesa::high_low_original::Multiply(geo, pair(input.weight));
+    output.gradient_dot.values[lane] = {dot.high, dot.low};
+    output.weighted_geometric.values[lane] = {weighted.high, weighted.low};
+  }
+  return output;
+}
+
+fesa::hex8_internal::GeometryPacketInput GeometryDyadicFixture() {
+  fesa::hex8_internal::GeometryPacketInput input{};
+  input.weight = {0.125, 0};
+  for (std::size_t j = 0; j < 3; ++j) {
+    input.gradient_a[j] = {static_cast<double>(j + 1) * 0.25, 0};
+    for (std::size_t l = 0; l < 3; ++l) {
+      // Deliberately asymmetric first-products and distinct b/component mapping.
+      input.gradient_stress[j][l] =
+          {static_cast<double>(1 + 3 * j + l) * 0.125, 0};
+      for (std::size_t lane = 0; lane < 4; ++lane)
+        input.gradient_b[l].values[lane] =
+            {static_cast<double>(1 + 4 * l + lane) * 0.0625, 0};
+    }
+  }
+  return input;
+}
+
+// Original observation repeats geo*weight for three independent accumulators.
+// Selected observation reuses the weighted pair but retains each checked Add.
+void ExpectGeometryPacketBitsAndFlags(
+    const fesa::hex8_internal::GeometryPacketInput& input, bool prior_status) {
+  SCOPED_TRACE(prior_status);
+  std::fenv_t original;
+  ASSERT_EQ(std::fegetenv(&original), 0);
+  const auto original_csr = _mm_getcsr();
+  struct Observation {
+    fesa::hex8_internal::GeometryPacketOutput output;
+    fesa::hex8_internal::PairPacket accumulators[3];
+    int flags;
+    unsigned int status;
+  };
+  const auto current_pair = [](fesa::hex8_internal::PacketPair value) {
+    return HighLow{value.high, value.low};
+  };
+  const auto observe = [&](bool original_order) {
+    EXPECT_EQ(std::fesetenv(&original), 0);
+    _mm_setcsr(original_csr & ~0x3fU);
+    EXPECT_EQ(std::feclearexcept(FE_ALL_EXCEPT), 0);
+    if (prior_status) {
+      EXPECT_EQ(std::feraiseexcept(FE_INEXACT), 0);
+      _mm_setcsr(_mm_getcsr() | 0x02U);
+    }
+    Observation value{};
+    if (!original_order)
+      fesa::hex8_internal::EvaluateGeometryPacketAvx2(input, &value.output);
+    for (std::size_t lane = 0; lane < 4; ++lane) {
+      HighLow geo;
+      if (original_order) {
+        HighLow dot;
+        for (std::size_t i = 0; i < 3; ++i)
+          dot = fesa::hex8_internal::AddPair(dot,
+              fesa::hex8_internal::MultiplyPair(current_pair(input.gradient_a[i]),
+                  current_pair(input.gradient_b[i].values[lane])));
+        for (std::size_t j = 0; j < 3; ++j)
+          for (std::size_t l = 0; l < 3; ++l)
+            geo = fesa::hex8_internal::AddPair(geo,
+                fesa::hex8_internal::MultiplyPair(
+                    current_pair(input.gradient_stress[j][l]),
+                    current_pair(input.gradient_b[l].values[lane])));
+        value.output.gradient_dot.values[lane] = {dot.high, dot.low};
+      }
+      const HighLow accumulators[3]{{},
+          {static_cast<double>(lane + 1), std::ldexp(1.0, -54)}, {-1, -0.0}};
+      for (std::size_t component = 0; component < 3; ++component) {
+        const auto weighted = original_order
+            ? fesa::hex8_internal::MultiplyPair(geo, current_pair(input.weight))
+            : current_pair(value.output.weighted_geometric.values[lane]);
+        if (original_order)
+          value.output.weighted_geometric.values[lane] =
+              {weighted.high, weighted.low};
+        const auto accumulated = fesa::hex8_internal::AddPair(
+            accumulators[component], weighted);
+        value.accumulators[component].values[lane] =
+            {accumulated.high, accumulated.low};
+      }
+    }
+    value.flags = std::fetestexcept(FE_ALL_EXCEPT);
+    value.status = _mm_getcsr() & 0x3fU;
+    return value;
+  };
+  const auto baseline = observe(true);
+  const auto selected = observe(false);
+  fesa::hex8_internal::GeometryPacketOutput scalar;
+  fesa::hex8_internal::EvaluateGeometryPacketScalar(input, &scalar);
+  const auto expected = FrozenGeometryPacket(input);
+  EXPECT_EQ(std::fesetenv(&original), 0);
+  _mm_setcsr(original_csr);
+  EXPECT_EQ(selected.flags, baseline.flags);
+  EXPECT_EQ(selected.status, baseline.status);
+  const auto same_bits = [](const fesa::hex8_internal::PairPacket& a,
+                            const fesa::hex8_internal::PairPacket& b) {
+    for (std::size_t lane = 0; lane < 4; ++lane) {
+      EXPECT_EQ(PairBits(a.values[lane].high), PairBits(b.values[lane].high));
+      EXPECT_EQ(PairBits(a.values[lane].low), PairBits(b.values[lane].low));
+    }
+  };
+  same_bits(selected.output.gradient_dot, expected.gradient_dot);
+  same_bits(selected.output.weighted_geometric, expected.weighted_geometric);
+  same_bits(scalar.gradient_dot, expected.gradient_dot);
+  same_bits(scalar.weighted_geometric, expected.weighted_geometric);
+  for (std::size_t component = 0; component < 3; ++component)
+    same_bits(selected.accumulators[component], baseline.accumulators[component]);
+}
+
+TEST(Hex8HighLow, GeometryPacketPreservesAsymmetricLanesAndUniformShapes) {
+  if (fesa::hex8_internal::SelectMaterialPacketEvaluator() ==
+      fesa::hex8_internal::EvaluateMaterialPacketScalar)
+    GTEST_SKIP() << "Geometry AVX2 masked environment is not selected.";
+  RecordProperty("private_geometry_packet_backend", "avx2");
+  static_assert(std::is_trivial_v<fesa::hex8_internal::GeometryPacketInput>);
+  static_assert(std::is_trivial_v<fesa::hex8_internal::GeometryPacketOutput>);
+  std::size_t checks = 0;
+  for (int shape = 0; shape < 4; ++shape) {
+    auto input = GeometryDyadicFixture();
+    for (std::size_t j = 0; j < 3; ++j) {
+      if ((shape & 1) != 0) {
+        input.gradient_a[j].low = std::ldexp(input.gradient_a[j].high, -55);
+        for (std::size_t l = 0; l < 3; ++l)
+          input.gradient_stress[j][l].low =
+              std::ldexp(input.gradient_stress[j][l].high, -55);
+      }
+      if ((shape & 2) != 0)
+        for (std::size_t lane = 0; lane < 4; ++lane)
+          input.gradient_b[j].values[lane].low =
+              std::ldexp(input.gradient_b[j].values[lane].high, -55);
+    }
+    ExpectGeometryPacketBitsAndFlags(input, false);
+    ExpectGeometryPacketBitsAndFlags(input, true);
+    checks += 2;
+  }
+  RecordProperty("avx2_geometry_uniform_bit_flag_checks", static_cast<int>(checks));
+}
+
+TEST(Hex8HighLow, GeometryPacketRetainsSpecialLanesAndExactTailBoundary) {
+  if (fesa::hex8_internal::SelectMaterialPacketEvaluator() ==
+      fesa::hex8_internal::EvaluateMaterialPacketScalar)
+    GTEST_SKIP() << "Geometry AVX2 masked environment is not selected.";
+  using fesa::hex8_internal::PacketPair;
+  const double tiny = std::numeric_limits<double>::denorm_min();
+  const double maximum = std::numeric_limits<double>::max();
+  const std::vector<PacketPair> special{{0, 0}, {-0.0, 0}, {1, -0.0}, {1, 1},
+      {tiny, 0}, {maximum, maximum},
+      {std::numeric_limits<double>::signaling_NaN(), 0},
+      {0, std::numeric_limits<double>::signaling_NaN()}};
+  std::size_t checks = 0;
+  for (const auto pair : special)
+    for (std::size_t lane = 0; lane < 4; ++lane) {
+      auto input = GeometryDyadicFixture();
+      input.gradient_b[1].values[lane] = pair;
+      ExpectGeometryPacketBitsAndFlags(input, false);
+      ExpectGeometryPacketBitsAndFlags(input, true);
+      checks += 2;
+    }
+  const auto odd = [](int exponent) {
+    return std::nextafter(std::ldexp(1.0, exponent),
+                          std::numeric_limits<double>::infinity());
+  };
+  for (int exponent : {-470, -471})
+    for (std::size_t lane = 0; lane < 4; ++lane) {
+      auto input = GeometryDyadicFixture();
+      input.gradient_stress[0][0] = {odd(-500), 0};
+      input.gradient_b[0].values[lane] = {odd(exponent), 0};
+      ExpectGeometryPacketBitsAndFlags(input, false);
+      checks++;
+    }
+  RecordProperty("avx2_geometry_special_tail_checks", static_cast<int>(checks));
+}
+
+TEST(Hex8HighLow, GeometryPacketPreservesOrderedOverflowAndPreparedInvalid) {
+  if (fesa::hex8_internal::SelectMaterialPacketEvaluator() ==
+      fesa::hex8_internal::EvaluateMaterialPacketScalar)
+    GTEST_SKIP() << "Geometry AVX2 masked environment is not selected.";
+  const double maximum = std::numeric_limits<double>::max();
+  std::size_t checks = 0;
+  for (bool overflow_order : {false, true}) {
+    SCOPED_TRACE(overflow_order);
+    fesa::hex8_internal::GeometryPacketInput input{};
+    input.weight = {0.5, 0};
+    for (std::size_t l = 0; l < 3; ++l)
+      for (std::size_t lane = 0; lane < 4; ++lane)
+        input.gradient_b[l].values[lane] = {1, 0};
+    input.gradient_stress[0][0] = {maximum, 0};
+    input.gradient_stress[0][1] = {overflow_order ? maximum : -maximum, 0};
+    input.gradient_stress[0][2] = {overflow_order ? -maximum : maximum, 0};
+    ExpectGeometryPacketBitsAndFlags(input, false);
+    ExpectGeometryPacketBitsAndFlags(input, true);
+    checks += 2;
+  }
+  auto input = GeometryDyadicFixture();
+  const auto invalid_first = fesa::high_low_original::Multiply(
+      {std::ldexp(1.0, -600), 0}, {std::ldexp(1.0, -500), 0});
+  input.gradient_stress[0][0] = {invalid_first.high, invalid_first.low};
+  for (std::size_t lane = 0; lane < 4; ++lane)
+    input.gradient_b[0].values[lane] = {std::ldexp(1.0, 600), 0};
+  ExpectGeometryPacketBitsAndFlags(input, false);
+  ExpectGeometryPacketBitsAndFlags(input, true);
+  checks += 2;
+  RecordProperty("avx2_geometry_ordered_invalid_checks", static_cast<int>(checks));
+}
+
+TEST(Hex8HighLow, GeometryPacketSumFallbackPreservesNearMaxAndSubnormalStatus) {
+  if (fesa::hex8_internal::SelectMaterialPacketEvaluator() ==
+      fesa::hex8_internal::EvaluateMaterialPacketScalar)
+    GTEST_SKIP() << "Geometry AVX2 masked environment is not selected.";
+  const double maximum = std::numeric_limits<double>::max();
+  const double normal = std::ldexp(1.0, -1022);
+  const double tiny = std::ldexp(1.0, -1074);
+  const double quarter = std::ldexp(1.0, 969);
+  std::size_t checks = 0;
+  for (double tail : {0.0, normal, tiny})
+    for (std::size_t bad = 0; bad < 4; ++bad) {
+      SCOPED_TRACE(tail);
+      SCOPED_TRACE(bad);
+      fesa::hex8_internal::GeometryPacketInput input{};
+      input.weight = {0.5, 0};
+      // Both pairs are normalized. Head sum is finite; middle contributes
+      // 2^970 and the joined sum overflows. Other lanes retain finite values.
+      input.gradient_stress[0][0] = {maximum, quarter};
+      input.gradient_stress[0][1] = {quarter, tail};
+      for (std::size_t lane = 0; lane < 4; ++lane) {
+        const double scale = lane == bad ? 1.0 : 0.5;
+        input.gradient_b[0].values[lane] = {scale, 0};
+        input.gradient_b[1].values[lane] = {scale, 0};
+        input.gradient_b[2].values[lane] = {1, 0};
+      }
+      ExpectGeometryPacketBitsAndFlags(input, false);
+      ExpectGeometryPacketBitsAndFlags(input, true);
+      checks += 2;
+    }
+  fesa::hex8_internal::GeometryPacketInput finite{};
+  finite.weight = {1, 0};
+  finite.gradient_stress[0][0] = {normal, 0};
+  finite.gradient_stress[0][1] = {normal + tiny, 0};
+  for (std::size_t lane = 0; lane < 4; ++lane) {
+    finite.gradient_b[0].values[lane] = {1, 0};
+    finite.gradient_b[1].values[lane] = {1, 0};
+  }
+  // TwoSum(N,N+d) has finite normal inputs and a subnormal error d.
+  ExpectGeometryPacketBitsAndFlags(finite, false);
+  ExpectGeometryPacketBitsAndFlags(finite, true);
+  checks += 2;
+  // Mixed prefix: finite subnormal-error lanes run before another lane's head
+  // overflow. The bridge must preserve the whole packet's status union.
+  for (std::size_t bad = 0; bad < 4; ++bad) {
+    SCOPED_TRACE(bad);
+    fesa::hex8_internal::GeometryPacketInput mixed{};
+    mixed.weight = {1, 0};
+    mixed.gradient_stress[0][2] = {normal, 0};
+    mixed.gradient_stress[1][2] = {normal + tiny, 0};
+    mixed.gradient_stress[2][0] = {maximum, 0};
+    mixed.gradient_stress[2][1] = {maximum, 0};
+    for (std::size_t lane = 0; lane < 4; ++lane) {
+      mixed.gradient_b[0].values[lane] = {lane == bad ? 1.0 : 0.0, 0};
+      mixed.gradient_b[1].values[lane] = {lane == bad ? 1.0 : 0.0, 0};
+      mixed.gradient_b[2].values[lane] = {lane == bad ? 0.0 : 1.0, 0};
+    }
+    ExpectGeometryPacketBitsAndFlags(mixed, false);
+    ExpectGeometryPacketBitsAndFlags(mixed, true);
+    checks += 2;
+  }
+  RecordProperty("avx2_geometry_sum_fallback_checks", static_cast<int>(checks));
+}
+#endif
+
 TEST(Hex8HighLow, PacketBackendSelectionRecoversAfterEnvironmentRestoration) {
  const auto initial = fesa::hex8_internal::SelectMaterialPacketEvaluator();
  const int rounding = std::fegetround();