perf: reuse owned shell Jet Hessians in accumulation
This commit is contained in:
@@ -630,6 +630,176 @@ __declspec(noinline) double Consume(const Jet& value) {
|
||||
return result + value.constant_derivatives + value.second.IsImplicitZero();
|
||||
}
|
||||
|
||||
// T3 evidence is separate from both immutable earlier snapshot formats. These
|
||||
// calls also exercise the original overload when built against the T1 source.
|
||||
std::string AccumulationSnapshots(bool reassociate) {
|
||||
const std::array<double, 12> boundary{
|
||||
0.0, -0.0, 1.0, -1.0, 0.1, -0.3, 1e16,
|
||||
std::numeric_limits<double>::denorm_min(),
|
||||
std::numeric_limits<double>::max(),
|
||||
std::numeric_limits<double>::infinity(),
|
||||
std::numeric_limits<double>::quiet_NaN(),
|
||||
std::numeric_limits<double>::signaling_NaN()};
|
||||
std::string out;
|
||||
const std::array<int, 4> rounding{FE_TONEAREST, FE_DOWNWARD, FE_UPWARD,
|
||||
FE_TOWARDZERO};
|
||||
for (const int mode : rounding) {
|
||||
for (const unsigned flush : {0u, 0x8040u}) {
|
||||
for (const unsigned sticky : {0u, 0x21u}) {
|
||||
for (std::size_t kind = 0; kind < boundary.size(); ++kind) {
|
||||
ResetFp();
|
||||
Jet left = Variable(boundary[kind], kind, -0.25);
|
||||
Jet right = Variable(0.5, (kind + 1) % 24, 0.125);
|
||||
if (kind % 3 == 0) left = Constant(boundary[kind]);
|
||||
if (kind % 3 == 1) {
|
||||
left.second.Data()[25] = boundary[kind];
|
||||
right.second.Data()[48] = -0.5;
|
||||
}
|
||||
left.mixed_first[3] = boundary[kind];
|
||||
left.mixed_second[4] = -0.0;
|
||||
const Jet saved_left = left;
|
||||
const Jet saved_right = right;
|
||||
const auto before_left = Snapshot("input", left, 0, 0, 0);
|
||||
const auto before_right = Snapshot("input", right, 0, 0, 0);
|
||||
const auto environment = [&] {
|
||||
ResetFp();
|
||||
std::fesetround(mode);
|
||||
_mm_setcsr((_mm_getcsr() & ~0x8040u) | flush | sticky);
|
||||
};
|
||||
const auto observe = [&](const char* name, auto expression) {
|
||||
environment();
|
||||
const auto before = FpState();
|
||||
const Jet result = expression();
|
||||
const int exceptions = std::fetestexcept(FE_ALL_EXCEPT);
|
||||
const unsigned status = _statusfp(), mxcsr = _mm_getcsr();
|
||||
out += std::to_string(mode) + ":" + std::to_string(flush) + ":" +
|
||||
std::to_string(sticky) + ":" + std::to_string(kind) + " " +
|
||||
before + " " + FpState() + " " +
|
||||
Snapshot(name, result, exceptions, status, mxcsr);
|
||||
};
|
||||
const JetVector3 a{left, right, Constant(-1)};
|
||||
const JetVector3 b{right, Constant(1), left};
|
||||
observe("dot", [&] {
|
||||
if (reassociate)
|
||||
return a[0] * b[0] + (a[1] * b[1] + a[2] * b[2]);
|
||||
return Dot(a, b);
|
||||
});
|
||||
for (std::size_t component = 0; component < 3; ++component)
|
||||
observe("cross", [&] { return Cross(a, b)[component]; });
|
||||
const JetMatrix3 matrix{{a, b, a}};
|
||||
const JetMatrix3 other{{b, a, b}};
|
||||
// Every matrix/vector result is consumed over the whole fixture;
|
||||
// boundary groups rotate the selected entry to keep evidence bounded.
|
||||
observe("matrix", [&] {
|
||||
return Multiply(matrix, other)[kind % 3][(kind / 3) % 3];
|
||||
});
|
||||
observe("matrix-vector", [&] {
|
||||
return Multiply(matrix, b)[kind % 3];
|
||||
});
|
||||
observe("frame", [&] { return FrameComponent(a, matrix, b); });
|
||||
observe("fixed-frame", [&] {
|
||||
return FrameComponent(ConstantVector(Vector3{0.5, -1, 0}), matrix,
|
||||
ConstantVector(Vector3{-0.0, 0.25, 1}));
|
||||
});
|
||||
observe("energy-accumulation", [&] {
|
||||
Jet energy{};
|
||||
for (std::size_t i = 0; i < 3; ++i)
|
||||
energy = std::move(energy) + a[i] * b[i];
|
||||
return energy;
|
||||
});
|
||||
observe("shared-rvalue", [&] {
|
||||
Jet value = left;
|
||||
return std::move(value) + right;
|
||||
});
|
||||
observe("same-object-rvalue", [&] {
|
||||
Jet value = right;
|
||||
return std::move(value) + value;
|
||||
});
|
||||
observe("unique-rvalue", [&] {
|
||||
Jet value = left * right;
|
||||
return std::move(value) + right;
|
||||
});
|
||||
observe("constant-product", [&] { return Constant(-0.5) * left; });
|
||||
observe("scalar-product", [&] { return -0.5 * left; });
|
||||
if (before_left != Snapshot("input", saved_left, 0, 0, 0) ||
|
||||
before_left != Snapshot("input", left, 0, 0, 0) ||
|
||||
before_right != Snapshot("input", saved_right, 0, 0, 0) ||
|
||||
before_right != Snapshot("input", right, 0, 0, 0))
|
||||
throw std::runtime_error("accumulation input alias modified");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ResetFp();
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string AccumulationAliasSnapshots(bool overwrite_input) {
|
||||
std::string out;
|
||||
for (int kind = 0; kind < 4; ++kind) {
|
||||
Jet value = Variable(kind == 0 ? -0.0 : 0.5, 0, -0.25) *
|
||||
Variable(0.75, 1, 0.5);
|
||||
if (kind == 1) value = value - value;
|
||||
if (kind == 2) value.second.Data()[0] =
|
||||
std::numeric_limits<double>::max();
|
||||
const auto before = Snapshot("live-input", value, 0, 0, 0);
|
||||
out += Observe("unique-same-object", [&] {
|
||||
if (overwrite_input) value.first[0] = 17; // Fixture-only negative control.
|
||||
return std::move(value) + value;
|
||||
});
|
||||
if (before != Snapshot("live-input", value, 0, 0, 0))
|
||||
throw std::runtime_error("same-object live operand modified");
|
||||
Jet survivor = value;
|
||||
out += Observe("shared-move", [&] {
|
||||
return std::move(value) + Constant(-1);
|
||||
});
|
||||
if (before != Snapshot("live-input", survivor, 0, 0, 0))
|
||||
throw std::runtime_error("surviving shared operand modified");
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
void AccumulationBench() {
|
||||
const Jet left = Variable(0.5, 0, 0.25) * Variable(0.75, 1, -0.5);
|
||||
const Jet right = Variable(0.25, 2, 0.125) * Variable(-0.5, 3, 0.25);
|
||||
const JetVector3 a{left, right, left}, b{right, left, right};
|
||||
const JetMatrix3 matrix{{a, b, a}}, other{{b, a, b}};
|
||||
const auto e1 = ConstantVector(Vector3{0.5, -1, 0.25});
|
||||
const auto e2 = ConstantVector(Vector3{-0.25, 0.5, 1});
|
||||
volatile double consumed = 0;
|
||||
for (int kind = 0; kind < 6; ++kind) {
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
for (int i = 0; i < 3000; ++i) {
|
||||
if (kind == 0) consumed = consumed + Consume(Dot(a, b));
|
||||
if (kind == 1) {
|
||||
const auto result = Cross(a, b);
|
||||
for (const auto& value : result) consumed = consumed + Consume(value);
|
||||
}
|
||||
if (kind == 2) {
|
||||
const auto result = Multiply(matrix, other);
|
||||
for (const auto& row : result)
|
||||
for (const auto& value : row) consumed = consumed + Consume(value);
|
||||
}
|
||||
if (kind == 3) {
|
||||
const auto result = Multiply(matrix, b);
|
||||
for (const auto& value : result) consumed = consumed + Consume(value);
|
||||
}
|
||||
if (kind == 4)
|
||||
consumed = consumed + Consume(FrameComponent(e1, matrix, e2));
|
||||
if (kind == 5) {
|
||||
Jet density{};
|
||||
for (std::size_t component = 0; component < 3; ++component)
|
||||
density = std::move(density) + a[component] * b[component];
|
||||
consumed = consumed + Consume(0.5 * density);
|
||||
}
|
||||
}
|
||||
std::cout << "accumulation-bench," << kind << ",3000,"
|
||||
<< std::chrono::duration<double>(
|
||||
std::chrono::steady_clock::now() - start).count()
|
||||
<< ",allocation-included,setup-excluded," << consumed << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
void Bench() {
|
||||
volatile double consumed = 0;
|
||||
for (int kind = 0; kind < 8; ++kind) {
|
||||
@@ -671,6 +841,39 @@ int main(int argc, char** argv) {
|
||||
fesa::Bench();
|
||||
return 0;
|
||||
}
|
||||
if (argc == 2 && std::string(argv[1]) == "--accumulation-bench") {
|
||||
fesa::AccumulationBench();
|
||||
return 0;
|
||||
}
|
||||
if (argc >= 3 && (std::string(argv[1]) == "--accumulation-write" ||
|
||||
std::string(argv[1]) == "--accumulation-compare")) {
|
||||
const bool alias = argc > 3 && std::string(argv[3]) == "--unique-alias";
|
||||
auto actual = alias ? fesa::AccumulationAliasSnapshots(
|
||||
argc > 4 && std::string(argv[4]) ==
|
||||
"--overwrite-input")
|
||||
: fesa::AccumulationSnapshots(
|
||||
argc > 3 && std::string(argv[3]) ==
|
||||
"--reassociate");
|
||||
if (std::string(argv[1]) == "--accumulation-write") {
|
||||
if (std::ifstream(argv[2], std::ios::binary))
|
||||
throw std::runtime_error("accumulation snapshot already exists");
|
||||
std::ofstream output(argv[2], std::ios::binary);
|
||||
output << actual;
|
||||
output.close();
|
||||
if (!output) throw std::runtime_error("accumulation snapshot write failure");
|
||||
} else {
|
||||
std::ifstream input(argv[2], std::ios::binary);
|
||||
const std::string expected((std::istreambuf_iterator<char>(input)), {});
|
||||
if (expected != actual)
|
||||
throw std::runtime_error("accumulation snapshot mismatch");
|
||||
}
|
||||
std::cout << argv[1]
|
||||
<< (alias ? " passed; 8 unique/same-object/shared alias cases"
|
||||
: " passed; 192 boundary/environment groups, Dot/"
|
||||
"Cross/matrix/vector/frame/accumulation/constants")
|
||||
<< ", 651 channels and FP state\n";
|
||||
return 0;
|
||||
}
|
||||
if (argc >= 3 && (std::string(argv[1]) == "--product-write" ||
|
||||
std::string(argv[1]) == "--product-compare")) {
|
||||
auto actual = fesa::ProductSnapshots(
|
||||
|
||||
Reference in New Issue
Block a user