perf(math): route ordinary products through MKL and preserve arithmetic

This commit is contained in:
KOKO\Mimi
2026-10-14 15:09:53 +09:00
parent b4fc1bb54b
commit 59c8e93b07
18 changed files with 793 additions and 169 deletions
+11
View File
@@ -5,6 +5,17 @@
namespace fesa::dense_blas_internal {
Matrix MultiplyTransposeLeft(const Matrix& left, const Matrix& right) {
if (left.Rows() != right.Rows()) {
throw std::invalid_argument{"Transpose product dimensions differ."};
}
Matrix transposed{left.Columns(), left.Rows()};
for (std::size_t row = 0; row < left.Rows(); ++row)
for (std::size_t column = 0; column < left.Columns(); ++column)
transposed(column, row) = left(row, column);
return transposed.Multiply(right);
}
Result<MKL_INT> ToMklSize(const std::size_t size) {
if (size > static_cast<std::size_t>((std::numeric_limits<MKL_INT>::max)())) {
return Result<MKL_INT>::Failure(Status::Failure(
+5
View File
@@ -6,6 +6,7 @@
#include <cstddef>
#include "fesa/core/status.h"
#include "fesa/math/matrix.h"
namespace fesa::dense_blas_internal {
@@ -13,6 +14,10 @@ namespace fesa::dense_blas_internal {
/// @return The converted length or an uncategorized overflow failure.
Result<MKL_INT> ToMklSize(std::size_t size);
/// @brief Computes left-transpose times right through ordinary dense MKL.
/// @throws std::invalid_argument if the row counts differ.
Matrix MultiplyTransposeLeft(const Matrix& left, const Matrix& right);
/// @brief Copies contiguous double values through the private BLAS backend.
/// @param source Readable storage for size values when size is nonzero.
/// @param size Number of contiguous values to copy.
+24 -10
View File
@@ -4,6 +4,8 @@
#include <cmath>
#include <cfenv>
#include <cstdint>
#include <cstring>
#include <initializer_list>
#include <limits>
#if defined(_M_X64) || defined(__SSE2__)
#include <xmmintrin.h>
@@ -56,9 +58,9 @@ inline bool IsNormalized(HighLow value) {
if (!IsFinite(value)) return false;
if (value.high == 0 && value.low == 0)
return !std::signbit(value.high) && !std::signbit(value.low);
const auto normalized = TwoSum(value.high, value.low);
return IsFinite(normalized) && normalized.high == value.high &&
normalized.low == value.low;
// If RN(high + low) == high, TwoSum's bridge is zero and its error is
// exactly low. The finite and canonical exact-zero guards remain above.
return value.high + value.low == value.high;
}
/// @brief Rounds the full physical value to one finite output value.
inline double Rounded(HighLow value) {
@@ -92,15 +94,27 @@ inline HighLow Subtract(HighLow left, HighLow right) {
/// @brief Returns the exponent of the least nonzero binary64 significand bit.
/// @pre value is finite and nonzero; gradual underflow is enabled.
inline int LowestBitExponent(double value) {
int exponent = 0;
const double fraction = std::frexp(std::abs(value), &exponent);
auto significand = static_cast<std::uint64_t>(std::ldexp(fraction, 53));
static_assert(sizeof(double) == sizeof(std::uint64_t) &&
std::numeric_limits<double>::is_iec559 &&
std::numeric_limits<double>::digits == 53);
std::uint64_t bits = 0;
std::memcpy(&bits, &value, sizeof(bits));
const auto encoded_exponent = (bits >> 52U) & 0x7ffU;
auto significand = bits & 0xfffffffffffffULL;
const int exponent = encoded_exponent == 0U
? -1074
: static_cast<int>(encoded_exponent) - 1075;
if (encoded_exponent != 0U) significand |= 1ULL << 52U;
int trailing = 0;
while ((significand & 1U) == 0U) {
significand >>= 1U;
++trailing;
// Strip large zero groups first without vendor/compiler intrinsics.
for (const int shift : {32, 16, 8, 4, 2, 1}) {
const auto mask = (1ULL << shift) - 1U;
if ((significand & mask) == 0U) {
significand >>= shift;
trailing += shift;
}
}
return exponent - 53 + trailing;
return exponent + trailing;
}
/// @brief Gives a product and its FMA residual when the dyadic tail fits.
/// @note Rejects underflow-risk exact tails, even if the rounded high is finite.
+74 -9
View File
@@ -8,11 +8,49 @@
#include <tuple>
#include <utility>
#include <mkl.h>
#include "fesa/fem/dof_manager.h"
#include "math/dense_blas_internal.h"
namespace fesa {
namespace {
/// @brief Checks the actual private backend integer width before conversion.
MKL_INT SparseSize(const std::size_t size) {
const auto converted = dense_blas_internal::ToMklSize(size);
if (!converted.HasValue()) {
throw std::length_error{"Sparse size exceeds the MKL integer range."};
}
return converted.Value();
}
/// @brief Releases a call-local sparse handle before its borrowed arrays die.
class SparseHandle {
public:
SparseHandle() = default;
SparseHandle(const SparseHandle&) = delete;
SparseHandle& operator=(const SparseHandle&) = delete;
~SparseHandle() {
if (handle != nullptr) static_cast<void>(mkl_sparse_destroy(handle));
}
/// @brief Reports normal release failure before publishing the candidate.
sparse_status_t Destroy() {
const auto owned = handle;
handle = nullptr;
return mkl_sparse_destroy(owned);
}
sparse_matrix_t handle{nullptr};
};
/// @brief Fails explicitly instead of substituting a scalar backend.
void CheckSparseStatus(const sparse_status_t status, const char* operation) {
if (status != SPARSE_STATUS_SUCCESS) {
throw std::runtime_error{std::string{"Sparse backend "} + operation +
" failed."};
}
}
/// @brief Builds a structured sparse-matrix model failure.
Status SparseFailure(const std::string& code, const std::string& identity,
const std::string& message) {
@@ -170,16 +208,43 @@ Vector SparseMatrix::Multiply(const Vector& rhs) const {
throw std::invalid_argument{
"Sparse matrix-vector multiplication has incompatible dimensions."};
}
Vector result{rows_};
for (std::size_t row = 0U; row < rows_; ++row) {
double value = 0.0;
for (std::size_t position = row_offsets_[row];
position < row_offsets_[row + 1U]; ++position) {
value += values_[position] * rhs[column_indices_[position]];
}
result[row] = value;
if (!Validate().IsOk()) {
throw std::invalid_argument{"Sparse multiplication requires valid CSR."};
}
const MKL_INT rows = SparseSize(rows_);
const MKL_INT columns = SparseSize(columns_);
static_cast<void>(SparseSize(values_.size()));
Vector result{rows_};
if (rows == 0 || columns == 0 || values_.empty()) return result;
// Each const call owns its converted arrays and handle. Neither copy/move
// nor concurrent calls retain pointers into another call's storage.
std::vector<MKL_INT> row_offsets;
std::vector<MKL_INT> column_indices;
row_offsets.reserve(row_offsets_.size());
column_indices.reserve(column_indices_.size());
for (const auto offset : row_offsets_) {
row_offsets.push_back(SparseSize(offset));
}
for (const auto column : column_indices_) {
column_indices.push_back(SparseSize(column));
}
auto values = values_;
SparseHandle sparse;
CheckSparseStatus(
mkl_sparse_d_create_csr(&sparse.handle, SPARSE_INDEX_BASE_ZERO, rows,
columns, row_offsets.data(),
row_offsets.data() + 1, column_indices.data(),
values.data()),
"create CSR");
const matrix_descr descriptor{SPARSE_MATRIX_TYPE_GENERAL,
SPARSE_FILL_MODE_FULL,
SPARSE_DIAG_NON_UNIT};
CheckSparseStatus(
mkl_sparse_d_mv(SPARSE_OPERATION_NON_TRANSPOSE, 1.0, sparse.handle,
descriptor, rhs.Data(), 0.0, result.Data()),
"multiply");
CheckSparseStatus(sparse.Destroy(), "destroy CSR");
return result;
}