perf(math): route ordinary products through MKL and preserve arithmetic
This commit is contained in:
@@ -5,6 +5,17 @@
|
||||
|
||||
namespace fesa::dense_blas_internal {
|
||||
|
||||
Matrix MultiplyTransposeLeft(const Matrix& left, const Matrix& right) {
|
||||
if (left.Rows() != right.Rows()) {
|
||||
throw std::invalid_argument{"Transpose product dimensions differ."};
|
||||
}
|
||||
Matrix transposed{left.Columns(), left.Rows()};
|
||||
for (std::size_t row = 0; row < left.Rows(); ++row)
|
||||
for (std::size_t column = 0; column < left.Columns(); ++column)
|
||||
transposed(column, row) = left(row, column);
|
||||
return transposed.Multiply(right);
|
||||
}
|
||||
|
||||
Result<MKL_INT> ToMklSize(const std::size_t size) {
|
||||
if (size > static_cast<std::size_t>((std::numeric_limits<MKL_INT>::max)())) {
|
||||
return Result<MKL_INT>::Failure(Status::Failure(
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
#include <cstddef>
|
||||
|
||||
#include "fesa/core/status.h"
|
||||
#include "fesa/math/matrix.h"
|
||||
|
||||
namespace fesa::dense_blas_internal {
|
||||
|
||||
@@ -13,6 +14,10 @@ namespace fesa::dense_blas_internal {
|
||||
/// @return The converted length or an uncategorized overflow failure.
|
||||
Result<MKL_INT> ToMklSize(std::size_t size);
|
||||
|
||||
/// @brief Computes left-transpose times right through ordinary dense MKL.
|
||||
/// @throws std::invalid_argument if the row counts differ.
|
||||
Matrix MultiplyTransposeLeft(const Matrix& left, const Matrix& right);
|
||||
|
||||
/// @brief Copies contiguous double values through the private BLAS backend.
|
||||
/// @param source Readable storage for size values when size is nonzero.
|
||||
/// @param size Number of contiguous values to copy.
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
#include <cmath>
|
||||
#include <cfenv>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <initializer_list>
|
||||
#include <limits>
|
||||
#if defined(_M_X64) || defined(__SSE2__)
|
||||
#include <xmmintrin.h>
|
||||
@@ -56,9 +58,9 @@ inline bool IsNormalized(HighLow value) {
|
||||
if (!IsFinite(value)) return false;
|
||||
if (value.high == 0 && value.low == 0)
|
||||
return !std::signbit(value.high) && !std::signbit(value.low);
|
||||
const auto normalized = TwoSum(value.high, value.low);
|
||||
return IsFinite(normalized) && normalized.high == value.high &&
|
||||
normalized.low == value.low;
|
||||
// If RN(high + low) == high, TwoSum's bridge is zero and its error is
|
||||
// exactly low. The finite and canonical exact-zero guards remain above.
|
||||
return value.high + value.low == value.high;
|
||||
}
|
||||
/// @brief Rounds the full physical value to one finite output value.
|
||||
inline double Rounded(HighLow value) {
|
||||
@@ -92,15 +94,27 @@ inline HighLow Subtract(HighLow left, HighLow right) {
|
||||
/// @brief Returns the exponent of the least nonzero binary64 significand bit.
|
||||
/// @pre value is finite and nonzero; gradual underflow is enabled.
|
||||
inline int LowestBitExponent(double value) {
|
||||
int exponent = 0;
|
||||
const double fraction = std::frexp(std::abs(value), &exponent);
|
||||
auto significand = static_cast<std::uint64_t>(std::ldexp(fraction, 53));
|
||||
static_assert(sizeof(double) == sizeof(std::uint64_t) &&
|
||||
std::numeric_limits<double>::is_iec559 &&
|
||||
std::numeric_limits<double>::digits == 53);
|
||||
std::uint64_t bits = 0;
|
||||
std::memcpy(&bits, &value, sizeof(bits));
|
||||
const auto encoded_exponent = (bits >> 52U) & 0x7ffU;
|
||||
auto significand = bits & 0xfffffffffffffULL;
|
||||
const int exponent = encoded_exponent == 0U
|
||||
? -1074
|
||||
: static_cast<int>(encoded_exponent) - 1075;
|
||||
if (encoded_exponent != 0U) significand |= 1ULL << 52U;
|
||||
int trailing = 0;
|
||||
while ((significand & 1U) == 0U) {
|
||||
significand >>= 1U;
|
||||
++trailing;
|
||||
// Strip large zero groups first without vendor/compiler intrinsics.
|
||||
for (const int shift : {32, 16, 8, 4, 2, 1}) {
|
||||
const auto mask = (1ULL << shift) - 1U;
|
||||
if ((significand & mask) == 0U) {
|
||||
significand >>= shift;
|
||||
trailing += shift;
|
||||
}
|
||||
}
|
||||
return exponent - 53 + trailing;
|
||||
return exponent + trailing;
|
||||
}
|
||||
/// @brief Gives a product and its FMA residual when the dyadic tail fits.
|
||||
/// @note Rejects underflow-risk exact tails, even if the rounded high is finite.
|
||||
|
||||
@@ -8,11 +8,49 @@
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
|
||||
#include <mkl.h>
|
||||
|
||||
#include "fesa/fem/dof_manager.h"
|
||||
#include "math/dense_blas_internal.h"
|
||||
|
||||
namespace fesa {
|
||||
namespace {
|
||||
|
||||
/// @brief Checks the actual private backend integer width before conversion.
|
||||
MKL_INT SparseSize(const std::size_t size) {
|
||||
const auto converted = dense_blas_internal::ToMklSize(size);
|
||||
if (!converted.HasValue()) {
|
||||
throw std::length_error{"Sparse size exceeds the MKL integer range."};
|
||||
}
|
||||
return converted.Value();
|
||||
}
|
||||
|
||||
/// @brief Releases a call-local sparse handle before its borrowed arrays die.
|
||||
class SparseHandle {
|
||||
public:
|
||||
SparseHandle() = default;
|
||||
SparseHandle(const SparseHandle&) = delete;
|
||||
SparseHandle& operator=(const SparseHandle&) = delete;
|
||||
~SparseHandle() {
|
||||
if (handle != nullptr) static_cast<void>(mkl_sparse_destroy(handle));
|
||||
}
|
||||
/// @brief Reports normal release failure before publishing the candidate.
|
||||
sparse_status_t Destroy() {
|
||||
const auto owned = handle;
|
||||
handle = nullptr;
|
||||
return mkl_sparse_destroy(owned);
|
||||
}
|
||||
sparse_matrix_t handle{nullptr};
|
||||
};
|
||||
|
||||
/// @brief Fails explicitly instead of substituting a scalar backend.
|
||||
void CheckSparseStatus(const sparse_status_t status, const char* operation) {
|
||||
if (status != SPARSE_STATUS_SUCCESS) {
|
||||
throw std::runtime_error{std::string{"Sparse backend "} + operation +
|
||||
" failed."};
|
||||
}
|
||||
}
|
||||
|
||||
/// @brief Builds a structured sparse-matrix model failure.
|
||||
Status SparseFailure(const std::string& code, const std::string& identity,
|
||||
const std::string& message) {
|
||||
@@ -170,16 +208,43 @@ Vector SparseMatrix::Multiply(const Vector& rhs) const {
|
||||
throw std::invalid_argument{
|
||||
"Sparse matrix-vector multiplication has incompatible dimensions."};
|
||||
}
|
||||
|
||||
Vector result{rows_};
|
||||
for (std::size_t row = 0U; row < rows_; ++row) {
|
||||
double value = 0.0;
|
||||
for (std::size_t position = row_offsets_[row];
|
||||
position < row_offsets_[row + 1U]; ++position) {
|
||||
value += values_[position] * rhs[column_indices_[position]];
|
||||
}
|
||||
result[row] = value;
|
||||
if (!Validate().IsOk()) {
|
||||
throw std::invalid_argument{"Sparse multiplication requires valid CSR."};
|
||||
}
|
||||
const MKL_INT rows = SparseSize(rows_);
|
||||
const MKL_INT columns = SparseSize(columns_);
|
||||
static_cast<void>(SparseSize(values_.size()));
|
||||
Vector result{rows_};
|
||||
if (rows == 0 || columns == 0 || values_.empty()) return result;
|
||||
|
||||
// Each const call owns its converted arrays and handle. Neither copy/move
|
||||
// nor concurrent calls retain pointers into another call's storage.
|
||||
std::vector<MKL_INT> row_offsets;
|
||||
std::vector<MKL_INT> column_indices;
|
||||
row_offsets.reserve(row_offsets_.size());
|
||||
column_indices.reserve(column_indices_.size());
|
||||
for (const auto offset : row_offsets_) {
|
||||
row_offsets.push_back(SparseSize(offset));
|
||||
}
|
||||
for (const auto column : column_indices_) {
|
||||
column_indices.push_back(SparseSize(column));
|
||||
}
|
||||
auto values = values_;
|
||||
SparseHandle sparse;
|
||||
CheckSparseStatus(
|
||||
mkl_sparse_d_create_csr(&sparse.handle, SPARSE_INDEX_BASE_ZERO, rows,
|
||||
columns, row_offsets.data(),
|
||||
row_offsets.data() + 1, column_indices.data(),
|
||||
values.data()),
|
||||
"create CSR");
|
||||
const matrix_descr descriptor{SPARSE_MATRIX_TYPE_GENERAL,
|
||||
SPARSE_FILL_MODE_FULL,
|
||||
SPARSE_DIAG_NON_UNIT};
|
||||
CheckSparseStatus(
|
||||
mkl_sparse_d_mv(SPARSE_OPERATION_NON_TRANSPOSE, 1.0, sparse.handle,
|
||||
descriptor, rhs.Data(), 0.0, result.Data()),
|
||||
"multiply");
|
||||
CheckSparseStatus(sparse.Destroy(), "destroy CSR");
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user