Emily Boudreaux
75cc638739
This commit uses global pre allocated work space to dramatically reduce memory usage and allocation time
869 lines
41 KiB
C++
869 lines
41 KiB
C++
module;
|
|
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <chrono>
|
|
#include <cmath>
|
|
#include <concepts>
|
|
#include <cstdint>
|
|
#include <limits>
|
|
#include <memory>
|
|
#include <stdexcept>
|
|
#include <type_traits>
|
|
#include <utility>
|
|
|
|
#include <mfem.hpp>
|
|
#include <mpi.h>
|
|
|
|
export module mean_field:solver.linear_backend;
|
|
|
|
export import :preconditioning.backend;
|
|
|
|
export namespace mean_field::solver {
|
|
enum class LinearSolveStatus : std::uint8_t {
|
|
converged,
|
|
maximum_iterations,
|
|
breakdown,
|
|
non_finite,
|
|
backend_failure
|
|
};
|
|
|
|
struct LinearSolveControl final {
|
|
double relativeTolerance{1.0e-8};
|
|
double absoluteTolerance{0.0};
|
|
int maximumIterations{100};
|
|
|
|
void Validate() const {
|
|
if (!std::isfinite(relativeTolerance) || relativeTolerance < 0.0) {
|
|
throw std::invalid_argument("A linear solve requires a finite, non-negative relative tolerance.");
|
|
}
|
|
if (!std::isfinite(absoluteTolerance) || absoluteTolerance < 0.0) {
|
|
throw std::invalid_argument("A linear solve requires a finite, non-negative absolute tolerance.");
|
|
}
|
|
if (maximumIterations <= 0) {
|
|
throw std::invalid_argument("A linear solve requires at least one permitted iteration.");
|
|
}
|
|
}
|
|
|
|
[[nodiscard]] double ConvergenceThreshold(const double globalRightHandSideNorm) const {
|
|
Validate();
|
|
if (!std::isfinite(globalRightHandSideNorm) || globalRightHandSideNorm < 0.0) {
|
|
throw std::invalid_argument("A linear solve requires a finite, non-negative right-hand-side norm.");
|
|
}
|
|
const double relativeThreshold = relativeTolerance * globalRightHandSideNorm;
|
|
if (!std::isfinite(relativeThreshold)) {
|
|
throw std::invalid_argument("The linear relative convergence threshold must be finite.");
|
|
}
|
|
return absoluteTolerance > relativeThreshold ? absoluteTolerance : relativeThreshold;
|
|
}
|
|
};
|
|
|
|
/*
|
|
* Numerical termination is data, not an exception. Implementations throw
|
|
* for invalid controls, configuration, dimensions, or violated lifetime
|
|
* contracts. All reported norms are communicator-global Euclidean norms.
|
|
* Solve uses the incoming correction as its initial guess and overwrites
|
|
* it with the final correction, so initialResidualNorm is ||b - A x_0||.
|
|
* Convergence remains relative to the right-hand side rather than the
|
|
* quality of a particular initial guess:
|
|
*
|
|
* ||b - A x|| <= max(absoluteTolerance,
|
|
* relativeTolerance * ||b||).
|
|
*
|
|
* For a zero right-hand side, relativeTrueResidualNorm is zero exactly
|
|
* when the true residual is zero and positive infinity otherwise. The
|
|
* true-residual fields are distinct from the backend's recurrence so
|
|
* callers never have to infer one from the other. This is deliberately
|
|
* fixed-size: recording a history is an optional backend concern whose
|
|
* storage must be owned and reserved by the prepared runtime, not allocated
|
|
* while Solve is active.
|
|
*/
|
|
struct LinearSolveReport final {
|
|
LinearSolveStatus status{LinearSolveStatus::backend_failure};
|
|
LinearSolveControl control{};
|
|
int iterations{0};
|
|
int restarts{0};
|
|
double rightHandSideNorm{0.0};
|
|
double initialResidualNorm{0.0};
|
|
double reportedResidualNorm{0.0};
|
|
double trueResidualNorm{0.0};
|
|
double relativeTrueResidualNorm{0.0};
|
|
// Includes MFEM's initial-guess residual application and the
|
|
// post-solve application used to verify the true residual.
|
|
std::uint64_t operatorApplications{0};
|
|
std::uint64_t inversePreconditionerApplications{0};
|
|
double solveSeconds{0.0};
|
|
// These totals include only completed applications. Operator time also
|
|
// includes the post-solve true-residual verification application.
|
|
double operatorSeconds{0.0};
|
|
double inversePreconditionerSeconds{0.0};
|
|
|
|
[[nodiscard]] bool Converged() const noexcept {
|
|
return status == LinearSolveStatus::converged;
|
|
}
|
|
};
|
|
|
|
struct LinearBackendConfigurationTag { };
|
|
|
|
template <typename Candidate>
|
|
concept LinearBackendConfiguration =
|
|
std::derived_from<std::remove_cvref_t<Candidate>, LinearBackendConfigurationTag> &&
|
|
std::move_constructible<std::remove_cvref_t<Candidate>> && requires {
|
|
requires std::same_as<
|
|
std::remove_cv_t<decltype(std::remove_cvref_t<Candidate>::supportedPreconditionerContract)>,
|
|
preconditioning::ApplicationContract>;
|
|
typename std::integral_constant<
|
|
preconditioning::ApplicationContract, std::remove_cvref_t<Candidate>::supportedPreconditionerContract>;
|
|
requires(
|
|
std::remove_cvref_t<Candidate>::supportedPreconditionerContract ==
|
|
preconditioning::ApplicationContract::stationary_linear ||
|
|
std::remove_cvref_t<Candidate>::supportedPreconditionerContract ==
|
|
preconditioning::ApplicationContract::flexible
|
|
);
|
|
};
|
|
|
|
} // namespace mean_field::solver
|
|
|
|
namespace mean_field::solver::detail {
|
|
template <typename Candidate>
|
|
concept StaticPreconditionerContractDeclared = requires { &std::remove_cvref_t<Candidate>::applicationContract; };
|
|
|
|
template <typename Candidate>
|
|
concept ExactStaticPreconditionerContract = requires {
|
|
requires std::same_as<
|
|
std::remove_cv_t<decltype(std::remove_cvref_t<Candidate>::applicationContract)>,
|
|
preconditioning::ApplicationContract>;
|
|
typename std::integral_constant<
|
|
preconditioning::ApplicationContract, std::remove_cvref_t<Candidate>::applicationContract>;
|
|
requires(
|
|
std::remove_cvref_t<Candidate>::applicationContract ==
|
|
preconditioning::ApplicationContract::stationary_linear ||
|
|
std::remove_cvref_t<Candidate>::applicationContract == preconditioning::ApplicationContract::flexible
|
|
);
|
|
};
|
|
|
|
template <typename Candidate, typename = void> struct StaticPreconditionerContract {
|
|
static constexpr bool declared = StaticPreconditionerContractDeclared<Candidate>;
|
|
static constexpr bool registered = false;
|
|
static constexpr preconditioning::ApplicationContract value =
|
|
preconditioning::ApplicationContract::stationary_linear;
|
|
};
|
|
|
|
template <typename Candidate>
|
|
struct StaticPreconditionerContract<Candidate, std::enable_if_t<ExactStaticPreconditionerContract<Candidate>>> {
|
|
static constexpr bool declared = true;
|
|
static constexpr auto value = std::remove_cvref_t<Candidate>::applicationContract;
|
|
static constexpr bool registered = true;
|
|
};
|
|
|
|
template <typename Candidate>
|
|
concept BackendPreconditionerContractDeclared = requires { typename std::remove_cvref_t<Candidate>::BackendType; };
|
|
|
|
template <typename Backend>
|
|
concept ExactRegisteredBackendPreconditionerContract = requires {
|
|
requires preconditioning::backend::Registered<std::remove_cvref_t<Backend>>;
|
|
requires std::same_as<
|
|
std::remove_cv_t<
|
|
decltype(preconditioning::backend::Traits<std::remove_cvref_t<Backend>>::applicationContract)>,
|
|
preconditioning::ApplicationContract>;
|
|
typename std::integral_constant<
|
|
preconditioning::ApplicationContract,
|
|
preconditioning::backend::Traits<std::remove_cvref_t<Backend>>::applicationContract>;
|
|
requires(
|
|
preconditioning::backend::Traits<std::remove_cvref_t<Backend>>::applicationContract ==
|
|
preconditioning::ApplicationContract::stationary_linear ||
|
|
preconditioning::backend::Traits<std::remove_cvref_t<Backend>>::applicationContract ==
|
|
preconditioning::ApplicationContract::flexible
|
|
);
|
|
};
|
|
|
|
template <typename Candidate>
|
|
concept ExactBackendPreconditionerContract =
|
|
BackendPreconditionerContractDeclared<Candidate> &&
|
|
ExactRegisteredBackendPreconditionerContract<typename std::remove_cvref_t<Candidate>::BackendType>;
|
|
|
|
template <typename Candidate, typename = void> struct BackendPreconditionerContract {
|
|
static constexpr bool declared = BackendPreconditionerContractDeclared<Candidate>;
|
|
static constexpr bool registered = false;
|
|
static constexpr preconditioning::ApplicationContract value =
|
|
preconditioning::ApplicationContract::stationary_linear;
|
|
};
|
|
|
|
template <typename Candidate>
|
|
struct BackendPreconditionerContract<Candidate, std::enable_if_t<ExactBackendPreconditionerContract<Candidate>>> {
|
|
private:
|
|
using Backend = typename std::remove_cvref_t<Candidate>::BackendType;
|
|
|
|
public:
|
|
static constexpr bool declared = true;
|
|
static constexpr bool registered = true;
|
|
static constexpr preconditioning::ApplicationContract value =
|
|
preconditioning::backend::Traits<std::remove_cvref_t<Backend>>::applicationContract;
|
|
};
|
|
|
|
template <typename Candidate> struct DirectPreconditionerContractAudit final {
|
|
private:
|
|
using StaticContract = StaticPreconditionerContract<Candidate>;
|
|
using BackendContract = BackendPreconditionerContract<Candidate>;
|
|
|
|
public:
|
|
static constexpr bool declarationsValid = (!StaticContract::declared || StaticContract::registered) &&
|
|
(!BackendContract::declared || BackendContract::registered);
|
|
static constexpr bool sourcesAgree = !StaticContract::registered || !BackendContract::registered ||
|
|
StaticContract::value == BackendContract::value;
|
|
static constexpr bool registered =
|
|
declarationsValid && sourcesAgree && (StaticContract::registered || BackendContract::registered);
|
|
static constexpr preconditioning::ApplicationContract value = [] {
|
|
if constexpr (StaticContract::registered) {
|
|
return StaticContract::value;
|
|
} else if constexpr (BackendContract::registered) {
|
|
return BackendContract::value;
|
|
} else {
|
|
return preconditioning::ApplicationContract::stationary_linear;
|
|
}
|
|
}();
|
|
};
|
|
|
|
template <typename Candidate>
|
|
concept PhysicalInversePreconditionerContractDeclared =
|
|
requires(const std::remove_cvref_t<Candidate> &candidate) { candidate.GetPhysicalInverse(); };
|
|
|
|
template <typename Candidate>
|
|
using PhysicalInverseType =
|
|
std::remove_cvref_t<decltype(std::declval<const std::remove_cvref_t<Candidate> &>().GetPhysicalInverse())>;
|
|
|
|
template <typename Candidate>
|
|
concept ExactPhysicalInversePreconditionerContract =
|
|
PhysicalInversePreconditionerContractDeclared<Candidate> &&
|
|
DirectPreconditionerContractAudit<PhysicalInverseType<Candidate>>::registered;
|
|
|
|
template <typename Candidate, typename = void> struct PhysicalInversePreconditionerContract {
|
|
static constexpr bool declared = PhysicalInversePreconditionerContractDeclared<Candidate>;
|
|
static constexpr bool registered = false;
|
|
static constexpr preconditioning::ApplicationContract value =
|
|
preconditioning::ApplicationContract::stationary_linear;
|
|
};
|
|
|
|
template <typename Candidate>
|
|
struct PhysicalInversePreconditionerContract<
|
|
Candidate,
|
|
std::enable_if_t<ExactPhysicalInversePreconditionerContract<Candidate>>> {
|
|
using PhysicalInverse = PhysicalInverseType<Candidate>;
|
|
using ContractAudit = DirectPreconditionerContractAudit<PhysicalInverse>;
|
|
|
|
static constexpr bool declared = true;
|
|
static constexpr bool registered = true;
|
|
static constexpr preconditioning::ApplicationContract value = ContractAudit::value;
|
|
};
|
|
|
|
template <typename Candidate> struct LinearPreconditionerContractAudit final {
|
|
private:
|
|
using StaticContract = StaticPreconditionerContract<Candidate>;
|
|
using BackendContract = BackendPreconditionerContract<Candidate>;
|
|
using PhysicalContract = PhysicalInversePreconditionerContract<Candidate>;
|
|
|
|
public:
|
|
static constexpr bool declarationsValid = (!StaticContract::declared || StaticContract::registered) &&
|
|
(!BackendContract::declared || BackendContract::registered) &&
|
|
(!PhysicalContract::declared || PhysicalContract::registered);
|
|
static constexpr bool sourcesAgree = (!StaticContract::registered || !BackendContract::registered ||
|
|
StaticContract::value == BackendContract::value) &&
|
|
(!StaticContract::registered || !PhysicalContract::registered ||
|
|
StaticContract::value == PhysicalContract::value) &&
|
|
(!BackendContract::registered || !PhysicalContract::registered ||
|
|
BackendContract::value == PhysicalContract::value);
|
|
static constexpr bool registered =
|
|
declarationsValid && sourcesAgree &&
|
|
(StaticContract::registered || BackendContract::registered || PhysicalContract::registered);
|
|
static constexpr preconditioning::ApplicationContract value = [] {
|
|
if constexpr (StaticContract::registered) {
|
|
return StaticContract::value;
|
|
} else if constexpr (BackendContract::registered) {
|
|
return BackendContract::value;
|
|
} else if constexpr (PhysicalContract::registered) {
|
|
return PhysicalContract::value;
|
|
} else {
|
|
return preconditioning::ApplicationContract::stationary_linear;
|
|
}
|
|
}();
|
|
};
|
|
} // namespace mean_field::solver::detail
|
|
|
|
export namespace mean_field::solver {
|
|
template <typename Candidate>
|
|
concept LinearPreconditionerApplicationContractAvailable =
|
|
detail::LinearPreconditionerContractAudit<std::remove_cvref_t<Candidate>>::registered;
|
|
|
|
template <LinearPreconditionerApplicationContractAvailable Candidate>
|
|
inline constexpr preconditioning::ApplicationContract linearPreconditionerApplicationContract =
|
|
detail::LinearPreconditionerContractAudit<std::remove_cvref_t<Candidate>>::value;
|
|
|
|
template <typename Configuration, typename Preconditioner>
|
|
concept LinearBackendPreconditionerCompatible =
|
|
LinearBackendConfiguration<Configuration> && LinearPreconditionerApplicationContractAvailable<Preconditioner> &&
|
|
(std::remove_cvref_t<Configuration>::supportedPreconditionerContract ==
|
|
preconditioning::ApplicationContract::flexible ||
|
|
linearPreconditionerApplicationContract<std::remove_cvref_t<Preconditioner>> ==
|
|
preconditioning::ApplicationContract::stationary_linear);
|
|
|
|
/*
|
|
* A prepared backend is bound once to the exact operator and inverse that
|
|
* its owner keeps at stable addresses and identifies the communicator on
|
|
* which it operates. The communicator supplied to preparation is borrowed;
|
|
* a backend may retain it or own a congruent duplicate. The handle returned
|
|
* by GetCommunicator is borrowed from the backend and must not be freed by
|
|
* the caller. Every prepared backend must be destroyed before MPI_Finalize.
|
|
* It owns its numerical workspaces; neither copyability nor movability is
|
|
* required. Solve treats a caller-provided, correctly sized correction
|
|
* vector as its initial guess and overwrites it with the final correction.
|
|
*/
|
|
template <typename Candidate, typename Operator, typename Preconditioner>
|
|
concept PreparedLinearBackendFor = std::derived_from<std::remove_cvref_t<Operator>, mfem::Operator> &&
|
|
std::derived_from<std::remove_cvref_t<Preconditioner>, mfem::Solver> &&
|
|
std::destructible<std::remove_cvref_t<Candidate>> &&
|
|
requires(
|
|
std::remove_cvref_t<Candidate> &prepared,
|
|
const std::remove_cvref_t<Candidate> &constantPrepared,
|
|
const mfem::Vector &rightHandSide,
|
|
mfem::Vector &correction,
|
|
const LinearSolveControl &control
|
|
) {
|
|
{
|
|
constantPrepared.GetOperator()
|
|
} -> std::same_as<const std::remove_cvref_t<Operator> &>;
|
|
{
|
|
constantPrepared.GetPreconditioner()
|
|
} -> std::same_as<const std::remove_cvref_t<Preconditioner> &>;
|
|
{ constantPrepared.GetCommunicator() } -> std::same_as<MPI_Comm>;
|
|
{ constantPrepared.IsReady() } -> std::same_as<bool>;
|
|
{ constantPrepared.RightHandSideSize() } -> std::same_as<int>;
|
|
{ constantPrepared.CorrectionSize() } -> std::same_as<int>;
|
|
{
|
|
prepared.Solve(rightHandSide, correction, control)
|
|
} -> std::same_as<LinearSolveReport>;
|
|
};
|
|
|
|
/*
|
|
* `prepareLinearBackend` is intentionally unqualified in this detection
|
|
* boundary. A third-party configuration supplies its overload beside the
|
|
* configuration type and ADL discovers it without a library registry.
|
|
*/
|
|
template <typename Configuration, typename Operator, typename Preconditioner>
|
|
concept LinearBackendRuntimeAvailableFor =
|
|
LinearBackendPreconditionerCompatible<Configuration, Preconditioner> &&
|
|
std::derived_from<std::remove_cvref_t<Operator>, mfem::Operator> &&
|
|
std::derived_from<std::remove_cvref_t<Preconditioner>, mfem::Solver> &&
|
|
requires(
|
|
std::remove_cvref_t<Configuration> configuration,
|
|
const std::remove_cvref_t<Operator> &operation,
|
|
std::remove_cvref_t<Preconditioner> &preconditioner,
|
|
MPI_Comm communicator
|
|
) {
|
|
requires std::same_as<
|
|
decltype(prepareLinearBackend(std::move(configuration), operation, preconditioner, communicator)),
|
|
std::remove_cvref_t<
|
|
decltype(prepareLinearBackend(std::move(configuration), operation, preconditioner, communicator))>>;
|
|
{
|
|
prepareLinearBackend(std::move(configuration), operation, preconditioner, communicator)
|
|
} -> PreparedLinearBackendFor<std::remove_cvref_t<Operator>, std::remove_cvref_t<Preconditioner>>;
|
|
};
|
|
|
|
template <LinearBackendConfiguration Configuration, typename Operator, typename Preconditioner>
|
|
requires LinearBackendRuntimeAvailableFor<Configuration, Operator, Preconditioner>
|
|
using PreparedLinearBackendType = std::remove_cvref_t<decltype(prepareLinearBackend(
|
|
std::declval<std::remove_cvref_t<Configuration> &&>(),
|
|
std::declval<const std::remove_cvref_t<Operator> &>(),
|
|
std::declval<std::remove_cvref_t<Preconditioner> &>(),
|
|
std::declval<MPI_Comm>()
|
|
))>;
|
|
} // namespace mean_field::solver
|
|
|
|
export namespace mean_field::solver::linear {
|
|
struct FGMRESOptions final {
|
|
int restartLength{50};
|
|
int printLevel{1};
|
|
|
|
void Validate() const {
|
|
if (restartLength <= 0) {
|
|
throw std::invalid_argument("MFEM FGMRES requires a positive restart length.");
|
|
}
|
|
if (printLevel < -1 || printLevel > 3) {
|
|
throw std::invalid_argument("MFEM FGMRES print level must be between -1 and 3.");
|
|
}
|
|
}
|
|
};
|
|
|
|
class FGMRES final : public LinearBackendConfigurationTag {
|
|
public:
|
|
static constexpr preconditioning::ApplicationContract supportedPreconditionerContract =
|
|
preconditioning::ApplicationContract::flexible;
|
|
|
|
FGMRES() = default;
|
|
|
|
explicit FGMRES(FGMRESOptions options) : m_options(std::move(options)) {
|
|
m_options.Validate();
|
|
}
|
|
|
|
[[nodiscard]] const FGMRESOptions &GetOptions() const noexcept {
|
|
return m_options;
|
|
}
|
|
|
|
private:
|
|
FGMRESOptions m_options{};
|
|
};
|
|
|
|
namespace detail {
|
|
template <typename Operation>
|
|
requires std::derived_from<std::remove_cvref_t<Operation>, mfem::Operator>
|
|
class CountedOperator final : public mfem::Operator {
|
|
public:
|
|
explicit CountedOperator(const Operation &operation)
|
|
: mfem::Operator(
|
|
operation.Height(),
|
|
operation.Width()
|
|
),
|
|
m_operation(std::addressof(operation)) {
|
|
}
|
|
|
|
void Mult(
|
|
const mfem::Vector &input,
|
|
mfem::Vector &output
|
|
) const override {
|
|
const auto start = std::chrono::steady_clock::now();
|
|
m_operation->Mult(input, output);
|
|
m_seconds += std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count();
|
|
++m_applications;
|
|
}
|
|
|
|
void Reset() const noexcept {
|
|
m_applications = 0;
|
|
m_seconds = 0.0;
|
|
}
|
|
|
|
[[nodiscard]] std::uint64_t Applications() const noexcept {
|
|
return m_applications;
|
|
}
|
|
|
|
[[nodiscard]] double Seconds() const noexcept {
|
|
return m_seconds;
|
|
}
|
|
|
|
private:
|
|
const Operation *m_operation;
|
|
mutable std::uint64_t m_applications{0};
|
|
mutable double m_seconds{0.0};
|
|
};
|
|
|
|
template <typename Operation, typename Preconditioner>
|
|
requires std::derived_from<std::remove_cvref_t<Operation>, mfem::Operator> &&
|
|
std::derived_from<std::remove_cvref_t<Preconditioner>, mfem::Solver>
|
|
class CountedPreconditioner final : public mfem::Solver {
|
|
public:
|
|
CountedPreconditioner(
|
|
const Operation &operation,
|
|
Preconditioner &preconditioner,
|
|
const CountedOperator<Operation> &countedOperation
|
|
)
|
|
: mfem::Solver(
|
|
preconditioner.Height(),
|
|
preconditioner.Width(),
|
|
false
|
|
),
|
|
m_operation(std::addressof(operation)),
|
|
m_preconditioner(std::addressof(preconditioner)),
|
|
m_countedOperation(std::addressof(countedOperation)) {
|
|
m_preconditioner->iterative_mode = false;
|
|
}
|
|
|
|
void SetOperator(const mfem::Operator &operation) override {
|
|
if (std::addressof(operation) != m_countedOperation) {
|
|
throw std::invalid_argument("The MFEM FGMRES preconditioner received an unexpected operator.");
|
|
}
|
|
m_preconditioner->SetOperator(*m_operation);
|
|
if (m_preconditioner->Height() != Height() || m_preconditioner->Width() != Width()) {
|
|
throw std::invalid_argument(
|
|
"The MFEM FGMRES preconditioner changed dimensions while binding its operator."
|
|
);
|
|
}
|
|
}
|
|
|
|
void Mult(
|
|
const mfem::Vector &input,
|
|
mfem::Vector &output
|
|
) const override {
|
|
const auto start = std::chrono::steady_clock::now();
|
|
m_preconditioner->Mult(input, output);
|
|
m_seconds += std::chrono::duration<double>(std::chrono::steady_clock::now() - start).count();
|
|
++m_applications;
|
|
}
|
|
|
|
void Reset() const noexcept {
|
|
m_applications = 0;
|
|
m_seconds = 0.0;
|
|
}
|
|
|
|
[[nodiscard]] std::uint64_t Applications() const noexcept {
|
|
return m_applications;
|
|
}
|
|
|
|
[[nodiscard]] double Seconds() const noexcept {
|
|
return m_seconds;
|
|
}
|
|
|
|
private:
|
|
const Operation *m_operation;
|
|
Preconditioner *m_preconditioner;
|
|
const CountedOperator<Operation> *m_countedOperation;
|
|
mutable std::uint64_t m_applications{0};
|
|
mutable double m_seconds{0.0};
|
|
};
|
|
|
|
[[nodiscard]] inline bool MpiIsUsable() noexcept {
|
|
int initialized = 0;
|
|
int finalized = 0;
|
|
return MPI_Initialized(&initialized) == MPI_SUCCESS && initialized != 0 &&
|
|
MPI_Finalized(&finalized) == MPI_SUCCESS && finalized == 0;
|
|
}
|
|
|
|
[[nodiscard]] inline bool AllRanksAgree(
|
|
const bool localValue,
|
|
const MPI_Comm communicator
|
|
) {
|
|
int local = localValue ? 1 : 0;
|
|
int global = 0;
|
|
if (MPI_Allreduce(&local, &global, 1, MPI_INT, MPI_MIN, communicator) != MPI_SUCCESS) {
|
|
throw std::runtime_error("MFEM FGMRES could not perform a communicator-wide validity check.");
|
|
}
|
|
return global != 0;
|
|
}
|
|
|
|
inline void RequireCollectivelyIdenticalConfiguration(
|
|
const FGMRESOptions &options,
|
|
const LinearSolveControl &control,
|
|
const MPI_Comm communicator
|
|
) {
|
|
const std::array<double, 2> localRealValues{control.relativeTolerance, control.absoluteTolerance};
|
|
std::array<double, 2> minimumRealValues{};
|
|
std::array<double, 2> maximumRealValues{};
|
|
const std::array<int, 3> localIntegerValues{
|
|
control.maximumIterations, options.restartLength, options.printLevel
|
|
};
|
|
std::array<int, 3> minimumIntegerValues{};
|
|
std::array<int, 3> maximumIntegerValues{};
|
|
|
|
if (MPI_Allreduce(
|
|
localRealValues.data(), minimumRealValues.data(), static_cast<int>(localRealValues.size()),
|
|
MPI_DOUBLE, MPI_MIN, communicator
|
|
) != MPI_SUCCESS ||
|
|
MPI_Allreduce(
|
|
localRealValues.data(), maximumRealValues.data(), static_cast<int>(localRealValues.size()),
|
|
MPI_DOUBLE, MPI_MAX, communicator
|
|
) != MPI_SUCCESS ||
|
|
MPI_Allreduce(
|
|
localIntegerValues.data(), minimumIntegerValues.data(), static_cast<int>(localIntegerValues.size()),
|
|
MPI_INT, MPI_MIN, communicator
|
|
) != MPI_SUCCESS ||
|
|
MPI_Allreduce(
|
|
localIntegerValues.data(), maximumIntegerValues.data(), static_cast<int>(localIntegerValues.size()),
|
|
MPI_INT, MPI_MAX, communicator
|
|
) != MPI_SUCCESS) {
|
|
throw std::runtime_error("MFEM FGMRES could not validate its distributed configuration.");
|
|
}
|
|
|
|
if (minimumRealValues != maximumRealValues || minimumIntegerValues != maximumIntegerValues) {
|
|
throw std::invalid_argument(
|
|
"MFEM FGMRES requires identical options and solve controls on every communicator rank."
|
|
);
|
|
}
|
|
}
|
|
|
|
[[nodiscard]] inline bool LocallyFinite(const mfem::Vector &values) {
|
|
for (int index = 0; index < values.Size(); ++index) {
|
|
if (!std::isfinite(values(index))) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
[[nodiscard]] inline double GlobalNorm(
|
|
const mfem::Vector &values,
|
|
const MPI_Comm communicator
|
|
) {
|
|
const double localNorm = values.Norml2();
|
|
const double localNormSquared = localNorm * localNorm;
|
|
double globalNormSquared = 0.0;
|
|
if (MPI_Allreduce(&localNormSquared, &globalNormSquared, 1, MPI_DOUBLE, MPI_SUM, communicator) !=
|
|
MPI_SUCCESS) {
|
|
throw std::runtime_error("MFEM FGMRES could not reduce a global vector norm.");
|
|
}
|
|
if (!std::isfinite(globalNormSquared) || globalNormSquared < 0.0) {
|
|
return std::numeric_limits<double>::quiet_NaN();
|
|
}
|
|
return std::sqrt(globalNormSquared);
|
|
}
|
|
|
|
[[nodiscard]] inline double MaximumRankValue(
|
|
const double localValue,
|
|
const MPI_Comm communicator
|
|
) {
|
|
double maximumValue = 0.0;
|
|
if (MPI_Allreduce(&localValue, &maximumValue, 1, MPI_DOUBLE, MPI_MAX, communicator) != MPI_SUCCESS) {
|
|
throw std::runtime_error("MFEM FGMRES could not reduce a communicator-wide timing measurement.");
|
|
}
|
|
return maximumValue;
|
|
}
|
|
|
|
template <typename Candidate> [[nodiscard]] bool RuntimeDependencyIsCurrent(const Candidate &candidate) {
|
|
if constexpr (requires {
|
|
{ candidate.IsCurrent() } -> std::same_as<bool>;
|
|
}) {
|
|
return candidate.IsCurrent();
|
|
} else if constexpr (requires {
|
|
{ candidate.IsPrepared() } -> std::same_as<bool>;
|
|
}) {
|
|
return candidate.IsPrepared();
|
|
} else {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
template <typename Operation, typename Preconditioner>
|
|
requires std::derived_from<std::remove_cvref_t<Operation>, mfem::Operator> &&
|
|
std::derived_from<std::remove_cvref_t<Preconditioner>, mfem::Solver>
|
|
class PreparedFGMRES final {
|
|
private:
|
|
using Clock = std::chrono::steady_clock;
|
|
|
|
public:
|
|
PreparedFGMRES(
|
|
FGMRESOptions options,
|
|
const Operation &operation,
|
|
Preconditioner &preconditioner,
|
|
const MPI_Comm communicator
|
|
)
|
|
: m_options(std::move(options)),
|
|
m_operation(std::addressof(operation)),
|
|
m_preconditioner(std::addressof(preconditioner)),
|
|
m_communicator(communicator),
|
|
m_countedOperation(operation),
|
|
m_countedPreconditioner(
|
|
operation,
|
|
preconditioner,
|
|
m_countedOperation
|
|
),
|
|
m_solver(communicator),
|
|
m_rightHandSide(operation.Height()),
|
|
m_operationAction(operation.Height()),
|
|
m_trueResidual(operation.Height()) {
|
|
m_options.Validate();
|
|
if (!MpiIsUsable()) {
|
|
throw std::logic_error("MFEM FGMRES requires initialized MPI that has not been finalized.");
|
|
}
|
|
if (m_communicator == MPI_COMM_NULL) {
|
|
throw std::invalid_argument("MFEM FGMRES requires a non-null MPI communicator.");
|
|
}
|
|
if (operation.Height() <= 0 || operation.Width() <= 0 || operation.Height() != operation.Width() ||
|
|
preconditioner.Height() != operation.Width() || preconditioner.Width() != operation.Height()) {
|
|
throw std::invalid_argument(
|
|
"MFEM FGMRES requires compatible square operator and preconditioner dimensions."
|
|
);
|
|
}
|
|
|
|
m_rightHandSide = 0.0;
|
|
m_operationAction = 0.0;
|
|
m_trueResidual = 0.0;
|
|
|
|
m_solver.SetPreconditioner(m_countedPreconditioner);
|
|
m_solver.SetOperator(m_countedOperation);
|
|
m_solver.SetKDim(m_options.restartLength);
|
|
m_solver.SetPrintLevel(m_options.printLevel);
|
|
m_solver.iterative_mode = true;
|
|
}
|
|
|
|
PreparedFGMRES(const PreparedFGMRES &) = delete;
|
|
PreparedFGMRES &operator=(const PreparedFGMRES &) = delete;
|
|
PreparedFGMRES(PreparedFGMRES &&) = delete;
|
|
PreparedFGMRES &operator=(PreparedFGMRES &&) = delete;
|
|
|
|
[[nodiscard]] const Operation &GetOperator() const noexcept {
|
|
return *m_operation;
|
|
}
|
|
|
|
[[nodiscard]] const Preconditioner &GetPreconditioner() const noexcept {
|
|
return *m_preconditioner;
|
|
}
|
|
|
|
[[nodiscard]] MPI_Comm GetCommunicator() const noexcept {
|
|
return m_communicator;
|
|
}
|
|
|
|
[[nodiscard]] bool IsReady() const {
|
|
return m_operation != nullptr && m_preconditioner != nullptr && m_communicator != MPI_COMM_NULL &&
|
|
m_operation->Height() == m_operation->Width() &&
|
|
m_preconditioner->Height() == m_operation->Width() &&
|
|
m_preconditioner->Width() == m_operation->Height() &&
|
|
m_rightHandSide.Size() == m_operation->Height() &&
|
|
m_operationAction.Size() == m_operation->Height() &&
|
|
m_trueResidual.Size() == m_operation->Height() && RuntimeDependencyIsCurrent(*m_operation) &&
|
|
RuntimeDependencyIsCurrent(*m_preconditioner);
|
|
}
|
|
|
|
[[nodiscard]] int RightHandSideSize() const noexcept {
|
|
return m_operation->Height();
|
|
}
|
|
|
|
[[nodiscard]] int CorrectionSize() const noexcept {
|
|
return m_operation->Width();
|
|
}
|
|
|
|
[[nodiscard]] LinearSolveReport Solve(
|
|
const mfem::Vector &rightHandSide,
|
|
mfem::Vector &correction,
|
|
const LinearSolveControl &control
|
|
) {
|
|
bool localConfigurationIsValid = true;
|
|
try {
|
|
m_options.Validate();
|
|
control.Validate();
|
|
} catch (const std::invalid_argument &) {
|
|
localConfigurationIsValid = false;
|
|
}
|
|
if (!AllRanksAgree(localConfigurationIsValid, m_communicator)) {
|
|
throw std::invalid_argument(
|
|
"MFEM FGMRES requires valid options and solve controls on every communicator rank."
|
|
);
|
|
}
|
|
if (!localConfigurationIsValid) {
|
|
throw std::invalid_argument("MFEM FGMRES received invalid options or solve controls.");
|
|
}
|
|
RequireCollectivelyIdenticalConfiguration(m_options, control, m_communicator);
|
|
if (!AllRanksAgree(IsReady(), m_communicator)) {
|
|
throw std::logic_error(
|
|
"MFEM FGMRES requires a complete, current prepared runtime on every communicator rank."
|
|
);
|
|
}
|
|
if (!AllRanksAgree(
|
|
rightHandSide.Size() == RightHandSideSize() && correction.Size() == CorrectionSize(),
|
|
m_communicator
|
|
)) {
|
|
throw std::invalid_argument(
|
|
"MFEM FGMRES received incompatible linear-system vectors on at least one rank."
|
|
);
|
|
}
|
|
if (!AllRanksAgree(LocallyFinite(rightHandSide), m_communicator) ||
|
|
!AllRanksAgree(LocallyFinite(correction), m_communicator)) {
|
|
throw std::invalid_argument(
|
|
"MFEM FGMRES requires finite right-hand side and initial-guess values."
|
|
);
|
|
}
|
|
|
|
m_rightHandSide = rightHandSide;
|
|
const double rightHandSideNorm = GlobalNorm(m_rightHandSide, m_communicator);
|
|
const double threshold = control.ConvergenceThreshold(rightHandSideNorm);
|
|
|
|
m_countedOperation.Reset();
|
|
m_countedPreconditioner.Reset();
|
|
m_solver.SetRelTol(0.0);
|
|
m_solver.SetAbsTol(threshold);
|
|
m_solver.SetMaxIter(control.maximumIterations);
|
|
|
|
const Clock::time_point start = Clock::now();
|
|
m_solver.Mult(m_rightHandSide, correction);
|
|
const double solveSeconds =
|
|
MaximumRankValue(std::chrono::duration<double>(Clock::now() - start).count(), m_communicator);
|
|
|
|
const bool correctionIsFinite = AllRanksAgree(LocallyFinite(correction), m_communicator);
|
|
double trueResidualNorm = std::numeric_limits<double>::quiet_NaN();
|
|
if (correctionIsFinite) {
|
|
m_countedOperation.Mult(correction, m_operationAction);
|
|
m_trueResidual = m_rightHandSide;
|
|
m_trueResidual -= m_operationAction;
|
|
if (AllRanksAgree(LocallyFinite(m_trueResidual), m_communicator)) {
|
|
trueResidualNorm = GlobalNorm(m_trueResidual, m_communicator);
|
|
}
|
|
}
|
|
|
|
const double initialResidualNorm = m_solver.GetInitialNorm();
|
|
const double reportedResidualNorm = m_solver.GetFinalNorm();
|
|
const std::uint64_t krylovIterationCount = m_countedPreconditioner.Applications();
|
|
if (krylovIterationCount > static_cast<std::uint64_t>(std::numeric_limits<int>::max())) {
|
|
throw std::overflow_error("MFEM FGMRES reported more Krylov iterations than can be represented.");
|
|
}
|
|
const int iterations = static_cast<int>(krylovIterationCount);
|
|
// A restart is an additional Krylov cycle entered after the
|
|
// initial cycle, not the residual check at a cycle boundary.
|
|
const int restarts = iterations > 0 ? (iterations - 1) / m_options.restartLength : 0;
|
|
const bool numericalValuesAreFinite = correctionIsFinite && std::isfinite(initialResidualNorm) &&
|
|
std::isfinite(reportedResidualNorm) &&
|
|
std::isfinite(trueResidualNorm);
|
|
|
|
LinearSolveStatus status = LinearSolveStatus::backend_failure;
|
|
if (!numericalValuesAreFinite) {
|
|
status = LinearSolveStatus::non_finite;
|
|
} else if (trueResidualNorm <= threshold) {
|
|
status = LinearSolveStatus::converged;
|
|
} else if (!m_solver.GetConverged() && iterations >= control.maximumIterations) {
|
|
status = LinearSolveStatus::maximum_iterations;
|
|
}
|
|
|
|
const double relativeTrueResidualNorm =
|
|
rightHandSideNorm > 0.0 ? trueResidualNorm / rightHandSideNorm
|
|
: (trueResidualNorm == 0.0 ? 0.0 : std::numeric_limits<double>::infinity());
|
|
const double operatorSeconds = MaximumRankValue(m_countedOperation.Seconds(), m_communicator);
|
|
const double inversePreconditionerSeconds =
|
|
MaximumRankValue(m_countedPreconditioner.Seconds(), m_communicator);
|
|
|
|
return {
|
|
.status = status,
|
|
.control = control,
|
|
.iterations = iterations,
|
|
.restarts = restarts,
|
|
.rightHandSideNorm = rightHandSideNorm,
|
|
.initialResidualNorm = initialResidualNorm,
|
|
.reportedResidualNorm = reportedResidualNorm,
|
|
.trueResidualNorm = trueResidualNorm,
|
|
.relativeTrueResidualNorm = relativeTrueResidualNorm,
|
|
.operatorApplications = m_countedOperation.Applications(),
|
|
.inversePreconditionerApplications = m_countedPreconditioner.Applications(),
|
|
.solveSeconds = solveSeconds,
|
|
.operatorSeconds = operatorSeconds,
|
|
.inversePreconditionerSeconds = inversePreconditionerSeconds
|
|
};
|
|
}
|
|
|
|
private:
|
|
FGMRESOptions m_options;
|
|
const Operation *m_operation;
|
|
Preconditioner *m_preconditioner;
|
|
MPI_Comm m_communicator;
|
|
CountedOperator<Operation> m_countedOperation;
|
|
CountedPreconditioner<Operation, Preconditioner> m_countedPreconditioner;
|
|
mfem::FGMRESSolver m_solver;
|
|
mfem::Vector m_rightHandSide;
|
|
mfem::Vector m_operationAction;
|
|
mfem::Vector m_trueResidual;
|
|
};
|
|
} // namespace detail
|
|
|
|
template <
|
|
typename Operation,
|
|
typename Preconditioner>
|
|
requires std::derived_from<
|
|
std::remove_cvref_t<Operation>,
|
|
mfem::Operator> &&
|
|
std::derived_from<
|
|
std::remove_cvref_t<Preconditioner>,
|
|
mfem::Solver>
|
|
[[nodiscard]] auto prepareLinearBackend(
|
|
FGMRES configuration,
|
|
const Operation &operation,
|
|
Preconditioner &preconditioner,
|
|
const MPI_Comm communicator
|
|
) {
|
|
return detail::PreparedFGMRES<Operation, Preconditioner>{
|
|
configuration.GetOptions(), operation, preconditioner, communicator
|
|
};
|
|
}
|
|
} // namespace mean_field::solver::linear
|