feat(preconditioner): major work on preconditioner system
first preconditioner MVP
This commit is contained in:
1218
experiments/full_stellar_preconditioning.cpp
Normal file
1218
experiments/full_stellar_preconditioning.cpp
Normal file
File diff suppressed because it is too large
Load Diff
591
experiments/gravity_preconditioning.cpp
Normal file
591
experiments/gravity_preconditioning.cpp
Normal file
@@ -0,0 +1,591 @@
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
#include <mfem.hpp>
|
||||
#include <mpi.h>
|
||||
|
||||
import experiment;
|
||||
import mean_field;
|
||||
import test_helpers;
|
||||
|
||||
namespace {
|
||||
using Clock = std::chrono::steady_clock;
|
||||
|
||||
namespace backend = mean_field::preconditioning::backend;
|
||||
namespace preconditioning = mean_field::preconditioning;
|
||||
|
||||
[[nodiscard]] const char *buildConfiguration() noexcept {
|
||||
#ifdef NDEBUG
|
||||
return "release";
|
||||
#else
|
||||
return "debug";
|
||||
#endif
|
||||
}
|
||||
|
||||
[[nodiscard]] double maximumRankSeconds(
|
||||
const Clock::time_point start,
|
||||
const MPI_Comm communicator
|
||||
) {
|
||||
const double localSeconds = std::chrono::duration<double>(Clock::now() - start).count();
|
||||
double maximumSeconds = 0.0;
|
||||
MPI_Allreduce(&localSeconds, &maximumSeconds, 1, MPI_DOUBLE, MPI_MAX, communicator);
|
||||
return maximumSeconds;
|
||||
}
|
||||
|
||||
[[nodiscard]] double globalNorm(
|
||||
const mfem::Vector &vector,
|
||||
const MPI_Comm communicator
|
||||
) {
|
||||
const double localSquared = vector * vector;
|
||||
double globalSquared = 0.0;
|
||||
MPI_Allreduce(&localSquared, &globalSquared, 1, MPI_DOUBLE, MPI_SUM, communicator);
|
||||
return std::sqrt(std::max(globalSquared, 0.0));
|
||||
}
|
||||
|
||||
[[nodiscard]] double globalDot(
|
||||
const mfem::Vector &left,
|
||||
const mfem::Vector &right,
|
||||
const MPI_Comm communicator
|
||||
) {
|
||||
const double localDot = left * right;
|
||||
double result = 0.0;
|
||||
MPI_Allreduce(&localDot, &result, 1, MPI_DOUBLE, MPI_SUM, communicator);
|
||||
return result;
|
||||
}
|
||||
|
||||
void announce(
|
||||
const MPI_Comm communicator,
|
||||
const std::string &message
|
||||
) {
|
||||
int rank = 0;
|
||||
MPI_Comm_rank(communicator, &rank);
|
||||
if (rank == 0) {
|
||||
std::cout << message << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
class ReducedGravityOperator final : public mfem::Operator {
|
||||
public:
|
||||
explicit ReducedGravityOperator(
|
||||
const mean_field::operators::context::gravity_field::GravityFieldGeometryContext &context
|
||||
)
|
||||
: mfem::Operator(
|
||||
context.GetMassOperator().GetFluxMap().reduced_size() +
|
||||
context.GetSourceOperator().GetPotentialMap().reduced_size()
|
||||
),
|
||||
m_mass(&context.GetMassOperator()),
|
||||
m_divergence(
|
||||
context.GetDivergenceOperator(),
|
||||
context.GetMassOperator().GetFluxMap(),
|
||||
context.GetSourceOperator().GetPotentialMap()
|
||||
),
|
||||
m_offsets(3),
|
||||
m_gradientWorkspace(context.GetMassOperator().GetFluxMap().reduced_size()) {
|
||||
m_offsets[0] = 0;
|
||||
m_offsets[1] = context.GetMassOperator().GetFluxMap().reduced_size();
|
||||
m_offsets[2] = Height();
|
||||
}
|
||||
|
||||
void Mult(
|
||||
const mfem::Vector &state,
|
||||
mfem::Vector &residual
|
||||
) const override {
|
||||
if (state.Size() != Width() || residual.Size() != Height()) {
|
||||
throw std::invalid_argument("The reduced gravity experiment requires preallocated compatible vectors.");
|
||||
}
|
||||
|
||||
const mfem::Vector gradient(
|
||||
const_cast<mfem::real_t *>(state.GetData()) + m_offsets[0], m_offsets[1] - m_offsets[0]
|
||||
);
|
||||
const mfem::Vector potential(
|
||||
const_cast<mfem::real_t *>(state.GetData()) + m_offsets[1], m_offsets[2] - m_offsets[1]
|
||||
);
|
||||
mfem::Vector gradientResidual(residual.GetData() + m_offsets[0], m_offsets[1] - m_offsets[0]);
|
||||
mfem::Vector potentialResidual(residual.GetData() + m_offsets[1], m_offsets[2] - m_offsets[1]);
|
||||
|
||||
m_mass->Mult(gradient, gradientResidual);
|
||||
m_divergence.MultTranspose(potential, m_gradientWorkspace);
|
||||
gradientResidual += m_gradientWorkspace;
|
||||
m_divergence.Mult(gradient, potentialResidual);
|
||||
}
|
||||
|
||||
private:
|
||||
const mfem::Operator *m_mass;
|
||||
preconditioning::ReducedGravityDivergenceOperator m_divergence;
|
||||
mfem::Array<int> m_offsets;
|
||||
mutable mfem::Vector m_gradientWorkspace;
|
||||
};
|
||||
|
||||
[[nodiscard]] std::map<
|
||||
std::string,
|
||||
std::string>
|
||||
commonParameters(
|
||||
const std::string &candidate,
|
||||
const std::string &measurement,
|
||||
const int dimension
|
||||
) {
|
||||
return {
|
||||
{"build_configuration", buildConfiguration()},
|
||||
{"candidate", candidate},
|
||||
{"experiment_schema", "p4_reduced_gravity_v1"},
|
||||
{"factorization", candidate},
|
||||
{"measurement", measurement},
|
||||
{"mesh_file", test_utils::setup_args().mesh_file},
|
||||
{"operator", "reduced_gravity_saddle_point"},
|
||||
{"preconditioned_product", "G M^-1"},
|
||||
{"root_dimension", std::to_string(dimension)}
|
||||
};
|
||||
}
|
||||
|
||||
void recordSpectrum(
|
||||
const std::string &candidate,
|
||||
const mean_field::solver::ArnoldiSpectralMeasurement &spectrum,
|
||||
const int dimension,
|
||||
const double setupSeconds
|
||||
) {
|
||||
experiment::record_experiment_result(
|
||||
"gravity_preconditioning_p4", candidate + "_spectrum",
|
||||
commonParameters(candidate, "arnoldi_summary", dimension),
|
||||
{{"setup_seconds_maximum_rank", setupSeconds},
|
||||
{"requested_dimension", static_cast<double>(spectrum.requestedDimension)},
|
||||
{"achieved_dimension", static_cast<double>(spectrum.achievedDimension)},
|
||||
{"operator_applications", static_cast<double>(spectrum.operatorApplications)},
|
||||
{"measurement_seconds_maximum_rank", spectrum.measurementSecondsMaximumRank},
|
||||
{"operator_application_seconds_maximum_rank", spectrum.operatorApplicationSecondsMaximumRank},
|
||||
{"projected_condition_proxy", spectrum.projectedConditionProxy},
|
||||
{"projected_largest_singular_value", spectrum.projectedLargestSingularValue},
|
||||
{"projected_smallest_singular_value", spectrum.projectedSmallestSingularValue},
|
||||
{"centroid_real_part", spectrum.centroidRealPart},
|
||||
{"rms_distance_from_one", spectrum.rmsDistanceFromOne},
|
||||
{"rms_cluster_radius", spectrum.rmsClusterRadius},
|
||||
{"minimum_magnitude", spectrum.minimumMagnitude},
|
||||
{"maximum_magnitude", spectrum.maximumMagnitude},
|
||||
{"minimum_real_part", spectrum.minimumRealPart},
|
||||
{"maximum_real_part", spectrum.maximumRealPart},
|
||||
{"maximum_absolute_imaginary_part", spectrum.maximumAbsoluteImaginaryPart},
|
||||
{"negative_real_part_count", static_cast<double>(spectrum.negativeRealPartCount)},
|
||||
{"converged_ritz_value_count", static_cast<double>(spectrum.convergedRitzValueCount)},
|
||||
{"conjugate_pair_defect", spectrum.conjugatePairDefect},
|
||||
{"projected_departure_from_normality", spectrum.projectedDepartureFromNormality},
|
||||
{"field_of_values_minimum_real_part", spectrum.projectedFieldOfValuesMinimumRealPart},
|
||||
{"field_of_values_maximum_real_part", spectrum.projectedFieldOfValuesMaximumRealPart}}
|
||||
);
|
||||
|
||||
for (std::size_t index = 0; index < spectrum.ritzValues.size(); ++index) {
|
||||
const auto &value = spectrum.ritzValues[index];
|
||||
experiment::record_experiment_result(
|
||||
"gravity_preconditioning_p4", candidate + "_ritz_" + std::to_string(index),
|
||||
commonParameters(candidate, "ritz_value", dimension),
|
||||
{{"ritz_index", static_cast<double>(index)},
|
||||
{"real_part", value.realPart},
|
||||
{"imaginary_part", value.imaginaryPart},
|
||||
{"magnitude", value.magnitude},
|
||||
{"distance_from_one", value.distanceFromOne},
|
||||
{"residual_estimate", value.residualEstimate},
|
||||
{"relative_residual_estimate", value.relativeResidualEstimate},
|
||||
{"converged", value.converged ? 1.0 : 0.0}}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
void measureCandidate(
|
||||
const std::string &candidate,
|
||||
mfem::Solver &inversePreconditioner,
|
||||
const double setupSeconds,
|
||||
const ReducedGravityOperator &gravityOperator,
|
||||
const mfem::Vector &rightHandSide,
|
||||
const mfem::Vector &arnoldiDirection,
|
||||
const MPI_Comm communicator
|
||||
) {
|
||||
constexpr int arnoldiDimension = 32;
|
||||
|
||||
mean_field::solver::InstrumentedOperator instrumentedGravity(gravityOperator);
|
||||
mean_field::solver::InstrumentedPreconditioner instrumentedPreconditioner(inversePreconditioner);
|
||||
mean_field::solver::ResidualHistoryMonitor monitor;
|
||||
mfem::FGMRESSolver krylov(communicator);
|
||||
krylov.SetPreconditioner(instrumentedPreconditioner);
|
||||
krylov.SetOperator(instrumentedGravity);
|
||||
krylov.SetMonitor(monitor);
|
||||
krylov.SetRelTol(1.0e-8);
|
||||
krylov.SetAbsTol(1.0e-12);
|
||||
krylov.SetMaxIter(100);
|
||||
krylov.SetKDim(30);
|
||||
krylov.SetPrintLevel(0);
|
||||
|
||||
mfem::Vector solution(gravityOperator.Width());
|
||||
solution = 0.0;
|
||||
announce(communicator, "P4 reduced gravity: solving with " + candidate);
|
||||
const Clock::time_point solveStart = Clock::now();
|
||||
krylov.Mult(rightHandSide, solution);
|
||||
const double solveSeconds = maximumRankSeconds(solveStart, communicator);
|
||||
|
||||
mfem::Vector reconstructed(rightHandSide.Size());
|
||||
gravityOperator.Mult(solution, reconstructed);
|
||||
reconstructed -= rightHandSide;
|
||||
const double relativeResidual =
|
||||
globalNorm(reconstructed, communicator) /
|
||||
std::max(globalNorm(rightHandSide, communicator), std::numeric_limits<double>::epsilon());
|
||||
|
||||
const auto jacobianStatistics = instrumentedGravity.GetStatistics();
|
||||
const auto preconditionerStatistics = instrumentedPreconditioner.GetStatistics();
|
||||
REQUIRE(std::isfinite(relativeResidual));
|
||||
experiment::record_experiment_result(
|
||||
"gravity_preconditioning_p4", candidate + "_linear_solve",
|
||||
commonParameters(candidate, "linear_solve", gravityOperator.Width()),
|
||||
{{"setup_seconds_maximum_rank", setupSeconds},
|
||||
{"solver_converged", krylov.GetConverged() ? 1.0 : 0.0},
|
||||
{"outer_iterations", static_cast<double>(krylov.GetNumIterations())},
|
||||
{"true_relative_residual", relativeResidual},
|
||||
{"solve_seconds_maximum_rank", solveSeconds},
|
||||
{"gravity_applications", static_cast<double>(jacobianStatistics.applications)},
|
||||
{"gravity_application_seconds", jacobianStatistics.totalSeconds},
|
||||
{"preconditioner_applications", static_cast<double>(preconditionerStatistics.applications)},
|
||||
{"preconditioner_application_seconds", preconditionerStatistics.totalSeconds},
|
||||
{"preconditioner_maximum_application_seconds", preconditionerStatistics.maximumSeconds}}
|
||||
);
|
||||
|
||||
instrumentedGravity.ResetStatistics();
|
||||
instrumentedPreconditioner.ResetStatistics();
|
||||
mean_field::solver::FixedRightPreconditionedOperator product(instrumentedGravity, instrumentedPreconditioner);
|
||||
announce(communicator, "P4 reduced gravity: measuring " + candidate + " Arnoldi spectrum");
|
||||
const auto spectrum = mean_field::solver::measureArnoldiSpectrum(
|
||||
product, arnoldiDirection, communicator,
|
||||
{.krylovDimension = arnoldiDimension,
|
||||
.breakdownRelativeTolerance = 1.0e-13,
|
||||
.ritzConvergenceRelativeTolerance = 1.0e-7,
|
||||
.reorthogonalize = true}
|
||||
);
|
||||
recordSpectrum(candidate, spectrum, gravityOperator.Width(), setupSeconds);
|
||||
}
|
||||
|
||||
template <
|
||||
preconditioning::GravityFactorizationPolicy Policy,
|
||||
backend::Registered MassBackend = backend::Diagonal>
|
||||
requires backend::Compatible<
|
||||
MassBackend,
|
||||
preconditioning::GravityMassInverseCharacteristics>
|
||||
void prepareAndMeasureTypedCandidate(
|
||||
const std::string &candidate,
|
||||
Policy policy,
|
||||
const mean_field::fem::FEM &finiteElements,
|
||||
const mean_field::operators::context::gravity_field::GravityFieldGeometryContext &geometryContext,
|
||||
const ReducedGravityOperator &gravityOperator,
|
||||
const mfem::Vector &rightHandSide,
|
||||
const mfem::Vector &arnoldiDirection,
|
||||
const MPI_Comm communicator,
|
||||
const int amgCycles = 1,
|
||||
MassBackend massBackend = {}
|
||||
) {
|
||||
const Clock::time_point setupStart = Clock::now();
|
||||
const auto block = preconditioning::GravityFieldBlock(
|
||||
std::move(massBackend), backend::HypreBoomerAMG{backend::FixedCycles{.cycles = amgCycles}}, policy
|
||||
);
|
||||
auto prepared = preconditioning::prepare(finiteElements, geometryContext, block);
|
||||
const double setupTime = maximumRankSeconds(setupStart, communicator);
|
||||
|
||||
const auto &massOperator = geometryContext.GetMassOperator();
|
||||
const mfem::Vector firstMassRightHandSide =
|
||||
gravity_prepared_test_utils::make_deterministic_vector(massOperator.Width(), 0.41);
|
||||
const mfem::Vector secondMassRightHandSide =
|
||||
gravity_prepared_test_utils::make_deterministic_vector(massOperator.Width(), 1.17);
|
||||
mfem::Vector firstMassAction(massOperator.Width());
|
||||
mfem::Vector secondMassAction(massOperator.Width());
|
||||
prepared.GetMassInverse().Mult(firstMassRightHandSide, firstMassAction);
|
||||
prepared.GetMassInverse().Mult(secondMassRightHandSide, secondMassAction);
|
||||
mfem::Vector recoveredMassRightHandSide(massOperator.Height());
|
||||
massOperator.Mult(firstMassAction, recoveredMassRightHandSide);
|
||||
recoveredMassRightHandSide -= firstMassRightHandSide;
|
||||
const double massRecoveryDefect =
|
||||
globalNorm(recoveredMassRightHandSide, communicator) / globalNorm(firstMassRightHandSide, communicator);
|
||||
const double firstSecond = globalDot(firstMassRightHandSide, secondMassAction, communicator);
|
||||
const double secondFirst = globalDot(secondMassRightHandSide, firstMassAction, communicator);
|
||||
const double massSymmetryDefect =
|
||||
std::abs(firstSecond - secondFirst) / std::max({1.0, std::abs(firstSecond), std::abs(secondFirst)});
|
||||
const double massPositiveRayleigh = globalDot(firstMassRightHandSide, firstMassAction, communicator) /
|
||||
std::max(
|
||||
globalDot(firstMassRightHandSide, firstMassRightHandSide, communicator),
|
||||
std::numeric_limits<double>::min()
|
||||
);
|
||||
|
||||
const auto &schurOperator = prepared.GetPotentialSchurSurrogate();
|
||||
const mfem::Vector schurRightHandSide =
|
||||
gravity_prepared_test_utils::make_deterministic_vector(schurOperator.Width(), 0.73);
|
||||
mfem::Vector schurAction(schurOperator.Width());
|
||||
prepared.GetPotentialSchurInverse().Mult(schurRightHandSide, schurAction);
|
||||
mfem::Vector recoveredSchurRightHandSide(schurOperator.Height());
|
||||
schurOperator.Mult(schurAction, recoveredSchurRightHandSide);
|
||||
recoveredSchurRightHandSide -= schurRightHandSide;
|
||||
const double schurRecoveryDefect =
|
||||
globalNorm(recoveredSchurRightHandSide, communicator) / globalNorm(schurRightHandSide, communicator);
|
||||
|
||||
experiment::record_experiment_result(
|
||||
"gravity_preconditioning_p4", candidate + "_block_quality",
|
||||
commonParameters(candidate, "block_inverse_quality", gravityOperator.Width()),
|
||||
{{"amg_cycles", static_cast<double>(amgCycles)},
|
||||
{"mass_inverse_recovery_defect", massRecoveryDefect},
|
||||
{"mass_inverse_symmetry_defect", massSymmetryDefect},
|
||||
{"mass_inverse_positive_rayleigh", massPositiveRayleigh},
|
||||
{"potential_schur_inverse_recovery_defect", schurRecoveryDefect}}
|
||||
);
|
||||
measureCandidate(
|
||||
candidate, prepared, setupTime, gravityOperator, rightHandSide, arnoldiDirection, communicator
|
||||
);
|
||||
}
|
||||
|
||||
[[nodiscard]] int firstReportedThresholdIteration(
|
||||
const std::vector<mean_field::solver::IterationResidualMeasurement> &history,
|
||||
const double initialNorm,
|
||||
const double relativeThreshold
|
||||
) {
|
||||
if (!std::isfinite(initialNorm) || initialNorm <= 0.0) {
|
||||
return -1;
|
||||
}
|
||||
for (const auto &sample : history) {
|
||||
if (std::abs(sample.reportedNorm) / initialNorm <= relativeThreshold) {
|
||||
return sample.iteration;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST_CASE(
|
||||
"Reduced Gravity P4 Factorization Comparison",
|
||||
"[preconditioning][gravity][diagnostics][experiment][spectrum]"
|
||||
) {
|
||||
const auto arguments = test_utils::setup_args();
|
||||
mean_field::fem::FEM finiteElements = mean_field::fem::setup_fem(arguments.mesh_file, arguments, 0);
|
||||
const MPI_Comm communicator = finiteElements.mesh->GetComm();
|
||||
using GeometryContext = mean_field::operators::context::gravity_field::GravityFieldGeometryContext;
|
||||
GeometryContext geometryContext(finiteElements, *finiteElements.domainMapperStateless);
|
||||
|
||||
mfem::Vector displacementTrue(finiteElements.displacementFes->GetTrueVSize());
|
||||
displacementTrue = 0.0;
|
||||
const mfem::Vector displacement = geometryContext.GetDisplacementMap().gather(displacementTrue);
|
||||
geometryContext.PreparePrimal(displacement, {.value = 1}, {.value = 1});
|
||||
|
||||
ReducedGravityOperator gravityOperator(geometryContext);
|
||||
const mfem::Vector exact = gravity_prepared_test_utils::make_deterministic_vector(gravityOperator.Width(), 0.37);
|
||||
mfem::Vector rightHandSide(gravityOperator.Height());
|
||||
gravityOperator.Mult(exact, rightHandSide);
|
||||
const mfem::Vector arnoldiDirection =
|
||||
gravity_prepared_test_utils::make_deterministic_vector(gravityOperator.Width(), 0.83);
|
||||
|
||||
const Clock::time_point legacySetupStart = Clock::now();
|
||||
mean_field::operators::ReducedGravityFieldPreconditioner legacy(finiteElements, geometryContext);
|
||||
const double legacySetupTime = maximumRankSeconds(legacySetupStart, communicator);
|
||||
measureCandidate(
|
||||
"legacy_block_diagonal", legacy, legacySetupTime, gravityOperator, rightHandSide, arnoldiDirection, communicator
|
||||
);
|
||||
|
||||
prepareAndMeasureTypedCandidate(
|
||||
"typed_block_diagonal", preconditioning::GravityBlockDiagonal{}, finiteElements, geometryContext,
|
||||
gravityOperator, rightHandSide, arnoldiDirection, communicator
|
||||
);
|
||||
prepareAndMeasureTypedCandidate(
|
||||
"lower_triangular", preconditioning::GravityLowerTriangular{}, finiteElements, geometryContext, gravityOperator,
|
||||
rightHandSide, arnoldiDirection, communicator
|
||||
);
|
||||
prepareAndMeasureTypedCandidate(
|
||||
"upper_triangular", preconditioning::GravityUpperTriangular{}, finiteElements, geometryContext, gravityOperator,
|
||||
rightHandSide, arnoldiDirection, communicator
|
||||
);
|
||||
prepareAndMeasureTypedCandidate(
|
||||
"approximate_ldu", preconditioning::GravityApproximateLDU{}, finiteElements, geometryContext, gravityOperator,
|
||||
rightHandSide, arnoldiDirection, communicator
|
||||
);
|
||||
}
|
||||
|
||||
TEST_CASE(
|
||||
"Reduced Gravity P4 Fixed AMG Cycle Sweep",
|
||||
"[preconditioning][gravity][diagnostics][experiment][amg_cycle_sweep]"
|
||||
) {
|
||||
const auto arguments = test_utils::setup_args();
|
||||
mean_field::fem::FEM finiteElements = mean_field::fem::setup_fem(arguments.mesh_file, arguments, 0);
|
||||
const MPI_Comm communicator = finiteElements.mesh->GetComm();
|
||||
using GeometryContext = mean_field::operators::context::gravity_field::GravityFieldGeometryContext;
|
||||
GeometryContext geometryContext(finiteElements, *finiteElements.domainMapperStateless);
|
||||
|
||||
mfem::Vector displacementTrue(finiteElements.displacementFes->GetTrueVSize());
|
||||
displacementTrue = 0.0;
|
||||
const mfem::Vector displacement = geometryContext.GetDisplacementMap().gather(displacementTrue);
|
||||
geometryContext.PreparePrimal(displacement, {.value = 1}, {.value = 1});
|
||||
|
||||
ReducedGravityOperator gravityOperator(geometryContext);
|
||||
const mfem::Vector exact = gravity_prepared_test_utils::make_deterministic_vector(gravityOperator.Width(), 0.37);
|
||||
mfem::Vector rightHandSide(gravityOperator.Height());
|
||||
gravityOperator.Mult(exact, rightHandSide);
|
||||
const mfem::Vector arnoldiDirection =
|
||||
gravity_prepared_test_utils::make_deterministic_vector(gravityOperator.Width(), 0.83);
|
||||
|
||||
for (const int cycles : {1, 2, 3, 4, 6, 8}) {
|
||||
prepareAndMeasureTypedCandidate(
|
||||
"approximate_ldu_amg_cycles_" + std::to_string(cycles), preconditioning::GravityApproximateLDU{},
|
||||
finiteElements, geometryContext, gravityOperator, rightHandSide, arnoldiDirection, communicator, cycles
|
||||
);
|
||||
}
|
||||
for (const int order : {2, 3, 4, 5}) {
|
||||
for (const int cycles : {1, 2, 3}) {
|
||||
prepareAndMeasureTypedCandidate(
|
||||
"approximate_ldu_chebyshev_" + std::to_string(order) + "_amg_cycles_" + std::to_string(cycles),
|
||||
preconditioning::GravityApproximateLDU{}, finiteElements, geometryContext, gravityOperator,
|
||||
rightHandSide, arnoldiDirection, communicator, cycles,
|
||||
backend::MatrixFreeChebyshev{.order = order, .powerIterations = 20}
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE(
|
||||
"Reduced Gravity P4 LDU Extended FGMRES Convergence",
|
||||
"[preconditioning][gravity][diagnostics][experiment][p4_followup][extended_solve]"
|
||||
) {
|
||||
constexpr int maximumIterations = 200;
|
||||
constexpr int restartDimension = 30;
|
||||
|
||||
const auto arguments = test_utils::setup_args();
|
||||
mean_field::fem::FEM finiteElements = mean_field::fem::setup_fem(arguments.mesh_file, arguments, 0);
|
||||
const MPI_Comm communicator = finiteElements.mesh->GetComm();
|
||||
using GeometryContext = mean_field::operators::context::gravity_field::GravityFieldGeometryContext;
|
||||
GeometryContext geometryContext(finiteElements, *finiteElements.domainMapperStateless);
|
||||
|
||||
mfem::Vector displacementTrue(finiteElements.displacementFes->GetTrueVSize());
|
||||
displacementTrue = 0.0;
|
||||
const mfem::Vector displacement = geometryContext.GetDisplacementMap().gather(displacementTrue);
|
||||
geometryContext.PreparePrimal(displacement, {.value = 1}, {.value = 1});
|
||||
|
||||
ReducedGravityOperator gravityOperator(geometryContext);
|
||||
const mfem::Vector exact = gravity_prepared_test_utils::make_deterministic_vector(gravityOperator.Width(), 0.37);
|
||||
mfem::Vector rightHandSide(gravityOperator.Height());
|
||||
gravityOperator.Mult(exact, rightHandSide);
|
||||
|
||||
const Clock::time_point setupStart = Clock::now();
|
||||
const auto block = preconditioning::GravityFieldBlock(
|
||||
backend::Diagonal{}, backend::HypreBoomerAMG{backend::FixedCycles{.cycles = 1}},
|
||||
preconditioning::GravityApproximateLDU{}
|
||||
);
|
||||
auto prepared = preconditioning::prepare(finiteElements, geometryContext, block);
|
||||
const double setupTime = maximumRankSeconds(setupStart, communicator);
|
||||
|
||||
mean_field::solver::InstrumentedOperator instrumentedGravity(gravityOperator);
|
||||
mean_field::solver::InstrumentedPreconditioner instrumentedPreconditioner(prepared);
|
||||
mean_field::solver::ResidualHistoryMonitor monitor;
|
||||
mfem::FGMRESSolver krylov(communicator);
|
||||
krylov.SetPreconditioner(instrumentedPreconditioner);
|
||||
krylov.SetOperator(instrumentedGravity);
|
||||
krylov.SetMonitor(monitor);
|
||||
krylov.SetRelTol(1.0e-8);
|
||||
krylov.SetAbsTol(1.0e-12);
|
||||
krylov.SetMaxIter(maximumIterations);
|
||||
krylov.SetKDim(restartDimension);
|
||||
krylov.SetPrintLevel(0);
|
||||
|
||||
mfem::Vector solution(gravityOperator.Width());
|
||||
solution = 0.0;
|
||||
announce(communicator, "P4 follow-up: running 200-iteration approximate-LDU FGMRES");
|
||||
const Clock::time_point solveStart = Clock::now();
|
||||
krylov.Mult(rightHandSide, solution);
|
||||
const double solveSeconds = maximumRankSeconds(solveStart, communicator);
|
||||
|
||||
mfem::Vector reconstructed(rightHandSide.Size());
|
||||
gravityOperator.Mult(solution, reconstructed);
|
||||
reconstructed -= rightHandSide;
|
||||
const double trueRelativeResidual =
|
||||
globalNorm(reconstructed, communicator) /
|
||||
std::max(globalNorm(rightHandSide, communicator), std::numeric_limits<double>::epsilon());
|
||||
const double initialNorm = std::abs(krylov.GetInitialNorm());
|
||||
const auto &history = monitor.GetHistory();
|
||||
const int iteration1e4 = firstReportedThresholdIteration(history, initialNorm, 1.0e-4);
|
||||
const int iteration1e6 = firstReportedThresholdIteration(history, initialNorm, 1.0e-6);
|
||||
const int iteration1e8 = firstReportedThresholdIteration(history, initialNorm, 1.0e-8);
|
||||
|
||||
REQUIRE(std::isfinite(trueRelativeResidual));
|
||||
REQUIRE_FALSE(history.empty());
|
||||
experiment::record_experiment_result(
|
||||
"gravity_preconditioning_p4_followup", "approximate_ldu_extended_linear_solve",
|
||||
commonParameters("approximate_ldu_extended", "linear_solve", gravityOperator.Width()),
|
||||
{{"maximum_iterations", static_cast<double>(maximumIterations)},
|
||||
{"restart_dimension", static_cast<double>(restartDimension)},
|
||||
{"setup_seconds_maximum_rank", setupTime},
|
||||
{"solver_converged", krylov.GetConverged() ? 1.0 : 0.0},
|
||||
{"outer_iterations", static_cast<double>(krylov.GetNumIterations())},
|
||||
{"reported_initial_residual_norm", initialNorm},
|
||||
{"reported_final_residual_norm", std::abs(krylov.GetFinalNorm())},
|
||||
{"reported_residual_reduction", initialNorm > 0.0 ? std::abs(krylov.GetFinalNorm()) / initialNorm : 0.0},
|
||||
{"reported_iteration_to_1e-4", static_cast<double>(iteration1e4)},
|
||||
{"reported_iteration_to_1e-6", static_cast<double>(iteration1e6)},
|
||||
{"reported_iteration_to_1e-8", static_cast<double>(iteration1e8)},
|
||||
{"true_relative_residual", trueRelativeResidual},
|
||||
{"solve_seconds_maximum_rank", solveSeconds},
|
||||
{"gravity_applications", static_cast<double>(instrumentedGravity.GetStatistics().applications)},
|
||||
{"gravity_application_seconds", instrumentedGravity.GetStatistics().totalSeconds},
|
||||
{"preconditioner_applications", static_cast<double>(instrumentedPreconditioner.GetStatistics().applications)},
|
||||
{"preconditioner_application_seconds", instrumentedPreconditioner.GetStatistics().totalSeconds}}
|
||||
);
|
||||
|
||||
for (std::size_t index = 0; index < history.size(); ++index) {
|
||||
const auto &sample = history[index];
|
||||
experiment::record_experiment_result(
|
||||
"gravity_preconditioning_p4_followup", "approximate_ldu_history_" + std::to_string(index),
|
||||
commonParameters("approximate_ldu_extended", "fgmres_residual_history", gravityOperator.Width()),
|
||||
{{"history_sample", static_cast<double>(index)},
|
||||
{"iteration", static_cast<double>(sample.iteration)},
|
||||
{"reported_residual_norm", sample.reportedNorm},
|
||||
{"reported_relative_residual", initialNorm > 0.0 ? std::abs(sample.reportedNorm) / initialNorm : 0.0},
|
||||
{"final_measurement", sample.final ? 1.0 : 0.0}}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE(
|
||||
"Reduced Gravity P4 LDU Extended Arnoldi Convergence",
|
||||
"[preconditioning][gravity][diagnostics][experiment][spectrum][p4_followup][extended_arnoldi]"
|
||||
) {
|
||||
constexpr int arnoldiDimension = 96;
|
||||
|
||||
const auto arguments = test_utils::setup_args();
|
||||
mean_field::fem::FEM finiteElements = mean_field::fem::setup_fem(arguments.mesh_file, arguments, 0);
|
||||
const MPI_Comm communicator = finiteElements.mesh->GetComm();
|
||||
using GeometryContext = mean_field::operators::context::gravity_field::GravityFieldGeometryContext;
|
||||
GeometryContext geometryContext(finiteElements, *finiteElements.domainMapperStateless);
|
||||
|
||||
mfem::Vector displacementTrue(finiteElements.displacementFes->GetTrueVSize());
|
||||
displacementTrue = 0.0;
|
||||
const mfem::Vector displacement = geometryContext.GetDisplacementMap().gather(displacementTrue);
|
||||
geometryContext.PreparePrimal(displacement, {.value = 1}, {.value = 1});
|
||||
|
||||
ReducedGravityOperator gravityOperator(geometryContext);
|
||||
const mfem::Vector arnoldiDirection =
|
||||
gravity_prepared_test_utils::make_deterministic_vector(gravityOperator.Width(), 0.83);
|
||||
|
||||
const Clock::time_point setupStart = Clock::now();
|
||||
const auto block = preconditioning::GravityFieldBlock(
|
||||
backend::Diagonal{}, backend::HypreBoomerAMG{backend::FixedCycles{.cycles = 1}},
|
||||
preconditioning::GravityApproximateLDU{}
|
||||
);
|
||||
auto prepared = preconditioning::prepare(finiteElements, geometryContext, block);
|
||||
const double setupTime = maximumRankSeconds(setupStart, communicator);
|
||||
|
||||
mean_field::solver::InstrumentedOperator instrumentedGravity(gravityOperator);
|
||||
mean_field::solver::InstrumentedPreconditioner instrumentedPreconditioner(prepared);
|
||||
mean_field::solver::FixedRightPreconditionedOperator product(instrumentedGravity, instrumentedPreconditioner);
|
||||
announce(communicator, "P4 follow-up: measuring the 96-vector approximate-LDU Arnoldi spectrum");
|
||||
const auto spectrum = mean_field::solver::measureArnoldiSpectrum(
|
||||
product, arnoldiDirection, communicator,
|
||||
{.krylovDimension = arnoldiDimension,
|
||||
.breakdownRelativeTolerance = 1.0e-13,
|
||||
.ritzConvergenceRelativeTolerance = 1.0e-7,
|
||||
.reorthogonalize = true}
|
||||
);
|
||||
|
||||
REQUIRE(spectrum.achievedDimension > 32);
|
||||
recordSpectrum("approximate_ldu_arnoldi_96", spectrum, gravityOperator.Width(), setupTime);
|
||||
}
|
||||
993
experiments/material_surface_preconditioning.cpp
Normal file
993
experiments/material_surface_preconditioning.cpp
Normal file
@@ -0,0 +1,993 @@
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <numbers>
|
||||
#include <ranges>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
#include <mfem.hpp>
|
||||
#include <mpi.h>
|
||||
|
||||
import experiment;
|
||||
import mean_field;
|
||||
import test_helpers;
|
||||
|
||||
namespace {
|
||||
using Clock = std::chrono::steady_clock;
|
||||
|
||||
namespace backend = mean_field::preconditioning::backend;
|
||||
namespace preconditioning = mean_field::preconditioning;
|
||||
namespace solver = mean_field::solver;
|
||||
|
||||
struct MaterialBlockMeasurements final {
|
||||
double density{0.0};
|
||||
double surface{0.0};
|
||||
double enthalpy{0.0};
|
||||
};
|
||||
|
||||
[[nodiscard]] const char *buildConfiguration() noexcept {
|
||||
#ifdef NDEBUG
|
||||
return "release";
|
||||
#else
|
||||
return "debug";
|
||||
#endif
|
||||
}
|
||||
|
||||
[[nodiscard]] double maximumRankSeconds(
|
||||
const Clock::time_point start,
|
||||
const MPI_Comm communicator
|
||||
) {
|
||||
const double localSeconds = std::chrono::duration<double>(Clock::now() - start).count();
|
||||
double maximumSeconds = 0.0;
|
||||
MPI_Allreduce(&localSeconds, &maximumSeconds, 1, MPI_DOUBLE, MPI_MAX, communicator);
|
||||
return maximumSeconds;
|
||||
}
|
||||
|
||||
[[nodiscard]] double globalNorm(
|
||||
const mfem::Vector &vector,
|
||||
const MPI_Comm communicator
|
||||
) {
|
||||
const double localSquaredNorm = vector * vector;
|
||||
double globalSquaredNorm = 0.0;
|
||||
MPI_Allreduce(&localSquaredNorm, &globalSquaredNorm, 1, MPI_DOUBLE, MPI_SUM, communicator);
|
||||
return std::sqrt(std::max(globalSquaredNorm, 0.0));
|
||||
}
|
||||
|
||||
void announce(
|
||||
const MPI_Comm communicator,
|
||||
const std::string &message
|
||||
) {
|
||||
int rank = 0;
|
||||
MPI_Comm_rank(communicator, &rank);
|
||||
if (rank == 0) {
|
||||
std::cout << "[P9 material-surface] " << message << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] mean_field::operators::StellarEquilibriumDependencies makeDependencies() {
|
||||
return {
|
||||
.discretization = {.identity = 10103, .revision = 1},
|
||||
.density = {.identity = 10111, .revision = 1},
|
||||
.surfaceDeformation = {.identity = 10133, .revision = 1},
|
||||
.gravityGradient = {.identity = 10139, .revision = 1},
|
||||
.gravityPotential = {.identity = 10141, .revision = 1},
|
||||
.enthalpy = {.identity = 10151, .revision = 1},
|
||||
.bernoulliConstant = {.identity = 10159, .revision = 1},
|
||||
.rotation = {.identity = 10163, .revision = 1},
|
||||
.targetMass = {.identity = 10169, .revision = 1}
|
||||
};
|
||||
}
|
||||
|
||||
[[nodiscard]] mean_field::physics::RigidRotation zeroRotation() {
|
||||
mfem::Vector angularVelocity(3);
|
||||
mfem::Vector center(3);
|
||||
angularVelocity = 0.0;
|
||||
center = 0.0;
|
||||
return {angularVelocity, center};
|
||||
}
|
||||
|
||||
[[nodiscard]] mfem::Vector blockBalancedDirection(
|
||||
const mfem::Array<int> &offsets,
|
||||
const double phase,
|
||||
const MPI_Comm communicator
|
||||
) {
|
||||
mfem::Vector direction(offsets.Last());
|
||||
direction = 0.0;
|
||||
for (int block = 0; block < offsets.Size() - 1; ++block) {
|
||||
mfem::Vector values(direction, offsets[block], offsets[block + 1] - offsets[block]);
|
||||
for (int index = 0; index < values.Size(); ++index) {
|
||||
const double ordinal = static_cast<double>(index + 1);
|
||||
values(index) = std::sin(0.371 * ordinal + phase + static_cast<double>(block)) +
|
||||
0.29 * std::cos(0.173 * ordinal - 0.5 * phase);
|
||||
}
|
||||
const double norm = globalNorm(values, communicator);
|
||||
REQUIRE(norm > 0.0);
|
||||
values /= norm;
|
||||
values.SyncAliasMemory(direction);
|
||||
}
|
||||
return direction;
|
||||
}
|
||||
|
||||
[[nodiscard]] MaterialBlockMeasurements blockNorms(
|
||||
const mfem::Vector &vector,
|
||||
const mfem::Array<int> &offsets,
|
||||
const MPI_Comm communicator
|
||||
) {
|
||||
REQUIRE(offsets.Size() == 4);
|
||||
const mfem::Vector density(const_cast<mfem::real_t *>(vector.GetData()) + offsets[0], offsets[1] - offsets[0]);
|
||||
const mfem::Vector surface(const_cast<mfem::real_t *>(vector.GetData()) + offsets[1], offsets[2] - offsets[1]);
|
||||
const mfem::Vector enthalpy(const_cast<mfem::real_t *>(vector.GetData()) + offsets[2], offsets[3] - offsets[2]);
|
||||
return {
|
||||
.density = globalNorm(density, communicator),
|
||||
.surface = globalNorm(surface, communicator),
|
||||
.enthalpy = globalNorm(enthalpy, communicator)
|
||||
};
|
||||
}
|
||||
|
||||
[[nodiscard]] MaterialBlockMeasurements relativeBlockNorms(
|
||||
const mfem::Vector &numerator,
|
||||
const mfem::Vector &denominator,
|
||||
const mfem::Array<int> &offsets,
|
||||
const MPI_Comm communicator
|
||||
) {
|
||||
const MaterialBlockMeasurements numeratorNorms = blockNorms(numerator, offsets, communicator);
|
||||
const MaterialBlockMeasurements denominatorNorms = blockNorms(denominator, offsets, communicator);
|
||||
constexpr double floor = 1.0e-300;
|
||||
return {
|
||||
.density = numeratorNorms.density / std::max(denominatorNorms.density, floor),
|
||||
.surface = numeratorNorms.surface / std::max(denominatorNorms.surface, floor),
|
||||
.enthalpy = numeratorNorms.enthalpy / std::max(denominatorNorms.enthalpy, floor)
|
||||
};
|
||||
}
|
||||
|
||||
[[nodiscard]] std::map<
|
||||
std::string,
|
||||
std::string>
|
||||
commonParameters(
|
||||
const std::string &candidate,
|
||||
const std::string &measurement,
|
||||
const int dimension
|
||||
) {
|
||||
return {
|
||||
{"build_configuration", buildConfiguration()},
|
||||
{"candidate", candidate},
|
||||
{"equation_of_state", "Polytrope(n=1)"},
|
||||
{"experiment_schema", "p9_material_surface_v1"},
|
||||
{"factorization", candidate},
|
||||
{"linearization_state", "projected_lane_emden"},
|
||||
{"measurement", measurement},
|
||||
{"mesh_file", test_utils::setup_args().mesh_file},
|
||||
{"operator", "restricted_material_surface_jacobian"},
|
||||
{"preconditioned_product", "A_material_surface M^-1"},
|
||||
{"root_dimension", std::to_string(dimension)},
|
||||
{"rotation", "zero"}
|
||||
};
|
||||
}
|
||||
|
||||
void recordSpectrum(
|
||||
const std::string &candidate,
|
||||
const solver::ArnoldiSpectralMeasurement &spectrum,
|
||||
const int dimension,
|
||||
const double setupSeconds
|
||||
) {
|
||||
experiment::record_experiment_result(
|
||||
"material_surface_preconditioning_p9", candidate + "_arnoldi_summary",
|
||||
commonParameters(candidate, "arnoldi_summary", dimension),
|
||||
{{"setup_seconds_maximum_rank", setupSeconds},
|
||||
{"requested_dimension", static_cast<double>(spectrum.requestedDimension)},
|
||||
{"achieved_dimension", static_cast<double>(spectrum.achievedDimension)},
|
||||
{"invariant_subspace_found", spectrum.invariantSubspaceFound ? 1.0 : 0.0},
|
||||
{"operator_applications", static_cast<double>(spectrum.operatorApplications)},
|
||||
{"measurement_seconds_maximum_rank", spectrum.measurementSecondsMaximumRank},
|
||||
{"operator_application_seconds_maximum_rank", spectrum.operatorApplicationSecondsMaximumRank},
|
||||
{"projected_condition_proxy", spectrum.projectedConditionProxy},
|
||||
{"projected_largest_singular_value", spectrum.projectedLargestSingularValue},
|
||||
{"projected_smallest_singular_value", spectrum.projectedSmallestSingularValue},
|
||||
{"centroid_real_part", spectrum.centroidRealPart},
|
||||
{"centroid_imaginary_part", spectrum.centroidImaginaryPart},
|
||||
{"rms_distance_from_one", spectrum.rmsDistanceFromOne},
|
||||
{"rms_cluster_radius", spectrum.rmsClusterRadius},
|
||||
{"minimum_magnitude", spectrum.minimumMagnitude},
|
||||
{"maximum_magnitude", spectrum.maximumMagnitude},
|
||||
{"minimum_real_part", spectrum.minimumRealPart},
|
||||
{"maximum_real_part", spectrum.maximumRealPart},
|
||||
{"maximum_absolute_imaginary_part", spectrum.maximumAbsoluteImaginaryPart},
|
||||
{"negative_real_part_count", static_cast<double>(spectrum.negativeRealPartCount)},
|
||||
{"converged_ritz_value_count", static_cast<double>(spectrum.convergedRitzValueCount)},
|
||||
{"conjugate_pair_defect", spectrum.conjugatePairDefect},
|
||||
{"projected_departure_from_normality", spectrum.projectedDepartureFromNormality},
|
||||
{"field_of_values_minimum_real_part", spectrum.projectedFieldOfValuesMinimumRealPart},
|
||||
{"field_of_values_maximum_real_part", spectrum.projectedFieldOfValuesMaximumRealPart}}
|
||||
);
|
||||
|
||||
std::vector<solver::RitzValueMeasurement> ordered = spectrum.ritzValues;
|
||||
std::ranges::sort(ordered, [](const auto &left, const auto &right) {
|
||||
if (left.realPart != right.realPart) {
|
||||
return left.realPart < right.realPart;
|
||||
}
|
||||
return left.imaginaryPart < right.imaginaryPart;
|
||||
});
|
||||
for (std::size_t index = 0; index < ordered.size(); ++index) {
|
||||
const auto &value = ordered[index];
|
||||
experiment::record_experiment_result(
|
||||
"material_surface_preconditioning_p9", candidate + "_ritz_" + std::to_string(index),
|
||||
commonParameters(candidate, "ritz_value", dimension),
|
||||
{{"ritz_index", static_cast<double>(index)},
|
||||
{"real_part", value.realPart},
|
||||
{"imaginary_part", value.imaginaryPart},
|
||||
{"magnitude", value.magnitude},
|
||||
{"distance_from_one", value.distanceFromOne},
|
||||
{"residual_estimate", value.residualEstimate},
|
||||
{"relative_residual_estimate", value.relativeResidualEstimate},
|
||||
{"converged", value.converged ? 1.0 : 0.0}}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Preconditioner>
|
||||
void measureCandidate(
|
||||
const std::string &candidate,
|
||||
Preconditioner &inversePreconditioner,
|
||||
const double setupSeconds,
|
||||
const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation,
|
||||
const mfem::Vector &exactCorrection,
|
||||
const mfem::Vector &rightHandSide,
|
||||
const mfem::Vector &arnoldiDirection,
|
||||
const MPI_Comm communicator,
|
||||
std::map<
|
||||
std::string,
|
||||
double> preparationMetrics = {}
|
||||
) {
|
||||
constexpr int maximumIterations = 40;
|
||||
constexpr int restartDimension = 20;
|
||||
constexpr int arnoldiDimension = 16;
|
||||
|
||||
solver::InstrumentedOperator instrumentedOperation(operation);
|
||||
solver::InstrumentedPreconditioner instrumentedPreconditioner(inversePreconditioner);
|
||||
solver::ResidualHistoryMonitor monitor;
|
||||
mfem::FGMRESSolver krylov(communicator);
|
||||
krylov.SetPreconditioner(instrumentedPreconditioner);
|
||||
krylov.SetOperator(instrumentedOperation);
|
||||
krylov.SetMonitor(monitor);
|
||||
krylov.SetRelTol(1.0e-8);
|
||||
krylov.SetAbsTol(1.0e-12);
|
||||
krylov.SetMaxIter(maximumIterations);
|
||||
krylov.SetKDim(restartDimension);
|
||||
krylov.SetPrintLevel(0);
|
||||
|
||||
mfem::Vector solution(operation.Width());
|
||||
solution = 0.0;
|
||||
announce(communicator, "solving manufactured system with " + candidate);
|
||||
const Clock::time_point solveStart = Clock::now();
|
||||
krylov.Mult(rightHandSide, solution);
|
||||
const double solveSeconds = maximumRankSeconds(solveStart, communicator);
|
||||
|
||||
mfem::Vector trueResidual(operation.Height());
|
||||
operation.Mult(solution, trueResidual);
|
||||
trueResidual -= rightHandSide;
|
||||
mfem::Vector solutionError(solution);
|
||||
solutionError -= exactCorrection;
|
||||
const double trueRelativeResidual =
|
||||
globalNorm(trueResidual, communicator) /
|
||||
std::max(globalNorm(rightHandSide, communicator), std::numeric_limits<double>::min());
|
||||
const double relativeSolutionError =
|
||||
globalNorm(solutionError, communicator) /
|
||||
std::max(globalNorm(exactCorrection, communicator), std::numeric_limits<double>::min());
|
||||
const MaterialBlockMeasurements relativeResidualBlocks =
|
||||
relativeBlockNorms(trueResidual, rightHandSide, operation.GetOffsets(), communicator);
|
||||
|
||||
mfem::Vector preconditionedDirection(operation.Height());
|
||||
inversePreconditioner.Mult(arnoldiDirection, preconditionedDirection);
|
||||
mfem::Vector defect(operation.Height());
|
||||
operation.Mult(preconditionedDirection, defect);
|
||||
defect -= arnoldiDirection;
|
||||
const MaterialBlockMeasurements defectBlocks = blockNorms(defect, operation.GetOffsets(), communicator);
|
||||
const double defectNorm =
|
||||
globalNorm(defect, communicator) /
|
||||
std::max(globalNorm(arnoldiDirection, communicator), std::numeric_limits<double>::min());
|
||||
|
||||
std::map<std::string, double> solveMetrics{
|
||||
{"setup_seconds_maximum_rank", setupSeconds},
|
||||
{"maximum_iterations", static_cast<double>(maximumIterations)},
|
||||
{"restart_dimension", static_cast<double>(restartDimension)},
|
||||
{"solver_converged", krylov.GetConverged() ? 1.0 : 0.0},
|
||||
{"outer_iterations", static_cast<double>(krylov.GetNumIterations())},
|
||||
{"reported_initial_residual_norm", std::abs(krylov.GetInitialNorm())},
|
||||
{"reported_final_residual_norm", std::abs(krylov.GetFinalNorm())},
|
||||
{"true_relative_residual", trueRelativeResidual},
|
||||
{"relative_solution_error", relativeSolutionError},
|
||||
{"density_relative_residual", relativeResidualBlocks.density},
|
||||
{"surface_relative_residual", relativeResidualBlocks.surface},
|
||||
{"enthalpy_relative_residual", relativeResidualBlocks.enthalpy},
|
||||
{"right_preconditioned_defect", defectNorm},
|
||||
{"density_defect_norm", defectBlocks.density},
|
||||
{"surface_defect_norm", defectBlocks.surface},
|
||||
{"enthalpy_defect_norm", defectBlocks.enthalpy},
|
||||
{"solve_seconds_maximum_rank", solveSeconds},
|
||||
{"jacobian_applications", static_cast<double>(instrumentedOperation.GetStatistics().applications)},
|
||||
{"jacobian_application_seconds", instrumentedOperation.GetStatistics().totalSeconds},
|
||||
{"preconditioner_applications",
|
||||
static_cast<double>(instrumentedPreconditioner.GetStatistics().applications)},
|
||||
{"preconditioner_application_seconds", instrumentedPreconditioner.GetStatistics().totalSeconds},
|
||||
{"preconditioner_maximum_application_seconds", instrumentedPreconditioner.GetStatistics().maximumSeconds}
|
||||
};
|
||||
solveMetrics.insert(preparationMetrics.begin(), preparationMetrics.end());
|
||||
experiment::record_experiment_result(
|
||||
"material_surface_preconditioning_p9", candidate + "_linear_solve",
|
||||
commonParameters(candidate, "manufactured_linear_solve", operation.Width()), std::move(solveMetrics)
|
||||
);
|
||||
|
||||
const double initialNorm = std::max(std::abs(krylov.GetInitialNorm()), 1.0e-300);
|
||||
const auto &history = monitor.GetHistory();
|
||||
for (std::size_t index = 0; index < history.size(); ++index) {
|
||||
const auto &sample = history[index];
|
||||
experiment::record_experiment_result(
|
||||
"material_surface_preconditioning_p9", candidate + "_history_" + std::to_string(index),
|
||||
commonParameters(candidate, "fgmres_residual_history", operation.Width()),
|
||||
{{"history_sample", static_cast<double>(index)},
|
||||
{"iteration", static_cast<double>(sample.iteration)},
|
||||
{"reported_residual_norm", sample.reportedNorm},
|
||||
{"reported_relative_residual", std::abs(sample.reportedNorm) / initialNorm},
|
||||
{"final_measurement", sample.final ? 1.0 : 0.0}}
|
||||
);
|
||||
}
|
||||
|
||||
instrumentedOperation.ResetStatistics();
|
||||
instrumentedPreconditioner.ResetStatistics();
|
||||
solver::FixedRightPreconditionedOperator product(instrumentedOperation, instrumentedPreconditioner);
|
||||
announce(communicator, "measuring " + candidate + " with 16-vector Arnoldi");
|
||||
const solver::ArnoldiSpectralMeasurement spectrum = solver::measureArnoldiSpectrum(
|
||||
product, arnoldiDirection, communicator,
|
||||
{.krylovDimension = arnoldiDimension,
|
||||
.breakdownRelativeTolerance = 1.0e-13,
|
||||
.ritzConvergenceRelativeTolerance = 1.0e-7,
|
||||
.reorthogonalize = true}
|
||||
);
|
||||
REQUIRE(std::isfinite(trueRelativeResidual));
|
||||
REQUIRE(std::isfinite(relativeSolutionError));
|
||||
REQUIRE(std::isfinite(defectNorm));
|
||||
REQUIRE(std::isfinite(spectrum.projectedConditionProxy));
|
||||
recordSpectrum(candidate, spectrum, operation.Width(), setupSeconds);
|
||||
|
||||
int rank = 0;
|
||||
MPI_Comm_rank(communicator, &rank);
|
||||
if (rank == 0) {
|
||||
std::cout << "[P9 material-surface] " << candidate << ": iterations=" << krylov.GetNumIterations()
|
||||
<< ", converged=" << (krylov.GetConverged() ? "yes" : "no")
|
||||
<< ", true residual=" << trueRelativeResidual << ", defect=" << defectNorm
|
||||
<< ", projected condition=" << spectrum.projectedConditionProxy << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
template <preconditioning::MaterialSurfaceFactorizationPolicy Policy>
|
||||
void prepareAndMeasure(
|
||||
const std::string &candidate,
|
||||
const Policy policy,
|
||||
const auto &problem,
|
||||
const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation,
|
||||
const mfem::Vector &exactCorrection,
|
||||
const mfem::Vector &rightHandSide,
|
||||
const mfem::Vector &arnoldiDirection,
|
||||
const MPI_Comm communicator,
|
||||
const preconditioning::MaterialSurfaceDiagonalOptions diagonalOptions = {}
|
||||
) {
|
||||
const Clock::time_point setupStart = Clock::now();
|
||||
auto block = preconditioning::materialSurfaceBlock(
|
||||
problem, backend::Diagonal{}, backend::Diagonal{}, policy, diagonalOptions
|
||||
);
|
||||
auto prepared = preconditioning::prepare(problem, block);
|
||||
const double setupTime = maximumRankSeconds(setupStart, communicator);
|
||||
const auto &density = prepared.GetDensityDiagonalQuality();
|
||||
const auto &surface = prepared.GetSurfaceDiagonalQuality();
|
||||
const auto &enthalpy = prepared.GetEnthalpyDiagonalQuality();
|
||||
const auto &calibration = prepared.GetSurfaceCalibration();
|
||||
measureCandidate(
|
||||
candidate, prepared, setupTime, operation, exactCorrection, rightHandSide, arnoldiDirection, communicator,
|
||||
{{"density_diagonal_minimum", density.minimumAbsoluteEntryBeforeRegularization},
|
||||
{"density_diagonal_maximum", density.maximumAbsoluteEntryBeforeRegularization},
|
||||
{"density_diagonal_floor", density.appliedFloor},
|
||||
{"density_regularized_entries", static_cast<double>(density.regularizedEntries)},
|
||||
{"surface_diagonal_minimum", surface.minimumAbsoluteEntryBeforeRegularization},
|
||||
{"surface_diagonal_maximum", surface.maximumAbsoluteEntryBeforeRegularization},
|
||||
{"surface_diagonal_floor", surface.appliedFloor},
|
||||
{"surface_regularized_entries", static_cast<double>(surface.regularizedEntries)},
|
||||
{"surface_calibration_target", static_cast<double>(calibration.target)},
|
||||
{"surface_calibration_probes", static_cast<double>(calibration.probeCount)},
|
||||
{"surface_calibration_objective", static_cast<double>(calibration.objective)},
|
||||
{"surface_calibration_scale", calibration.scale},
|
||||
{"surface_calibration_inverse_multiplier", calibration.inverseMultiplier},
|
||||
{"surface_calibration_numerator", calibration.leastSquaresNumerator},
|
||||
{"surface_calibration_denominator", calibration.leastSquaresDenominator},
|
||||
{"enthalpy_diagonal_minimum", enthalpy.minimumAbsoluteEntryBeforeRegularization},
|
||||
{"enthalpy_diagonal_maximum", enthalpy.maximumAbsoluteEntryBeforeRegularization},
|
||||
{"enthalpy_diagonal_floor", enthalpy.appliedFloor},
|
||||
{"enthalpy_regularized_entries", static_cast<double>(enthalpy.regularizedEntries)}}
|
||||
);
|
||||
}
|
||||
|
||||
template <preconditioning::MaterialSurfaceFactorizationPolicy Policy>
|
||||
void prepareAndMeasureH1(
|
||||
const std::string &candidate,
|
||||
const Policy policy,
|
||||
const int fixedAMGCycles,
|
||||
const int calibrationProbeCount,
|
||||
const auto &problem,
|
||||
const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation,
|
||||
const mfem::Vector &exactCorrection,
|
||||
const mfem::Vector &rightHandSide,
|
||||
const mfem::Vector &arnoldiDirection,
|
||||
const MPI_Comm communicator
|
||||
) {
|
||||
REQUIRE(fixedAMGCycles > 0);
|
||||
REQUIRE(calibrationProbeCount >= 3);
|
||||
|
||||
const Clock::time_point setupStart = Clock::now();
|
||||
auto block = preconditioning::materialSurfaceBlock(
|
||||
problem, backend::Diagonal{}, backend::HypreBoomerAMG{backend::FixedCycles{.cycles = fixedAMGCycles}},
|
||||
policy,
|
||||
preconditioning::SurfaceH1MassStiffness{
|
||||
.calibration = {
|
||||
.target = preconditioning::SurfaceRieszCalibrationTarget::surface_jacobian,
|
||||
.probeCount = calibrationProbeCount
|
||||
}
|
||||
}
|
||||
);
|
||||
auto prepared = preconditioning::prepare(problem, std::move(block));
|
||||
const double setupTime = maximumRankSeconds(setupStart, communicator);
|
||||
|
||||
const auto &density = prepared.GetDensityDiagonalQuality();
|
||||
const auto &enthalpy = prepared.GetEnthalpyDiagonalQuality();
|
||||
const auto &fit = prepared.GetSurfaceFit();
|
||||
measureCandidate(
|
||||
candidate, prepared, setupTime, operation, exactCorrection, rightHandSide, arnoldiDirection, communicator,
|
||||
{{"density_diagonal_minimum", density.minimumAbsoluteEntryBeforeRegularization},
|
||||
{"density_diagonal_maximum", density.maximumAbsoluteEntryBeforeRegularization},
|
||||
{"density_diagonal_floor", density.appliedFloor},
|
||||
{"density_regularized_entries", static_cast<double>(density.regularizedEntries)},
|
||||
{"surface_h1_calibration_target", static_cast<double>(fit.target)},
|
||||
{"surface_h1_calibration_probes", static_cast<double>(fit.probeCount)},
|
||||
{"surface_h1_fit_sign", fit.sign},
|
||||
{"surface_h1_mass_coefficient", fit.massCoefficient},
|
||||
{"surface_h1_stiffness_coefficient", fit.stiffnessCoefficient},
|
||||
{"surface_h1_fit_relative_residual", fit.relativeResidual},
|
||||
{"surface_h1_fit_relative_gram_determinant", fit.relativeGramDeterminant},
|
||||
{"surface_amg_fixed_cycles", static_cast<double>(fixedAMGCycles)},
|
||||
{"enthalpy_diagonal_minimum", enthalpy.minimumAbsoluteEntryBeforeRegularization},
|
||||
{"enthalpy_diagonal_maximum", enthalpy.maximumAbsoluteEntryBeforeRegularization},
|
||||
{"enthalpy_diagonal_floor", enthalpy.appliedFloor},
|
||||
{"enthalpy_regularized_entries", static_cast<double>(enthalpy.regularizedEntries)}}
|
||||
);
|
||||
|
||||
const auto &surfaceBackendStatistics = prepared.GetSurfaceBackend().GetStatistics();
|
||||
const auto &factorizationStatistics = prepared.GetFactorization().GetStatistics();
|
||||
const auto &preparationStatistics = prepared.GetStatistics();
|
||||
auto parameters = commonParameters(candidate, "surface_h1_backend_statistics", operation.Width());
|
||||
parameters["experiment_schema"] = "p9_material_surface_h1_v1";
|
||||
parameters["surface_surrogate"] = "h1_mass_plus_tangential_stiffness";
|
||||
parameters["surface_calibration_target"] = "surface_jacobian";
|
||||
parameters["surface_calibration_probes"] = std::to_string(calibrationProbeCount);
|
||||
parameters["surface_amg_fixed_cycles"] = std::to_string(fixedAMGCycles);
|
||||
experiment::record_experiment_result(
|
||||
"material_surface_preconditioning_p9", candidate + "_surface_h1_backend_statistics", std::move(parameters),
|
||||
{{"setup_seconds_maximum_rank", setupTime},
|
||||
{"surface_h1_fit_sign", fit.sign},
|
||||
{"surface_h1_mass_coefficient", fit.massCoefficient},
|
||||
{"surface_h1_stiffness_coefficient", fit.stiffnessCoefficient},
|
||||
{"surface_h1_fit_relative_residual", fit.relativeResidual},
|
||||
{"surface_h1_fit_relative_gram_determinant", fit.relativeGramDeterminant},
|
||||
{"surface_backend_setups", static_cast<double>(surfaceBackendStatistics.setups)},
|
||||
{"surface_backend_applications", static_cast<double>(surfaceBackendStatistics.applications)},
|
||||
{"surface_backend_inner_iterations", static_cast<double>(surfaceBackendStatistics.innerIterations)},
|
||||
{"surface_backend_last_inner_iterations",
|
||||
static_cast<double>(surfaceBackendStatistics.lastInnerIterations)},
|
||||
{"factorization_applications", static_cast<double>(factorizationStatistics.applications)},
|
||||
{"surface_inverse_applications", static_cast<double>(factorizationStatistics.surfaceInverseApplications)},
|
||||
{"block_setups", static_cast<double>(preparationStatistics.setups)},
|
||||
{"surface_jacobian_probes", static_cast<double>(preparationStatistics.surfaceJacobianProbes)},
|
||||
{"surface_h1_assemblies", static_cast<double>(preparationStatistics.surfaceH1Assemblies)}}
|
||||
);
|
||||
}
|
||||
|
||||
enum class SurfaceProbeMode { constant, ordered_low, alternating_high, deterministic_mixed };
|
||||
|
||||
[[nodiscard]] const char *surfaceProbeModeName(const SurfaceProbeMode mode) noexcept {
|
||||
switch (mode) {
|
||||
case SurfaceProbeMode::constant:
|
||||
return "constant";
|
||||
case SurfaceProbeMode::ordered_low:
|
||||
return "ordered_low";
|
||||
case SurfaceProbeMode::alternating_high:
|
||||
return "alternating_high";
|
||||
case SurfaceProbeMode::deterministic_mixed:
|
||||
return "deterministic_mixed";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
[[nodiscard]] mfem::Vector normalizedSurfaceProbe(
|
||||
const int localSize,
|
||||
const SurfaceProbeMode mode,
|
||||
const MPI_Comm communicator
|
||||
) {
|
||||
int globalSize = 0;
|
||||
int offset = 0;
|
||||
MPI_Allreduce(&localSize, &globalSize, 1, MPI_INT, MPI_SUM, communicator);
|
||||
MPI_Exscan(&localSize, &offset, 1, MPI_INT, MPI_SUM, communicator);
|
||||
int rank = 0;
|
||||
MPI_Comm_rank(communicator, &rank);
|
||||
if (rank == 0) {
|
||||
offset = 0;
|
||||
}
|
||||
REQUIRE(globalSize > 0);
|
||||
|
||||
mfem::Vector probe(localSize);
|
||||
for (int index = 0; index < localSize; ++index) {
|
||||
const int globalIndex = offset + index;
|
||||
const double position = (static_cast<double>(globalIndex) + 0.5) / static_cast<double>(globalSize);
|
||||
switch (mode) {
|
||||
case SurfaceProbeMode::constant:
|
||||
probe(index) = 1.0;
|
||||
break;
|
||||
case SurfaceProbeMode::ordered_low:
|
||||
probe(index) = std::cos(std::numbers::pi_v<double> * position);
|
||||
break;
|
||||
case SurfaceProbeMode::alternating_high:
|
||||
probe(index) = globalIndex % 2 == 0 ? 1.0 : -1.0;
|
||||
break;
|
||||
case SurfaceProbeMode::deterministic_mixed:
|
||||
probe(index) = 0.41 * std::cos(std::numbers::pi_v<double> * position) +
|
||||
std::sin(5.0 * std::numbers::pi_v<double> * position) +
|
||||
0.23 * (globalIndex % 2 == 0 ? 1.0 : -1.0);
|
||||
break;
|
||||
}
|
||||
}
|
||||
const double norm = globalNorm(probe, communicator);
|
||||
REQUIRE(norm > 0.0);
|
||||
probe /= norm;
|
||||
return probe;
|
||||
}
|
||||
|
||||
void applyParameterOverrides(
|
||||
std::map<
|
||||
std::string,
|
||||
std::string> ¶meters,
|
||||
const std::map<
|
||||
std::string,
|
||||
std::string> &overrides
|
||||
) {
|
||||
for (const auto &[key, value] : overrides) {
|
||||
parameters.insert_or_assign(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename SurfaceInverse>
|
||||
void recordSurfaceInverseRecovery(
|
||||
const std::string &candidate,
|
||||
SurfaceInverse &surfaceInverse,
|
||||
const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation,
|
||||
const MPI_Comm communicator,
|
||||
const std::map<
|
||||
std::string,
|
||||
std::string> ¶meterOverrides
|
||||
) {
|
||||
constexpr std::array modes{
|
||||
SurfaceProbeMode::constant, SurfaceProbeMode::ordered_low, SurfaceProbeMode::alternating_high,
|
||||
SurfaceProbeMode::deterministic_mixed
|
||||
};
|
||||
const int surfaceSize = operation.GetOffsets()[2] - operation.GetOffsets()[1];
|
||||
REQUIRE(surfaceInverse.Width() == surfaceSize);
|
||||
REQUIRE(surfaceInverse.Height() == surfaceSize);
|
||||
for (const SurfaceProbeMode mode : modes) {
|
||||
const mfem::Vector probe = normalizedSurfaceProbe(surfaceSize, mode, communicator);
|
||||
mfem::Vector surfaceAction(surfaceSize);
|
||||
mfem::Vector recovered(surfaceSize);
|
||||
operation.ApplySurfaceToSurface(probe, surfaceAction);
|
||||
const Clock::time_point inverseStart = Clock::now();
|
||||
surfaceInverse.Mult(surfaceAction, recovered);
|
||||
const double inverseSeconds = maximumRankSeconds(inverseStart, communicator);
|
||||
|
||||
mfem::Vector recoveryError(recovered);
|
||||
recoveryError -= probe;
|
||||
const double probeNorm = globalNorm(probe, communicator);
|
||||
const double actionNorm = globalNorm(surfaceAction, communicator);
|
||||
const double recoveredNorm = globalNorm(recovered, communicator);
|
||||
const double relativeError = globalNorm(recoveryError, communicator) / probeNorm;
|
||||
constexpr double nonzeroFloor = 1.0e-300;
|
||||
|
||||
auto parameters = commonParameters(candidate, "surface_inverse_recovery", operation.Width());
|
||||
applyParameterOverrides(parameters, parameterOverrides);
|
||||
parameters["surface_probe_mode"] = surfaceProbeModeName(mode);
|
||||
experiment::record_experiment_result(
|
||||
"material_surface_preconditioning_p9", candidate + "_" + surfaceProbeModeName(mode),
|
||||
std::move(parameters),
|
||||
{{"surface_probe_norm", probeNorm},
|
||||
{"surface_action_norm", actionNorm},
|
||||
{"surface_recovered_norm", recoveredNorm},
|
||||
{"surface_recovery_relative_error", relativeError},
|
||||
{"surface_operator_gain", actionNorm / probeNorm},
|
||||
{"surface_inverse_gain", recoveredNorm / std::max(actionNorm, nonzeroFloor)},
|
||||
{"surface_recovered_gain", recoveredNorm / probeNorm},
|
||||
{"surface_inverse_seconds_maximum_rank", inverseSeconds}}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename PreparedPreconditioner>
|
||||
void recordBalancedPreconditionedDefect(
|
||||
const std::string &candidate,
|
||||
PreparedPreconditioner &prepared,
|
||||
const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation,
|
||||
const MPI_Comm communicator,
|
||||
const std::map<
|
||||
std::string,
|
||||
std::string> ¶meterOverrides
|
||||
) {
|
||||
const mfem::Vector direction = blockBalancedDirection(operation.GetOffsets(), 1.37, communicator);
|
||||
mfem::Vector correction(operation.Width());
|
||||
mfem::Vector defect(operation.Height());
|
||||
const Clock::time_point applicationStart = Clock::now();
|
||||
prepared.Mult(direction, correction);
|
||||
const double applicationSeconds = maximumRankSeconds(applicationStart, communicator);
|
||||
operation.Mult(correction, defect);
|
||||
defect -= direction;
|
||||
const MaterialBlockMeasurements blockDefects = blockNorms(defect, operation.GetOffsets(), communicator);
|
||||
const double relativeDefect = globalNorm(defect, communicator) /
|
||||
std::max(globalNorm(direction, communicator), std::numeric_limits<double>::min());
|
||||
|
||||
auto parameters = commonParameters(candidate, "balanced_right_preconditioned_defect", operation.Width());
|
||||
applyParameterOverrides(parameters, parameterOverrides);
|
||||
experiment::record_experiment_result(
|
||||
"material_surface_preconditioning_p9", candidate + "_balanced_defect", std::move(parameters),
|
||||
{{"right_preconditioned_defect", relativeDefect},
|
||||
{"density_defect_norm", blockDefects.density},
|
||||
{"surface_defect_norm", blockDefects.surface},
|
||||
{"enthalpy_defect_norm", blockDefects.enthalpy},
|
||||
{"preconditioner_application_seconds_maximum_rank", applicationSeconds}}
|
||||
);
|
||||
}
|
||||
|
||||
void measureH1CalibrationFloor(
|
||||
const std::string &candidate,
|
||||
const std::string &relativeMassCoefficientFloorLabel,
|
||||
const double relativeMassCoefficientFloor,
|
||||
const auto &problem,
|
||||
const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation,
|
||||
const MPI_Comm communicator
|
||||
) {
|
||||
const Clock::time_point setupStart = Clock::now();
|
||||
auto block = preconditioning::materialSurfaceBlock(
|
||||
problem, backend::Diagonal{}, backend::HypreBoomerAMG{backend::FixedCycles{.cycles = 1}},
|
||||
preconditioning::ApproximateMaterialSurfaceLDU{},
|
||||
preconditioning::SurfaceH1MassStiffness{
|
||||
.calibration =
|
||||
{.target = preconditioning::SurfaceRieszCalibrationTarget::surface_jacobian, .probeCount = 4},
|
||||
.relativeMassCoefficientFloor = relativeMassCoefficientFloor
|
||||
}
|
||||
);
|
||||
auto prepared = preconditioning::prepare(problem, std::move(block));
|
||||
const double setupTime = maximumRankSeconds(setupStart, communicator);
|
||||
const auto &fit = prepared.GetSurfaceFit();
|
||||
|
||||
const std::map<std::string, std::string> parameters{
|
||||
{"experiment_schema", "p9_material_surface_h1_tuning_v1"},
|
||||
{"surface_surrogate", "h1_mass_plus_tangential_stiffness"},
|
||||
{"surface_calibration_target", "surface_jacobian"},
|
||||
{"surface_calibration_probes", "4"},
|
||||
{"surface_amg_fixed_cycles", "1"},
|
||||
{"relative_mass_coefficient_floor", relativeMassCoefficientFloorLabel}
|
||||
};
|
||||
recordSurfaceInverseRecovery(candidate, prepared.GetSurfaceInverse(), operation, communicator, parameters);
|
||||
recordBalancedPreconditionedDefect(candidate, prepared, operation, communicator, parameters);
|
||||
|
||||
const auto &backendStatistics = prepared.GetSurfaceBackend().GetStatistics();
|
||||
const auto &factorizationStatistics = prepared.GetFactorization().GetStatistics();
|
||||
const auto &preparationStatistics = prepared.GetStatistics();
|
||||
auto summaryParameters = commonParameters(candidate, "surface_h1_floor_summary", operation.Width());
|
||||
applyParameterOverrides(summaryParameters, parameters);
|
||||
experiment::record_experiment_result(
|
||||
"material_surface_preconditioning_p9", candidate + "_summary", std::move(summaryParameters),
|
||||
{{"setup_seconds_maximum_rank", setupTime},
|
||||
{"relative_mass_coefficient_floor", relativeMassCoefficientFloor},
|
||||
{"surface_h1_fit_sign", fit.sign},
|
||||
{"surface_h1_mass_coefficient", fit.massCoefficient},
|
||||
{"surface_h1_stiffness_coefficient", fit.stiffnessCoefficient},
|
||||
{"surface_h1_fit_relative_residual", fit.relativeResidual},
|
||||
{"surface_h1_fit_relative_gram_determinant", fit.relativeGramDeterminant},
|
||||
{"surface_backend_setups", static_cast<double>(backendStatistics.setups)},
|
||||
{"surface_backend_applications", static_cast<double>(backendStatistics.applications)},
|
||||
{"surface_backend_inner_iterations", static_cast<double>(backendStatistics.innerIterations)},
|
||||
{"surface_backend_last_inner_iterations", static_cast<double>(backendStatistics.lastInnerIterations)},
|
||||
{"factorization_applications", static_cast<double>(factorizationStatistics.applications)},
|
||||
{"surface_inverse_applications", static_cast<double>(factorizationStatistics.surfaceInverseApplications)},
|
||||
{"surface_jacobian_probes", static_cast<double>(preparationStatistics.surfaceJacobianProbes)},
|
||||
{"surface_h1_assemblies", static_cast<double>(preparationStatistics.surfaceH1Assemblies)}}
|
||||
);
|
||||
}
|
||||
|
||||
void measureScalarSurfaceControl(
|
||||
const std::string &candidate,
|
||||
const preconditioning::SurfaceRieszCalibrationObjective objective,
|
||||
const int calibrationProbeCount,
|
||||
const auto &problem,
|
||||
const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation,
|
||||
const MPI_Comm communicator
|
||||
) {
|
||||
REQUIRE(calibrationProbeCount > 0);
|
||||
const preconditioning::MaterialSurfaceDiagonalOptions calibration{
|
||||
.surfaceCalibration = {
|
||||
.target = preconditioning::SurfaceRieszCalibrationTarget::surface_jacobian,
|
||||
.probeCount = calibrationProbeCount,
|
||||
.objective = objective
|
||||
}
|
||||
};
|
||||
const Clock::time_point setupStart = Clock::now();
|
||||
auto block = preconditioning::materialSurfaceBlock(
|
||||
problem, backend::Diagonal{}, backend::Diagonal{}, preconditioning::ApproximateMaterialSurfaceLDU{},
|
||||
calibration
|
||||
);
|
||||
auto prepared = preconditioning::prepare(problem, std::move(block));
|
||||
auto directSurfaceInverse = backend::prepare(backend::Diagonal{}, prepared.GetSurfaceDiagonal());
|
||||
const double setupTime = maximumRankSeconds(setupStart, communicator);
|
||||
const auto &calibrationData = prepared.GetSurfaceCalibration();
|
||||
|
||||
const std::map<std::string, std::string> parameters{
|
||||
{"experiment_schema", "p9_material_surface_h1_tuning_v1"},
|
||||
{"surface_surrogate", "scalar_mass_diagonal"},
|
||||
{"surface_calibration_target", "surface_jacobian"},
|
||||
{"surface_calibration_probes", std::to_string(calibrationProbeCount)},
|
||||
{"surface_calibration_objective",
|
||||
objective == preconditioning::SurfaceRieszCalibrationObjective::operator_action
|
||||
? "operator_action"
|
||||
: "right_preconditioned_action"}
|
||||
};
|
||||
recordSurfaceInverseRecovery(candidate, directSurfaceInverse, operation, communicator, parameters);
|
||||
recordBalancedPreconditionedDefect(candidate, prepared, operation, communicator, parameters);
|
||||
|
||||
const auto &directStatistics = directSurfaceInverse.GetStatistics();
|
||||
const auto &factorizationStatistics = prepared.GetFactorization().GetStatistics();
|
||||
auto summaryParameters = commonParameters(candidate, "scalar_surface_control_summary", operation.Width());
|
||||
applyParameterOverrides(summaryParameters, parameters);
|
||||
experiment::record_experiment_result(
|
||||
"material_surface_preconditioning_p9", candidate + "_summary", std::move(summaryParameters),
|
||||
{{"setup_seconds_maximum_rank", setupTime},
|
||||
{"surface_calibration_scale", calibrationData.scale},
|
||||
{"surface_calibration_inverse_multiplier", calibrationData.inverseMultiplier},
|
||||
{"surface_calibration_numerator", calibrationData.leastSquaresNumerator},
|
||||
{"surface_calibration_denominator", calibrationData.leastSquaresDenominator},
|
||||
{"surface_backend_setups", static_cast<double>(directStatistics.setups)},
|
||||
{"surface_backend_applications", static_cast<double>(directStatistics.applications)},
|
||||
{"factorization_applications", static_cast<double>(factorizationStatistics.applications)},
|
||||
{"surface_inverse_applications", static_cast<double>(factorizationStatistics.surfaceInverseApplications)}}
|
||||
);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST_CASE(
|
||||
"Material Surface P9 Numerical Factorization Comparison",
|
||||
"[preconditioning][material_surface][diagnostics][experiment][spectrum][p9][p9_baseline]"
|
||||
) {
|
||||
using namespace mean_field;
|
||||
|
||||
const utils::Args arguments = test_utils::setup_args();
|
||||
fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0);
|
||||
REQUIRE(finiteElements.okay());
|
||||
const MPI_Comm communicator = finiteElements.mesh->GetComm();
|
||||
|
||||
constexpr double radius = utils::RADIUS;
|
||||
constexpr double mass = utils::MASS;
|
||||
const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v<double>;
|
||||
const double centralDensity = std::numbers::pi_v<double> * mass / (4.0 * radius * radius * radius);
|
||||
auto model = model::StellarModel(
|
||||
eos::Polytrope({.n = 1.0, .K = polytropicConstant}),
|
||||
surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}),
|
||||
integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}),
|
||||
constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}})
|
||||
);
|
||||
auto problem = equilibrium::discretize(model, finiteElements);
|
||||
auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 1024}));
|
||||
problem.Prepare(projected.values, makeDependencies(), zeroRotation());
|
||||
|
||||
const auto &physical = problem.GetPreparedOperator().GetPhysicalOperator();
|
||||
preconditioning::MaterialSurfaceJacobianOperator materialSurfaceOperator(physical);
|
||||
const auto exactCorrection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.23, communicator);
|
||||
mfem::Vector rightHandSide(materialSurfaceOperator.Height());
|
||||
materialSurfaceOperator.Mult(exactCorrection, rightHandSide);
|
||||
const auto arnoldiDirection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.79, communicator);
|
||||
|
||||
solver::IdentityPreconditioner identity(materialSurfaceOperator.Width());
|
||||
measureCandidate(
|
||||
"identity", identity, 0.0, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection,
|
||||
communicator
|
||||
);
|
||||
prepareAndMeasure(
|
||||
"block_diagonal", preconditioning::MaterialSurfaceBlockDiagonal{}, problem, materialSurfaceOperator,
|
||||
exactCorrection, rightHandSide, arnoldiDirection, communicator
|
||||
);
|
||||
prepareAndMeasure(
|
||||
"material_independent_surface", preconditioning::CoupledMaterialIndependentSurface{}, problem,
|
||||
materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator
|
||||
);
|
||||
prepareAndMeasure(
|
||||
"material_then_surface", preconditioning::MaterialThenSurfaceTriangular{}, problem, materialSurfaceOperator,
|
||||
exactCorrection, rightHandSide, arnoldiDirection, communicator
|
||||
);
|
||||
prepareAndMeasure(
|
||||
"surface_then_material", preconditioning::SurfaceThenMaterialTriangular{}, problem, materialSurfaceOperator,
|
||||
exactCorrection, rightHandSide, arnoldiDirection, communicator
|
||||
);
|
||||
prepareAndMeasure(
|
||||
"approximate_ldu", preconditioning::ApproximateMaterialSurfaceLDU{}, problem, materialSurfaceOperator,
|
||||
exactCorrection, rightHandSide, arnoldiDirection, communicator
|
||||
);
|
||||
}
|
||||
|
||||
TEST_CASE(
|
||||
"Material Surface P9 Calibrated LDU Comparison",
|
||||
"[preconditioning][material_surface][diagnostics][experiment][spectrum][p9][p9_refinement]"
|
||||
) {
|
||||
using namespace mean_field;
|
||||
|
||||
const utils::Args arguments = test_utils::setup_args();
|
||||
fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0);
|
||||
REQUIRE(finiteElements.okay());
|
||||
const MPI_Comm communicator = finiteElements.mesh->GetComm();
|
||||
|
||||
constexpr double radius = utils::RADIUS;
|
||||
constexpr double mass = utils::MASS;
|
||||
const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v<double>;
|
||||
const double centralDensity = std::numbers::pi_v<double> * mass / (4.0 * radius * radius * radius);
|
||||
auto model = model::StellarModel(
|
||||
eos::Polytrope({.n = 1.0, .K = polytropicConstant}),
|
||||
surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}),
|
||||
integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}),
|
||||
constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}})
|
||||
);
|
||||
auto problem = equilibrium::discretize(model, finiteElements);
|
||||
auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 1024}));
|
||||
problem.Prepare(projected.values, makeDependencies(), zeroRotation());
|
||||
|
||||
const auto &physical = problem.GetPreparedOperator().GetPhysicalOperator();
|
||||
preconditioning::MaterialSurfaceJacobianOperator materialSurfaceOperator(physical);
|
||||
const auto exactCorrection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.23, communicator);
|
||||
mfem::Vector rightHandSide(materialSurfaceOperator.Height());
|
||||
materialSurfaceOperator.Mult(exactCorrection, rightHandSide);
|
||||
const auto arnoldiDirection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.79, communicator);
|
||||
|
||||
constexpr preconditioning::MaterialSurfaceDiagonalOptions surfaceJacobianCalibration{
|
||||
.surfaceCalibration = {
|
||||
.target = preconditioning::SurfaceRieszCalibrationTarget::surface_jacobian, .probeCount = 4
|
||||
}
|
||||
};
|
||||
constexpr preconditioning::MaterialSurfaceDiagonalOptions surfaceSchurCalibration{
|
||||
.surfaceCalibration = {
|
||||
.target = preconditioning::SurfaceRieszCalibrationTarget::approximate_material_schur, .probeCount = 4
|
||||
}
|
||||
};
|
||||
|
||||
prepareAndMeasure(
|
||||
"surface_then_material_calibrated_aqq", preconditioning::SurfaceThenMaterialTriangular{}, problem,
|
||||
materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator,
|
||||
surfaceJacobianCalibration
|
||||
);
|
||||
prepareAndMeasure(
|
||||
"approximate_ldu", preconditioning::ApproximateMaterialSurfaceLDU{}, problem, materialSurfaceOperator,
|
||||
exactCorrection, rightHandSide, arnoldiDirection, communicator
|
||||
);
|
||||
prepareAndMeasure(
|
||||
"approximate_ldu_calibrated_aqq", preconditioning::ApproximateMaterialSurfaceLDU{}, problem,
|
||||
materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator,
|
||||
surfaceJacobianCalibration
|
||||
);
|
||||
prepareAndMeasure(
|
||||
"approximate_ldu_calibrated_schur", preconditioning::ApproximateMaterialSurfaceLDU{}, problem,
|
||||
materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator, surfaceSchurCalibration
|
||||
);
|
||||
}
|
||||
|
||||
TEST_CASE(
|
||||
"Material Surface P9 Frequency-Aware Surface Refinement",
|
||||
"[preconditioning][material_surface][diagnostics][experiment][spectrum][p9][p9_h1_refinement]"
|
||||
) {
|
||||
using namespace mean_field;
|
||||
|
||||
const utils::Args arguments = test_utils::setup_args();
|
||||
fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0);
|
||||
REQUIRE(finiteElements.okay());
|
||||
const MPI_Comm communicator = finiteElements.mesh->GetComm();
|
||||
|
||||
constexpr double radius = utils::RADIUS;
|
||||
constexpr double mass = utils::MASS;
|
||||
const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v<double>;
|
||||
const double centralDensity = std::numbers::pi_v<double> * mass / (4.0 * radius * radius * radius);
|
||||
auto model = model::StellarModel(
|
||||
eos::Polytrope({.n = 1.0, .K = polytropicConstant}),
|
||||
surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}),
|
||||
integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}),
|
||||
constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}})
|
||||
);
|
||||
auto problem = equilibrium::discretize(model, finiteElements);
|
||||
auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 1024}));
|
||||
problem.Prepare(projected.values, makeDependencies(), zeroRotation());
|
||||
|
||||
const auto &physical = problem.GetPreparedOperator().GetPhysicalOperator();
|
||||
preconditioning::MaterialSurfaceJacobianOperator materialSurfaceOperator(physical);
|
||||
const auto exactCorrection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.23, communicator);
|
||||
mfem::Vector rightHandSide(materialSurfaceOperator.Height());
|
||||
materialSurfaceOperator.Mult(exactCorrection, rightHandSide);
|
||||
const auto arnoldiDirection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.79, communicator);
|
||||
|
||||
constexpr int calibrationProbeCount = 4;
|
||||
prepareAndMeasureH1(
|
||||
"h1_aqq_surface_then_material_amg1", preconditioning::SurfaceThenMaterialTriangular{}, 1, calibrationProbeCount,
|
||||
problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator
|
||||
);
|
||||
prepareAndMeasureH1(
|
||||
"h1_aqq_approximate_ldu_amg1", preconditioning::ApproximateMaterialSurfaceLDU{}, 1, calibrationProbeCount,
|
||||
problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator
|
||||
);
|
||||
prepareAndMeasureH1(
|
||||
"h1_aqq_approximate_ldu_amg2", preconditioning::ApproximateMaterialSurfaceLDU{}, 2, calibrationProbeCount,
|
||||
problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator
|
||||
);
|
||||
}
|
||||
|
||||
TEST_CASE(
|
||||
"Material Surface P9 H1 Calibration Floor Tuning",
|
||||
"[preconditioning][material_surface][diagnostics][experiment][p9][p9_h1_tuning]"
|
||||
) {
|
||||
using namespace mean_field;
|
||||
|
||||
const utils::Args arguments = test_utils::setup_args();
|
||||
fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0);
|
||||
REQUIRE(finiteElements.okay());
|
||||
const MPI_Comm communicator = finiteElements.mesh->GetComm();
|
||||
|
||||
constexpr double radius = utils::RADIUS;
|
||||
constexpr double mass = utils::MASS;
|
||||
const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v<double>;
|
||||
const double centralDensity = std::numbers::pi_v<double> * mass / (4.0 * radius * radius * radius);
|
||||
auto model = model::StellarModel(
|
||||
eos::Polytrope({.n = 1.0, .K = polytropicConstant}),
|
||||
surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}),
|
||||
integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}),
|
||||
constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}})
|
||||
);
|
||||
auto problem = equilibrium::discretize(model, finiteElements);
|
||||
auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 1024}));
|
||||
problem.Prepare(projected.values, makeDependencies(), zeroRotation());
|
||||
|
||||
const auto &physical = problem.GetPreparedOperator().GetPhysicalOperator();
|
||||
preconditioning::MaterialSurfaceJacobianOperator materialSurfaceOperator(physical);
|
||||
|
||||
constexpr std::array floorCases{
|
||||
std::pair{"1e-10", 1.0e-10}, std::pair{"1e-4", 1.0e-4}, std::pair{"1e-2", 1.0e-2}, std::pair{"1e-1", 1.0e-1},
|
||||
std::pair{"1", 1.0}
|
||||
};
|
||||
for (const auto &[label, floor] : floorCases) {
|
||||
measureH1CalibrationFloor(
|
||||
std::string("h1_aqq_floor_") + label, label, floor, problem, materialSurfaceOperator, communicator
|
||||
);
|
||||
}
|
||||
measureScalarSurfaceControl(
|
||||
"scalar_aqq_operator_calibrated_diagonal_control",
|
||||
preconditioning::SurfaceRieszCalibrationObjective::operator_action, 4, problem, materialSurfaceOperator,
|
||||
communicator
|
||||
);
|
||||
constexpr std::array inverseProbeCounts{1, 2, 4, 8, 16};
|
||||
for (const int probeCount : inverseProbeCounts) {
|
||||
measureScalarSurfaceControl(
|
||||
"scalar_aqq_right_calibrated_diagonal_" + std::to_string(probeCount) + "_probes",
|
||||
preconditioning::SurfaceRieszCalibrationObjective::right_preconditioned_action, probeCount, problem,
|
||||
materialSurfaceOperator, communicator
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user