tboudreaux/MeanField
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(preconditioner): major work on preconditioner system

Browse Source 
first preconditioner MVP
This commit is contained in:
Emily Boudreaux
2026-09-04 07:54:10 -04:00
parent 25510008dd
commit 71423d543f
61 changed files with 15920 additions and 422 deletions
Download Patch File Download Diff File Expand all files Collapse all files

292
tests/mpi/distributed_execution.cpp Normal file
View File

@@ -0,0 +1,292 @@
#include <algorithm>
#include <catch2/catch_test_macros.hpp>
#include <cmath>
#include <limits>
#include <mfem.hpp>
#include <vector>
import mean_field;
import test_helpers;
namespace {
bool vector_is_finite(const mfem::Vector &vector) {
for (int index = 0; index < vector.Size(); ++index) {
if (!std::isfinite(vector(index))) {
return false;
}
}
return true;
}
mfem::Vector make_deterministic_vector(
const int size,
const double phase
) {
mfem::Vector vector(size);
for (int index = 0; index < size; ++index) {
const double coordinate = static_cast<double>(index + 1);
vector(index) = std::sin(phase + 0.017 * coordinate) + 0.25 * std::cos(0.031 * coordinate);
}
return vector;
}
double global_dot(
const mfem::Vector &left,
const mfem::Vector &right,
const MPI_Comm communicator
) {
REQUIRE(left.Size() == right.Size());
const double local = left * right;
double global = 0.0;
REQUIRE(MPI_Allreduce(&local, &global, 1, MPI_DOUBLE, MPI_SUM, communicator) == MPI_SUCCESS);
return global;
}
double global_norm(
const mfem::Vector &vector,
const MPI_Comm communicator
) {
return std::sqrt(global_dot(vector, vector, communicator));
}
} // namespace
TEST_CASE(
"MPI Runtime Preserves World And Split Communicator Membership",
"[mpi][distributed][unit]"
) {
int rank = 0;
int size = 1;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
std::vector<int> ranks(static_cast<std::size_t>(size), -1);
MPI_Allgather(&rank, 1, MPI_INT, ranks.data(), 1, MPI_INT, MPI_COMM_WORLD);
CHECK(size >= 2);
for (int expected = 0; expected < size; ++expected) {
CHECK(ranks[expected] == expected);
}
MPI_Comm parity_communicator = MPI_COMM_NULL;
MPI_Comm_split(MPI_COMM_WORLD, rank % 2, rank, &parity_communicator);
int parity_size = 0;
MPI_Comm_size(parity_communicator, &parity_size);
const int expected_parity_size = (size + 1 - rank % 2) / 2;
CHECK(parity_size == expected_parity_size);
MPI_Comm_free(&parity_communicator);
}
TEST_CASE(
"MPI FEM Setup Partitions Every Element Exactly Once",
"[mpi][distributed][mesh][integration]"
) {
const mean_field::utils::Args args = test_utils::setup_args();
const mean_field::fem::FEM f = mean_field::fem::setup_fem(args.mesh_file, args, 0);
const long long local_elements = f.mesh->GetNE();
long long global_elements = 0;
long long minimum_elements = 0;
MPI_Allreduce(&local_elements, &global_elements, 1, MPI_LONG_LONG, MPI_SUM, f.mesh->GetComm());
MPI_Allreduce(&local_elements, &minimum_elements, 1, MPI_LONG_LONG, MPI_MIN, f.mesh->GetComm());
CHECK(global_elements == f.smesh.mesh->GetNE());
CHECK(minimum_elements > 0);
CHECK(f.logicalReferenceMesh->GetNE() == f.mesh->GetNE());
}
TEST_CASE(
"MPI Prepared Gravity Operators Preserve Global Algebraic Identities",
"[mpi][distributed][gravity][operators][unit]"
) {
const auto args = test_utils::setup_args();
mean_field::fem::FEM f = mean_field::fem::setup_fem(args.mesh_file, args, 0);
using GeometryContext = mean_field::operators::context::gravity_field::GravityFieldGeometryContext;
GeometryContext geometry_context(f, *f.domainMapperStateless);
mfem::Vector displacement_true(f.displacementFes->GetTrueVSize());
displacement_true = 0.0;
const mfem::Vector displacement = geometry_context.GetDisplacementMap().gather(displacement_true);
geometry_context.PreparePrimal(displacement, {0}, {0});
const mfem::Operator &mass = geometry_context.GetMassOperator();
const mfem::Vector first = make_deterministic_vector(mass.Width(), 0.17);
const mfem::Vector second = make_deterministic_vector(mass.Width(), 0.83);
mfem::Vector combination(first);
combination *= 1.7;
combination.Add(-0.4, second);
mfem::Vector first_action;
mfem::Vector second_action;
mfem::Vector combination_action;
mass.Mult(first, first_action);
mass.Mult(second, second_action);
mass.Mult(combination, combination_action);
mfem::Vector expected_combination(first_action);
expected_combination *= 1.7;
expected_combination.Add(-0.4, second_action);
mfem::Vector linearity_difference(combination_action);
linearity_difference -= expected_combination;
const MPI_Comm communicator = f.mesh->GetComm();
const double symmetry_scale = std::max(
{std::abs(global_dot(first, second_action, communicator)),
std::abs(global_dot(second, first_action, communicator)), std::numeric_limits<double>::epsilon()}
);
const double symmetry_error =
std::abs(global_dot(first, second_action, communicator) - global_dot(second, first_action, communicator)) /
symmetry_scale;
const double linearity_error =
global_norm(linearity_difference, communicator) /
std::max(global_norm(expected_combination, communicator), std::numeric_limits<double>::epsilon());
CHECK(symmetry_error <= 2.0e-12);
CHECK(linearity_error <= 2.0e-12);
const mfem::Operator &divergence = geometry_context.GetDivergenceOperator();
const mfem::Operator &transpose_divergence = geometry_context.GetTransposeDivergenceOperator();
const mfem::Vector flux = make_deterministic_vector(divergence.Width(), 0.41);
const mfem::Vector potential = make_deterministic_vector(divergence.Height(), 0.67);
mfem::Vector divergence_action;
mfem::Vector transpose_action;
divergence.Mult(flux, divergence_action);
transpose_divergence.Mult(potential, transpose_action);
const double forward_product = global_dot(potential, divergence_action, communicator);
const double transpose_product = global_dot(flux, transpose_action, communicator);
const double adjoint_scale =
std::max({std::abs(forward_product), std::abs(transpose_product), std::numeric_limits<double>::epsilon()});
const double adjoint_error = std::abs(forward_product - transpose_product) / adjoint_scale;
CHECK(adjoint_error <= 2.0e-12);
}
TEST_CASE(
"MPI Coupled Gravity LDU Is Stationary Linear And Does Not Reprepare Geometry",
"[mpi][distributed][gravity][preconditioning][integration]"
) {
const auto args = test_utils::setup_args();
mean_field::fem::FEM f = mean_field::fem::setup_fem(args.mesh_file, args, 0);
using GeometryContext = mean_field::operators::context::gravity_field::GravityFieldGeometryContext;
GeometryContext geometryContext(f, *f.domainMapperStateless);
mfem::Vector displacementTrue(f.displacementFes->GetTrueVSize());
displacementTrue = 0.0;
const mfem::Vector displacement = geometryContext.GetDisplacementMap().gather(displacementTrue);
geometryContext.PreparePrimal(displacement, {.value = 1}, {.value = 1});
namespace backend = mean_field::preconditioning::backend;
namespace preconditioning = mean_field::preconditioning;
const auto block = preconditioning::GravityFieldBlock(
backend::Diagonal{}, backend::HypreBoomerAMG{backend::FixedCycles{.cycles = 1}},
preconditioning::GravityApproximateLDU{}
);
auto prepared = preconditioning::prepare(f, geometryContext, block);
const mfem::Vector first = make_deterministic_vector(prepared.Width(), 0.23);
const mfem::Vector second = make_deterministic_vector(prepared.Width(), 0.79);
mfem::Vector combined(first);
combined *= 1.3;
combined.Add(-0.45, second);
mfem::Vector firstAction(prepared.Height());
mfem::Vector secondAction(prepared.Height());
mfem::Vector combinedAction(prepared.Height());
mfem::Vector repeatedAction(prepared.Height());
firstAction = 0.0;
secondAction = 0.0;
combinedAction = 0.0;
repeatedAction = 0.0;
const std::uint64_t massPreparations = geometryContext.GetMassOperator().GetPreparationCount();
const std::uint64_t sourcePreparations = geometryContext.GetSourceOperator().GetPreparationCount();
double *const combinedStorage = combinedAction.GetData();
prepared.Mult(first, firstAction);
prepared.Mult(second, secondAction);
prepared.Mult(combined, combinedAction);
prepared.Mult(first, repeatedAction);
mfem::Vector expectedCombined(firstAction);
expectedCombined *= 1.3;
expectedCombined.Add(-0.45, secondAction);
const MPI_Comm communicator = f.mesh->GetComm();
mfem::Vector linearityDifference(combinedAction);
linearityDifference -= expectedCombined;
mfem::Vector determinismDifference(repeatedAction);
determinismDifference -= firstAction;
const double linearityError =
global_norm(linearityDifference, communicator) /
std::max(global_norm(expectedCombined, communicator), std::numeric_limits<double>::epsilon());
CHECK(vector_is_finite(combinedAction));
CHECK(linearityError <= 5.0e-12);
CHECK(global_norm(determinismDifference, communicator) <= 5.0e-14);
CHECK(combinedAction.GetData() == combinedStorage);
CHECK(geometryContext.GetMassOperator().GetPreparationCount() == massPreparations);
CHECK(geometryContext.GetSourceOperator().GetPreparationCount() == sourcePreparations);
CHECK(prepared.GetFactorization().GetStatistics().applications == 4);
}
TEST_CASE(
"MPI Gravity Analysis And Solve Produce Finite Distributed Fields",
"[mpi][distributed][gravity][integration]"
) {
auto args = test_utils::setup_args();
mean_field::fem::FEM f = mean_field::fem::setup_fem(args.mesh_file, args, 0);
*f.displacement = 0.0;
using DomainSchema = mean_field::utils::domain::CoreEnvelopeVacuumDomainSchema;
mfem::Vector attribute_density(f.smesh.mesh->attributes.Max());
attribute_density = 0.0;
for (int index = 0; index < f.smesh.mesh->attributes.Size(); ++index) {
const int attribute = f.smesh.mesh->attributes[index];
if (DomainSchema::template attribute_belongs_to<mean_field::utils::domain::Stellar>(attribute)) {
attribute_density(attribute - 1) = 1.0;
}
}
mfem::PWConstCoefficient density_coefficient(attribute_density);
mfem::ParGridFunction density(f.densityFes.get());
density.ProjectCoefficient(density_coefficient);
mean_field::analysis::conserve_mass(f, density, mean_field::utils::MASS);
const double integrated_mass = mean_field::analysis::domain_integrate_grid_function(
f, density, mean_field::utils::DOMAINS::STELLAR, mean_field::mapping::COORDINATE_SPACE::PHYSICAL
);
f.com = mean_field::analysis::get_com(f, density);
f.Q = mean_field::physics::compute_quadrupole_moment_tensor(f, density, f.com);
const mean_field::physics::GravitySolution solution = mean_field::physics::solve_gravity_field(
f,
mean_field::physics::GravitySolveOptions{
.relativeTolerance = 1.0e-12, .absoluteTolerance = 1.0e-15, .maximumIterations = 1000
},
density, *f.displacement
);
mfem::Vector flux_true;
mfem::Vector potential_true;
solution.gradPhi.GetTrueDofs(flux_true);
solution.phi.GetTrueDofs(potential_true);
const int local_finite = vector_is_finite(flux_true) && vector_is_finite(potential_true) ? 1 : 0;
int globally_finite = 0;
MPI_Allreduce(&local_finite, &globally_finite, 1, MPI_INT, MPI_MIN, f.mesh->GetComm());
const double local_norms[2]{flux_true * flux_true, potential_true * potential_true};
double global_norms[2]{};
MPI_Allreduce(local_norms, global_norms, 2, MPI_DOUBLE, MPI_SUM, f.mesh->GetComm());
CHECK(globally_finite == 1);
CHECK(std::abs(integrated_mass - mean_field::utils::MASS) <= 1.0e-12 * mean_field::utils::MASS);
CHECK(global_norms[0] > std::numeric_limits<double>::min());
CHECK(global_norms[1] > std::numeric_limits<double>::min());
}

58
tests/mpi/mpi_test_main.cpp Normal file
View File

@@ -0,0 +1,58 @@
#include <catch2/catch_session.hpp>
#include <cstdint>
#include <iostream>
#include <mfem.hpp>
#include <string>
#include <vector>
int main(
int argc,
char *argv[]
) {
mfem::Mpi::Init(argc, argv);
const int rank = mfem::Mpi::WorldRank();
std::vector<std::string> arguments;
arguments.reserve(static_cast<std::size_t>(argc) + 6);
for (int index = 0; index < argc; ++index) {
arguments.emplace_back(argv[index]);
}
arguments.emplace_back("--order");
arguments.emplace_back("lex");
arguments.emplace_back("--rng-seed");
arguments.emplace_back("184467");
if (rank != 0) {
arguments.emplace_back("--out");
arguments.emplace_back("/dev/null");
}
std::vector<const char *> catch_arguments;
catch_arguments.reserve(arguments.size());
for (const std::string &argument : arguments) {
catch_arguments.push_back(argument.c_str());
}
int local_result = 0;
{
Catch::Session session;
if (const int parse_result =
session.applyCommandLine(static_cast<int>(catch_arguments.size()), catch_arguments.data());
parse_result != 0) {
local_result = parse_result;
} else {
local_result = session.run();
}
}
int global_result = 0;
MPI_Allreduce(&local_result, &global_result, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
if (rank == 0 && global_result != 0 && local_result == 0) {
std::cerr << "At least one non-root MPI rank reported a test failure.\n";
}
mfem::Mpi::Finalize();
return global_result;
}

66
tests/mpi/profiling.cpp Normal file
View File

@@ -0,0 +1,66 @@
#include "profile.h"
#include <algorithm>
#include <catch2/catch_test_macros.hpp>
#include <sstream>
#include <string>
#include <vector>
namespace {
const mean_field::profiling::DistributedStatistics *find_region(
const std::vector<mean_field::profiling::DistributedStatistics> &statistics,
const std::string &label
) {
const auto iterator =
std::ranges::find(statistics, label, &mean_field::profiling::DistributedStatistics::label);
return iterator != statistics.end() ? &*iterator : nullptr;
}
} // namespace
TEST_CASE(
"MPI Profiling Aggregates Rank-Local Label Sets Without Collective Divergence",
"[mpi][profiling][distributed]"
) {
int rank = 0;
int size = 1;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
mean_field::profiling::Registry &registry = mean_field::profiling::Registry::Get();
registry.Reset();
registry.Record("common-region", static_cast<double>(rank + 1));
registry.AddCount("common-region", static_cast<std::uint64_t>(10 + rank));
const std::string local_label = "rank-" + std::to_string(rank) + "-only";
registry.Record(local_label, 0.125 * static_cast<double>(rank + 1));
const auto aggregate = registry.Aggregate(MPI_COMM_WORLD);
const auto *common = find_region(aggregate, "common-region");
CHECK(common != nullptr);
if (common != nullptr) {
CHECK(common->minimum_samples == 1);
CHECK(common->maximum_samples == 1);
CHECK(common->minimum_work_units == 10);
CHECK(common->maximum_work_units == static_cast<std::uint64_t>(9 + size));
CHECK(common->global_minimum_seconds == 1.0);
CHECK(common->global_maximum_seconds == static_cast<double>(size));
}
for (int owner = 0; owner < size; ++owner) {
const auto *local = find_region(aggregate, "rank-" + std::to_string(owner) + "-only");
CHECK(local != nullptr);
if (local != nullptr) {
CHECK(local->minimum_samples == 0);
CHECK(local->maximum_samples == 1);
}
}
std::ostringstream csv;
registry.PrintCsv(MPI_COMM_WORLD, csv);
if (rank == 0) {
CHECK(csv.str().find("common-region") != std::string::npos);
CHECK(csv.str().find("," + std::to_string(size) + "\n") != std::string::npos);
} else {
CHECK(csv.str().empty());
}
}