diff --git a/CMakeLists.txt b/CMakeLists.txt index edccfb0..1c94f17 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,6 +5,9 @@ set(CMAKE_CXX_STANDARD 23) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) +option(MEAN_FIELD_ENABLE_PROFILING "Enable low-overhead scoped profiling instrumentation" OFF) +option(MEAN_FIELD_ENABLE_IPO "Enable interprocedural optimization in release builds" ON) + set(MEAN_FIELD_UNIFORM_POLYNOMIAL_ORDER_INCREMENT 0 CACHE STRING "Uniform increment applied to every registered finite-element family order") if (NOT MEAN_FIELD_UNIFORM_POLYNOMIAL_ORDER_INCREMENT MATCHES "^[0-9]+$") @@ -44,6 +47,7 @@ add_library(mean_field) target_compile_definitions(mean_field PUBLIC MEAN_FIELD_UNIFORM_POLYNOMIAL_ORDER_INCREMENT=${MEAN_FIELD_UNIFORM_POLYNOMIAL_ORDER_INCREMENT} + MEAN_FIELD_ENABLE_PROFILING=$ ) target_include_directories(mean_field @@ -53,6 +57,7 @@ target_include_directories(mean_field target_sources(mean_field PRIVATE + libmeanfield/impl/profile.cpp libmeanfield/impl/analysis/integral.cpp libmeanfield/impl/fem.cpp libmeanfield/impl/mapping/coefficients.cpp @@ -96,11 +101,21 @@ target_sources(mean_field libmeanfield/impl/seed/lane_emden.cpp libmeanfield/impl/seed/stellar_equilibrium_projection.cpp libmeanfield/impl/solver/preconditioning_diagnostics.cpp + libmeanfield/impl/preconditioning/gravity_field.cpp libmeanfield/impl/operators/prepared_mass_normalization.cpp libmeanfield/impl/operators/prepared_central_density_stellar_equilibrium.cpp libmeanfield/impl/operators/prepared_stellar_equilibrium.cpp ) +if (MEAN_FIELD_ENABLE_IPO) + include(CheckIPOSupported) + check_ipo_supported(RESULT mean_field_ipo_supported OUTPUT mean_field_ipo_error LANGUAGES CXX) + if (NOT mean_field_ipo_supported) + message(FATAL_ERROR "MEAN_FIELD_ENABLE_IPO was requested, but the compiler does not support it: ${mean_field_ipo_error}") + endif () + set_property(TARGET mean_field PROPERTY INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) +endif () + target_sources(mean_field PUBLIC FILE_SET CXX_MODULES FILES @@ -131,6 +146,16 @@ target_sources(mean_field libmeanfield/interface/quadrature/mfem.cppm libmeanfield/interface/solver/fields.cppm libmeanfield/interface/solver/preconditioning_diagnostics.cppm + libmeanfield/interface/preconditioning/backend.cppm + libmeanfield/interface/preconditioning/backend_implementations.cppm + libmeanfield/interface/preconditioning/gravity_field.cppm + libmeanfield/interface/preconditioning/material_surface.cppm + libmeanfield/interface/preconditioning/plan.cppm + libmeanfield/interface/preconditioning/stellar_equilibrium.cppm + libmeanfield/interface/preconditioning/stellar_structure.cppm + libmeanfield/interface/preconditioning/specification_border.cppm + libmeanfield/interface/preconditioning/equilibrium_coordinates.cppm + libmeanfield/interface/preconditioning/preconditioning.cppm libmeanfield/interface/operators/gravity_field.cppm libmeanfield/interface/operators/gravity_field_jacobian.cppm libmeanfield/interface/operators/kernels/gravity_kernels.cppm @@ -177,6 +202,7 @@ target_sources(mean_field libmeanfield/interface/surface/dependencies.cppm libmeanfield/interface/surface/compiled.cppm libmeanfield/interface/surface/compiler.cppm + libmeanfield/interface/material/thermodynamic_equations.cppm libmeanfield/interface/deformation/descriptors.cppm libmeanfield/interface/deformation/surface_prescription.cppm libmeanfield/interface/deformation/nodal_radial_surface.cppm @@ -240,6 +266,7 @@ add_executable(tests tests/mapping/domain_mapper.cpp tests/mapping/compactification/kelvin.cpp tests/utils/blocks.cpp + tests/utils/profiling.cpp tests/operators/gravity_field.cpp tests/mapping/hdiv_mass_tensor.cpp tests/operators/prepared_hdiv_mass.cpp @@ -252,6 +279,7 @@ add_executable(tests tests/physics/equation_of_state_consumer_contracts.cpp tests/physics/polytropic_eos_relations.cpp tests/physics/equation_of_state_runtime_view.cpp + tests/material/thermodynamic_equation_compilation.cpp tests/surface/constant_surface_compilation.cpp tests/operators/kernels/barotropic_closure_kernels.cpp tests/operators/prepared_barotropic_closure.cpp @@ -293,11 +321,32 @@ add_executable(tests tests/field/field_registry.cpp tests/field/field_mfem.cpp tests/field/field_dof_map.cpp + tests/preconditioning/plan.cpp + tests/preconditioning/backends.cpp + tests/preconditioning/gravity_field.cpp + tests/preconditioning/material_surface.cpp + tests/preconditioning/stellar_structure.cpp + tests/preconditioning/specification_border.cpp + tests/preconditioning/equilibrium_coordinates.cpp + tests/preconditioning/stellar_equilibrium.cpp + tests/user-api/stellar_equilibrium.cpp tests/solver/preconditioning_diagnostics.cpp ) target_link_libraries(tests PRIVATE mean_field test_mod Catch2::Catch2 Boost::boost) +add_executable(mpi_tests + tests/mpi/mpi_test_main.cpp + tests/mpi/distributed_execution.cpp + tests/mpi/profiling.cpp +) +target_link_libraries(mpi_tests PRIVATE mean_field test_mod Catch2::Catch2 Boost::boost) + +if (MEAN_FIELD_ENABLE_IPO) + set_property(TARGET tests PROPERTY INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) + set_property(TARGET mpi_tests PROPERTY INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) +endif () + add_library(experiment_mod) target_sources(experiment_mod PUBLIC @@ -313,7 +362,10 @@ target_link_libraries(experiment_mod add_executable(experiments experiments/experiment_main.cpp + experiments/full_stellar_preconditioning.cpp experiments/gravity_accuracy_budget.cpp + experiments/gravity_preconditioning.cpp + experiments/material_surface_preconditioning.cpp experiments/preconditioning_diagnostics.cpp ) @@ -335,6 +387,12 @@ target_link_libraries(stellar_null_space_experiments Boost::boost ) +if (MEAN_FIELD_ENABLE_IPO) + foreach (mean_field_ipo_target IN ITEMS test_mod experiment_mod experiments stellar_null_space_experiments) + set_property(TARGET ${mean_field_ipo_target} PROPERTY INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) + endforeach () +endif () + include (CTest) include (Catch) catch_discover_tests( @@ -342,3 +400,32 @@ catch_discover_tests( experiments WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" ) + +foreach (mean_field_mpi_ranks IN ITEMS 2 4) + add_test( + NAME mpi_${mean_field_mpi_ranks}_ranks + COMMAND + ${MPIEXEC_EXECUTABLE} + ${MPIEXEC_NUMPROC_FLAG} ${mean_field_mpi_ranks} + ${MPIEXEC_PREFLAGS} + $ + ${MPIEXEC_POSTFLAGS} + "[mpi]" + ) + set_tests_properties( + mpi_${mean_field_mpi_ranks}_ranks + PROPERTIES + LABELS "mpi;distributed" + PROCESSORS ${mean_field_mpi_ranks} + RESOURCE_LOCK mean_field_mpi + TIMEOUT 180 + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" + ) +endforeach () + +add_custom_target( + check_mpi + COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure --label-regex "mpi" + DEPENDS mpi_tests + USES_TERMINAL +) diff --git a/experiments/full_stellar_preconditioning.cpp b/experiments/full_stellar_preconditioning.cpp new file mode 100644 index 0000000..e7a563a --- /dev/null +++ b/experiments/full_stellar_preconditioning.cpp @@ -0,0 +1,1218 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +import experiment; +import mean_field; +import test_helpers; + +namespace { + using Clock = std::chrono::steady_clock; + + namespace backend = mean_field::preconditioning::backend; + namespace preconditioning = mean_field::preconditioning; + namespace solver = mean_field::solver; + + struct CandidateDescription final { + std::string name; + std::string materialFactorization; + std::string surfaceCalibration; + std::string stellarStructureFactorization; + std::string gravityFactorization; + std::string specificationBorder; + }; + + struct SetupTimings final { + double finiteElementSeconds{0.0}; + double laneEmdenCalibrationSeconds{0.0}; + double problemConstructionSeconds{0.0}; + double seedProjectionSeconds{0.0}; + double operatorPreparationSeconds{0.0}; + double manufacturedRightHandSideSeconds{0.0}; + }; + + struct NewtonCandidateResult final { + CandidateDescription candidate; + mfem::Vector correction; + mfem::Vector linearAction; + }; + + [[nodiscard]] const char *buildConfiguration() noexcept { +#ifdef NDEBUG + return "release"; +#else + return "debug"; +#endif + } + + [[nodiscard]] double maximumRankSeconds( + const Clock::time_point start, + const MPI_Comm communicator + ) { + const double localSeconds = std::chrono::duration(Clock::now() - start).count(); + double maximumSeconds = 0.0; + MPI_Allreduce(&localSeconds, &maximumSeconds, 1, MPI_DOUBLE, MPI_MAX, communicator); + return maximumSeconds; + } + + [[nodiscard]] double globalNorm( + const mfem::Vector &vector, + const MPI_Comm communicator + ) { + const double localSquared = vector * vector; + double globalSquared = 0.0; + MPI_Allreduce(&localSquared, &globalSquared, 1, MPI_DOUBLE, MPI_SUM, communicator); + return std::sqrt(std::max(globalSquared, 0.0)); + } + + void announce( + const MPI_Comm communicator, + const std::string &message + ) { + int rank = 0; + MPI_Comm_rank(communicator, &rank); + if (rank == 0) { + std::cout << "[P10 full stellar] " << message << std::endl; + } + } + + class ArnoldiProgressOperator final : public mfem::Operator { + public: + ArnoldiProgressOperator( + const mfem::Operator &operation, + const MPI_Comm communicator, + const int expectedApplications + ) + : mfem::Operator( + operation.Height(), + operation.Width() + ), + m_operation(&operation), + m_communicator(communicator), + m_expectedApplications(expectedApplications) { + } + + void Mult( + const mfem::Vector &input, + mfem::Vector &output + ) const override { + m_operation->Mult(input, output); + ++m_completedApplications; + if (m_completedApplications == 1 || m_completedApplications == m_expectedApplications || + m_completedApplications % 4 == 0) { + announce( + m_communicator, "Arnoldi progress " + std::to_string(m_completedApplications) + "/" + + std::to_string(m_expectedApplications) + ); + } + } + + private: + const mfem::Operator *m_operation; + MPI_Comm m_communicator; + int m_expectedApplications; + mutable int m_completedApplications{0}; + }; + + [[nodiscard]] mean_field::operators::StellarEquilibriumDependencies makeDependencies() { + return { + .discretization = {.identity = 111103, .revision = 1}, + .density = {.identity = 111109, .revision = 1}, + .surfaceDeformation = {.identity = 111119, .revision = 1}, + .gravityGradient = {.identity = 111121, .revision = 1}, + .gravityPotential = {.identity = 111127, .revision = 1}, + .enthalpy = {.identity = 111143, .revision = 1}, + .bernoulliConstant = {.identity = 111149, .revision = 1}, + .rotation = {.identity = 111151, .revision = 1}, + .targetMass = {.identity = 111157, .revision = 1} + }; + } + + [[nodiscard]] mean_field::physics::RigidRotation zeroRotation() { + mfem::Vector angularVelocity(3); + mfem::Vector center(3); + angularVelocity = 0.0; + center = 0.0; + return {angularVelocity, center}; + } + + [[nodiscard]] mfem::Vector blockBalancedDirection( + const int size, + const std::span blocks, + const double phase, + const MPI_Comm communicator + ) { + mfem::Vector direction(size); + direction = 0.0; + for (const auto &block : blocks) { + REQUIRE(block.offset >= 0); + REQUIRE(block.size > 0); + REQUIRE(block.offset + block.size <= size); + mfem::Vector values(direction.GetData() + block.offset, block.size); + for (int index = 0; index < values.Size(); ++index) { + const double ordinal = static_cast(index + 1); + values(index) = + std::sin(0.6180339887498948 * ordinal + phase + static_cast(block.canonicalIndex + 1)) + + 0.31 * std::cos(0.1732050807568877 * ordinal - phase); + } + const double norm = globalNorm(values, communicator); + REQUIRE(norm > 0.0); + values /= norm; + } + return direction; + } + + [[nodiscard]] std::map< + std::string, + std::string> + commonParameters( + const CandidateDescription &candidate, + const std::string &measurement, + const int dimension + ) { + return { + {"build_configuration", buildConfiguration()}, + {"candidate", candidate.name}, + {"equation_of_state", "Polytrope(n=3)"}, + {"experiment_schema", "p9_p10_full_stellar_v1"}, + {"gravity_factorization", candidate.gravityFactorization}, + {"linearization_state", "projected_lane_emden"}, + {"manufactured_rhs", "J_times_value_block_balanced_correction"}, + {"material_factorization", candidate.materialFactorization}, + {"measurement", measurement}, + {"mesh_file", test_utils::setup_args().mesh_file}, + {"operator", "canonical_bordered_stellar_jacobian"}, + {"preconditioned_product", "J M^-1"}, + {"residual_arnoldi_seed", "residual_block_balanced"}, + {"root_dimension", std::to_string(dimension)}, + {"rotation", "zero"}, + {"specification_border", candidate.specificationBorder}, + {"stellar_structure_factorization", candidate.stellarStructureFactorization}, + {"surface_calibration", candidate.surfaceCalibration} + }; + } + + [[nodiscard]] std::map< + std::string, + std::string> + newtonParameters( + const CandidateDescription &candidate, + const std::string &measurement, + const int dimension + ) { + auto parameters = commonParameters(candidate, measurement, dimension); + parameters["experiment_schema"] = "p10_physical_newton_rhs_v1"; + parameters["manufactured_rhs"] = "none"; + parameters["right_hand_side"] = "negative_nonlinear_residual"; + parameters["preconditioned_product"] = "not_measured"; + parameters["residual_arnoldi_seed"] = "not_applicable"; + return parameters; + } + + void incrementStateRevisions(mean_field::operators::StellarEquilibriumDependencies &dependencies) { + ++dependencies.density.revision; + ++dependencies.surfaceDeformation.revision; + ++dependencies.gravityGradient.revision; + ++dependencies.gravityPotential.revision; + ++dependencies.enthalpy.revision; + ++dependencies.bernoulliConstant.revision; + ++dependencies.rotation.revision; + ++dependencies.targetMass.revision; + } + + [[nodiscard]] const mean_field::operators::RootBlockDescriptor &findBlock( + const std::span blocks, + const std::string_view stableId + ) { + const auto iterator = + std::ranges::find(blocks, stableId, &mean_field::operators::RootBlockDescriptor::stableId); + REQUIRE(iterator != blocks.end()); + return *iterator; + } + + [[nodiscard]] int firstThresholdIteration( + const std::vector &history, + const double initialNorm, + const double threshold + ) { + if (!std::isfinite(initialNorm) || initialNorm <= 0.0) { + return -1; + } + for (const auto &sample : history) { + if (std::abs(sample.reportedNorm) / initialNorm <= threshold) { + return sample.iteration; + } + } + return -1; + } + + void recordSpectrum( + const CandidateDescription &candidate, + const solver::ArnoldiSpectralMeasurement &spectrum, + const int dimension, + const double setupSeconds, + const solver::OperatorApplicationStatistics &jacobianStatistics, + const solver::OperatorApplicationStatistics &preconditionerStatistics + ) { + experiment::record_experiment_result( + "stellar_preconditioning_p10", candidate.name + "_arnoldi_summary", + commonParameters(candidate, "arnoldi_summary", dimension), + {{"preconditioner_setup_seconds_maximum_rank", setupSeconds}, + {"requested_dimension", static_cast(spectrum.requestedDimension)}, + {"achieved_dimension", static_cast(spectrum.achievedDimension)}, + {"invariant_subspace_found", spectrum.invariantSubspaceFound ? 1.0 : 0.0}, + {"operator_applications", static_cast(spectrum.operatorApplications)}, + {"measurement_seconds_maximum_rank", spectrum.measurementSecondsMaximumRank}, + {"operator_application_seconds_maximum_rank", spectrum.operatorApplicationSecondsMaximumRank}, + {"operator_maximum_application_seconds_maximum_rank", + spectrum.operatorMaximumApplicationSecondsMaximumRank}, + {"nonapplication_seconds_maximum_rank", spectrum.nonApplicationSecondsMaximumRank}, + {"measured_jacobian_applications", static_cast(jacobianStatistics.applications)}, + {"measured_jacobian_application_seconds", jacobianStatistics.totalSeconds}, + {"measured_jacobian_maximum_application_seconds", jacobianStatistics.maximumSeconds}, + {"measured_preconditioner_applications", static_cast(preconditionerStatistics.applications)}, + {"measured_preconditioner_application_seconds", preconditionerStatistics.totalSeconds}, + {"measured_preconditioner_maximum_application_seconds", preconditionerStatistics.maximumSeconds}, + {"projected_condition_proxy", spectrum.projectedConditionProxy}, + {"projected_largest_singular_value", spectrum.projectedLargestSingularValue}, + {"projected_smallest_singular_value", spectrum.projectedSmallestSingularValue}, + {"centroid_real_part", spectrum.centroidRealPart}, + {"centroid_imaginary_part", spectrum.centroidImaginaryPart}, + {"rms_distance_from_one", spectrum.rmsDistanceFromOne}, + {"rms_cluster_radius", spectrum.rmsClusterRadius}, + {"minimum_magnitude", spectrum.minimumMagnitude}, + {"maximum_magnitude", spectrum.maximumMagnitude}, + {"minimum_real_part", spectrum.minimumRealPart}, + {"maximum_real_part", spectrum.maximumRealPart}, + {"maximum_absolute_imaginary_part", spectrum.maximumAbsoluteImaginaryPart}, + {"negative_real_part_count", static_cast(spectrum.negativeRealPartCount)}, + {"converged_ritz_value_count", static_cast(spectrum.convergedRitzValueCount)}, + {"conjugate_pair_defect", spectrum.conjugatePairDefect}, + {"projected_departure_from_normality", spectrum.projectedDepartureFromNormality}, + {"field_of_values_minimum_real_part", spectrum.projectedFieldOfValuesMinimumRealPart}, + {"field_of_values_maximum_real_part", spectrum.projectedFieldOfValuesMaximumRealPart}} + ); + + std::vector ordered = spectrum.ritzValues; + std::ranges::sort(ordered, [](const auto &left, const auto &right) { + if (left.realPart != right.realPart) { + return left.realPart < right.realPart; + } + return left.imaginaryPart < right.imaginaryPart; + }); + for (std::size_t index = 0; index < ordered.size(); ++index) { + const auto &value = ordered[index]; + experiment::record_experiment_result( + "stellar_preconditioning_p10", candidate.name + "_ritz_" + std::to_string(index), + commonParameters(candidate, "ritz_value", dimension), + {{"ritz_index", static_cast(index)}, + {"real_part", value.realPart}, + {"imaginary_part", value.imaginaryPart}, + {"magnitude", value.magnitude}, + {"distance_from_one", value.distanceFromOne}, + {"residual_estimate", value.residualEstimate}, + {"relative_residual_estimate", value.relativeResidualEstimate}, + {"converged", value.converged ? 1.0 : 0.0}} + ); + } + } + + template < + typename Preconditioner, + typename Problem> + void measureCandidate( + const CandidateDescription &candidate, + Preconditioner &inversePreconditioner, + const double preconditionerSetupSeconds, + const Problem &problem, + const mfem::Vector &exactCorrection, + const mfem::Vector &rightHandSide, + const mfem::Vector &arnoldiDirection, + const SetupTimings &setupTimings, + const MPI_Comm communicator + ) { + constexpr int maximumIterations = 36; + constexpr int restartDimension = 18; + constexpr int arnoldiDimension = 12; + + const mfem::Operator &jacobian = problem.GetLinearizationOperator(); + solver::InstrumentedOperator instrumentedJacobian(jacobian); + solver::InstrumentedPreconditioner instrumentedPreconditioner(inversePreconditioner); + solver::ResidualHistoryMonitor monitor; + mfem::FGMRESSolver krylov(communicator); + krylov.SetPreconditioner(instrumentedPreconditioner); + krylov.SetOperator(instrumentedJacobian); + krylov.SetMonitor(monitor); + krylov.SetRelTol(1.0e-8); + krylov.SetAbsTol(1.0e-12); + krylov.SetMaxIter(maximumIterations); + krylov.SetKDim(restartDimension); + krylov.SetPrintLevel(0); + + mfem::Vector solution(problem.StateSize()); + solution = 0.0; + announce(communicator, "solving the manufactured system with " + candidate.name); + const Clock::time_point solveStart = Clock::now(); + krylov.Mult(rightHandSide, solution); + const double localSolveSeconds = std::chrono::duration(Clock::now() - solveStart).count(); + + const solver::LinearSolveMeasurement solve = solver::measureLinearSolve( + krylov, jacobian, rightHandSide, solution, problem.GetManifest().residualBlocks(), + instrumentedJacobian.GetStatistics(), instrumentedPreconditioner.GetStatistics(), + instrumentedPreconditioner.GetLifecycleStatistics(), monitor, localSolveSeconds, communicator + ); + mfem::Vector correctionError(solution); + correctionError -= exactCorrection; + const double relativeCorrectionError = + globalNorm(correctionError, communicator) / + std::max(globalNorm(exactCorrection, communicator), std::numeric_limits::min()); + + const double reportedInitial = std::max(std::abs(krylov.GetInitialNorm()), 1.0e-300); + const int iteration1e2 = firstThresholdIteration(solve.reportedResidualHistory, reportedInitial, 1.0e-2); + const int iteration1e4 = firstThresholdIteration(solve.reportedResidualHistory, reportedInitial, 1.0e-4); + const int iteration1e6 = firstThresholdIteration(solve.reportedResidualHistory, reportedInitial, 1.0e-6); + const int iteration1e8 = firstThresholdIteration(solve.reportedResidualHistory, reportedInitial, 1.0e-8); + + REQUIRE(std::isfinite(solve.directResidual.relativeResidual)); + REQUIRE(std::isfinite(relativeCorrectionError)); + std::map metrics{ + {"maximum_iterations", static_cast(maximumIterations)}, + {"restart_dimension", static_cast(restartDimension)}, + {"solver_converged", solve.solverConverged ? 1.0 : 0.0}, + {"outer_iterations", static_cast(solve.outerIterations)}, + {"reported_initial_residual_norm", solve.solverReportedInitialNorm}, + {"reported_final_residual_norm", solve.solverReportedFinalNorm}, + {"reported_residual_reduction", solve.solverReportedResidualReduction}, + {"reported_iteration_to_1e-2", static_cast(iteration1e2)}, + {"reported_iteration_to_1e-4", static_cast(iteration1e4)}, + {"reported_iteration_to_1e-6", static_cast(iteration1e6)}, + {"reported_iteration_to_1e-8", static_cast(iteration1e8)}, + {"rhs_norm", solve.directResidual.rightHandSideNorm}, + {"true_residual_norm", solve.directResidual.trueResidualNorm}, + {"true_relative_residual", solve.directResidual.relativeResidual}, + {"relative_correction_error", relativeCorrectionError}, + {"true_residual_digits_per_jacobian_application", solve.trueResidualDigitsReducedPerJacobianApplication}, + {"solve_seconds_maximum_rank", solve.solveSecondsMaximumRank}, + {"jacobian_applications", static_cast(solve.jacobian.applications)}, + {"jacobian_application_seconds", solve.jacobian.totalSeconds}, + {"jacobian_maximum_application_seconds", solve.jacobian.maximumSeconds}, + {"preconditioner_applications", static_cast(solve.inversePreconditioner.applications)}, + {"preconditioner_application_seconds", solve.inversePreconditioner.totalSeconds}, + {"preconditioner_maximum_application_seconds", solve.inversePreconditioner.maximumSeconds}, + {"preconditioner_setups", static_cast(solve.inversePreconditionerLifecycle.setups)}, + {"preconditioner_setup_seconds_in_solver", solve.inversePreconditionerLifecycle.setupSeconds}, + {"preconditioner_setup_seconds_maximum_rank", preconditionerSetupSeconds}, + {"finite_element_setup_seconds", setupTimings.finiteElementSeconds}, + {"lane_emden_calibration_seconds", setupTimings.laneEmdenCalibrationSeconds}, + {"problem_construction_seconds", setupTimings.problemConstructionSeconds}, + {"seed_projection_seconds", setupTimings.seedProjectionSeconds}, + {"operator_preparation_seconds", setupTimings.operatorPreparationSeconds}, + {"manufactured_rhs_seconds", setupTimings.manufacturedRightHandSideSeconds} + }; + for (const auto &block : solve.directResidual.blocks) { + const std::string prefix = "residual_block." + block.stableId; + metrics[prefix + ".size"] = static_cast(block.size); + metrics[prefix + ".descriptor_scale"] = block.descriptorScale; + metrics[prefix + ".rhs_norm"] = block.rightHandSideNorm; + metrics[prefix + ".true_norm"] = block.trueResidualNorm; + metrics[prefix + ".block_relative_residual"] = block.blockRelativeResidual; + metrics[prefix + ".scaled_rhs_norm"] = block.scaledRightHandSideNorm; + metrics[prefix + ".scaled_true_norm"] = block.scaledTrueResidualNorm; + metrics[prefix + ".global_relative_contribution"] = block.contributionToGlobalRelativeResidual; + metrics[prefix + ".fraction_global_squared_residual"] = block.fractionOfGlobalSquaredResidualNorm; + } + const double correctionErrorNorm = globalNorm(correctionError, communicator); + for (const auto &block : problem.GetManifest().valueBlocks()) { + const mfem::Vector exactBlock( + const_cast(exactCorrection.GetData()) + block.offset, block.size + ); + const mfem::Vector errorBlock(correctionError.GetData() + block.offset, block.size); + const double exactBlockNorm = globalNorm(exactBlock, communicator); + const double errorBlockNorm = globalNorm(errorBlock, communicator); + const std::string prefix = "correction_block." + std::string(block.stableId); + metrics[prefix + ".size"] = static_cast(block.size); + metrics[prefix + ".descriptor_scale"] = block.scale; + metrics[prefix + ".exact_norm"] = exactBlockNorm; + metrics[prefix + ".error_norm"] = errorBlockNorm; + metrics[prefix + ".block_relative_error"] = + errorBlockNorm / std::max(exactBlockNorm, std::numeric_limits::min()); + metrics[prefix + ".scaled_exact_norm"] = exactBlockNorm / block.scale; + metrics[prefix + ".scaled_error_norm"] = errorBlockNorm / block.scale; + metrics[prefix + ".fraction_global_squared_error"] = + correctionErrorNorm > 0.0 + ? errorBlockNorm * errorBlockNorm / (correctionErrorNorm * correctionErrorNorm) + : 0.0; + } + experiment::record_experiment_result( + "stellar_preconditioning_p10", candidate.name + "_linear_solve", + commonParameters(candidate, "manufactured_linear_solve", problem.StateSize()), std::move(metrics) + ); + + for (std::size_t index = 0; index < solve.reportedResidualHistory.size(); ++index) { + const auto &sample = solve.reportedResidualHistory[index]; + experiment::record_experiment_result( + "stellar_preconditioning_p10", candidate.name + "_history_" + std::to_string(index), + commonParameters(candidate, "fgmres_residual_history", problem.StateSize()), + {{"history_sample", static_cast(index)}, + {"iteration", static_cast(sample.iteration)}, + {"reported_residual_norm", sample.reportedNorm}, + {"reported_relative_residual", std::abs(sample.reportedNorm) / reportedInitial}, + {"final_measurement", sample.final ? 1.0 : 0.0}} + ); + } + + instrumentedJacobian.ResetStatistics(); + instrumentedPreconditioner.ResetStatistics(); + solver::FixedRightPreconditionedOperator product(instrumentedJacobian, instrumentedPreconditioner); + ArnoldiProgressOperator progress(product, communicator, arnoldiDimension); + announce(communicator, "measuring " + candidate.name + " with 12-vector Arnoldi"); + const solver::ArnoldiSpectralMeasurement spectrum = solver::measureArnoldiSpectrum( + progress, arnoldiDirection, communicator, + {.krylovDimension = arnoldiDimension, + .breakdownRelativeTolerance = 1.0e-13, + .ritzConvergenceRelativeTolerance = 1.0e-7, + .reorthogonalize = true} + ); + REQUIRE(spectrum.achievedDimension > 0); + recordSpectrum( + candidate, spectrum, problem.StateSize(), preconditionerSetupSeconds, instrumentedJacobian.GetStatistics(), + instrumentedPreconditioner.GetStatistics() + ); + + int rank = 0; + MPI_Comm_rank(communicator, &rank); + if (rank == 0) { + std::cout << "[P10 full stellar] " << candidate.name << ": iterations=" << solve.outerIterations + << ", converged=" << (solve.solverConverged ? "yes" : "no") + << ", true residual=" << solve.directResidual.relativeResidual + << ", correction error=" << relativeCorrectionError + << ", projected condition=" << spectrum.projectedConditionProxy << '\n'; + } + } + + template + void prepareAndMeasureDefault( + const CandidateDescription &candidate, + const Problem &problem, + const mfem::Vector &exactCorrection, + const mfem::Vector &rightHandSide, + const mfem::Vector &arnoldiDirection, + const SetupTimings &setupTimings, + const MPI_Comm communicator + ) { + const Clock::time_point setupStart = Clock::now(); + auto block = preconditioning::makePreconditioner(problem); + auto prepared = preconditioning::prepare(problem, std::move(block)); + const double setupSeconds = maximumRankSeconds(setupStart, communicator); + measureCandidate( + candidate, prepared, setupSeconds, problem, exactCorrection, rightHandSide, arnoldiDirection, setupTimings, + communicator + ); + } + + template < + preconditioning::MaterialSurfaceFactorizationPolicy MaterialPolicy, + preconditioning::StellarStructureFactorizationPolicy StructurePolicy, + typename Problem, + backend::Registered GravityMassBackend = backend::Diagonal> + requires backend::Compatible< + GravityMassBackend, + preconditioning::GravityMassInverseCharacteristics> + void prepareAndMeasureComposed( + const CandidateDescription &candidate, + const Problem &problem, + const MaterialPolicy materialPolicy, + const StructurePolicy structurePolicy, + const preconditioning::MaterialSurfaceDiagonalOptions materialOptions, + const mfem::Vector &exactCorrection, + const mfem::Vector &rightHandSide, + const mfem::Vector &arnoldiDirection, + const SetupTimings &setupTimings, + const MPI_Comm communicator, + GravityMassBackend gravityMassBackend = {}, + const int gravityAmgCycles = 1 + ) { + const Clock::time_point setupStart = Clock::now(); + auto material = preconditioning::materialSurfaceBlock( + problem, backend::Diagonal{}, backend::Diagonal{}, materialPolicy, materialOptions + ); + using FixedAMG = backend::HypreBoomerAMG; + auto gravity = preconditioning::GravityFieldBlock( + std::move(gravityMassBackend), FixedAMG{backend::FixedCycles{.cycles = gravityAmgCycles}}, + preconditioning::GravityApproximateLDU{} + ); + auto structure = + preconditioning::stellarStructureBlock(problem, std::move(material), std::move(gravity), structurePolicy); + auto block = preconditioning::specificationBorderBlock(problem, std::move(structure), backend::DenseDirect{}); + auto prepared = preconditioning::prepare(problem, std::move(block)); + const double setupSeconds = maximumRankSeconds(setupStart, communicator); + measureCandidate( + candidate, prepared, setupSeconds, problem, exactCorrection, rightHandSide, arnoldiDirection, setupTimings, + communicator + ); + } + + template < + typename Preconditioner, + typename Problem> + [[nodiscard]] NewtonCandidateResult solveNewtonCandidate( + const CandidateDescription &candidate, + Preconditioner &inversePreconditioner, + const double preconditionerSetupSeconds, + const Problem &problem, + const mfem::Vector &baseResidual, + const mfem::Vector &rightHandSide, + const MPI_Comm communicator + ) { + constexpr int maximumIterations = 48; + constexpr int restartDimension = 20; + + const mfem::Operator &jacobian = problem.GetLinearizationOperator(); + solver::InstrumentedOperator instrumentedJacobian(jacobian); + solver::InstrumentedPreconditioner instrumentedPreconditioner(inversePreconditioner); + solver::ResidualHistoryMonitor monitor; + mfem::FGMRESSolver krylov(communicator); + krylov.SetPreconditioner(instrumentedPreconditioner); + krylov.SetOperator(instrumentedJacobian); + krylov.SetMonitor(monitor); + krylov.SetRelTol(1.0e-8); + krylov.SetAbsTol(1.0e-12); + krylov.SetMaxIter(maximumIterations); + krylov.SetKDim(restartDimension); + krylov.SetPrintLevel(0); + + mfem::Vector correction(problem.StateSize()); + correction = 0.0; + announce(communicator, "solving the physical Newton system with " + candidate.name); + const Clock::time_point solveStart = Clock::now(); + krylov.Mult(rightHandSide, correction); + const double localSolveSeconds = std::chrono::duration(Clock::now() - solveStart).count(); + + const solver::LinearSolveMeasurement solve = solver::measureLinearSolve( + krylov, jacobian, rightHandSide, correction, problem.GetManifest().residualBlocks(), + instrumentedJacobian.GetStatistics(), instrumentedPreconditioner.GetStatistics(), + instrumentedPreconditioner.GetLifecycleStatistics(), monitor, localSolveSeconds, communicator + ); + mfem::Vector linearAction(problem.EquationSize()); + jacobian.Mult(correction, linearAction); + mfem::Vector predictedResidual(baseResidual); + predictedResidual += linearAction; + + const double baseResidualNorm = globalNorm(baseResidual, communicator); + const double correctionNorm = globalNorm(correction, communicator); + const double predictedResidualNorm = globalNorm(predictedResidual, communicator); + REQUIRE(std::isfinite(solve.directResidual.relativeResidual)); + REQUIRE(std::isfinite(correctionNorm)); + REQUIRE(std::isfinite(predictedResidualNorm)); + + std::map metrics{ + {"maximum_iterations", static_cast(maximumIterations)}, + {"restart_dimension", static_cast(restartDimension)}, + {"solver_converged", solve.solverConverged ? 1.0 : 0.0}, + {"outer_iterations", static_cast(solve.outerIterations)}, + {"reported_initial_residual_norm", solve.solverReportedInitialNorm}, + {"reported_final_residual_norm", solve.solverReportedFinalNorm}, + {"reported_residual_reduction", solve.solverReportedResidualReduction}, + {"base_nonlinear_residual_norm", baseResidualNorm}, + {"rhs_norm", solve.directResidual.rightHandSideNorm}, + {"true_linear_residual_norm", solve.directResidual.trueResidualNorm}, + {"true_linear_relative_residual", solve.directResidual.relativeResidual}, + {"predicted_full_step_residual_norm", predictedResidualNorm}, + {"predicted_full_step_relative_residual", + predictedResidualNorm / std::max(baseResidualNorm, std::numeric_limits::min())}, + {"correction_norm", correctionNorm}, + {"solve_seconds_maximum_rank", solve.solveSecondsMaximumRank}, + {"jacobian_applications", static_cast(solve.jacobian.applications)}, + {"jacobian_application_seconds", solve.jacobian.totalSeconds}, + {"preconditioner_applications", static_cast(solve.inversePreconditioner.applications)}, + {"preconditioner_application_seconds", solve.inversePreconditioner.totalSeconds}, + {"preconditioner_setup_seconds_maximum_rank", preconditionerSetupSeconds} + }; + for (const auto &block : solve.directResidual.blocks) { + const std::string prefix = "linear_residual_block." + block.stableId; + metrics[prefix + ".rhs_norm"] = block.rightHandSideNorm; + metrics[prefix + ".true_norm"] = block.trueResidualNorm; + metrics[prefix + ".block_relative_residual"] = block.blockRelativeResidual; + metrics[prefix + ".global_relative_contribution"] = block.contributionToGlobalRelativeResidual; + } + for (const auto &block : problem.GetManifest().valueBlocks()) { + const mfem::Vector correctionBlock(correction.GetData() + block.offset, block.size); + const std::string prefix = "correction_block." + std::string(block.stableId); + metrics[prefix + ".norm"] = globalNorm(correctionBlock, communicator); + metrics[prefix + ".descriptor_scale"] = block.scale; + metrics[prefix + ".scaled_norm"] = metrics[prefix + ".norm"] / block.scale; + } + experiment::record_experiment_result( + "stellar_preconditioning_p10_newton", candidate.name + "_linear_solve", + newtonParameters(candidate, "physical_newton_linear_solve", problem.StateSize()), std::move(metrics) + ); + + const double reportedInitial = std::max(std::abs(krylov.GetInitialNorm()), 1.0e-300); + for (std::size_t index = 0; index < solve.reportedResidualHistory.size(); ++index) { + const auto &sample = solve.reportedResidualHistory[index]; + experiment::record_experiment_result( + "stellar_preconditioning_p10_newton", candidate.name + "_history_" + std::to_string(index), + newtonParameters(candidate, "fgmres_residual_history", problem.StateSize()), + {{"history_sample", static_cast(index)}, + {"iteration", static_cast(sample.iteration)}, + {"reported_residual_norm", sample.reportedNorm}, + {"reported_relative_residual", std::abs(sample.reportedNorm) / reportedInitial}, + {"final_measurement", sample.final ? 1.0 : 0.0}} + ); + } + + return {.candidate = candidate, .correction = std::move(correction), .linearAction = std::move(linearAction)}; + } + + template < + preconditioning::StellarStructureFactorizationPolicy StructurePolicy, + typename Problem> + [[nodiscard]] NewtonCandidateResult prepareAndSolveNewtonComposed( + const CandidateDescription &candidate, + const Problem &problem, + const StructurePolicy structurePolicy, + const preconditioning::MaterialSurfaceDiagonalOptions materialOptions, + const mfem::Vector &baseResidual, + const mfem::Vector &rightHandSide, + const MPI_Comm communicator + ) { + const Clock::time_point setupStart = Clock::now(); + auto material = preconditioning::materialSurfaceBlock( + problem, backend::Diagonal{}, backend::Diagonal{}, preconditioning::SurfaceThenMaterialTriangular{}, + materialOptions + ); + using FixedAMG = backend::HypreBoomerAMG; + auto gravity = preconditioning::GravityFieldBlock( + backend::Diagonal{}, FixedAMG{backend::FixedCycles{.cycles = 1}}, preconditioning::GravityApproximateLDU{} + ); + auto structure = + preconditioning::stellarStructureBlock(problem, std::move(material), std::move(gravity), structurePolicy); + auto block = preconditioning::specificationBorderBlock(problem, std::move(structure), backend::DenseDirect{}); + auto prepared = preconditioning::prepare(problem, std::move(block)); + const double setupSeconds = maximumRankSeconds(setupStart, communicator); + return solveNewtonCandidate( + candidate, prepared, setupSeconds, problem, baseResidual, rightHandSide, communicator + ); + } +} // namespace + +TEST_CASE( + "Full Stellar P9 P10 Canonical Preconditioner Comparison", + "[preconditioning][diagnostics][experiment][spectrum][p9][p10][full_system]" +) { + using namespace mean_field; + + constexpr int radialSampleCount = 4096; + const Clock::time_point finiteElementStart = Clock::now(); + const utils::Args arguments = test_utils::setup_args(); + fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0); + REQUIRE(finiteElements.okay()); + const MPI_Comm communicator = finiteElements.mesh->GetComm(); + int communicatorSize = 0; + MPI_Comm_size(communicator, &communicatorSize); + REQUIRE(communicatorSize == 1); + + SetupTimings setupTimings; + setupTimings.finiteElementSeconds = maximumRankSeconds(finiteElementStart, communicator); + + constexpr double stellarRadius = utils::RADIUS; + constexpr double targetMass = utils::MASS; + const Clock::time_point calibrationStart = Clock::now(); + const seed::DimensionlessLaneEmdenSolution profile = seed::integrateLaneEmden(3.0, 10.0); + REQUIRE(profile.firstZeroCoordinate.has_value()); + const double surfaceCoordinate = *profile.firstZeroCoordinate; + const double surfaceDerivative = profile.thetaDerivative(profile.thetaDerivative.Size() - 1); + const double dimensionlessMass = -surfaceCoordinate * surfaceCoordinate * surfaceDerivative; + REQUIRE(dimensionlessMass > 0.0); + const double massScale = targetMass / (4.0 * std::numbers::pi_v * dimensionlessMass); + const double polytropicConstant = std::numbers::pi_v * utils::G * std::pow(massScale, 2.0 / 3.0); + const double radialScale = stellarRadius / surfaceCoordinate; + const double centralDensity = + std::pow(polytropicConstant / (std::numbers::pi_v * utils::G * radialScale * radialScale), 1.5); + setupTimings.laneEmdenCalibrationSeconds = maximumRankSeconds(calibrationStart, communicator); + + const Clock::time_point constructionStart = Clock::now(); + auto model = model::StellarModel( + eos::Polytrope({.n = 3.0, .K = polytropicConstant}), + surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), + integral::FixedTotalMass({.Mtotal = dimensions::MassValue{targetMass}}), + constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}}) + ); + auto problem = equilibrium::discretize(model, finiteElements); + setupTimings.problemConstructionSeconds = maximumRankSeconds(constructionStart, communicator); + + announce(communicator, "projecting the n=3 Lane-Emden state"); + const Clock::time_point projectionStart = Clock::now(); + auto projected = + seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = radialSampleCount})); + setupTimings.seedProjectionSeconds = maximumRankSeconds(projectionStart, communicator); + + announce(communicator, "preparing the canonical bordered stellar Jacobian"); + const Clock::time_point preparationStart = Clock::now(); + const auto preparation = problem.Prepare(projected.values, makeDependencies(), zeroRotation()); + REQUIRE(preparation.assembledResidual); + setupTimings.operatorPreparationSeconds = maximumRankSeconds(preparationStart, communicator); + + const mfem::Operator &jacobian = problem.GetLinearizationOperator(); + const mfem::Vector exactCorrection = + blockBalancedDirection(problem.StateSize(), problem.GetManifest().valueBlocks(), 0.23, communicator); + mfem::Vector rightHandSide(problem.EquationSize()); + const Clock::time_point rightHandSideStart = Clock::now(); + jacobian.Mult(exactCorrection, rightHandSide); + setupTimings.manufacturedRightHandSideSeconds = maximumRankSeconds(rightHandSideStart, communicator); + REQUIRE(std::isfinite(globalNorm(rightHandSide, communicator))); + const mfem::Vector arnoldiDirection = + blockBalancedDirection(problem.EquationSize(), problem.GetManifest().residualBlocks(), 0.79, communicator); + + solver::IdentityPreconditioner identity(problem.StateSize()); + measureCandidate( + {.name = "identity", + .materialFactorization = "identity", + .surfaceCalibration = "none", + .stellarStructureFactorization = "identity", + .gravityFactorization = "identity", + .specificationBorder = "identity"}, + identity, 0.0, problem, exactCorrection, rightHandSide, arnoldiDirection, setupTimings, communicator + ); + + prepareAndMeasureDefault( + {.name = "current_default", + .materialFactorization = "surface_then_material_triangular", + .surfaceCalibration = "none", + .stellarStructureFactorization = "independent_subsystems", + .gravityFactorization = "approximate_ldu", + .specificationBorder = "compiled_dense_schur"}, + problem, exactCorrection, rightHandSide, arnoldiDirection, setupTimings, communicator + ); + + // P9 found that fitting the surface Riesz scale to A_qq gives the same + // measured material-surface numerics as fitting the approximate Schur + // complement while reducing calibration setup by roughly forty percent. + // Keep both successful material factorizations explicit here: calibrated + // surface-first is cheaper, while calibrated material LDU gave the best + // isolated residual. + constexpr preconditioning::MaterialSurfaceDiagonalOptions surfaceJacobianCalibratedOptions{ + .surfaceCalibration = { + .target = preconditioning::SurfaceRieszCalibrationTarget::surface_jacobian, .probeCount = 4 + } + }; + + prepareAndMeasureComposed( + {.name = "surface_then_material_aqq_calibrated_independent", + .materialFactorization = "surface_then_material_triangular", + .surfaceCalibration = "surface_jacobian_4_probes", + .stellarStructureFactorization = "independent_subsystems", + .gravityFactorization = "approximate_ldu", + .specificationBorder = "compiled_dense_schur"}, + problem, preconditioning::SurfaceThenMaterialTriangular{}, preconditioning::IndependentStellarSubsystems{}, + surfaceJacobianCalibratedOptions, exactCorrection, rightHandSide, arnoldiDirection, setupTimings, communicator + ); + prepareAndMeasureComposed( + {.name = "material_ldu_aqq_calibrated_independent", + .materialFactorization = "approximate_material_surface_ldu", + .surfaceCalibration = "surface_jacobian_4_probes", + .stellarStructureFactorization = "independent_subsystems", + .gravityFactorization = "approximate_ldu", + .specificationBorder = "compiled_dense_schur"}, + problem, preconditioning::ApproximateMaterialSurfaceLDU{}, preconditioning::IndependentStellarSubsystems{}, + surfaceJacobianCalibratedOptions, exactCorrection, rightHandSide, arnoldiDirection, setupTimings, communicator + ); + prepareAndMeasureComposed( + {.name = "surface_then_material_aqq_calibrated_then_gravity", + .materialFactorization = "surface_then_material_triangular", + .surfaceCalibration = "surface_jacobian_4_probes", + .stellarStructureFactorization = "material_then_gravity_triangular", + .gravityFactorization = "approximate_ldu", + .specificationBorder = "compiled_dense_schur"}, + problem, preconditioning::SurfaceThenMaterialTriangular{}, preconditioning::MaterialThenGravityTriangular{}, + surfaceJacobianCalibratedOptions, exactCorrection, rightHandSide, arnoldiDirection, setupTimings, communicator + ); + prepareAndMeasureComposed( + {.name = "gravity_then_surface_then_material_aqq_calibrated", + .materialFactorization = "surface_then_material_triangular", + .surfaceCalibration = "surface_jacobian_4_probes", + .stellarStructureFactorization = "gravity_then_material_triangular", + .gravityFactorization = "approximate_ldu", + .specificationBorder = "compiled_dense_schur"}, + problem, preconditioning::SurfaceThenMaterialTriangular{}, preconditioning::GravityThenMaterialTriangular{}, + surfaceJacobianCalibratedOptions, exactCorrection, rightHandSide, arnoldiDirection, setupTimings, communicator + ); + prepareAndMeasureComposed( + {.name = "surface_then_material_aqq_calibrated_stellar_approximate_ldu", + .materialFactorization = "surface_then_material_triangular", + .surfaceCalibration = "surface_jacobian_4_probes", + .stellarStructureFactorization = "approximate_stellar_block_ldu", + .gravityFactorization = "approximate_ldu", + .specificationBorder = "compiled_dense_schur"}, + problem, preconditioning::SurfaceThenMaterialTriangular{}, preconditioning::ApproximateStellarBlockLDU{}, + surfaceJacobianCalibratedOptions, exactCorrection, rightHandSide, arnoldiDirection, setupTimings, communicator + ); + prepareAndMeasureComposed( + {.name = "material_ldu_aqq_calibrated_stellar_approximate_ldu", + .materialFactorization = "approximate_material_surface_ldu", + .surfaceCalibration = "surface_jacobian_4_probes", + .stellarStructureFactorization = "approximate_stellar_block_ldu", + .gravityFactorization = "approximate_ldu", + .specificationBorder = "compiled_dense_schur"}, + problem, preconditioning::ApproximateMaterialSurfaceLDU{}, preconditioning::ApproximateStellarBlockLDU{}, + surfaceJacobianCalibratedOptions, exactCorrection, rightHandSide, arnoldiDirection, setupTimings, communicator + ); +} + +TEST_CASE( + "Full Stellar P10 Gravity Backend Finalists", + "[preconditioning][diagnostics][experiment][p10][full_system][gravity_backend_finalists]" +) { + using namespace mean_field; + + constexpr int radialSampleCount = 4096; + const Clock::time_point finiteElementStart = Clock::now(); + const utils::Args arguments = test_utils::setup_args(); + fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0); + REQUIRE(finiteElements.okay()); + const MPI_Comm communicator = finiteElements.mesh->GetComm(); + int communicatorSize = 0; + MPI_Comm_size(communicator, &communicatorSize); + REQUIRE(communicatorSize == 1); + + SetupTimings setupTimings; + setupTimings.finiteElementSeconds = maximumRankSeconds(finiteElementStart, communicator); + + constexpr double stellarRadius = utils::RADIUS; + constexpr double targetMass = utils::MASS; + const Clock::time_point calibrationStart = Clock::now(); + const seed::DimensionlessLaneEmdenSolution profile = seed::integrateLaneEmden(3.0, 10.0); + REQUIRE(profile.firstZeroCoordinate.has_value()); + const double surfaceCoordinate = *profile.firstZeroCoordinate; + const double surfaceDerivative = profile.thetaDerivative(profile.thetaDerivative.Size() - 1); + const double dimensionlessMass = -surfaceCoordinate * surfaceCoordinate * surfaceDerivative; + REQUIRE(dimensionlessMass > 0.0); + const double massScale = targetMass / (4.0 * std::numbers::pi_v * dimensionlessMass); + const double polytropicConstant = std::numbers::pi_v * utils::G * std::pow(massScale, 2.0 / 3.0); + const double radialScale = stellarRadius / surfaceCoordinate; + const double centralDensity = + std::pow(polytropicConstant / (std::numbers::pi_v * utils::G * radialScale * radialScale), 1.5); + setupTimings.laneEmdenCalibrationSeconds = maximumRankSeconds(calibrationStart, communicator); + + const Clock::time_point constructionStart = Clock::now(); + auto model = model::StellarModel( + eos::Polytrope({.n = 3.0, .K = polytropicConstant}), + surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), + integral::FixedTotalMass({.Mtotal = dimensions::MassValue{targetMass}}), + constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}}) + ); + auto problem = equilibrium::discretize(model, finiteElements); + setupTimings.problemConstructionSeconds = maximumRankSeconds(constructionStart, communicator); + + announce(communicator, "projecting the n=3 Lane-Emden state for gravity backend finalists"); + const Clock::time_point projectionStart = Clock::now(); + auto projected = + seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = radialSampleCount})); + setupTimings.seedProjectionSeconds = maximumRankSeconds(projectionStart, communicator); + + const Clock::time_point preparationStart = Clock::now(); + const auto preparation = problem.Prepare(projected.values, makeDependencies(), zeroRotation()); + REQUIRE(preparation.assembledResidual); + setupTimings.operatorPreparationSeconds = maximumRankSeconds(preparationStart, communicator); + + const mfem::Operator &jacobian = problem.GetLinearizationOperator(); + const mfem::Vector exactCorrection = + blockBalancedDirection(problem.StateSize(), problem.GetManifest().valueBlocks(), 0.23, communicator); + mfem::Vector rightHandSide(problem.EquationSize()); + const Clock::time_point rightHandSideStart = Clock::now(); + jacobian.Mult(exactCorrection, rightHandSide); + setupTimings.manufacturedRightHandSideSeconds = maximumRankSeconds(rightHandSideStart, communicator); + const mfem::Vector arnoldiDirection = + blockBalancedDirection(problem.EquationSize(), problem.GetManifest().residualBlocks(), 0.79, communicator); + constexpr preconditioning::MaterialSurfaceDiagonalOptions materialOptions{}; + + prepareAndMeasureComposed( + {.name = "current_structure_diagonal_mass_amg2", + .materialFactorization = "surface_then_material_triangular", + .surfaceCalibration = "none", + .stellarStructureFactorization = "independent_subsystems", + .gravityFactorization = "approximate_ldu_diagonal_mass_amg2", + .specificationBorder = "compiled_dense_schur"}, + problem, preconditioning::SurfaceThenMaterialTriangular{}, preconditioning::IndependentStellarSubsystems{}, + materialOptions, exactCorrection, rightHandSide, arnoldiDirection, setupTimings, communicator, + backend::Diagonal{}, 2 + ); + prepareAndMeasureComposed( + {.name = "current_structure_chebyshev3_mass_amg2", + .materialFactorization = "surface_then_material_triangular", + .surfaceCalibration = "none", + .stellarStructureFactorization = "independent_subsystems", + .gravityFactorization = "approximate_ldu_chebyshev3_mass_amg2", + .specificationBorder = "compiled_dense_schur"}, + problem, preconditioning::SurfaceThenMaterialTriangular{}, preconditioning::IndependentStellarSubsystems{}, + materialOptions, exactCorrection, rightHandSide, arnoldiDirection, setupTimings, communicator, + backend::MatrixFreeChebyshev{.order = 3, .powerIterations = 20}, 2 + ); + prepareAndMeasureComposed( + {.name = "current_structure_chebyshev4_mass_amg3", + .materialFactorization = "surface_then_material_triangular", + .surfaceCalibration = "none", + .stellarStructureFactorization = "independent_subsystems", + .gravityFactorization = "approximate_ldu_chebyshev4_mass_amg3", + .specificationBorder = "compiled_dense_schur"}, + problem, preconditioning::SurfaceThenMaterialTriangular{}, preconditioning::IndependentStellarSubsystems{}, + materialOptions, exactCorrection, rightHandSide, arnoldiDirection, setupTimings, communicator, + backend::MatrixFreeChebyshev{.order = 4, .powerIterations = 20}, 3 + ); + prepareAndMeasureComposed( + {.name = "current_structure_chebyshev5_mass_amg3", + .materialFactorization = "surface_then_material_triangular", + .surfaceCalibration = "none", + .stellarStructureFactorization = "independent_subsystems", + .gravityFactorization = "approximate_ldu_chebyshev5_mass_amg3", + .specificationBorder = "compiled_dense_schur"}, + problem, preconditioning::SurfaceThenMaterialTriangular{}, preconditioning::IndependentStellarSubsystems{}, + materialOptions, exactCorrection, rightHandSide, arnoldiDirection, setupTimings, communicator, + backend::MatrixFreeChebyshev{.order = 5, .powerIterations = 20}, 3 + ); +} + +TEST_CASE( + "Full Stellar Physical Newton Right Hand Side And Damped Trial States", + "[preconditioning][diagnostics][experiment][p10][full_system][physical_newton_rhs]" +) { + using namespace mean_field; + + constexpr int radialSampleCount = 4096; + const utils::Args arguments = test_utils::setup_args(); + fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0); + REQUIRE(finiteElements.okay()); + const MPI_Comm communicator = finiteElements.mesh->GetComm(); + int communicatorSize = 0; + MPI_Comm_size(communicator, &communicatorSize); + REQUIRE(communicatorSize == 1); + + constexpr double stellarRadius = utils::RADIUS; + constexpr double targetMass = utils::MASS; + const seed::DimensionlessLaneEmdenSolution profile = seed::integrateLaneEmden(3.0, 10.0); + REQUIRE(profile.firstZeroCoordinate.has_value()); + const double surfaceCoordinate = *profile.firstZeroCoordinate; + const double surfaceDerivative = profile.thetaDerivative(profile.thetaDerivative.Size() - 1); + const double dimensionlessMass = -surfaceCoordinate * surfaceCoordinate * surfaceDerivative; + REQUIRE(dimensionlessMass > 0.0); + const double massScale = targetMass / (4.0 * std::numbers::pi_v * dimensionlessMass); + const double polytropicConstant = std::numbers::pi_v * utils::G * std::pow(massScale, 2.0 / 3.0); + const double radialScale = stellarRadius / surfaceCoordinate; + const double centralDensity = + std::pow(polytropicConstant / (std::numbers::pi_v * utils::G * radialScale * radialScale), 1.5); + + auto model = model::StellarModel( + eos::Polytrope({.n = 3.0, .K = polytropicConstant}), + surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), + integral::FixedTotalMass({.Mtotal = dimensions::MassValue{targetMass}}), + constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}}) + ); + auto problem = equilibrium::discretize(model, finiteElements); + auto projected = + seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = radialSampleCount})); + auto dependencies = makeDependencies(); + const auto rotation = zeroRotation(); + const auto preparation = problem.Prepare(projected.values, dependencies, rotation); + REQUIRE(preparation.assembledResidual); + + mfem::Vector baseResidual; + problem.BuildResidual(baseResidual); + const double baseResidualNorm = globalNorm(baseResidual, communicator); + REQUIRE(std::isfinite(baseResidualNorm)); + REQUIRE(baseResidualNorm > 0.0); + mfem::Vector rightHandSide(baseResidual); + rightHandSide *= -1.0; + + constexpr preconditioning::MaterialSurfaceDiagonalOptions uncalibratedMaterialOptions{}; + constexpr preconditioning::MaterialSurfaceDiagonalOptions rightCalibratedMaterialOptions{ + .surfaceCalibration = { + .target = preconditioning::SurfaceRieszCalibrationTarget::surface_jacobian, + .probeCount = 4, + .objective = preconditioning::SurfaceRieszCalibrationObjective::right_preconditioned_action + } + }; + + std::vector candidates; + candidates.reserve(4); + solver::IdentityPreconditioner identity(problem.StateSize()); + candidates.push_back(solveNewtonCandidate( + {.name = "identity", + .materialFactorization = "identity", + .surfaceCalibration = "none", + .stellarStructureFactorization = "identity", + .gravityFactorization = "identity", + .specificationBorder = "identity"}, + identity, 0.0, problem, baseResidual, rightHandSide, communicator + )); + candidates.push_back(prepareAndSolveNewtonComposed( + {.name = "current_default", + .materialFactorization = "surface_then_material_triangular", + .surfaceCalibration = "none", + .stellarStructureFactorization = "independent_subsystems", + .gravityFactorization = "approximate_ldu", + .specificationBorder = "compiled_dense_schur"}, + problem, preconditioning::IndependentStellarSubsystems{}, uncalibratedMaterialOptions, baseResidual, + rightHandSide, communicator + )); + candidates.push_back(prepareAndSolveNewtonComposed( + {.name = "surface_then_material_uncalibrated_then_gravity", + .materialFactorization = "surface_then_material_triangular", + .surfaceCalibration = "none", + .stellarStructureFactorization = "material_then_gravity_triangular", + .gravityFactorization = "approximate_ldu", + .specificationBorder = "compiled_dense_schur"}, + problem, preconditioning::MaterialThenGravityTriangular{}, uncalibratedMaterialOptions, baseResidual, + rightHandSide, communicator + )); + candidates.push_back(prepareAndSolveNewtonComposed( + {.name = "surface_then_material_right_calibrated_then_gravity", + .materialFactorization = "surface_then_material_triangular", + .surfaceCalibration = "surface_jacobian_right_action_4_probes", + .stellarStructureFactorization = "material_then_gravity_triangular", + .gravityFactorization = "approximate_ldu", + .specificationBorder = "compiled_dense_schur"}, + problem, preconditioning::MaterialThenGravityTriangular{}, rightCalibratedMaterialOptions, baseResidual, + rightHandSide, communicator + )); + + // All four corrections above were obtained while the problem remained at + // the identical projected Lane-Emden base point. Only now do we mutate the + // prepared state to measure the nonlinear quality of each correction. + constexpr std::array dampingFactors{1.0, 0.5, 0.25, 0.125, 0.0625, 0.03125, 0.015625}; + const auto valueBlocks = problem.GetManifest().valueBlocks(); + const auto residualBlocks = problem.GetManifest().residualBlocks(); + const auto &surfaceBlock = findBlock(valueBlocks, "surface_deformation"); + const auto &densityBlock = findBlock(valueBlocks, "density"); + const auto &physical = problem.GetPreparedOperator().GetPhysicalOperator(); + const auto &domainDeformation = physical.GetDomainDeformation(); + mfem::Vector volumeDisplacement(domainDeformation.volumeDisplacementSize()); + + for (const NewtonCandidateResult &candidate : candidates) { + for (const double alpha : dampingFactors) { + incrementStateRevisions(dependencies); + mfem::Vector trialState(projected.values); + trialState.Add(alpha, candidate.correction); + + int localStateIsFinite = 1; + for (int index = 0; index < trialState.Size(); ++index) { + if (!std::isfinite(trialState(index))) { + localStateIsFinite = 0; + } + } + int stateIsFinite = 0; + MPI_Allreduce(&localStateIsFinite, &stateIsFinite, 1, MPI_INT, MPI_MIN, communicator); + + const mfem::Vector density(trialState.GetData() + densityBlock.offset, densityBlock.size); + double localMinimumDensity = std::numeric_limits::infinity(); + for (int index = 0; index < density.Size(); ++index) { + localMinimumDensity = std::min(localMinimumDensity, density(index)); + } + double minimumDensity = 0.0; + MPI_Allreduce(&localMinimumDensity, &minimumDensity, 1, MPI_DOUBLE, MPI_MIN, communicator); + + const mfem::Vector surfaceParameters(trialState.GetData() + surfaceBlock.offset, surfaceBlock.size); + domainDeformation.buildVolumeDisplacement(surfaceParameters, volumeDisplacement); + const deformation::DomainDeformationGeometryReport geometry = + domainDeformation.inspectMappedGeometry(volumeDisplacement); + const bool geometryIsValid = geometry.isOrientationPreserving(); + const bool densityIsValid = std::isfinite(minimumDensity) && minimumDensity >= 0.0; + const bool trialIsValid = stateIsFinite != 0 && geometryIsValid && densityIsValid; + + mfem::Vector predictedResidual(baseResidual); + predictedResidual.Add(alpha, candidate.linearAction); + const double predictedNorm = globalNorm(predictedResidual, communicator); + std::map metrics{ + {"alpha", alpha}, + {"dependency_revision", static_cast(dependencies.density.revision)}, + {"state_is_finite", stateIsFinite != 0 ? 1.0 : 0.0}, + {"density_is_nonnegative", densityIsValid ? 1.0 : 0.0}, + {"minimum_density_dof", minimumDensity}, + {"geometry_is_orientation_preserving", geometryIsValid ? 1.0 : 0.0}, + {"minimum_mapping_jacobian_determinant", geometry.minimumJacobianDeterminant}, + {"trial_is_valid", trialIsValid ? 1.0 : 0.0}, + {"base_residual_norm", baseResidualNorm}, + {"correction_norm", globalNorm(candidate.correction, communicator)}, + {"damped_correction_norm", alpha * globalNorm(candidate.correction, communicator)}, + {"predicted_residual_norm", predictedNorm}, + {"predicted_relative_residual", predictedNorm / baseResidualNorm}, + {"predicted_fractional_reduction", 1.0 - predictedNorm / baseResidualNorm} + }; + + auto parameters = + newtonParameters(candidate.candidate, "damped_physical_newton_trial", problem.StateSize()); + parameters["trial_status"] = trialIsValid ? "prevalidated" : "rejected_before_residual_evaluation"; + if (!trialIsValid) { + experiment::record_experiment_result( + "stellar_preconditioning_p10_newton", candidate.candidate.name + "_alpha_" + std::to_string(alpha), + std::move(parameters), std::move(metrics) + ); + continue; + } + + try { + problem.Prepare(trialState, dependencies, rotation); + mfem::Vector actualResidual; + problem.BuildResidual(actualResidual); + const double actualNorm = globalNorm(actualResidual, communicator); + mfem::Vector nonlinearRemainder(actualResidual); + nonlinearRemainder -= predictedResidual; + const double nonlinearRemainderNorm = globalNorm(nonlinearRemainder, communicator); + mfem::Vector residualDeparture(actualResidual); + residualDeparture -= baseResidual; + const double residualDepartureNorm = globalNorm(residualDeparture, communicator); + + metrics["residual_evaluated"] = 1.0; + metrics["actual_residual_norm"] = actualNorm; + metrics["actual_relative_residual"] = actualNorm / baseResidualNorm; + metrics["actual_fractional_reduction"] = 1.0 - actualNorm / baseResidualNorm; + metrics["nonlinear_remainder_norm"] = nonlinearRemainderNorm; + metrics["relative_nonlinear_remainder"] = + nonlinearRemainderNorm / std::max(residualDepartureNorm, std::numeric_limits::min()); + metrics["actual_to_predicted_norm_ratio"] = + actualNorm / std::max(predictedNorm, std::numeric_limits::min()); + + for (const auto &block : residualBlocks) { + const mfem::Vector baseBlock(baseResidual.GetData() + block.offset, block.size); + const mfem::Vector predictedBlock(predictedResidual.GetData() + block.offset, block.size); + const mfem::Vector actualBlock(actualResidual.GetData() + block.offset, block.size); + const mfem::Vector remainderBlock(nonlinearRemainder.GetData() + block.offset, block.size); + const double baseBlockNorm = globalNorm(baseBlock, communicator); + const std::string prefix = "residual_block." + std::string(block.stableId); + metrics[prefix + ".base_norm"] = baseBlockNorm; + metrics[prefix + ".predicted_norm"] = globalNorm(predictedBlock, communicator); + metrics[prefix + ".actual_norm"] = globalNorm(actualBlock, communicator); + metrics[prefix + ".actual_ratio"] = + metrics[prefix + ".actual_norm"] / std::max(baseBlockNorm, std::numeric_limits::min()); + metrics[prefix + ".remainder_norm"] = globalNorm(remainderBlock, communicator); + } + parameters["trial_status"] = "evaluated"; + } catch (const std::exception &) { + metrics["residual_evaluated"] = 0.0; + parameters["trial_status"] = "residual_evaluation_threw"; + } + + experiment::record_experiment_result( + "stellar_preconditioning_p10_newton", candidate.candidate.name + "_alpha_" + std::to_string(alpha), + std::move(parameters), std::move(metrics) + ); + } + } +} diff --git a/experiments/gravity_preconditioning.cpp b/experiments/gravity_preconditioning.cpp new file mode 100644 index 0000000..fc764d2 --- /dev/null +++ b/experiments/gravity_preconditioning.cpp @@ -0,0 +1,591 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +import experiment; +import mean_field; +import test_helpers; + +namespace { + using Clock = std::chrono::steady_clock; + + namespace backend = mean_field::preconditioning::backend; + namespace preconditioning = mean_field::preconditioning; + + [[nodiscard]] const char *buildConfiguration() noexcept { +#ifdef NDEBUG + return "release"; +#else + return "debug"; +#endif + } + + [[nodiscard]] double maximumRankSeconds( + const Clock::time_point start, + const MPI_Comm communicator + ) { + const double localSeconds = std::chrono::duration(Clock::now() - start).count(); + double maximumSeconds = 0.0; + MPI_Allreduce(&localSeconds, &maximumSeconds, 1, MPI_DOUBLE, MPI_MAX, communicator); + return maximumSeconds; + } + + [[nodiscard]] double globalNorm( + const mfem::Vector &vector, + const MPI_Comm communicator + ) { + const double localSquared = vector * vector; + double globalSquared = 0.0; + MPI_Allreduce(&localSquared, &globalSquared, 1, MPI_DOUBLE, MPI_SUM, communicator); + return std::sqrt(std::max(globalSquared, 0.0)); + } + + [[nodiscard]] double globalDot( + const mfem::Vector &left, + const mfem::Vector &right, + const MPI_Comm communicator + ) { + const double localDot = left * right; + double result = 0.0; + MPI_Allreduce(&localDot, &result, 1, MPI_DOUBLE, MPI_SUM, communicator); + return result; + } + + void announce( + const MPI_Comm communicator, + const std::string &message + ) { + int rank = 0; + MPI_Comm_rank(communicator, &rank); + if (rank == 0) { + std::cout << message << std::endl; + } + } + + class ReducedGravityOperator final : public mfem::Operator { + public: + explicit ReducedGravityOperator( + const mean_field::operators::context::gravity_field::GravityFieldGeometryContext &context + ) + : mfem::Operator( + context.GetMassOperator().GetFluxMap().reduced_size() + + context.GetSourceOperator().GetPotentialMap().reduced_size() + ), + m_mass(&context.GetMassOperator()), + m_divergence( + context.GetDivergenceOperator(), + context.GetMassOperator().GetFluxMap(), + context.GetSourceOperator().GetPotentialMap() + ), + m_offsets(3), + m_gradientWorkspace(context.GetMassOperator().GetFluxMap().reduced_size()) { + m_offsets[0] = 0; + m_offsets[1] = context.GetMassOperator().GetFluxMap().reduced_size(); + m_offsets[2] = Height(); + } + + void Mult( + const mfem::Vector &state, + mfem::Vector &residual + ) const override { + if (state.Size() != Width() || residual.Size() != Height()) { + throw std::invalid_argument("The reduced gravity experiment requires preallocated compatible vectors."); + } + + const mfem::Vector gradient( + const_cast(state.GetData()) + m_offsets[0], m_offsets[1] - m_offsets[0] + ); + const mfem::Vector potential( + const_cast(state.GetData()) + m_offsets[1], m_offsets[2] - m_offsets[1] + ); + mfem::Vector gradientResidual(residual.GetData() + m_offsets[0], m_offsets[1] - m_offsets[0]); + mfem::Vector potentialResidual(residual.GetData() + m_offsets[1], m_offsets[2] - m_offsets[1]); + + m_mass->Mult(gradient, gradientResidual); + m_divergence.MultTranspose(potential, m_gradientWorkspace); + gradientResidual += m_gradientWorkspace; + m_divergence.Mult(gradient, potentialResidual); + } + + private: + const mfem::Operator *m_mass; + preconditioning::ReducedGravityDivergenceOperator m_divergence; + mfem::Array m_offsets; + mutable mfem::Vector m_gradientWorkspace; + }; + + [[nodiscard]] std::map< + std::string, + std::string> + commonParameters( + const std::string &candidate, + const std::string &measurement, + const int dimension + ) { + return { + {"build_configuration", buildConfiguration()}, + {"candidate", candidate}, + {"experiment_schema", "p4_reduced_gravity_v1"}, + {"factorization", candidate}, + {"measurement", measurement}, + {"mesh_file", test_utils::setup_args().mesh_file}, + {"operator", "reduced_gravity_saddle_point"}, + {"preconditioned_product", "G M^-1"}, + {"root_dimension", std::to_string(dimension)} + }; + } + + void recordSpectrum( + const std::string &candidate, + const mean_field::solver::ArnoldiSpectralMeasurement &spectrum, + const int dimension, + const double setupSeconds + ) { + experiment::record_experiment_result( + "gravity_preconditioning_p4", candidate + "_spectrum", + commonParameters(candidate, "arnoldi_summary", dimension), + {{"setup_seconds_maximum_rank", setupSeconds}, + {"requested_dimension", static_cast(spectrum.requestedDimension)}, + {"achieved_dimension", static_cast(spectrum.achievedDimension)}, + {"operator_applications", static_cast(spectrum.operatorApplications)}, + {"measurement_seconds_maximum_rank", spectrum.measurementSecondsMaximumRank}, + {"operator_application_seconds_maximum_rank", spectrum.operatorApplicationSecondsMaximumRank}, + {"projected_condition_proxy", spectrum.projectedConditionProxy}, + {"projected_largest_singular_value", spectrum.projectedLargestSingularValue}, + {"projected_smallest_singular_value", spectrum.projectedSmallestSingularValue}, + {"centroid_real_part", spectrum.centroidRealPart}, + {"rms_distance_from_one", spectrum.rmsDistanceFromOne}, + {"rms_cluster_radius", spectrum.rmsClusterRadius}, + {"minimum_magnitude", spectrum.minimumMagnitude}, + {"maximum_magnitude", spectrum.maximumMagnitude}, + {"minimum_real_part", spectrum.minimumRealPart}, + {"maximum_real_part", spectrum.maximumRealPart}, + {"maximum_absolute_imaginary_part", spectrum.maximumAbsoluteImaginaryPart}, + {"negative_real_part_count", static_cast(spectrum.negativeRealPartCount)}, + {"converged_ritz_value_count", static_cast(spectrum.convergedRitzValueCount)}, + {"conjugate_pair_defect", spectrum.conjugatePairDefect}, + {"projected_departure_from_normality", spectrum.projectedDepartureFromNormality}, + {"field_of_values_minimum_real_part", spectrum.projectedFieldOfValuesMinimumRealPart}, + {"field_of_values_maximum_real_part", spectrum.projectedFieldOfValuesMaximumRealPart}} + ); + + for (std::size_t index = 0; index < spectrum.ritzValues.size(); ++index) { + const auto &value = spectrum.ritzValues[index]; + experiment::record_experiment_result( + "gravity_preconditioning_p4", candidate + "_ritz_" + std::to_string(index), + commonParameters(candidate, "ritz_value", dimension), + {{"ritz_index", static_cast(index)}, + {"real_part", value.realPart}, + {"imaginary_part", value.imaginaryPart}, + {"magnitude", value.magnitude}, + {"distance_from_one", value.distanceFromOne}, + {"residual_estimate", value.residualEstimate}, + {"relative_residual_estimate", value.relativeResidualEstimate}, + {"converged", value.converged ? 1.0 : 0.0}} + ); + } + } + + void measureCandidate( + const std::string &candidate, + mfem::Solver &inversePreconditioner, + const double setupSeconds, + const ReducedGravityOperator &gravityOperator, + const mfem::Vector &rightHandSide, + const mfem::Vector &arnoldiDirection, + const MPI_Comm communicator + ) { + constexpr int arnoldiDimension = 32; + + mean_field::solver::InstrumentedOperator instrumentedGravity(gravityOperator); + mean_field::solver::InstrumentedPreconditioner instrumentedPreconditioner(inversePreconditioner); + mean_field::solver::ResidualHistoryMonitor monitor; + mfem::FGMRESSolver krylov(communicator); + krylov.SetPreconditioner(instrumentedPreconditioner); + krylov.SetOperator(instrumentedGravity); + krylov.SetMonitor(monitor); + krylov.SetRelTol(1.0e-8); + krylov.SetAbsTol(1.0e-12); + krylov.SetMaxIter(100); + krylov.SetKDim(30); + krylov.SetPrintLevel(0); + + mfem::Vector solution(gravityOperator.Width()); + solution = 0.0; + announce(communicator, "P4 reduced gravity: solving with " + candidate); + const Clock::time_point solveStart = Clock::now(); + krylov.Mult(rightHandSide, solution); + const double solveSeconds = maximumRankSeconds(solveStart, communicator); + + mfem::Vector reconstructed(rightHandSide.Size()); + gravityOperator.Mult(solution, reconstructed); + reconstructed -= rightHandSide; + const double relativeResidual = + globalNorm(reconstructed, communicator) / + std::max(globalNorm(rightHandSide, communicator), std::numeric_limits::epsilon()); + + const auto jacobianStatistics = instrumentedGravity.GetStatistics(); + const auto preconditionerStatistics = instrumentedPreconditioner.GetStatistics(); + REQUIRE(std::isfinite(relativeResidual)); + experiment::record_experiment_result( + "gravity_preconditioning_p4", candidate + "_linear_solve", + commonParameters(candidate, "linear_solve", gravityOperator.Width()), + {{"setup_seconds_maximum_rank", setupSeconds}, + {"solver_converged", krylov.GetConverged() ? 1.0 : 0.0}, + {"outer_iterations", static_cast(krylov.GetNumIterations())}, + {"true_relative_residual", relativeResidual}, + {"solve_seconds_maximum_rank", solveSeconds}, + {"gravity_applications", static_cast(jacobianStatistics.applications)}, + {"gravity_application_seconds", jacobianStatistics.totalSeconds}, + {"preconditioner_applications", static_cast(preconditionerStatistics.applications)}, + {"preconditioner_application_seconds", preconditionerStatistics.totalSeconds}, + {"preconditioner_maximum_application_seconds", preconditionerStatistics.maximumSeconds}} + ); + + instrumentedGravity.ResetStatistics(); + instrumentedPreconditioner.ResetStatistics(); + mean_field::solver::FixedRightPreconditionedOperator product(instrumentedGravity, instrumentedPreconditioner); + announce(communicator, "P4 reduced gravity: measuring " + candidate + " Arnoldi spectrum"); + const auto spectrum = mean_field::solver::measureArnoldiSpectrum( + product, arnoldiDirection, communicator, + {.krylovDimension = arnoldiDimension, + .breakdownRelativeTolerance = 1.0e-13, + .ritzConvergenceRelativeTolerance = 1.0e-7, + .reorthogonalize = true} + ); + recordSpectrum(candidate, spectrum, gravityOperator.Width(), setupSeconds); + } + + template < + preconditioning::GravityFactorizationPolicy Policy, + backend::Registered MassBackend = backend::Diagonal> + requires backend::Compatible< + MassBackend, + preconditioning::GravityMassInverseCharacteristics> + void prepareAndMeasureTypedCandidate( + const std::string &candidate, + Policy policy, + const mean_field::fem::FEM &finiteElements, + const mean_field::operators::context::gravity_field::GravityFieldGeometryContext &geometryContext, + const ReducedGravityOperator &gravityOperator, + const mfem::Vector &rightHandSide, + const mfem::Vector &arnoldiDirection, + const MPI_Comm communicator, + const int amgCycles = 1, + MassBackend massBackend = {} + ) { + const Clock::time_point setupStart = Clock::now(); + const auto block = preconditioning::GravityFieldBlock( + std::move(massBackend), backend::HypreBoomerAMG{backend::FixedCycles{.cycles = amgCycles}}, policy + ); + auto prepared = preconditioning::prepare(finiteElements, geometryContext, block); + const double setupTime = maximumRankSeconds(setupStart, communicator); + + const auto &massOperator = geometryContext.GetMassOperator(); + const mfem::Vector firstMassRightHandSide = + gravity_prepared_test_utils::make_deterministic_vector(massOperator.Width(), 0.41); + const mfem::Vector secondMassRightHandSide = + gravity_prepared_test_utils::make_deterministic_vector(massOperator.Width(), 1.17); + mfem::Vector firstMassAction(massOperator.Width()); + mfem::Vector secondMassAction(massOperator.Width()); + prepared.GetMassInverse().Mult(firstMassRightHandSide, firstMassAction); + prepared.GetMassInverse().Mult(secondMassRightHandSide, secondMassAction); + mfem::Vector recoveredMassRightHandSide(massOperator.Height()); + massOperator.Mult(firstMassAction, recoveredMassRightHandSide); + recoveredMassRightHandSide -= firstMassRightHandSide; + const double massRecoveryDefect = + globalNorm(recoveredMassRightHandSide, communicator) / globalNorm(firstMassRightHandSide, communicator); + const double firstSecond = globalDot(firstMassRightHandSide, secondMassAction, communicator); + const double secondFirst = globalDot(secondMassRightHandSide, firstMassAction, communicator); + const double massSymmetryDefect = + std::abs(firstSecond - secondFirst) / std::max({1.0, std::abs(firstSecond), std::abs(secondFirst)}); + const double massPositiveRayleigh = globalDot(firstMassRightHandSide, firstMassAction, communicator) / + std::max( + globalDot(firstMassRightHandSide, firstMassRightHandSide, communicator), + std::numeric_limits::min() + ); + + const auto &schurOperator = prepared.GetPotentialSchurSurrogate(); + const mfem::Vector schurRightHandSide = + gravity_prepared_test_utils::make_deterministic_vector(schurOperator.Width(), 0.73); + mfem::Vector schurAction(schurOperator.Width()); + prepared.GetPotentialSchurInverse().Mult(schurRightHandSide, schurAction); + mfem::Vector recoveredSchurRightHandSide(schurOperator.Height()); + schurOperator.Mult(schurAction, recoveredSchurRightHandSide); + recoveredSchurRightHandSide -= schurRightHandSide; + const double schurRecoveryDefect = + globalNorm(recoveredSchurRightHandSide, communicator) / globalNorm(schurRightHandSide, communicator); + + experiment::record_experiment_result( + "gravity_preconditioning_p4", candidate + "_block_quality", + commonParameters(candidate, "block_inverse_quality", gravityOperator.Width()), + {{"amg_cycles", static_cast(amgCycles)}, + {"mass_inverse_recovery_defect", massRecoveryDefect}, + {"mass_inverse_symmetry_defect", massSymmetryDefect}, + {"mass_inverse_positive_rayleigh", massPositiveRayleigh}, + {"potential_schur_inverse_recovery_defect", schurRecoveryDefect}} + ); + measureCandidate( + candidate, prepared, setupTime, gravityOperator, rightHandSide, arnoldiDirection, communicator + ); + } + + [[nodiscard]] int firstReportedThresholdIteration( + const std::vector &history, + const double initialNorm, + const double relativeThreshold + ) { + if (!std::isfinite(initialNorm) || initialNorm <= 0.0) { + return -1; + } + for (const auto &sample : history) { + if (std::abs(sample.reportedNorm) / initialNorm <= relativeThreshold) { + return sample.iteration; + } + } + return -1; + } +} // namespace + +TEST_CASE( + "Reduced Gravity P4 Factorization Comparison", + "[preconditioning][gravity][diagnostics][experiment][spectrum]" +) { + const auto arguments = test_utils::setup_args(); + mean_field::fem::FEM finiteElements = mean_field::fem::setup_fem(arguments.mesh_file, arguments, 0); + const MPI_Comm communicator = finiteElements.mesh->GetComm(); + using GeometryContext = mean_field::operators::context::gravity_field::GravityFieldGeometryContext; + GeometryContext geometryContext(finiteElements, *finiteElements.domainMapperStateless); + + mfem::Vector displacementTrue(finiteElements.displacementFes->GetTrueVSize()); + displacementTrue = 0.0; + const mfem::Vector displacement = geometryContext.GetDisplacementMap().gather(displacementTrue); + geometryContext.PreparePrimal(displacement, {.value = 1}, {.value = 1}); + + ReducedGravityOperator gravityOperator(geometryContext); + const mfem::Vector exact = gravity_prepared_test_utils::make_deterministic_vector(gravityOperator.Width(), 0.37); + mfem::Vector rightHandSide(gravityOperator.Height()); + gravityOperator.Mult(exact, rightHandSide); + const mfem::Vector arnoldiDirection = + gravity_prepared_test_utils::make_deterministic_vector(gravityOperator.Width(), 0.83); + + const Clock::time_point legacySetupStart = Clock::now(); + mean_field::operators::ReducedGravityFieldPreconditioner legacy(finiteElements, geometryContext); + const double legacySetupTime = maximumRankSeconds(legacySetupStart, communicator); + measureCandidate( + "legacy_block_diagonal", legacy, legacySetupTime, gravityOperator, rightHandSide, arnoldiDirection, communicator + ); + + prepareAndMeasureTypedCandidate( + "typed_block_diagonal", preconditioning::GravityBlockDiagonal{}, finiteElements, geometryContext, + gravityOperator, rightHandSide, arnoldiDirection, communicator + ); + prepareAndMeasureTypedCandidate( + "lower_triangular", preconditioning::GravityLowerTriangular{}, finiteElements, geometryContext, gravityOperator, + rightHandSide, arnoldiDirection, communicator + ); + prepareAndMeasureTypedCandidate( + "upper_triangular", preconditioning::GravityUpperTriangular{}, finiteElements, geometryContext, gravityOperator, + rightHandSide, arnoldiDirection, communicator + ); + prepareAndMeasureTypedCandidate( + "approximate_ldu", preconditioning::GravityApproximateLDU{}, finiteElements, geometryContext, gravityOperator, + rightHandSide, arnoldiDirection, communicator + ); +} + +TEST_CASE( + "Reduced Gravity P4 Fixed AMG Cycle Sweep", + "[preconditioning][gravity][diagnostics][experiment][amg_cycle_sweep]" +) { + const auto arguments = test_utils::setup_args(); + mean_field::fem::FEM finiteElements = mean_field::fem::setup_fem(arguments.mesh_file, arguments, 0); + const MPI_Comm communicator = finiteElements.mesh->GetComm(); + using GeometryContext = mean_field::operators::context::gravity_field::GravityFieldGeometryContext; + GeometryContext geometryContext(finiteElements, *finiteElements.domainMapperStateless); + + mfem::Vector displacementTrue(finiteElements.displacementFes->GetTrueVSize()); + displacementTrue = 0.0; + const mfem::Vector displacement = geometryContext.GetDisplacementMap().gather(displacementTrue); + geometryContext.PreparePrimal(displacement, {.value = 1}, {.value = 1}); + + ReducedGravityOperator gravityOperator(geometryContext); + const mfem::Vector exact = gravity_prepared_test_utils::make_deterministic_vector(gravityOperator.Width(), 0.37); + mfem::Vector rightHandSide(gravityOperator.Height()); + gravityOperator.Mult(exact, rightHandSide); + const mfem::Vector arnoldiDirection = + gravity_prepared_test_utils::make_deterministic_vector(gravityOperator.Width(), 0.83); + + for (const int cycles : {1, 2, 3, 4, 6, 8}) { + prepareAndMeasureTypedCandidate( + "approximate_ldu_amg_cycles_" + std::to_string(cycles), preconditioning::GravityApproximateLDU{}, + finiteElements, geometryContext, gravityOperator, rightHandSide, arnoldiDirection, communicator, cycles + ); + } + for (const int order : {2, 3, 4, 5}) { + for (const int cycles : {1, 2, 3}) { + prepareAndMeasureTypedCandidate( + "approximate_ldu_chebyshev_" + std::to_string(order) + "_amg_cycles_" + std::to_string(cycles), + preconditioning::GravityApproximateLDU{}, finiteElements, geometryContext, gravityOperator, + rightHandSide, arnoldiDirection, communicator, cycles, + backend::MatrixFreeChebyshev{.order = order, .powerIterations = 20} + ); + } + } +} + +TEST_CASE( + "Reduced Gravity P4 LDU Extended FGMRES Convergence", + "[preconditioning][gravity][diagnostics][experiment][p4_followup][extended_solve]" +) { + constexpr int maximumIterations = 200; + constexpr int restartDimension = 30; + + const auto arguments = test_utils::setup_args(); + mean_field::fem::FEM finiteElements = mean_field::fem::setup_fem(arguments.mesh_file, arguments, 0); + const MPI_Comm communicator = finiteElements.mesh->GetComm(); + using GeometryContext = mean_field::operators::context::gravity_field::GravityFieldGeometryContext; + GeometryContext geometryContext(finiteElements, *finiteElements.domainMapperStateless); + + mfem::Vector displacementTrue(finiteElements.displacementFes->GetTrueVSize()); + displacementTrue = 0.0; + const mfem::Vector displacement = geometryContext.GetDisplacementMap().gather(displacementTrue); + geometryContext.PreparePrimal(displacement, {.value = 1}, {.value = 1}); + + ReducedGravityOperator gravityOperator(geometryContext); + const mfem::Vector exact = gravity_prepared_test_utils::make_deterministic_vector(gravityOperator.Width(), 0.37); + mfem::Vector rightHandSide(gravityOperator.Height()); + gravityOperator.Mult(exact, rightHandSide); + + const Clock::time_point setupStart = Clock::now(); + const auto block = preconditioning::GravityFieldBlock( + backend::Diagonal{}, backend::HypreBoomerAMG{backend::FixedCycles{.cycles = 1}}, + preconditioning::GravityApproximateLDU{} + ); + auto prepared = preconditioning::prepare(finiteElements, geometryContext, block); + const double setupTime = maximumRankSeconds(setupStart, communicator); + + mean_field::solver::InstrumentedOperator instrumentedGravity(gravityOperator); + mean_field::solver::InstrumentedPreconditioner instrumentedPreconditioner(prepared); + mean_field::solver::ResidualHistoryMonitor monitor; + mfem::FGMRESSolver krylov(communicator); + krylov.SetPreconditioner(instrumentedPreconditioner); + krylov.SetOperator(instrumentedGravity); + krylov.SetMonitor(monitor); + krylov.SetRelTol(1.0e-8); + krylov.SetAbsTol(1.0e-12); + krylov.SetMaxIter(maximumIterations); + krylov.SetKDim(restartDimension); + krylov.SetPrintLevel(0); + + mfem::Vector solution(gravityOperator.Width()); + solution = 0.0; + announce(communicator, "P4 follow-up: running 200-iteration approximate-LDU FGMRES"); + const Clock::time_point solveStart = Clock::now(); + krylov.Mult(rightHandSide, solution); + const double solveSeconds = maximumRankSeconds(solveStart, communicator); + + mfem::Vector reconstructed(rightHandSide.Size()); + gravityOperator.Mult(solution, reconstructed); + reconstructed -= rightHandSide; + const double trueRelativeResidual = + globalNorm(reconstructed, communicator) / + std::max(globalNorm(rightHandSide, communicator), std::numeric_limits::epsilon()); + const double initialNorm = std::abs(krylov.GetInitialNorm()); + const auto &history = monitor.GetHistory(); + const int iteration1e4 = firstReportedThresholdIteration(history, initialNorm, 1.0e-4); + const int iteration1e6 = firstReportedThresholdIteration(history, initialNorm, 1.0e-6); + const int iteration1e8 = firstReportedThresholdIteration(history, initialNorm, 1.0e-8); + + REQUIRE(std::isfinite(trueRelativeResidual)); + REQUIRE_FALSE(history.empty()); + experiment::record_experiment_result( + "gravity_preconditioning_p4_followup", "approximate_ldu_extended_linear_solve", + commonParameters("approximate_ldu_extended", "linear_solve", gravityOperator.Width()), + {{"maximum_iterations", static_cast(maximumIterations)}, + {"restart_dimension", static_cast(restartDimension)}, + {"setup_seconds_maximum_rank", setupTime}, + {"solver_converged", krylov.GetConverged() ? 1.0 : 0.0}, + {"outer_iterations", static_cast(krylov.GetNumIterations())}, + {"reported_initial_residual_norm", initialNorm}, + {"reported_final_residual_norm", std::abs(krylov.GetFinalNorm())}, + {"reported_residual_reduction", initialNorm > 0.0 ? std::abs(krylov.GetFinalNorm()) / initialNorm : 0.0}, + {"reported_iteration_to_1e-4", static_cast(iteration1e4)}, + {"reported_iteration_to_1e-6", static_cast(iteration1e6)}, + {"reported_iteration_to_1e-8", static_cast(iteration1e8)}, + {"true_relative_residual", trueRelativeResidual}, + {"solve_seconds_maximum_rank", solveSeconds}, + {"gravity_applications", static_cast(instrumentedGravity.GetStatistics().applications)}, + {"gravity_application_seconds", instrumentedGravity.GetStatistics().totalSeconds}, + {"preconditioner_applications", static_cast(instrumentedPreconditioner.GetStatistics().applications)}, + {"preconditioner_application_seconds", instrumentedPreconditioner.GetStatistics().totalSeconds}} + ); + + for (std::size_t index = 0; index < history.size(); ++index) { + const auto &sample = history[index]; + experiment::record_experiment_result( + "gravity_preconditioning_p4_followup", "approximate_ldu_history_" + std::to_string(index), + commonParameters("approximate_ldu_extended", "fgmres_residual_history", gravityOperator.Width()), + {{"history_sample", static_cast(index)}, + {"iteration", static_cast(sample.iteration)}, + {"reported_residual_norm", sample.reportedNorm}, + {"reported_relative_residual", initialNorm > 0.0 ? std::abs(sample.reportedNorm) / initialNorm : 0.0}, + {"final_measurement", sample.final ? 1.0 : 0.0}} + ); + } +} + +TEST_CASE( + "Reduced Gravity P4 LDU Extended Arnoldi Convergence", + "[preconditioning][gravity][diagnostics][experiment][spectrum][p4_followup][extended_arnoldi]" +) { + constexpr int arnoldiDimension = 96; + + const auto arguments = test_utils::setup_args(); + mean_field::fem::FEM finiteElements = mean_field::fem::setup_fem(arguments.mesh_file, arguments, 0); + const MPI_Comm communicator = finiteElements.mesh->GetComm(); + using GeometryContext = mean_field::operators::context::gravity_field::GravityFieldGeometryContext; + GeometryContext geometryContext(finiteElements, *finiteElements.domainMapperStateless); + + mfem::Vector displacementTrue(finiteElements.displacementFes->GetTrueVSize()); + displacementTrue = 0.0; + const mfem::Vector displacement = geometryContext.GetDisplacementMap().gather(displacementTrue); + geometryContext.PreparePrimal(displacement, {.value = 1}, {.value = 1}); + + ReducedGravityOperator gravityOperator(geometryContext); + const mfem::Vector arnoldiDirection = + gravity_prepared_test_utils::make_deterministic_vector(gravityOperator.Width(), 0.83); + + const Clock::time_point setupStart = Clock::now(); + const auto block = preconditioning::GravityFieldBlock( + backend::Diagonal{}, backend::HypreBoomerAMG{backend::FixedCycles{.cycles = 1}}, + preconditioning::GravityApproximateLDU{} + ); + auto prepared = preconditioning::prepare(finiteElements, geometryContext, block); + const double setupTime = maximumRankSeconds(setupStart, communicator); + + mean_field::solver::InstrumentedOperator instrumentedGravity(gravityOperator); + mean_field::solver::InstrumentedPreconditioner instrumentedPreconditioner(prepared); + mean_field::solver::FixedRightPreconditionedOperator product(instrumentedGravity, instrumentedPreconditioner); + announce(communicator, "P4 follow-up: measuring the 96-vector approximate-LDU Arnoldi spectrum"); + const auto spectrum = mean_field::solver::measureArnoldiSpectrum( + product, arnoldiDirection, communicator, + {.krylovDimension = arnoldiDimension, + .breakdownRelativeTolerance = 1.0e-13, + .ritzConvergenceRelativeTolerance = 1.0e-7, + .reorthogonalize = true} + ); + + REQUIRE(spectrum.achievedDimension > 32); + recordSpectrum("approximate_ldu_arnoldi_96", spectrum, gravityOperator.Width(), setupTime); +} diff --git a/experiments/material_surface_preconditioning.cpp b/experiments/material_surface_preconditioning.cpp new file mode 100644 index 0000000..2fe4e41 --- /dev/null +++ b/experiments/material_surface_preconditioning.cpp @@ -0,0 +1,993 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +import experiment; +import mean_field; +import test_helpers; + +namespace { + using Clock = std::chrono::steady_clock; + + namespace backend = mean_field::preconditioning::backend; + namespace preconditioning = mean_field::preconditioning; + namespace solver = mean_field::solver; + + struct MaterialBlockMeasurements final { + double density{0.0}; + double surface{0.0}; + double enthalpy{0.0}; + }; + + [[nodiscard]] const char *buildConfiguration() noexcept { +#ifdef NDEBUG + return "release"; +#else + return "debug"; +#endif + } + + [[nodiscard]] double maximumRankSeconds( + const Clock::time_point start, + const MPI_Comm communicator + ) { + const double localSeconds = std::chrono::duration(Clock::now() - start).count(); + double maximumSeconds = 0.0; + MPI_Allreduce(&localSeconds, &maximumSeconds, 1, MPI_DOUBLE, MPI_MAX, communicator); + return maximumSeconds; + } + + [[nodiscard]] double globalNorm( + const mfem::Vector &vector, + const MPI_Comm communicator + ) { + const double localSquaredNorm = vector * vector; + double globalSquaredNorm = 0.0; + MPI_Allreduce(&localSquaredNorm, &globalSquaredNorm, 1, MPI_DOUBLE, MPI_SUM, communicator); + return std::sqrt(std::max(globalSquaredNorm, 0.0)); + } + + void announce( + const MPI_Comm communicator, + const std::string &message + ) { + int rank = 0; + MPI_Comm_rank(communicator, &rank); + if (rank == 0) { + std::cout << "[P9 material-surface] " << message << std::endl; + } + } + + [[nodiscard]] mean_field::operators::StellarEquilibriumDependencies makeDependencies() { + return { + .discretization = {.identity = 10103, .revision = 1}, + .density = {.identity = 10111, .revision = 1}, + .surfaceDeformation = {.identity = 10133, .revision = 1}, + .gravityGradient = {.identity = 10139, .revision = 1}, + .gravityPotential = {.identity = 10141, .revision = 1}, + .enthalpy = {.identity = 10151, .revision = 1}, + .bernoulliConstant = {.identity = 10159, .revision = 1}, + .rotation = {.identity = 10163, .revision = 1}, + .targetMass = {.identity = 10169, .revision = 1} + }; + } + + [[nodiscard]] mean_field::physics::RigidRotation zeroRotation() { + mfem::Vector angularVelocity(3); + mfem::Vector center(3); + angularVelocity = 0.0; + center = 0.0; + return {angularVelocity, center}; + } + + [[nodiscard]] mfem::Vector blockBalancedDirection( + const mfem::Array &offsets, + const double phase, + const MPI_Comm communicator + ) { + mfem::Vector direction(offsets.Last()); + direction = 0.0; + for (int block = 0; block < offsets.Size() - 1; ++block) { + mfem::Vector values(direction, offsets[block], offsets[block + 1] - offsets[block]); + for (int index = 0; index < values.Size(); ++index) { + const double ordinal = static_cast(index + 1); + values(index) = std::sin(0.371 * ordinal + phase + static_cast(block)) + + 0.29 * std::cos(0.173 * ordinal - 0.5 * phase); + } + const double norm = globalNorm(values, communicator); + REQUIRE(norm > 0.0); + values /= norm; + values.SyncAliasMemory(direction); + } + return direction; + } + + [[nodiscard]] MaterialBlockMeasurements blockNorms( + const mfem::Vector &vector, + const mfem::Array &offsets, + const MPI_Comm communicator + ) { + REQUIRE(offsets.Size() == 4); + const mfem::Vector density(const_cast(vector.GetData()) + offsets[0], offsets[1] - offsets[0]); + const mfem::Vector surface(const_cast(vector.GetData()) + offsets[1], offsets[2] - offsets[1]); + const mfem::Vector enthalpy(const_cast(vector.GetData()) + offsets[2], offsets[3] - offsets[2]); + return { + .density = globalNorm(density, communicator), + .surface = globalNorm(surface, communicator), + .enthalpy = globalNorm(enthalpy, communicator) + }; + } + + [[nodiscard]] MaterialBlockMeasurements relativeBlockNorms( + const mfem::Vector &numerator, + const mfem::Vector &denominator, + const mfem::Array &offsets, + const MPI_Comm communicator + ) { + const MaterialBlockMeasurements numeratorNorms = blockNorms(numerator, offsets, communicator); + const MaterialBlockMeasurements denominatorNorms = blockNorms(denominator, offsets, communicator); + constexpr double floor = 1.0e-300; + return { + .density = numeratorNorms.density / std::max(denominatorNorms.density, floor), + .surface = numeratorNorms.surface / std::max(denominatorNorms.surface, floor), + .enthalpy = numeratorNorms.enthalpy / std::max(denominatorNorms.enthalpy, floor) + }; + } + + [[nodiscard]] std::map< + std::string, + std::string> + commonParameters( + const std::string &candidate, + const std::string &measurement, + const int dimension + ) { + return { + {"build_configuration", buildConfiguration()}, + {"candidate", candidate}, + {"equation_of_state", "Polytrope(n=1)"}, + {"experiment_schema", "p9_material_surface_v1"}, + {"factorization", candidate}, + {"linearization_state", "projected_lane_emden"}, + {"measurement", measurement}, + {"mesh_file", test_utils::setup_args().mesh_file}, + {"operator", "restricted_material_surface_jacobian"}, + {"preconditioned_product", "A_material_surface M^-1"}, + {"root_dimension", std::to_string(dimension)}, + {"rotation", "zero"} + }; + } + + void recordSpectrum( + const std::string &candidate, + const solver::ArnoldiSpectralMeasurement &spectrum, + const int dimension, + const double setupSeconds + ) { + experiment::record_experiment_result( + "material_surface_preconditioning_p9", candidate + "_arnoldi_summary", + commonParameters(candidate, "arnoldi_summary", dimension), + {{"setup_seconds_maximum_rank", setupSeconds}, + {"requested_dimension", static_cast(spectrum.requestedDimension)}, + {"achieved_dimension", static_cast(spectrum.achievedDimension)}, + {"invariant_subspace_found", spectrum.invariantSubspaceFound ? 1.0 : 0.0}, + {"operator_applications", static_cast(spectrum.operatorApplications)}, + {"measurement_seconds_maximum_rank", spectrum.measurementSecondsMaximumRank}, + {"operator_application_seconds_maximum_rank", spectrum.operatorApplicationSecondsMaximumRank}, + {"projected_condition_proxy", spectrum.projectedConditionProxy}, + {"projected_largest_singular_value", spectrum.projectedLargestSingularValue}, + {"projected_smallest_singular_value", spectrum.projectedSmallestSingularValue}, + {"centroid_real_part", spectrum.centroidRealPart}, + {"centroid_imaginary_part", spectrum.centroidImaginaryPart}, + {"rms_distance_from_one", spectrum.rmsDistanceFromOne}, + {"rms_cluster_radius", spectrum.rmsClusterRadius}, + {"minimum_magnitude", spectrum.minimumMagnitude}, + {"maximum_magnitude", spectrum.maximumMagnitude}, + {"minimum_real_part", spectrum.minimumRealPart}, + {"maximum_real_part", spectrum.maximumRealPart}, + {"maximum_absolute_imaginary_part", spectrum.maximumAbsoluteImaginaryPart}, + {"negative_real_part_count", static_cast(spectrum.negativeRealPartCount)}, + {"converged_ritz_value_count", static_cast(spectrum.convergedRitzValueCount)}, + {"conjugate_pair_defect", spectrum.conjugatePairDefect}, + {"projected_departure_from_normality", spectrum.projectedDepartureFromNormality}, + {"field_of_values_minimum_real_part", spectrum.projectedFieldOfValuesMinimumRealPart}, + {"field_of_values_maximum_real_part", spectrum.projectedFieldOfValuesMaximumRealPart}} + ); + + std::vector ordered = spectrum.ritzValues; + std::ranges::sort(ordered, [](const auto &left, const auto &right) { + if (left.realPart != right.realPart) { + return left.realPart < right.realPart; + } + return left.imaginaryPart < right.imaginaryPart; + }); + for (std::size_t index = 0; index < ordered.size(); ++index) { + const auto &value = ordered[index]; + experiment::record_experiment_result( + "material_surface_preconditioning_p9", candidate + "_ritz_" + std::to_string(index), + commonParameters(candidate, "ritz_value", dimension), + {{"ritz_index", static_cast(index)}, + {"real_part", value.realPart}, + {"imaginary_part", value.imaginaryPart}, + {"magnitude", value.magnitude}, + {"distance_from_one", value.distanceFromOne}, + {"residual_estimate", value.residualEstimate}, + {"relative_residual_estimate", value.relativeResidualEstimate}, + {"converged", value.converged ? 1.0 : 0.0}} + ); + } + } + + template + void measureCandidate( + const std::string &candidate, + Preconditioner &inversePreconditioner, + const double setupSeconds, + const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation, + const mfem::Vector &exactCorrection, + const mfem::Vector &rightHandSide, + const mfem::Vector &arnoldiDirection, + const MPI_Comm communicator, + std::map< + std::string, + double> preparationMetrics = {} + ) { + constexpr int maximumIterations = 40; + constexpr int restartDimension = 20; + constexpr int arnoldiDimension = 16; + + solver::InstrumentedOperator instrumentedOperation(operation); + solver::InstrumentedPreconditioner instrumentedPreconditioner(inversePreconditioner); + solver::ResidualHistoryMonitor monitor; + mfem::FGMRESSolver krylov(communicator); + krylov.SetPreconditioner(instrumentedPreconditioner); + krylov.SetOperator(instrumentedOperation); + krylov.SetMonitor(monitor); + krylov.SetRelTol(1.0e-8); + krylov.SetAbsTol(1.0e-12); + krylov.SetMaxIter(maximumIterations); + krylov.SetKDim(restartDimension); + krylov.SetPrintLevel(0); + + mfem::Vector solution(operation.Width()); + solution = 0.0; + announce(communicator, "solving manufactured system with " + candidate); + const Clock::time_point solveStart = Clock::now(); + krylov.Mult(rightHandSide, solution); + const double solveSeconds = maximumRankSeconds(solveStart, communicator); + + mfem::Vector trueResidual(operation.Height()); + operation.Mult(solution, trueResidual); + trueResidual -= rightHandSide; + mfem::Vector solutionError(solution); + solutionError -= exactCorrection; + const double trueRelativeResidual = + globalNorm(trueResidual, communicator) / + std::max(globalNorm(rightHandSide, communicator), std::numeric_limits::min()); + const double relativeSolutionError = + globalNorm(solutionError, communicator) / + std::max(globalNorm(exactCorrection, communicator), std::numeric_limits::min()); + const MaterialBlockMeasurements relativeResidualBlocks = + relativeBlockNorms(trueResidual, rightHandSide, operation.GetOffsets(), communicator); + + mfem::Vector preconditionedDirection(operation.Height()); + inversePreconditioner.Mult(arnoldiDirection, preconditionedDirection); + mfem::Vector defect(operation.Height()); + operation.Mult(preconditionedDirection, defect); + defect -= arnoldiDirection; + const MaterialBlockMeasurements defectBlocks = blockNorms(defect, operation.GetOffsets(), communicator); + const double defectNorm = + globalNorm(defect, communicator) / + std::max(globalNorm(arnoldiDirection, communicator), std::numeric_limits::min()); + + std::map solveMetrics{ + {"setup_seconds_maximum_rank", setupSeconds}, + {"maximum_iterations", static_cast(maximumIterations)}, + {"restart_dimension", static_cast(restartDimension)}, + {"solver_converged", krylov.GetConverged() ? 1.0 : 0.0}, + {"outer_iterations", static_cast(krylov.GetNumIterations())}, + {"reported_initial_residual_norm", std::abs(krylov.GetInitialNorm())}, + {"reported_final_residual_norm", std::abs(krylov.GetFinalNorm())}, + {"true_relative_residual", trueRelativeResidual}, + {"relative_solution_error", relativeSolutionError}, + {"density_relative_residual", relativeResidualBlocks.density}, + {"surface_relative_residual", relativeResidualBlocks.surface}, + {"enthalpy_relative_residual", relativeResidualBlocks.enthalpy}, + {"right_preconditioned_defect", defectNorm}, + {"density_defect_norm", defectBlocks.density}, + {"surface_defect_norm", defectBlocks.surface}, + {"enthalpy_defect_norm", defectBlocks.enthalpy}, + {"solve_seconds_maximum_rank", solveSeconds}, + {"jacobian_applications", static_cast(instrumentedOperation.GetStatistics().applications)}, + {"jacobian_application_seconds", instrumentedOperation.GetStatistics().totalSeconds}, + {"preconditioner_applications", + static_cast(instrumentedPreconditioner.GetStatistics().applications)}, + {"preconditioner_application_seconds", instrumentedPreconditioner.GetStatistics().totalSeconds}, + {"preconditioner_maximum_application_seconds", instrumentedPreconditioner.GetStatistics().maximumSeconds} + }; + solveMetrics.insert(preparationMetrics.begin(), preparationMetrics.end()); + experiment::record_experiment_result( + "material_surface_preconditioning_p9", candidate + "_linear_solve", + commonParameters(candidate, "manufactured_linear_solve", operation.Width()), std::move(solveMetrics) + ); + + const double initialNorm = std::max(std::abs(krylov.GetInitialNorm()), 1.0e-300); + const auto &history = monitor.GetHistory(); + for (std::size_t index = 0; index < history.size(); ++index) { + const auto &sample = history[index]; + experiment::record_experiment_result( + "material_surface_preconditioning_p9", candidate + "_history_" + std::to_string(index), + commonParameters(candidate, "fgmres_residual_history", operation.Width()), + {{"history_sample", static_cast(index)}, + {"iteration", static_cast(sample.iteration)}, + {"reported_residual_norm", sample.reportedNorm}, + {"reported_relative_residual", std::abs(sample.reportedNorm) / initialNorm}, + {"final_measurement", sample.final ? 1.0 : 0.0}} + ); + } + + instrumentedOperation.ResetStatistics(); + instrumentedPreconditioner.ResetStatistics(); + solver::FixedRightPreconditionedOperator product(instrumentedOperation, instrumentedPreconditioner); + announce(communicator, "measuring " + candidate + " with 16-vector Arnoldi"); + const solver::ArnoldiSpectralMeasurement spectrum = solver::measureArnoldiSpectrum( + product, arnoldiDirection, communicator, + {.krylovDimension = arnoldiDimension, + .breakdownRelativeTolerance = 1.0e-13, + .ritzConvergenceRelativeTolerance = 1.0e-7, + .reorthogonalize = true} + ); + REQUIRE(std::isfinite(trueRelativeResidual)); + REQUIRE(std::isfinite(relativeSolutionError)); + REQUIRE(std::isfinite(defectNorm)); + REQUIRE(std::isfinite(spectrum.projectedConditionProxy)); + recordSpectrum(candidate, spectrum, operation.Width(), setupSeconds); + + int rank = 0; + MPI_Comm_rank(communicator, &rank); + if (rank == 0) { + std::cout << "[P9 material-surface] " << candidate << ": iterations=" << krylov.GetNumIterations() + << ", converged=" << (krylov.GetConverged() ? "yes" : "no") + << ", true residual=" << trueRelativeResidual << ", defect=" << defectNorm + << ", projected condition=" << spectrum.projectedConditionProxy << '\n'; + } + } + + template + void prepareAndMeasure( + const std::string &candidate, + const Policy policy, + const auto &problem, + const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation, + const mfem::Vector &exactCorrection, + const mfem::Vector &rightHandSide, + const mfem::Vector &arnoldiDirection, + const MPI_Comm communicator, + const preconditioning::MaterialSurfaceDiagonalOptions diagonalOptions = {} + ) { + const Clock::time_point setupStart = Clock::now(); + auto block = preconditioning::materialSurfaceBlock( + problem, backend::Diagonal{}, backend::Diagonal{}, policy, diagonalOptions + ); + auto prepared = preconditioning::prepare(problem, block); + const double setupTime = maximumRankSeconds(setupStart, communicator); + const auto &density = prepared.GetDensityDiagonalQuality(); + const auto &surface = prepared.GetSurfaceDiagonalQuality(); + const auto &enthalpy = prepared.GetEnthalpyDiagonalQuality(); + const auto &calibration = prepared.GetSurfaceCalibration(); + measureCandidate( + candidate, prepared, setupTime, operation, exactCorrection, rightHandSide, arnoldiDirection, communicator, + {{"density_diagonal_minimum", density.minimumAbsoluteEntryBeforeRegularization}, + {"density_diagonal_maximum", density.maximumAbsoluteEntryBeforeRegularization}, + {"density_diagonal_floor", density.appliedFloor}, + {"density_regularized_entries", static_cast(density.regularizedEntries)}, + {"surface_diagonal_minimum", surface.minimumAbsoluteEntryBeforeRegularization}, + {"surface_diagonal_maximum", surface.maximumAbsoluteEntryBeforeRegularization}, + {"surface_diagonal_floor", surface.appliedFloor}, + {"surface_regularized_entries", static_cast(surface.regularizedEntries)}, + {"surface_calibration_target", static_cast(calibration.target)}, + {"surface_calibration_probes", static_cast(calibration.probeCount)}, + {"surface_calibration_objective", static_cast(calibration.objective)}, + {"surface_calibration_scale", calibration.scale}, + {"surface_calibration_inverse_multiplier", calibration.inverseMultiplier}, + {"surface_calibration_numerator", calibration.leastSquaresNumerator}, + {"surface_calibration_denominator", calibration.leastSquaresDenominator}, + {"enthalpy_diagonal_minimum", enthalpy.minimumAbsoluteEntryBeforeRegularization}, + {"enthalpy_diagonal_maximum", enthalpy.maximumAbsoluteEntryBeforeRegularization}, + {"enthalpy_diagonal_floor", enthalpy.appliedFloor}, + {"enthalpy_regularized_entries", static_cast(enthalpy.regularizedEntries)}} + ); + } + + template + void prepareAndMeasureH1( + const std::string &candidate, + const Policy policy, + const int fixedAMGCycles, + const int calibrationProbeCount, + const auto &problem, + const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation, + const mfem::Vector &exactCorrection, + const mfem::Vector &rightHandSide, + const mfem::Vector &arnoldiDirection, + const MPI_Comm communicator + ) { + REQUIRE(fixedAMGCycles > 0); + REQUIRE(calibrationProbeCount >= 3); + + const Clock::time_point setupStart = Clock::now(); + auto block = preconditioning::materialSurfaceBlock( + problem, backend::Diagonal{}, backend::HypreBoomerAMG{backend::FixedCycles{.cycles = fixedAMGCycles}}, + policy, + preconditioning::SurfaceH1MassStiffness{ + .calibration = { + .target = preconditioning::SurfaceRieszCalibrationTarget::surface_jacobian, + .probeCount = calibrationProbeCount + } + } + ); + auto prepared = preconditioning::prepare(problem, std::move(block)); + const double setupTime = maximumRankSeconds(setupStart, communicator); + + const auto &density = prepared.GetDensityDiagonalQuality(); + const auto &enthalpy = prepared.GetEnthalpyDiagonalQuality(); + const auto &fit = prepared.GetSurfaceFit(); + measureCandidate( + candidate, prepared, setupTime, operation, exactCorrection, rightHandSide, arnoldiDirection, communicator, + {{"density_diagonal_minimum", density.minimumAbsoluteEntryBeforeRegularization}, + {"density_diagonal_maximum", density.maximumAbsoluteEntryBeforeRegularization}, + {"density_diagonal_floor", density.appliedFloor}, + {"density_regularized_entries", static_cast(density.regularizedEntries)}, + {"surface_h1_calibration_target", static_cast(fit.target)}, + {"surface_h1_calibration_probes", static_cast(fit.probeCount)}, + {"surface_h1_fit_sign", fit.sign}, + {"surface_h1_mass_coefficient", fit.massCoefficient}, + {"surface_h1_stiffness_coefficient", fit.stiffnessCoefficient}, + {"surface_h1_fit_relative_residual", fit.relativeResidual}, + {"surface_h1_fit_relative_gram_determinant", fit.relativeGramDeterminant}, + {"surface_amg_fixed_cycles", static_cast(fixedAMGCycles)}, + {"enthalpy_diagonal_minimum", enthalpy.minimumAbsoluteEntryBeforeRegularization}, + {"enthalpy_diagonal_maximum", enthalpy.maximumAbsoluteEntryBeforeRegularization}, + {"enthalpy_diagonal_floor", enthalpy.appliedFloor}, + {"enthalpy_regularized_entries", static_cast(enthalpy.regularizedEntries)}} + ); + + const auto &surfaceBackendStatistics = prepared.GetSurfaceBackend().GetStatistics(); + const auto &factorizationStatistics = prepared.GetFactorization().GetStatistics(); + const auto &preparationStatistics = prepared.GetStatistics(); + auto parameters = commonParameters(candidate, "surface_h1_backend_statistics", operation.Width()); + parameters["experiment_schema"] = "p9_material_surface_h1_v1"; + parameters["surface_surrogate"] = "h1_mass_plus_tangential_stiffness"; + parameters["surface_calibration_target"] = "surface_jacobian"; + parameters["surface_calibration_probes"] = std::to_string(calibrationProbeCount); + parameters["surface_amg_fixed_cycles"] = std::to_string(fixedAMGCycles); + experiment::record_experiment_result( + "material_surface_preconditioning_p9", candidate + "_surface_h1_backend_statistics", std::move(parameters), + {{"setup_seconds_maximum_rank", setupTime}, + {"surface_h1_fit_sign", fit.sign}, + {"surface_h1_mass_coefficient", fit.massCoefficient}, + {"surface_h1_stiffness_coefficient", fit.stiffnessCoefficient}, + {"surface_h1_fit_relative_residual", fit.relativeResidual}, + {"surface_h1_fit_relative_gram_determinant", fit.relativeGramDeterminant}, + {"surface_backend_setups", static_cast(surfaceBackendStatistics.setups)}, + {"surface_backend_applications", static_cast(surfaceBackendStatistics.applications)}, + {"surface_backend_inner_iterations", static_cast(surfaceBackendStatistics.innerIterations)}, + {"surface_backend_last_inner_iterations", + static_cast(surfaceBackendStatistics.lastInnerIterations)}, + {"factorization_applications", static_cast(factorizationStatistics.applications)}, + {"surface_inverse_applications", static_cast(factorizationStatistics.surfaceInverseApplications)}, + {"block_setups", static_cast(preparationStatistics.setups)}, + {"surface_jacobian_probes", static_cast(preparationStatistics.surfaceJacobianProbes)}, + {"surface_h1_assemblies", static_cast(preparationStatistics.surfaceH1Assemblies)}} + ); + } + + enum class SurfaceProbeMode { constant, ordered_low, alternating_high, deterministic_mixed }; + + [[nodiscard]] const char *surfaceProbeModeName(const SurfaceProbeMode mode) noexcept { + switch (mode) { + case SurfaceProbeMode::constant: + return "constant"; + case SurfaceProbeMode::ordered_low: + return "ordered_low"; + case SurfaceProbeMode::alternating_high: + return "alternating_high"; + case SurfaceProbeMode::deterministic_mixed: + return "deterministic_mixed"; + } + return "unknown"; + } + + [[nodiscard]] mfem::Vector normalizedSurfaceProbe( + const int localSize, + const SurfaceProbeMode mode, + const MPI_Comm communicator + ) { + int globalSize = 0; + int offset = 0; + MPI_Allreduce(&localSize, &globalSize, 1, MPI_INT, MPI_SUM, communicator); + MPI_Exscan(&localSize, &offset, 1, MPI_INT, MPI_SUM, communicator); + int rank = 0; + MPI_Comm_rank(communicator, &rank); + if (rank == 0) { + offset = 0; + } + REQUIRE(globalSize > 0); + + mfem::Vector probe(localSize); + for (int index = 0; index < localSize; ++index) { + const int globalIndex = offset + index; + const double position = (static_cast(globalIndex) + 0.5) / static_cast(globalSize); + switch (mode) { + case SurfaceProbeMode::constant: + probe(index) = 1.0; + break; + case SurfaceProbeMode::ordered_low: + probe(index) = std::cos(std::numbers::pi_v * position); + break; + case SurfaceProbeMode::alternating_high: + probe(index) = globalIndex % 2 == 0 ? 1.0 : -1.0; + break; + case SurfaceProbeMode::deterministic_mixed: + probe(index) = 0.41 * std::cos(std::numbers::pi_v * position) + + std::sin(5.0 * std::numbers::pi_v * position) + + 0.23 * (globalIndex % 2 == 0 ? 1.0 : -1.0); + break; + } + } + const double norm = globalNorm(probe, communicator); + REQUIRE(norm > 0.0); + probe /= norm; + return probe; + } + + void applyParameterOverrides( + std::map< + std::string, + std::string> ¶meters, + const std::map< + std::string, + std::string> &overrides + ) { + for (const auto &[key, value] : overrides) { + parameters.insert_or_assign(key, value); + } + } + + template + void recordSurfaceInverseRecovery( + const std::string &candidate, + SurfaceInverse &surfaceInverse, + const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation, + const MPI_Comm communicator, + const std::map< + std::string, + std::string> ¶meterOverrides + ) { + constexpr std::array modes{ + SurfaceProbeMode::constant, SurfaceProbeMode::ordered_low, SurfaceProbeMode::alternating_high, + SurfaceProbeMode::deterministic_mixed + }; + const int surfaceSize = operation.GetOffsets()[2] - operation.GetOffsets()[1]; + REQUIRE(surfaceInverse.Width() == surfaceSize); + REQUIRE(surfaceInverse.Height() == surfaceSize); + for (const SurfaceProbeMode mode : modes) { + const mfem::Vector probe = normalizedSurfaceProbe(surfaceSize, mode, communicator); + mfem::Vector surfaceAction(surfaceSize); + mfem::Vector recovered(surfaceSize); + operation.ApplySurfaceToSurface(probe, surfaceAction); + const Clock::time_point inverseStart = Clock::now(); + surfaceInverse.Mult(surfaceAction, recovered); + const double inverseSeconds = maximumRankSeconds(inverseStart, communicator); + + mfem::Vector recoveryError(recovered); + recoveryError -= probe; + const double probeNorm = globalNorm(probe, communicator); + const double actionNorm = globalNorm(surfaceAction, communicator); + const double recoveredNorm = globalNorm(recovered, communicator); + const double relativeError = globalNorm(recoveryError, communicator) / probeNorm; + constexpr double nonzeroFloor = 1.0e-300; + + auto parameters = commonParameters(candidate, "surface_inverse_recovery", operation.Width()); + applyParameterOverrides(parameters, parameterOverrides); + parameters["surface_probe_mode"] = surfaceProbeModeName(mode); + experiment::record_experiment_result( + "material_surface_preconditioning_p9", candidate + "_" + surfaceProbeModeName(mode), + std::move(parameters), + {{"surface_probe_norm", probeNorm}, + {"surface_action_norm", actionNorm}, + {"surface_recovered_norm", recoveredNorm}, + {"surface_recovery_relative_error", relativeError}, + {"surface_operator_gain", actionNorm / probeNorm}, + {"surface_inverse_gain", recoveredNorm / std::max(actionNorm, nonzeroFloor)}, + {"surface_recovered_gain", recoveredNorm / probeNorm}, + {"surface_inverse_seconds_maximum_rank", inverseSeconds}} + ); + } + } + + template + void recordBalancedPreconditionedDefect( + const std::string &candidate, + PreparedPreconditioner &prepared, + const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation, + const MPI_Comm communicator, + const std::map< + std::string, + std::string> ¶meterOverrides + ) { + const mfem::Vector direction = blockBalancedDirection(operation.GetOffsets(), 1.37, communicator); + mfem::Vector correction(operation.Width()); + mfem::Vector defect(operation.Height()); + const Clock::time_point applicationStart = Clock::now(); + prepared.Mult(direction, correction); + const double applicationSeconds = maximumRankSeconds(applicationStart, communicator); + operation.Mult(correction, defect); + defect -= direction; + const MaterialBlockMeasurements blockDefects = blockNorms(defect, operation.GetOffsets(), communicator); + const double relativeDefect = globalNorm(defect, communicator) / + std::max(globalNorm(direction, communicator), std::numeric_limits::min()); + + auto parameters = commonParameters(candidate, "balanced_right_preconditioned_defect", operation.Width()); + applyParameterOverrides(parameters, parameterOverrides); + experiment::record_experiment_result( + "material_surface_preconditioning_p9", candidate + "_balanced_defect", std::move(parameters), + {{"right_preconditioned_defect", relativeDefect}, + {"density_defect_norm", blockDefects.density}, + {"surface_defect_norm", blockDefects.surface}, + {"enthalpy_defect_norm", blockDefects.enthalpy}, + {"preconditioner_application_seconds_maximum_rank", applicationSeconds}} + ); + } + + void measureH1CalibrationFloor( + const std::string &candidate, + const std::string &relativeMassCoefficientFloorLabel, + const double relativeMassCoefficientFloor, + const auto &problem, + const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation, + const MPI_Comm communicator + ) { + const Clock::time_point setupStart = Clock::now(); + auto block = preconditioning::materialSurfaceBlock( + problem, backend::Diagonal{}, backend::HypreBoomerAMG{backend::FixedCycles{.cycles = 1}}, + preconditioning::ApproximateMaterialSurfaceLDU{}, + preconditioning::SurfaceH1MassStiffness{ + .calibration = + {.target = preconditioning::SurfaceRieszCalibrationTarget::surface_jacobian, .probeCount = 4}, + .relativeMassCoefficientFloor = relativeMassCoefficientFloor + } + ); + auto prepared = preconditioning::prepare(problem, std::move(block)); + const double setupTime = maximumRankSeconds(setupStart, communicator); + const auto &fit = prepared.GetSurfaceFit(); + + const std::map parameters{ + {"experiment_schema", "p9_material_surface_h1_tuning_v1"}, + {"surface_surrogate", "h1_mass_plus_tangential_stiffness"}, + {"surface_calibration_target", "surface_jacobian"}, + {"surface_calibration_probes", "4"}, + {"surface_amg_fixed_cycles", "1"}, + {"relative_mass_coefficient_floor", relativeMassCoefficientFloorLabel} + }; + recordSurfaceInverseRecovery(candidate, prepared.GetSurfaceInverse(), operation, communicator, parameters); + recordBalancedPreconditionedDefect(candidate, prepared, operation, communicator, parameters); + + const auto &backendStatistics = prepared.GetSurfaceBackend().GetStatistics(); + const auto &factorizationStatistics = prepared.GetFactorization().GetStatistics(); + const auto &preparationStatistics = prepared.GetStatistics(); + auto summaryParameters = commonParameters(candidate, "surface_h1_floor_summary", operation.Width()); + applyParameterOverrides(summaryParameters, parameters); + experiment::record_experiment_result( + "material_surface_preconditioning_p9", candidate + "_summary", std::move(summaryParameters), + {{"setup_seconds_maximum_rank", setupTime}, + {"relative_mass_coefficient_floor", relativeMassCoefficientFloor}, + {"surface_h1_fit_sign", fit.sign}, + {"surface_h1_mass_coefficient", fit.massCoefficient}, + {"surface_h1_stiffness_coefficient", fit.stiffnessCoefficient}, + {"surface_h1_fit_relative_residual", fit.relativeResidual}, + {"surface_h1_fit_relative_gram_determinant", fit.relativeGramDeterminant}, + {"surface_backend_setups", static_cast(backendStatistics.setups)}, + {"surface_backend_applications", static_cast(backendStatistics.applications)}, + {"surface_backend_inner_iterations", static_cast(backendStatistics.innerIterations)}, + {"surface_backend_last_inner_iterations", static_cast(backendStatistics.lastInnerIterations)}, + {"factorization_applications", static_cast(factorizationStatistics.applications)}, + {"surface_inverse_applications", static_cast(factorizationStatistics.surfaceInverseApplications)}, + {"surface_jacobian_probes", static_cast(preparationStatistics.surfaceJacobianProbes)}, + {"surface_h1_assemblies", static_cast(preparationStatistics.surfaceH1Assemblies)}} + ); + } + + void measureScalarSurfaceControl( + const std::string &candidate, + const preconditioning::SurfaceRieszCalibrationObjective objective, + const int calibrationProbeCount, + const auto &problem, + const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation, + const MPI_Comm communicator + ) { + REQUIRE(calibrationProbeCount > 0); + const preconditioning::MaterialSurfaceDiagonalOptions calibration{ + .surfaceCalibration = { + .target = preconditioning::SurfaceRieszCalibrationTarget::surface_jacobian, + .probeCount = calibrationProbeCount, + .objective = objective + } + }; + const Clock::time_point setupStart = Clock::now(); + auto block = preconditioning::materialSurfaceBlock( + problem, backend::Diagonal{}, backend::Diagonal{}, preconditioning::ApproximateMaterialSurfaceLDU{}, + calibration + ); + auto prepared = preconditioning::prepare(problem, std::move(block)); + auto directSurfaceInverse = backend::prepare(backend::Diagonal{}, prepared.GetSurfaceDiagonal()); + const double setupTime = maximumRankSeconds(setupStart, communicator); + const auto &calibrationData = prepared.GetSurfaceCalibration(); + + const std::map parameters{ + {"experiment_schema", "p9_material_surface_h1_tuning_v1"}, + {"surface_surrogate", "scalar_mass_diagonal"}, + {"surface_calibration_target", "surface_jacobian"}, + {"surface_calibration_probes", std::to_string(calibrationProbeCount)}, + {"surface_calibration_objective", + objective == preconditioning::SurfaceRieszCalibrationObjective::operator_action + ? "operator_action" + : "right_preconditioned_action"} + }; + recordSurfaceInverseRecovery(candidate, directSurfaceInverse, operation, communicator, parameters); + recordBalancedPreconditionedDefect(candidate, prepared, operation, communicator, parameters); + + const auto &directStatistics = directSurfaceInverse.GetStatistics(); + const auto &factorizationStatistics = prepared.GetFactorization().GetStatistics(); + auto summaryParameters = commonParameters(candidate, "scalar_surface_control_summary", operation.Width()); + applyParameterOverrides(summaryParameters, parameters); + experiment::record_experiment_result( + "material_surface_preconditioning_p9", candidate + "_summary", std::move(summaryParameters), + {{"setup_seconds_maximum_rank", setupTime}, + {"surface_calibration_scale", calibrationData.scale}, + {"surface_calibration_inverse_multiplier", calibrationData.inverseMultiplier}, + {"surface_calibration_numerator", calibrationData.leastSquaresNumerator}, + {"surface_calibration_denominator", calibrationData.leastSquaresDenominator}, + {"surface_backend_setups", static_cast(directStatistics.setups)}, + {"surface_backend_applications", static_cast(directStatistics.applications)}, + {"factorization_applications", static_cast(factorizationStatistics.applications)}, + {"surface_inverse_applications", static_cast(factorizationStatistics.surfaceInverseApplications)}} + ); + } +} // namespace + +TEST_CASE( + "Material Surface P9 Numerical Factorization Comparison", + "[preconditioning][material_surface][diagnostics][experiment][spectrum][p9][p9_baseline]" +) { + using namespace mean_field; + + const utils::Args arguments = test_utils::setup_args(); + fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0); + REQUIRE(finiteElements.okay()); + const MPI_Comm communicator = finiteElements.mesh->GetComm(); + + constexpr double radius = utils::RADIUS; + constexpr double mass = utils::MASS; + const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v; + const double centralDensity = std::numbers::pi_v * mass / (4.0 * radius * radius * radius); + auto model = model::StellarModel( + eos::Polytrope({.n = 1.0, .K = polytropicConstant}), + surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), + integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}), + constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}}) + ); + auto problem = equilibrium::discretize(model, finiteElements); + auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 1024})); + problem.Prepare(projected.values, makeDependencies(), zeroRotation()); + + const auto &physical = problem.GetPreparedOperator().GetPhysicalOperator(); + preconditioning::MaterialSurfaceJacobianOperator materialSurfaceOperator(physical); + const auto exactCorrection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.23, communicator); + mfem::Vector rightHandSide(materialSurfaceOperator.Height()); + materialSurfaceOperator.Mult(exactCorrection, rightHandSide); + const auto arnoldiDirection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.79, communicator); + + solver::IdentityPreconditioner identity(materialSurfaceOperator.Width()); + measureCandidate( + "identity", identity, 0.0, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, + communicator + ); + prepareAndMeasure( + "block_diagonal", preconditioning::MaterialSurfaceBlockDiagonal{}, problem, materialSurfaceOperator, + exactCorrection, rightHandSide, arnoldiDirection, communicator + ); + prepareAndMeasure( + "material_independent_surface", preconditioning::CoupledMaterialIndependentSurface{}, problem, + materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator + ); + prepareAndMeasure( + "material_then_surface", preconditioning::MaterialThenSurfaceTriangular{}, problem, materialSurfaceOperator, + exactCorrection, rightHandSide, arnoldiDirection, communicator + ); + prepareAndMeasure( + "surface_then_material", preconditioning::SurfaceThenMaterialTriangular{}, problem, materialSurfaceOperator, + exactCorrection, rightHandSide, arnoldiDirection, communicator + ); + prepareAndMeasure( + "approximate_ldu", preconditioning::ApproximateMaterialSurfaceLDU{}, problem, materialSurfaceOperator, + exactCorrection, rightHandSide, arnoldiDirection, communicator + ); +} + +TEST_CASE( + "Material Surface P9 Calibrated LDU Comparison", + "[preconditioning][material_surface][diagnostics][experiment][spectrum][p9][p9_refinement]" +) { + using namespace mean_field; + + const utils::Args arguments = test_utils::setup_args(); + fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0); + REQUIRE(finiteElements.okay()); + const MPI_Comm communicator = finiteElements.mesh->GetComm(); + + constexpr double radius = utils::RADIUS; + constexpr double mass = utils::MASS; + const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v; + const double centralDensity = std::numbers::pi_v * mass / (4.0 * radius * radius * radius); + auto model = model::StellarModel( + eos::Polytrope({.n = 1.0, .K = polytropicConstant}), + surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), + integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}), + constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}}) + ); + auto problem = equilibrium::discretize(model, finiteElements); + auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 1024})); + problem.Prepare(projected.values, makeDependencies(), zeroRotation()); + + const auto &physical = problem.GetPreparedOperator().GetPhysicalOperator(); + preconditioning::MaterialSurfaceJacobianOperator materialSurfaceOperator(physical); + const auto exactCorrection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.23, communicator); + mfem::Vector rightHandSide(materialSurfaceOperator.Height()); + materialSurfaceOperator.Mult(exactCorrection, rightHandSide); + const auto arnoldiDirection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.79, communicator); + + constexpr preconditioning::MaterialSurfaceDiagonalOptions surfaceJacobianCalibration{ + .surfaceCalibration = { + .target = preconditioning::SurfaceRieszCalibrationTarget::surface_jacobian, .probeCount = 4 + } + }; + constexpr preconditioning::MaterialSurfaceDiagonalOptions surfaceSchurCalibration{ + .surfaceCalibration = { + .target = preconditioning::SurfaceRieszCalibrationTarget::approximate_material_schur, .probeCount = 4 + } + }; + + prepareAndMeasure( + "surface_then_material_calibrated_aqq", preconditioning::SurfaceThenMaterialTriangular{}, problem, + materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator, + surfaceJacobianCalibration + ); + prepareAndMeasure( + "approximate_ldu", preconditioning::ApproximateMaterialSurfaceLDU{}, problem, materialSurfaceOperator, + exactCorrection, rightHandSide, arnoldiDirection, communicator + ); + prepareAndMeasure( + "approximate_ldu_calibrated_aqq", preconditioning::ApproximateMaterialSurfaceLDU{}, problem, + materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator, + surfaceJacobianCalibration + ); + prepareAndMeasure( + "approximate_ldu_calibrated_schur", preconditioning::ApproximateMaterialSurfaceLDU{}, problem, + materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator, surfaceSchurCalibration + ); +} + +TEST_CASE( + "Material Surface P9 Frequency-Aware Surface Refinement", + "[preconditioning][material_surface][diagnostics][experiment][spectrum][p9][p9_h1_refinement]" +) { + using namespace mean_field; + + const utils::Args arguments = test_utils::setup_args(); + fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0); + REQUIRE(finiteElements.okay()); + const MPI_Comm communicator = finiteElements.mesh->GetComm(); + + constexpr double radius = utils::RADIUS; + constexpr double mass = utils::MASS; + const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v; + const double centralDensity = std::numbers::pi_v * mass / (4.0 * radius * radius * radius); + auto model = model::StellarModel( + eos::Polytrope({.n = 1.0, .K = polytropicConstant}), + surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), + integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}), + constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}}) + ); + auto problem = equilibrium::discretize(model, finiteElements); + auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 1024})); + problem.Prepare(projected.values, makeDependencies(), zeroRotation()); + + const auto &physical = problem.GetPreparedOperator().GetPhysicalOperator(); + preconditioning::MaterialSurfaceJacobianOperator materialSurfaceOperator(physical); + const auto exactCorrection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.23, communicator); + mfem::Vector rightHandSide(materialSurfaceOperator.Height()); + materialSurfaceOperator.Mult(exactCorrection, rightHandSide); + const auto arnoldiDirection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.79, communicator); + + constexpr int calibrationProbeCount = 4; + prepareAndMeasureH1( + "h1_aqq_surface_then_material_amg1", preconditioning::SurfaceThenMaterialTriangular{}, 1, calibrationProbeCount, + problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator + ); + prepareAndMeasureH1( + "h1_aqq_approximate_ldu_amg1", preconditioning::ApproximateMaterialSurfaceLDU{}, 1, calibrationProbeCount, + problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator + ); + prepareAndMeasureH1( + "h1_aqq_approximate_ldu_amg2", preconditioning::ApproximateMaterialSurfaceLDU{}, 2, calibrationProbeCount, + problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator + ); +} + +TEST_CASE( + "Material Surface P9 H1 Calibration Floor Tuning", + "[preconditioning][material_surface][diagnostics][experiment][p9][p9_h1_tuning]" +) { + using namespace mean_field; + + const utils::Args arguments = test_utils::setup_args(); + fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0); + REQUIRE(finiteElements.okay()); + const MPI_Comm communicator = finiteElements.mesh->GetComm(); + + constexpr double radius = utils::RADIUS; + constexpr double mass = utils::MASS; + const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v; + const double centralDensity = std::numbers::pi_v * mass / (4.0 * radius * radius * radius); + auto model = model::StellarModel( + eos::Polytrope({.n = 1.0, .K = polytropicConstant}), + surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), + integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}), + constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}}) + ); + auto problem = equilibrium::discretize(model, finiteElements); + auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 1024})); + problem.Prepare(projected.values, makeDependencies(), zeroRotation()); + + const auto &physical = problem.GetPreparedOperator().GetPhysicalOperator(); + preconditioning::MaterialSurfaceJacobianOperator materialSurfaceOperator(physical); + + constexpr std::array floorCases{ + std::pair{"1e-10", 1.0e-10}, std::pair{"1e-4", 1.0e-4}, std::pair{"1e-2", 1.0e-2}, std::pair{"1e-1", 1.0e-1}, + std::pair{"1", 1.0} + }; + for (const auto &[label, floor] : floorCases) { + measureH1CalibrationFloor( + std::string("h1_aqq_floor_") + label, label, floor, problem, materialSurfaceOperator, communicator + ); + } + measureScalarSurfaceControl( + "scalar_aqq_operator_calibrated_diagonal_control", + preconditioning::SurfaceRieszCalibrationObjective::operator_action, 4, problem, materialSurfaceOperator, + communicator + ); + constexpr std::array inverseProbeCounts{1, 2, 4, 8, 16}; + for (const int probeCount : inverseProbeCounts) { + measureScalarSurfaceControl( + "scalar_aqq_right_calibrated_diagonal_" + std::to_string(probeCount) + "_probes", + preconditioning::SurfaceRieszCalibrationObjective::right_preconditioned_action, probeCount, problem, + materialSurfaceOperator, communicator + ); + } +} diff --git a/libmeanfield/impl/analysis/integral.cpp b/libmeanfield/impl/analysis/integral.cpp index 0890216..3c4179c 100644 --- a/libmeanfield/impl/analysis/integral.cpp +++ b/libmeanfield/impl/analysis/integral.cpp @@ -1,4 +1,5 @@ module; +#include "profile.h" #include #include @@ -62,6 +63,8 @@ namespace mean_field::analysis { utils::DOMAINS domain, mapping::COORDINATE_SPACE coord_space ) { + MEAN_FIELD_PROFILE_SCOPE_WARMUP("analysis::domain_integrate_grid_function", 0); + mfem::LinearForm lf(fem.densityFes.get()); mfem::GridFunctionCoefficient gf_c(&gf); double local_integral; @@ -107,11 +110,15 @@ namespace mean_field::analysis { const fem::FEM &fem, const mfem::GridFunction &rho ) { - const int dim = fem.mesh->Dimension(); + MEAN_FIELD_PROFILE_SCOPE_WARMUP("analysis::get_com", 0); + + std::uint64_t mapping_evaluations = 0; + const int dim = fem.mesh->Dimension(); mapping::GridFunctionMappingEvaluator mapping_evaluator( *fem.domainMapperStateless, *fem.displacement, *fem.compactificationCoordinate ); mfem::Vector local_com(dim); + mapping::VolumeMappingContext mapping_context; local_com = 0.0; double local_mass = 0.0; @@ -127,11 +134,11 @@ namespace mean_field::analysis { const mfem::IntegrationPoint &ip = ir.IntPoint(j); trans->SetIntPoint(&ip); - mapping::VolumeMappingContext mapping_context; MFEM_VERIFY( mapping_evaluator.EvaluateVolume(*trans, ip, mapping_context) == mapping::MappingStatus::valid, "Center-of-mass integration encountered an invalid mapping." ); + ++mapping_evaluations; const double weight = mapping_context.quadrature.weight; double rho_val = rho.GetValue(i, ip); @@ -146,13 +153,23 @@ namespace mean_field::analysis { } } - double global_mass = 0.0; + MEAN_FIELD_PROFILE_COUNT("analysis::get_com mapping evaluations", mapping_evaluations); + + mfem::Vector local_integrals(dim + 1); + mfem::Vector global_integrals(dim + 1); + local_integrals(0) = local_mass; + for (int d = 0; d < dim; ++d) { + local_integrals(d + 1) = local_com(d); + } + MPI_Allreduce( + local_integrals.GetData(), global_integrals.GetData(), dim + 1, MPI_DOUBLE, MPI_SUM, fem.mesh->GetComm() + ); + + const double global_mass = global_integrals(0); mfem::Vector global_com(dim); - MPI_Comm comm = fem.mesh->GetComm(); - - MPI_Allreduce(&local_mass, &global_mass, 1, MPI_DOUBLE, MPI_SUM, comm); - - MPI_Allreduce(local_com.GetData(), global_com.GetData(), dim, MPI_DOUBLE, MPI_SUM, comm); + for (int d = 0; d < dim; ++d) { + global_com(d) = global_integrals(d + 1); + } if (global_mass > 1e-18) { global_com /= global_mass; @@ -168,6 +185,8 @@ namespace mean_field::analysis { mfem::GridFunction &rho, const double target_mass ) { + MEAN_FIELD_PROFILE_SCOPE_WARMUP("analysis::conserve_mass", 0); + if (const double current_mass = domain_integrate_grid_function(fem, rho, utils::DOMAINS::STELLAR); current_mass > 1e-15) rho *= (target_mass / current_mass); @@ -177,6 +196,8 @@ namespace mean_field::analysis { const fem::FEM &fem, const mfem::GridFunction &rho ) { + MEAN_FIELD_PROFILE_SCOPE_WARMUP("analysis::get_moment_of_inertia", 0); + auto s2_func = [](const mfem::Vector &x) { return std::pow(x(0), 2) + std::pow(x(1), 2); }; std::unique_ptr s2_coeff; @@ -227,6 +248,8 @@ namespace mean_field::analysis { const mapping::COORDINATE_SPACE coordinate_space, const utils::DOMAINS domain ) { + MEAN_FIELD_PROFILE_SCOPE_WARMUP("analysis::get_mesh_volume", 0); + mfem::ParMesh &mesh = *fem.mesh; const bool physical = (coordinate_space == mapping::COORDINATE_SPACE::PHYSICAL); @@ -238,6 +261,7 @@ namespace mean_field::analysis { mapping::GridFunctionMappingEvaluator mapping_evaluator( *fem.domainMapperStateless, *fem.displacement, *fem.compactificationCoordinate ); + mapping::VolumeMappingContext mapping_context; for (int e = 0; e < mesh.GetNE(); ++e) { const int attr = mesh.GetAttribute(e); @@ -259,12 +283,11 @@ namespace mean_field::analysis { double dV = ip.weight * T->Weight(); if (physical) { - mapping::VolumeMappingContext context; MFEM_VERIFY( - mapping_evaluator.EvaluateVolume(*T, ip, context) == mapping::MappingStatus::valid, + mapping_evaluator.EvaluateVolume(*T, ip, mapping_context) == mapping::MappingStatus::valid, "Mesh-volume integration encountered an invalid mapping." ); - dV = context.quadrature.weight; + dV = mapping_context.quadrature.weight; } local_volume += dV; diff --git a/libmeanfield/impl/integrators/centrifugal.cpp b/libmeanfield/impl/integrators/centrifugal.cpp index 582aed6..e2644f1 100644 --- a/libmeanfield/impl/integrators/centrifugal.cpp +++ b/libmeanfield/impl/integrators/centrifugal.cpp @@ -58,8 +58,8 @@ namespace mean_field::integrators { } mfem::Vector shape_v(dof_v), shape_rho(dof_rho); - mfem::Vector x_phys(dim); mfem::Vector a(dim), b(dim); + mapping::VolumeMappingContext mapping_context; MFEM_VERIFY( m_ir, "CentrifugalForceIntegrator must be configured with an " @@ -72,24 +72,29 @@ namespace mean_field::integrators { const mfem::IntegrationPoint &ip = ir->IntPoint(q); Tr.SetIntPoint(&ip); - auto [J_inv, detJ, weight] = m_mapping.GetQuadratureContext(Tr, ip); + const mapping::MappingStatus mapping_status = m_mapping.EvaluateVolume(Tr, ip, mapping_context); + MFEM_VERIFY( + mapping_status == mapping::MappingStatus::valid, + "Centrifugal-force assembly encountered an invalid volume mapping." + ); + const double weight = mapping_context.quadrature.weight; fe_v->CalcShape(ip, shape_v); fe_rho->CalcShape(ip, shape_rho); - m_mapping.GetPhysicalPoint(Tr, ip, x_phys); + const mfem::Vector &x_phys = mapping_context.mapping.physical_position; // ω x r - a(0) = m_omega(1) * x_phys(2) - m_omega(2) * x_phys(1); - a(1) = m_omega(2) * x_phys(0) - m_omega(0) * x_phys(2); - a(2) = m_omega(0) * x_phys(1) - m_omega(1) * x_phys(0); + a(0) = m_omega(1) * x_phys(2) - m_omega(2) * x_phys(1); + a(1) = m_omega(2) * x_phys(0) - m_omega(0) * x_phys(2); + a(2) = m_omega(0) * x_phys(1) - m_omega(1) * x_phys(0); // ω x (ω x r) [centrifugal acceleration] - b(0) = m_omega(1) * a(2) - m_omega(2) * a(1); - b(1) = m_omega(2) * a(0) - m_omega(0) * a(2); - b(2) = m_omega(0) * a(1) - m_omega(1) * a(0); + b(0) = m_omega(1) * a(2) - m_omega(2) * a(1); + b(1) = m_omega(2) * a(0) - m_omega(0) * a(2); + b(2) = m_omega(0) * a(1) - m_omega(1) * a(0); - double rho_val = 0.0; + double rho_val = 0.0; for (int i = 0; i < dof_rho; ++i) { rho_val += rho_dofs(i) * shape_rho(i); } @@ -135,8 +140,8 @@ namespace mean_field::integrators { return; mfem::Vector shape_v(dof_v), shape_rho(dof_rho); - mfem::Vector x_phys(dim); mfem::Vector a(dim), b(dim); + mapping::VolumeMappingContext mapping_context; const mfem::IntegrationRule *ir = &mfem::IntRules.Get(fe_v->GetGeomType(), 2 * fe_v->GetOrder()); @@ -144,22 +149,27 @@ namespace mean_field::integrators { const mfem::IntegrationPoint &ip = ir->IntPoint(q); Tr.SetIntPoint(&ip); - auto [J_inv, detJ, weight] = m_mapping.GetQuadratureContext(Tr, ip); + const mapping::MappingStatus mapping_status = m_mapping.EvaluateVolume(Tr, ip, mapping_context); + MFEM_VERIFY( + mapping_status == mapping::MappingStatus::valid, + "Centrifugal-force Jacobian assembly encountered an invalid volume mapping." + ); + const double weight = mapping_context.quadrature.weight; fe_v->CalcShape(ip, shape_v); fe_rho->CalcShape(ip, shape_rho); - m_mapping.GetPhysicalPoint(Tr, ip, x_phys); + const mfem::Vector &x_phys = mapping_context.mapping.physical_position; // ω x r - a(0) = m_omega(1) * x_phys(2) - m_omega(2) * x_phys(1); - a(1) = m_omega(2) * x_phys(0) - m_omega(0) * x_phys(2); - a(2) = m_omega(0) * x_phys(1) - m_omega(1) * x_phys(0); + a(0) = m_omega(1) * x_phys(2) - m_omega(2) * x_phys(1); + a(1) = m_omega(2) * x_phys(0) - m_omega(0) * x_phys(2); + a(2) = m_omega(0) * x_phys(1) - m_omega(1) * x_phys(0); // ω x (ω x r) [centrifugal acceleration] - b(0) = m_omega(1) * a(2) - m_omega(2) * a(1); - b(1) = m_omega(2) * a(0) - m_omega(0) * a(2); - b(2) = m_omega(0) * a(1) - m_omega(1) * a(0); + b(0) = m_omega(1) * a(2) - m_omega(2) * a(1); + b(1) = m_omega(2) * a(0) - m_omega(0) * a(2); + b(2) = m_omega(0) * a(1) - m_omega(1) * a(0); // dR_dv_i_c / drho_j = φ_i * φ_j * b_c for (int i = 0; i < dof_v; ++i) { diff --git a/libmeanfield/impl/operators/contexts/gravity_field_context.cpp b/libmeanfield/impl/operators/contexts/gravity_field_context.cpp index 5e29068..4077877 100644 --- a/libmeanfield/impl/operators/contexts/gravity_field_context.cpp +++ b/libmeanfield/impl/operators/contexts/gravity_field_context.cpp @@ -9,7 +9,132 @@ import :operators.context.gravity_field; namespace { using DomainSchema = mean_field::utils::domain::CoreEnvelopeVacuumDomainSchema; - [[nodiscard]] std::unique_ptr make_divergence_operator(const mean_field::fem::FEM &f) { + void true_to_local( + const mfem::ParFiniteElementSpace &finite_element_space, + const mfem::Vector &true_vector, + mfem::Vector &local_vector + ) { + MFEM_VERIFY( + true_vector.Size() == finite_element_space.GetTrueVSize(), + "True-DOF operator received an input vector with the wrong size." + ); + + local_vector.SetSize(finite_element_space.GetVSize()); + const mfem::Operator *prolongation = finite_element_space.GetProlongationMatrix(); + if (prolongation != nullptr) { + prolongation->Mult(true_vector, local_vector); + } else { + local_vector = true_vector; + } + } + + void local_to_true( + const mfem::ParFiniteElementSpace &finite_element_space, + const mfem::Vector &local_vector, + mfem::Vector &true_vector + ) { + MFEM_VERIFY( + local_vector.Size() == finite_element_space.GetVSize(), + "True-DOF operator produced a local vector with the wrong size." + ); + + true_vector.SetSize(finite_element_space.GetTrueVSize()); + const mfem::Operator *prolongation = finite_element_space.GetProlongationMatrix(); + if (prolongation != nullptr) { + prolongation->MultTranspose(local_vector, true_vector); + } else { + true_vector = local_vector; + } + } + + bool communicator_has_single_rank(const MPI_Comm communicator) { + int size = 0; + MFEM_VERIFY(MPI_Comm_size(communicator, &size) == MPI_SUCCESS, "Failed to query the MPI communicator size."); + MFEM_VERIFY(size > 0, "The MPI communicator must contain at least one rank."); + return size == 1; + } + + class TrueDofParMixedBilinearFormOperator final : public mfem::Operator { + public: + TrueDofParMixedBilinearFormOperator( + const mfem::ParFiniteElementSpace &trial_space, + const mfem::ParFiniteElementSpace &test_space, + std::unique_ptr local_form + ) + : Operator( + test_space.GetTrueVSize(), + trial_space.GetTrueVSize() + ), + m_trial_space(trial_space), + m_test_space(test_space), + m_local_form(std::move(local_form)), + m_single_rank(communicator_has_single_rank(trial_space.GetComm())) { + int communicators_compare = MPI_UNEQUAL; + MFEM_VERIFY( + MPI_Comm_compare(trial_space.GetComm(), test_space.GetComm(), &communicators_compare) == MPI_SUCCESS, + "Failed to compare mixed-operator MPI communicators." + ); + MFEM_VERIFY( + communicators_compare == MPI_IDENT || communicators_compare == MPI_CONGRUENT, + "True-DOF mixed operator requires congruent trial and test communicators." + ); + MFEM_VERIFY(m_local_form != nullptr, "True-DOF mixed operator requires a local bilinear form."); + MFEM_VERIFY( + m_local_form->Width() == m_trial_space.GetVSize(), + "True-DOF mixed operator received an incompatible trial space." + ); + MFEM_VERIFY( + m_local_form->Height() == m_test_space.GetVSize(), + "True-DOF mixed operator received an incompatible test space." + ); + } + + void Mult( + const mfem::Vector &input, + mfem::Vector &output + ) const override { + MFEM_VERIFY(input.Size() == Width(), "True-DOF mixed operator received an input with the wrong size."); + + if (m_single_rank) [[likely]] { + output.SetSize(Height()); + m_local_form->Mult(input, output); + return; + } + + true_to_local(m_trial_space, input, m_trial_local); + m_test_local.SetSize(m_test_space.GetVSize()); + m_local_form->Mult(m_trial_local, m_test_local); + local_to_true(m_test_space, m_test_local, output); + } + + void MultTranspose( + const mfem::Vector &input, + mfem::Vector &output + ) const override { + MFEM_VERIFY(input.Size() == Height(), "True-DOF mixed transpose received an input with the wrong size."); + + if (m_single_rank) [[likely]] { + output.SetSize(Width()); + m_local_form->MultTranspose(input, output); + return; + } + + true_to_local(m_test_space, input, m_test_local); + m_trial_local.SetSize(m_trial_space.GetVSize()); + m_local_form->MultTranspose(m_test_local, m_trial_local); + local_to_true(m_trial_space, m_trial_local, output); + } + + private: + const mfem::ParFiniteElementSpace &m_trial_space; + const mfem::ParFiniteElementSpace &m_test_space; + std::unique_ptr m_local_form; + mutable mfem::Vector m_trial_local; + mutable mfem::Vector m_test_local; + bool m_single_rank; + }; + + [[nodiscard]] std::unique_ptr make_divergence_operator(const mean_field::fem::FEM &f) { auto divergence = std::make_unique(f.gravityFluxFes.get(), f.gravityPotentialFes.get()); @@ -29,7 +154,9 @@ namespace { divergence->AddDomainIntegrator(integrator.release()); divergence->Assemble(); - return divergence; + return std::make_unique( + *f.gravityFluxFes, *f.gravityPotentialFes, std::move(divergence) + ); } void validate_displacement( @@ -167,6 +294,25 @@ namespace mean_field::operators::context::gravity_field { const mfem::Vector &displacement, const DiscretizationRevision discretization_revision, const DisplacementRevision displacement_revision + ) { + return PrepareImpl( + displacement, discretization_revision, displacement_revision, PreparationMode::linearization + ); + } + + GravityFieldGeometryPreparation GravityFieldGeometryContext::PreparePrimal( + const mfem::Vector &displacement, + const DiscretizationRevision discretization_revision, + const DisplacementRevision displacement_revision + ) { + return PrepareImpl(displacement, discretization_revision, displacement_revision, PreparationMode::primal); + } + + GravityFieldGeometryPreparation GravityFieldGeometryContext::PrepareImpl( + const mfem::Vector &displacement, + const DiscretizationRevision discretization_revision, + const DisplacementRevision displacement_revision, + const PreparationMode mode ) { validate_displacement(m_displacement_map, displacement); @@ -185,21 +331,38 @@ namespace mean_field::operators::context::gravity_field { const bool discretization_changed = !m_is_prepared || discretization_revision != m_discretization_revision; const bool displacement_changed = !m_is_prepared || displacement_revision != m_displacement_revision; + const bool requires_variation = mode == PreparationMode::linearization; + const bool variation_upgrade = requires_variation && !m_variation_state_prepared; GravityFieldGeometryPreparation preparation; - if (!discretization_changed && !displacement_changed) { + if (!discretization_changed && !displacement_changed && !variation_upgrade) { return preparation; } + const auto prepare_mass = [&](PreparedMappedHDivMassOperator &mass_operator) { + if (requires_variation) { + mass_operator.Prepare(displacement); + } else { + mass_operator.PreparePrimal(displacement); + } + }; + const auto prepare_source = [&](PreparedMappedGravitySourceOperator &source_operator) { + if (requires_variation) { + source_operator.Prepare(displacement); + } else { + source_operator.PreparePrimal(displacement); + } + }; + if (discretization_changed) { auto mass_operator = std::make_unique(m_fem, m_domain_mapper); auto source_operator = std::make_unique(m_fem, m_domain_mapper); auto divergence_operator = make_divergence_operator(m_fem); auto transpose_divergence_operator = std::make_unique(divergence_operator.get()); - mass_operator->Prepare(displacement); - source_operator->Prepare(displacement); + prepare_mass(*mass_operator); + prepare_source(*source_operator); m_mass_operator = std::move(mass_operator); m_source_operator = std::move(source_operator); @@ -220,8 +383,8 @@ namespace mean_field::operators::context::gravity_field { "operator." ); - m_mass_operator->Prepare(displacement); - m_source_operator->Prepare(displacement); + prepare_mass(*m_mass_operator); + prepare_source(*m_source_operator); preparation.rebuilt_mass_operator = true; preparation.rebuilt_source_operator = true; @@ -232,8 +395,9 @@ namespace mean_field::operators::context::gravity_field { m_discretization_revision = discretization_revision; m_displacement_revision = displacement_revision; m_is_prepared = true; + m_variation_state_prepared = requires_variation; - preparation.refreshed_variation_state = true; + preparation.refreshed_variation_state = requires_variation; return preparation; } diff --git a/libmeanfield/impl/operators/gravity_field.cpp b/libmeanfield/impl/operators/gravity_field.cpp index 7b08f2b..7cf994b 100644 --- a/libmeanfield/impl/operators/gravity_field.cpp +++ b/libmeanfield/impl/operators/gravity_field.cpp @@ -346,20 +346,32 @@ namespace mean_field::operators { make_residual_view(action, m_residual_offsets, gravity_poisson_residual_block); const field::FieldDofMap &flux_map = geometry_context.GetMassOperator().GetFluxMap(); const field::FieldDofMap &potential_map = geometry_context.GetSourceOperator().GetPotentialMap(); - mfem::Vector potential_true(potential_map.full_size()); - mfem::Vector transpose_divergence_action_true(flux_map.full_size()); - mfem::Vector transpose_divergence_action(flux_map.reduced_size()); - mfem::Vector gradient_true(flux_map.full_size()); - mfem::Vector divergence_action_true(potential_map.full_size()); geometry_context.GetMassOperator().Mult(gravity_gradient, gravity_gradient_action); - potential_map.scatter(gravity_potential, potential_true); - geometry_context.GetTransposeDivergenceOperator().Mult(potential_true, transpose_divergence_action_true); - flux_map.gather(transpose_divergence_action_true, transpose_divergence_action); - gravity_gradient_action += transpose_divergence_action; - flux_map.scatter(gravity_gradient, gradient_true); - geometry_context.GetDivergenceOperator().Mult(gradient_true, divergence_action_true); - potential_map.gather(divergence_action_true, gravity_poisson_action); + + if (flux_map.is_identity() && potential_map.is_identity()) [[likely]] { + m_transpose_divergence_action_true.SetSize(flux_map.full_size()); + geometry_context.GetTransposeDivergenceOperator().Mult( + gravity_potential, m_transpose_divergence_action_true + ); + gravity_gradient_action += m_transpose_divergence_action_true; + geometry_context.GetDivergenceOperator().Mult(gravity_gradient, gravity_poisson_action); + return; + } + + m_potential_true.SetSize(potential_map.full_size()); + m_transpose_divergence_action_true.SetSize(flux_map.full_size()); + m_transpose_divergence_action.SetSize(flux_map.reduced_size()); + m_gradient_true.SetSize(flux_map.full_size()); + m_divergence_action_true.SetSize(potential_map.full_size()); + + potential_map.scatter(gravity_potential, m_potential_true); + geometry_context.GetTransposeDivergenceOperator().Mult(m_potential_true, m_transpose_divergence_action_true); + flux_map.gather(m_transpose_divergence_action_true, m_transpose_divergence_action); + gravity_gradient_action += m_transpose_divergence_action; + flux_map.scatter(gravity_gradient, m_gradient_true); + geometry_context.GetDivergenceOperator().Mult(m_gradient_true, m_divergence_action_true); + potential_map.gather(m_divergence_action_true, gravity_poisson_action); } void GravityFieldOperator::ApplyDensitySource( @@ -529,7 +541,7 @@ namespace mean_field::operators { ++displacement_revision.value; } - m_gravity_field_geometry_context.Prepare(displacement, discretization_revision, displacement_revision); + m_gravity_field_geometry_context.PreparePrimal(displacement, discretization_revision, displacement_revision); m_displacement = displacement; } diff --git a/libmeanfield/impl/operators/gravity_field_jacobian.cpp b/libmeanfield/impl/operators/gravity_field_jacobian.cpp index 818f9f9..3989e51 100644 --- a/libmeanfield/impl/operators/gravity_field_jacobian.cpp +++ b/libmeanfield/impl/operators/gravity_field_jacobian.cpp @@ -267,6 +267,9 @@ namespace mean_field::operators { gravity_poisson_action -= source_action; gravity_poisson_action -= source_variation_action; + + gravity_gradient_action.SyncAliasMemory(action); + gravity_poisson_action.SyncAliasMemory(action); } const context::gravity_field::GravityFieldLinearizationContext & diff --git a/libmeanfield/impl/operators/prepared_barotropic_closure.cpp b/libmeanfield/impl/operators/prepared_barotropic_closure.cpp index e7dcb9c..01abd79 100644 --- a/libmeanfield/impl/operators/prepared_barotropic_closure.cpp +++ b/libmeanfield/impl/operators/prepared_barotropic_closure.cpp @@ -469,6 +469,35 @@ namespace mean_field::operators { m_densityMap.gather(m_fullResidual, residual); } + void PreparedBarotropicClosureOperator::AssembleDensityJacobianDiagonal(mfem::Vector &diagonal) const { + VerifyPrepared(); + + mfem::Vector localDiagonal(m_fem.densityFes->GetVSize()); + localDiagonal = 0.0; + mfem::Vector elementDiagonal; + + for (const ElementPAData &data : m_elements) { + MFEM_VERIFY( + data.densityDofTransformation == nullptr, + "Density mass-diagonal assembly currently requires scalar L2 element DOFs without a DOF transform." + ); + elementDiagonal.SetSize(data.densityDofs.Size()); + elementDiagonal = 0.0; + for (int trialDof = 0; trialDof < data.densityDofs.Size(); ++trialDof) { + for (int quadraturePoint = 0; quadraturePoint < data.quadratureWeights.Size(); ++quadraturePoint) { + const double basis = data.densityBasis(quadraturePoint, trialDof); + elementDiagonal(trialDof) += data.quadratureWeights(quadraturePoint) * basis * basis; + } + } + localDiagonal.AddElementVector(data.densityDofs, elementDiagonal); + } + + mfem::Vector trueDiagonal; + local_to_true(*m_fem.densityFes, localDiagonal, trueDiagonal); + diagonal.SetSize(m_densityMap.reduced_size()); + m_densityMap.gather(trueDiagonal, diagonal); + } + void PreparedBarotropicClosureOperator::Mult( const mfem::Vector &densityVariation, const mfem::Vector &enthalpyVariation, diff --git a/libmeanfield/impl/operators/prepared_gravity_source.cpp b/libmeanfield/impl/operators/prepared_gravity_source.cpp index 045a9da..37948ea 100644 --- a/libmeanfield/impl/operators/prepared_gravity_source.cpp +++ b/libmeanfield/impl/operators/prepared_gravity_source.cpp @@ -1,4 +1,5 @@ module; +#include "profile.h" #include #include #include @@ -304,6 +305,19 @@ namespace mean_field::operators { } void PreparedMappedGravitySourceOperator::Prepare(const mfem::Vector &displacement) { + MEAN_FIELD_PROFILE_SCOPE_WARMUP("PreparedMappedGravitySourceOperator::Prepare linearization", 0); + PrepareImpl(displacement, PreparationMode::linearization); + } + + void PreparedMappedGravitySourceOperator::PreparePrimal(const mfem::Vector &displacement) { + MEAN_FIELD_PROFILE_SCOPE_WARMUP("PreparedMappedGravitySourceOperator::Prepare primal", 0); + PrepareImpl(displacement, PreparationMode::primal); + } + + void PreparedMappedGravitySourceOperator::PrepareImpl( + const mfem::Vector &displacement, + const PreparationMode mode + ) { MFEM_VERIFY( displacement.Size() == m_displacement_map.reduced_size(), "PreparedMappedGravitySourceOperator received a displacement " @@ -318,7 +332,8 @@ namespace mean_field::operators { ); } - m_is_prepared = false; + m_is_prepared = false; + m_has_variation_data = false; m_displacement_true.SetSize(m_displacement_map.full_size()); m_displacement_map.scatter(displacement, m_displacement_true); m_elements.clear(); @@ -343,8 +358,10 @@ namespace mean_field::operators { data.potential_dof_transformation = m_fem.gravityPotentialFes->GetElementDofs(element_id, data.potential_dofs); - data.displacement_dof_transformation = - m_fem.displacementFes->GetElementVDofs(element_id, data.displacement_dofs); + if (mode == PreparationMode::linearization) { + data.displacement_dof_transformation = + m_fem.displacementFes->GetElementVDofs(element_id, data.displacement_dofs); + } const mfem::FiniteElement &density_element = *m_fem.densityFes->GetFE(element_id); @@ -367,7 +384,9 @@ namespace mean_field::operators { data.potential_basis.SetSize(quadrature_point_count, potential_dof_count); const int dimension = m_fem.mesh->Dimension(); - data.inverse_element_jacobians.SetSize(quadrature_point_count, dimension * dimension); + if (mode == PreparationMode::linearization) { + data.inverse_element_jacobians.SetSize(quadrature_point_count, dimension * dimension); + } data.quadrature_data.SetSize(quadrature_point_count); @@ -395,11 +414,13 @@ namespace mean_field::operators { const double coefficient_value = source_coefficient.Eval(transformation, integration_point); - const mfem::DenseMatrix &inverse_element_jacobian = source_coefficient.GetInverseElementJacobian(); - for (int row = 0; row < dimension; ++row) { - for (int column = 0; column < dimension; ++column) { - data.inverse_element_jacobians(quadrature_point, row * dimension + column) = - inverse_element_jacobian(row, column); + if (mode == PreparationMode::linearization) { + const mfem::DenseMatrix &inverse_element_jacobian = source_coefficient.GetInverseElementJacobian(); + for (int row = 0; row < dimension; ++row) { + for (int column = 0; column < dimension; ++column) { + data.inverse_element_jacobians(quadrature_point, row * dimension + column) = + inverse_element_jacobian(row, column); + } } } @@ -420,13 +441,16 @@ namespace mean_field::operators { MFEM_VERIFY(!m_elements.empty(), "PreparedMappedGravitySourceOperator found no stellar elements."); - m_is_prepared = true; + m_is_prepared = true; + m_has_variation_data = mode == PreparationMode::linearization; ++m_preparation_count; } void PreparedMappedGravitySourceOperator::Mult( const mfem::Vector &density, mfem::Vector &action ) const { + MEAN_FIELD_PROFILE_SCOPE("PreparedMappedGravitySourceOperator::Mult"); + MFEM_VERIFY( m_is_prepared, "PreparedMappedGravitySourceOperator must be prepared before " "Mult is called." @@ -440,49 +464,47 @@ namespace mean_field::operators { m_density_true.SetSize(m_density_map.full_size()); m_density_map.scatter(density, m_density_true); - mfem::Vector density_local; + true_to_local(*m_fem.densityFes, m_density_true, m_density_local); - true_to_local(*m_fem.densityFes, m_density_true, density_local); - - mfem::Vector local_action(m_fem.gravityPotentialFes->GetVSize()); - local_action = 0.0; - - mfem::Vector element_density; - mfem::Vector quadrature_density; - mfem::Vector element_action; + m_local_action.SetSize(m_fem.gravityPotentialFes->GetVSize()); + m_local_action = 0.0; for (const ElementPAData &data : m_elements) { - density_local.GetSubVector(data.density_dofs, element_density); + m_density_local.GetSubVector(data.density_dofs, m_element_input); if (data.density_dof_transformation != nullptr) { - data.density_dof_transformation->InvTransformPrimal(element_density); + data.density_dof_transformation->InvTransformPrimal(m_element_input); } - quadrature_density.SetSize(data.quadrature_data.Size()); + m_quadrature_action.SetSize(data.quadrature_data.Size()); // B_density * x_e - data.density_basis.Mult(element_density, quadrature_density); + data.density_basis.Mult(m_element_input, m_quadrature_action); // D * B_density * x_e - for (int q = 0; q < quadrature_density.Size(); ++q) { - quadrature_density(q) *= data.quadrature_data(q); + for (int q = 0; q < m_quadrature_action.Size(); ++q) { + m_quadrature_action(q) *= data.quadrature_data(q); } - element_action.SetSize(data.potential_dofs.Size()); + m_element_action.SetSize(data.potential_dofs.Size()); // B_potential^T * D * B_density * x_e - data.potential_basis.MultTranspose(quadrature_density, element_action); + data.potential_basis.MultTranspose(m_quadrature_action, m_element_action); if (data.potential_dof_transformation != nullptr) { - data.potential_dof_transformation->TransformDual(element_action); + data.potential_dof_transformation->TransformDual(m_element_action); } - local_action.AddElementVector(data.potential_dofs, element_action); + m_local_action.AddElementVector(data.potential_dofs, m_element_action); } - local_to_true(*m_fem.gravityPotentialFes, local_action, m_action_true); - action.SetSize(Height()); - m_potential_map.gather(m_action_true, action); + if (m_potential_map.is_identity()) { + local_to_true(*m_fem.gravityPotentialFes, m_local_action, action); + } else { + local_to_true(*m_fem.gravityPotentialFes, m_local_action, m_action_true); + action.SetSize(Height()); + m_potential_map.gather(m_action_true, action); + } } void PreparedMappedGravitySourceOperator::MultDisplacementVariationTrue( @@ -494,6 +516,11 @@ namespace mean_field::operators { m_is_prepared, "PreparedMappedGravitySourceOperator must be prepared before applying a displacement variation." ); + MFEM_VERIFY( + m_has_variation_data, + "PreparedMappedGravitySourceOperator requires linearization preparation before applying a displacement " + "variation." + ); MFEM_VERIFY( densityTrue.Size() == m_fem.densityFes->GetTrueVSize(), "The full density vector has the wrong size." ); @@ -591,47 +618,44 @@ namespace mean_field::operators { "with the wrong size." ); - m_potential_true.SetSize(m_potential_map.full_size()); - m_potential_map.scatter(potential, m_potential_true); - - mfem::Vector potential_local; - - true_to_local(*m_fem.gravityPotentialFes, m_potential_true, potential_local); - - mfem::Vector local_action(m_fem.densityFes->GetVSize()); - local_action = 0.0; - - mfem::Vector element_potential; - mfem::Vector quadrature_potential; - mfem::Vector element_action; - - for (const ElementPAData &data : m_elements) { - potential_local.GetSubVector(data.potential_dofs, element_potential); - - if (data.potential_dof_transformation != nullptr) { - data.potential_dof_transformation->InvTransformPrimal(element_potential); - } - - quadrature_potential.SetSize(data.quadrature_data.Size()); - - data.potential_basis.Mult(element_potential, quadrature_potential); - - for (int q = 0; q < quadrature_potential.Size(); ++q) { - quadrature_potential(q) *= data.quadrature_data(q); - } - - element_action.SetSize(data.density_dofs.Size()); - - data.density_basis.MultTranspose(quadrature_potential, element_action); - - if (data.density_dof_transformation != nullptr) { - data.density_dof_transformation->TransformDual(element_action); - } - - local_action.AddElementVector(data.density_dofs, element_action); + if (m_potential_map.is_identity()) { + true_to_local(*m_fem.gravityPotentialFes, potential, m_potential_local); + } else { + m_potential_true.SetSize(m_potential_map.full_size()); + m_potential_map.scatter(potential, m_potential_true); + true_to_local(*m_fem.gravityPotentialFes, m_potential_true, m_potential_local); } - local_to_true(*m_fem.densityFes, local_action, m_action_true); + m_local_action.SetSize(m_fem.densityFes->GetVSize()); + m_local_action = 0.0; + + for (const ElementPAData &data : m_elements) { + m_potential_local.GetSubVector(data.potential_dofs, m_element_input); + + if (data.potential_dof_transformation != nullptr) { + data.potential_dof_transformation->InvTransformPrimal(m_element_input); + } + + m_quadrature_action.SetSize(data.quadrature_data.Size()); + + data.potential_basis.Mult(m_element_input, m_quadrature_action); + + for (int q = 0; q < m_quadrature_action.Size(); ++q) { + m_quadrature_action(q) *= data.quadrature_data(q); + } + + m_element_action.SetSize(data.density_dofs.Size()); + + data.density_basis.MultTranspose(m_quadrature_action, m_element_action); + + if (data.density_dof_transformation != nullptr) { + data.density_dof_transformation->TransformDual(m_element_action); + } + + m_local_action.AddElementVector(data.density_dofs, m_element_action); + } + + local_to_true(*m_fem.densityFes, m_local_action, m_action_true); action.SetSize(Width()); m_density_map.gather(m_action_true, action); } @@ -639,6 +663,10 @@ namespace mean_field::operators { return m_is_prepared; } + bool PreparedMappedGravitySourceOperator::HasVariationData() const noexcept { + return m_has_variation_data; + } + std::uint64_t PreparedMappedGravitySourceOperator::GetPreparationCount() const noexcept { return m_preparation_count; } diff --git a/libmeanfield/impl/operators/prepared_hdiv_mass.cpp b/libmeanfield/impl/operators/prepared_hdiv_mass.cpp index 330bd8f..85710b8 100644 --- a/libmeanfield/impl/operators/prepared_hdiv_mass.cpp +++ b/libmeanfield/impl/operators/prepared_hdiv_mass.cpp @@ -1,4 +1,5 @@ module; +#include "profile.h" #include #include #include @@ -19,6 +20,13 @@ namespace { .reduced_size(); } + bool communicator_has_single_rank(const MPI_Comm communicator) { + int size = 0; + MFEM_VERIFY(MPI_Comm_size(communicator, &size) == MPI_SUCCESS, "Failed to query the MPI communicator size."); + MFEM_VERIFY(size > 0, "The MPI communicator must contain at least one rank."); + return size == 1; + } + void true_to_local( const mfem::ParFiniteElementSpace &finite_element_space, const mfem::Vector &true_vector, @@ -361,7 +369,8 @@ namespace mean_field::operators { field::Displacement, DomainSchema>(*f.displacementFes) ), - m_variationWorkspace(domain_mapper.GetDimension()) { + m_variationWorkspace(domain_mapper.GetDimension()), + m_single_rank(communicator_has_single_rank(f.gravityFluxFes->GetComm())) { MFEM_VERIFY(f.mesh != nullptr, "PreparedMappedHDivMassOperator requires a mesh."); MFEM_VERIFY( f.gravityFluxFes != nullptr, "PreparedMappedHDivMassOperator requires the " @@ -409,6 +418,8 @@ namespace mean_field::operators { } void PreparedMappedHDivMassOperator::PrepareVariationData() { + MEAN_FIELD_PROFILE_SCOPE_WARMUP("PreparedMappedHDivMassOperator::PrepareVariationData", 0); + m_variationElements.clear(); m_variationElements.reserve(m_fem.mesh->GetNE()); @@ -481,6 +492,19 @@ namespace mean_field::operators { } void PreparedMappedHDivMassOperator::Prepare(const mfem::Vector &displacement) { + MEAN_FIELD_PROFILE_SCOPE_WARMUP("PreparedMappedHDivMassOperator::Prepare linearization", 0); + PrepareImpl(displacement, PreparationMode::linearization); + } + + void PreparedMappedHDivMassOperator::PreparePrimal(const mfem::Vector &displacement) { + MEAN_FIELD_PROFILE_SCOPE_WARMUP("PreparedMappedHDivMassOperator::Prepare primal", 0); + PrepareImpl(displacement, PreparationMode::primal); + } + + void PreparedMappedHDivMassOperator::PrepareImpl( + const mfem::Vector &displacement, + const PreparationMode mode + ) { MFEM_VERIFY( displacement.Size() == m_displacement_map.reduced_size(), "PreparedMappedHDivMassOperator received a displacement vector " @@ -496,6 +520,9 @@ namespace mean_field::operators { ); } + m_is_prepared = false; + m_has_variation_data = false; + m_displacement_true.SetSize(m_displacement_map.full_size()); m_displacement_map.scatter(displacement, m_displacement_true); @@ -541,7 +568,12 @@ namespace mean_field::operators { m_stellar_mass_form->Assemble(); m_vacuum_mass_form->Assemble(); - PrepareVariationData(); + if (mode == PreparationMode::linearization) { + PrepareVariationData(); + m_has_variation_data = true; + } else { + m_variationElements.clear(); + } m_is_prepared = true; ++m_preparation_count; @@ -551,6 +583,8 @@ namespace mean_field::operators { const mfem::Vector &gravity_gradient, mfem::Vector &action ) const { + MEAN_FIELD_PROFILE_SCOPE("PreparedMappedHDivMassOperator::Mult"); + MFEM_VERIFY( m_is_prepared, "PreparedMappedHDivMassOperator must be prepared " "before Mult is called." @@ -564,15 +598,39 @@ namespace mean_field::operators { "with the wrong size." ); - m_flux_true.SetSize(m_flux_map.full_size()); - m_action_true.SetSize(m_flux_map.full_size()); - m_domain_action_true.SetSize(m_flux_map.full_size()); - m_flux_map.scatter(gravity_gradient, m_flux_true); - m_stellar_mass_form->Mult(m_flux_true, m_action_true); - m_vacuum_mass_form->Mult(m_flux_true, m_domain_action_true); - m_action_true += m_domain_action_true; - action.SetSize(Height()); - m_flux_map.gather(m_action_true, action); + const mfem::Vector *gravity_gradient_true = &gravity_gradient; + if (!m_flux_map.is_identity()) [[unlikely]] { + m_flux_true.SetSize(m_flux_map.full_size()); + m_flux_map.scatter(gravity_gradient, m_flux_true); + gravity_gradient_true = &m_flux_true; + } + + mfem::Vector *action_true = &action; + if (!m_flux_map.is_identity()) [[unlikely]] { + m_action_true.SetSize(m_flux_map.full_size()); + action_true = &m_action_true; + } + + if (m_single_rank) [[likely]] { + action_true->SetSize(m_flux_map.full_size()); + m_domain_action_true.SetSize(m_flux_map.full_size()); + m_stellar_mass_form->Mult(*gravity_gradient_true, *action_true); + m_vacuum_mass_form->Mult(*gravity_gradient_true, m_domain_action_true); + *action_true += m_domain_action_true; + } else { + true_to_local(*m_fem.gravityFluxFes, *gravity_gradient_true, m_flux_local); + m_action_local.SetSize(m_fem.gravityFluxFes->GetVSize()); + m_domain_action_local.SetSize(m_fem.gravityFluxFes->GetVSize()); + m_stellar_mass_form->Mult(m_flux_local, m_action_local); + m_vacuum_mass_form->Mult(m_flux_local, m_domain_action_local); + m_action_local += m_domain_action_local; + local_to_true(*m_fem.gravityFluxFes, m_action_local, *action_true); + } + + if (!m_flux_map.is_identity()) [[unlikely]] { + action.SetSize(Height()); + m_flux_map.gather(m_action_true, action); + } } void PreparedMappedHDivMassOperator::MultDisplacementVariationTrue( @@ -583,6 +641,11 @@ namespace mean_field::operators { MFEM_VERIFY( m_is_prepared, "PreparedMappedHDivMassOperator must be prepared before applying a displacement variation." ); + MFEM_VERIFY( + m_has_variation_data, + "PreparedMappedHDivMassOperator requires linearization preparation before applying a displacement " + "variation." + ); MFEM_VERIFY( gravityGradientTrue.Size() == m_fem.gravityFluxFes->GetTrueVSize(), "The full gravity-gradient vector has the wrong size." @@ -707,6 +770,10 @@ namespace mean_field::operators { return m_is_prepared; } + bool PreparedMappedHDivMassOperator::HasVariationData() const noexcept { + return m_has_variation_data; + } + std::uint64_t PreparedMappedHDivMassOperator::GetPreparationCount() const noexcept { return m_preparation_count; } diff --git a/libmeanfield/impl/operators/prepared_hydrostatic_equilibrium.cpp b/libmeanfield/impl/operators/prepared_hydrostatic_equilibrium.cpp index 7bb8236..10eb935 100644 --- a/libmeanfield/impl/operators/prepared_hydrostatic_equilibrium.cpp +++ b/libmeanfield/impl/operators/prepared_hydrostatic_equilibrium.cpp @@ -787,6 +787,36 @@ namespace mean_field::operators { ++m_algebraicJacobianStatistics.enthalpyApplications; } + void PreparedHydrostaticEquilibriumOperator::AssembleEnthalpyJacobianDiagonal(mfem::Vector &diagonal) const { + VerifyPrepared(); + + mfem::Vector localDiagonal(m_fem.enthalpyFes->GetVSize()); + localDiagonal = 0.0; + mfem::Vector elementDiagonal; + + for (const ElementPAData &data : m_elements) { + MFEM_VERIFY( + data.enthalpyDofTransformation == nullptr, + "Enthalpy mass-diagonal assembly currently requires scalar H1 element DOFs without a DOF transform." + ); + MFEM_VERIFY( + data.enthalpyJacobian.Height() == data.enthalpyDofs.Size() && + data.enthalpyJacobian.Width() == data.enthalpyDofs.Size(), + "The prepared enthalpy Jacobian block is not square on an element." + ); + elementDiagonal.SetSize(data.enthalpyDofs.Size()); + for (int dof = 0; dof < data.enthalpyDofs.Size(); ++dof) { + elementDiagonal(dof) = data.enthalpyJacobian(dof, dof); + } + localDiagonal.AddElementVector(data.enthalpyDofs, elementDiagonal); + } + + mfem::Vector trueDiagonal; + local_to_true(*m_fem.enthalpyFes, localDiagonal, trueDiagonal); + diagonal.SetSize(m_context.GetEnthalpyMap().reduced_size()); + m_context.GetEnthalpyMap().gather(trueDiagonal, diagonal); + } + void PreparedHydrostaticEquilibriumOperator::ApplyGravityPotentialJacobianAction( const mfem::Vector &gravityPotentialVariation, mfem::Vector &action diff --git a/libmeanfield/impl/physics/gravity.cpp b/libmeanfield/impl/physics/gravity.cpp index ad65a88..6dac0bc 100644 --- a/libmeanfield/impl/physics/gravity.cpp +++ b/libmeanfield/impl/physics/gravity.cpp @@ -1,5 +1,6 @@ module; #include "mfem.hpp" +#include "profile.h" #include #include @@ -11,6 +12,8 @@ namespace mean_field::physics { const mfem::GridFunction &rho, const mfem::Vector &com ) { + MEAN_FIELD_PROFILE_SCOPE_WARMUP("analysis::quadrupole", 0); + const int dim = fem.mesh->Dimension(); mfem::DenseMatrix local_Q(dim, dim); local_Q = 0.0; @@ -18,6 +21,9 @@ namespace mean_field::physics { mapping::GridFunctionMappingEvaluator mapping_evaluator( *fem.domainMapperStateless, *fem.displacement, *fem.compactificationCoordinate ); + std::uint64_t mapping_evaluations = 0; + mapping::VolumeMappingContext mapping_context; + mfem::Vector x_prime(dim); for (int i = 0; i < fem.mesh->GetNE(); ++i) { if (!DomainSchema::template attribute_belongs_to(fem.mesh->GetAttribute(i))) @@ -36,19 +42,18 @@ namespace mean_field::physics { const mfem::IntegrationPoint &ip = ir.IntPoint(j); trans->SetIntPoint(&ip); - mapping::VolumeMappingContext mapping_context; MFEM_VERIFY( mapping_evaluator.EvaluateVolume(*trans, ip, mapping_context) == mapping::MappingStatus::valid, "Quadrupole integration encountered an invalid mapping." ); + ++mapping_evaluations; const double weight = mapping_context.quadrature.weight; const double rho_val = rho.GetValue(i, ip); const mfem::Vector &phys_point = mapping_context.mapping.physical_position; - mfem::Vector x_prime(dim); - double r_sq = 0.0; + double r_sq = 0.0; for (int d = 0; d < dim; ++d) { x_prime(d) = phys_point(d) - com(d); @@ -65,6 +70,8 @@ namespace mean_field::physics { } } + MEAN_FIELD_PROFILE_COUNT("analysis::quadrupole mapping evaluations", mapping_evaluations); + mfem::DenseMatrix global_Q(dim, dim); MPI_Allreduce(local_Q.GetData(), global_Q.GetData(), dim * dim, MPI_DOUBLE, MPI_SUM, fem.mesh->GetComm()); @@ -106,6 +113,8 @@ namespace mean_field::physics { const mfem::GridFunction &rho, const mfem::GridFunction &displacement ) { + MEAN_FIELD_PROFILE_SCOPE_WARMUP("physics::solve_gravity_field", 0); + MFEM_VERIFY(f.mesh != nullptr, "Gravity initialization requires a parallel mesh."); MFEM_VERIFY(f.densityFes != nullptr, "Gravity initialization requires the density finite-element space."); MFEM_VERIFY( @@ -150,15 +159,23 @@ namespace mean_field::physics { constexpr auto gravity_poisson_residual_block = utils::blocks::get_residual_block
(utils::blocks::gravity_field.poisson_term); - using DomainSchema = utils::domain::CoreEnvelopeVacuumDomainSchema; - const field::FieldDofGridFunctionAdapter density_adapter = - field::make_field_dof_grid_function_adapter(*f.densityFes); - const field::FieldDofGridFunctionAdapter displacement_adapter = - field::make_field_dof_grid_function_adapter(*f.displacementFes); - const field::FieldDofGridFunctionAdapter gravity_flux_adapter = - field::make_field_dof_grid_function_adapter(*f.gravityFluxFes); - const field::FieldDofGridFunctionAdapter gravity_potential_adapter = - field::make_field_dof_grid_function_adapter(*f.gravityPotentialFes); + using DomainSchema = utils::domain::CoreEnvelopeVacuumDomainSchema; + const field::FieldDofGridFunctionAdapter density_adapter = MEAN_FIELD_PROFILE_EVALUATE_WARMUP( + "gravity solve: density map", 0, + field::make_field_dof_grid_function_adapter(*f.densityFes) + ); + const field::FieldDofGridFunctionAdapter displacement_adapter = MEAN_FIELD_PROFILE_EVALUATE_WARMUP( + "gravity solve: displacement map", 0, + field::make_field_dof_grid_function_adapter(*f.displacementFes) + ); + const field::FieldDofGridFunctionAdapter gravity_flux_adapter = MEAN_FIELD_PROFILE_EVALUATE_WARMUP( + "gravity solve: flux map", 0, + field::make_field_dof_grid_function_adapter(*f.gravityFluxFes) + ); + const field::FieldDofGridFunctionAdapter gravity_potential_adapter = MEAN_FIELD_PROFILE_EVALUATE_WARMUP( + "gravity solve: potential map", 0, + field::make_field_dof_grid_function_adapter(*f.gravityPotentialFes) + ); const field::FieldDofMap &density_map = density_adapter.dof_map(); const field::FieldDofMap &displacement_map = displacement_adapter.dof_map(); @@ -174,34 +191,56 @@ namespace mean_field::physics { gravity_flux_map.reduced_size(), gravity_potential_map.reduced_size() }; - const utils::blocks::form_layout layout(value_sizes, residual_sizes); - - const mfem::Vector density = density_adapter.gather(rho); - const mfem::Vector reduced_displacement = displacement_adapter.gather(displacement); - - operators::context::gravity_field::GravityFieldLinearizationContext linearization_context( - f, *f.domainMapperStateless + const utils::blocks::form_layout layout = MEAN_FIELD_PROFILE_EVALUATE_WARMUP( + "gravity solve: block layout", 0, utils::blocks::form_layout(value_sizes, residual_sizes) ); - operators::GravityFieldJacobianOperator gravity_jacobian( - f, *f.domainMapperStateless, linearization_context, layout.value_offsets(), layout.residual_offsets() + const mfem::Vector density = + MEAN_FIELD_PROFILE_EVALUATE_WARMUP("gravity solve: gather density", 0, density_adapter.gather(rho)); + const mfem::Vector reduced_displacement = MEAN_FIELD_PROFILE_EVALUATE_WARMUP( + "gravity solve: gather displacement", 0, displacement_adapter.gather(displacement) ); - operators::GravityFieldOperator gravity_operator( - f, *f.domainMapperStateless, linearization_context, layout.value_offsets(), gravity_jacobian + operators::context::gravity_field::GravityFieldLinearizationContext linearization_context = + MEAN_FIELD_PROFILE_EVALUATE_WARMUP( + "gravity solve: linearization context", 0, + operators::context::gravity_field::GravityFieldLinearizationContext(f, *f.domainMapperStateless) + ); + + operators::GravityFieldJacobianOperator gravity_jacobian = MEAN_FIELD_PROFILE_EVALUATE_WARMUP( + "gravity solve: jacobian operator", 0, + operators::GravityFieldJacobianOperator( + f, *f.domainMapperStateless, linearization_context, layout.value_offsets(), layout.residual_offsets() + ) ); - operators::context::gravity_field::GravityFieldGeometryContext reduced_geometry_context( - f, *f.domainMapperStateless + operators::GravityFieldOperator gravity_operator = MEAN_FIELD_PROFILE_EVALUATE_WARMUP( + "gravity solve: nonlinear operator", 0, + operators::GravityFieldOperator( + f, *f.domainMapperStateless, linearization_context, layout.value_offsets(), gravity_jacobian + ) ); - operators::ReducedGravityFieldOperator reduced_operator( - gravity_operator, reduced_geometry_context, reduced_displacement + operators::context::gravity_field::GravityFieldGeometryContext reduced_geometry_context = + MEAN_FIELD_PROFILE_EVALUATE_WARMUP( + "gravity solve: reduced geometry context", 0, + operators::context::gravity_field::GravityFieldGeometryContext(f, *f.domainMapperStateless) + ); + + operators::ReducedGravityFieldOperator reduced_operator = MEAN_FIELD_PROFILE_EVALUATE_WARMUP( + "gravity solve: reduced operator", 0, + operators::ReducedGravityFieldOperator(gravity_operator, reduced_geometry_context, reduced_displacement) + ); + + operators::ReducedGravityFieldPreconditioner reduced_preconditioner = MEAN_FIELD_PROFILE_EVALUATE_WARMUP( + "gravity solve: preconditioner construction", 0, + operators::ReducedGravityFieldPreconditioner(f, reduced_geometry_context) ); - operators::ReducedGravityFieldPreconditioner reduced_preconditioner(f, reduced_geometry_context); mfem::Vector right_hand_side; - reduced_operator.BuildRightHandSide(density, right_hand_side); + MEAN_FIELD_PROFILE_CALL_WARMUP( + "gravity solve: right-hand side", 0, reduced_operator.BuildRightHandSide(density, right_hand_side) + ); MFEM_VERIFY( right_hand_side.Size() == reduced_operator.Height(), @@ -219,14 +258,18 @@ namespace mean_field::physics { minres.SetMaxIter(options.maximumIterations); // minres.SetPrintLevel(args.verbose ? 1 : 0); minres.SetPrintLevel(0); - minres.Mult(right_hand_side, gravity_state); + MEAN_FIELD_PROFILE_CALL_WARMUP("gravity solve: MINRES", 0, minres.Mult(right_hand_side, gravity_state)); + MEAN_FIELD_PROFILE_COUNT("gravity solve: MINRES iterations", minres.GetNumIterations()); MFEM_VERIFY(minres.GetConverged(), "The reduced gravity solve failed to converge."); GravitySolution solution(f); - gravity_flux_adapter.scatter(gravity_state.GetBlock(gravity_gradient_residual_block), solution.gradPhi); - gravity_potential_adapter.scatter(gravity_state.GetBlock(gravity_poisson_residual_block), solution.phi); + MEAN_FIELD_PROFILE_CALL_WARMUP( + "gravity solve: scatter solution", 0, + gravity_flux_adapter.scatter(gravity_state.GetBlock(gravity_gradient_residual_block), solution.gradPhi); + gravity_potential_adapter.scatter(gravity_state.GetBlock(gravity_poisson_residual_block), solution.phi) + ); return solution; } diff --git a/libmeanfield/impl/preconditioning/gravity_field.cpp b/libmeanfield/impl/preconditioning/gravity_field.cpp new file mode 100644 index 0000000..bd5de2f --- /dev/null +++ b/libmeanfield/impl/preconditioning/gravity_field.cpp @@ -0,0 +1,77 @@ +module; + +#include +#include +#include +#include + +module mean_field; + +import :preconditioning.gravity_field; + +namespace mean_field::preconditioning { + std::unique_ptr assembleGravityDivergenceSurrogate(const fem::FEM &f) { + if (f.mesh == nullptr || f.gravityFluxFes == nullptr || f.gravityPotentialFes == nullptr || + f.quadratureFactory == nullptr) { + throw std::invalid_argument( + "The gravity divergence surrogate requires its mesh, gravity spaces, and quadrature policy." + ); + } + + mfem::ParMixedBilinearForm divergence(f.gravityFluxFes.get(), f.gravityPotentialFes.get()); + auto integrator = std::make_unique(); + + const mfem::FiniteElement &trialElement = *f.gravityFluxFes->GetTypicalFE(); + const mfem::FiniteElement &testElement = *f.gravityPotentialFes->GetTypicalFE(); + const mfem::ElementTransformation &transformation = *f.mesh->GetElementTransformation(0); + + f.quadratureFactory->configure_gravity_divergence( + *integrator, quadrature::QuadratureRole::preconditioner, trialElement, testElement, transformation, + utils::DOMAINS::ALL, quadrature::MappingKind::none + ); + + divergence.AddDomainIntegrator(integrator.release()); + divergence.Assemble(); + divergence.Finalize(); + + std::unique_ptr assembled(divergence.ParallelAssemble()); + if (assembled == nullptr) { + throw std::runtime_error("MFEM did not assemble the gravity divergence surrogate."); + } + return assembled; + } + + std::unique_ptr assembleGravityPotentialSchurSurrogate( + const fem::FEM &f, + const mfem::Vector &trueMassDiagonal + ) { + if (f.gravityFluxFes == nullptr || trueMassDiagonal.Size() != f.gravityFluxFes->GetTrueVSize()) { + throw std::invalid_argument( + "The gravity Schur surrogate requires one mass-diagonal entry per true gravity-gradient DOF." + ); + } + + mfem::Vector inverseMassDiagonal(trueMassDiagonal); + for (int index = 0; index < inverseMassDiagonal.Size(); ++index) { + const double entry = inverseMassDiagonal(index); + if (!std::isfinite(entry) || entry <= 0.0) { + throw std::invalid_argument( + "The gravity Schur surrogate encountered a non-positive or non-finite mass diagonal." + ); + } + inverseMassDiagonal(index) = 1.0 / entry; + } + + std::unique_ptr divergence = assembleGravityDivergenceSurrogate(f); + std::unique_ptr inverseMassDivergenceTranspose(divergence->Transpose()); + inverseMassDivergenceTranspose->ScaleRows(inverseMassDiagonal); + + std::unique_ptr schur( + mfem::ParMult(divergence.get(), inverseMassDivergenceTranspose.get()) + ); + if (schur == nullptr) { + throw std::runtime_error("MFEM did not assemble the gravity potential-Schur surrogate."); + } + return schur; + } +} // namespace mean_field::preconditioning diff --git a/libmeanfield/impl/profile.cpp b/libmeanfield/impl/profile.cpp new file mode 100644 index 0000000..67362c0 --- /dev/null +++ b/libmeanfield/impl/profile.cpp @@ -0,0 +1,530 @@ +#include "profile.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + struct MpiContext { + bool active{false}; + int rank{0}; + int size{1}; + }; + + void check_mpi( + const int result, + const std::string_view operation + ) { + if (result == MPI_SUCCESS) { + return; + } + + std::array buffer{}; + int length = 0; + MPI_Error_string(result, buffer.data(), &length); + + throw std::runtime_error( + "MPI profiling operation '" + std::string(operation) + + "' failed: " + std::string(buffer.data(), static_cast(length)) + ); + } + + [[nodiscard]] MpiContext get_mpi_context(const MPI_Comm communicator) { + int initialized = 0; + check_mpi(MPI_Initialized(&initialized), "MPI_Initialized"); + + if (initialized == 0) { + return {}; + } + + int finalized = 0; + check_mpi(MPI_Finalized(&finalized), "MPI_Finalized"); + + if (finalized != 0) { + return {}; + } + + if (communicator == MPI_COMM_NULL) { + throw std::invalid_argument("Profiling aggregation requires a valid MPI communicator."); + } + + MpiContext context{.active = true}; + check_mpi(MPI_Comm_rank(communicator, &context.rank), "MPI_Comm_rank"); + check_mpi(MPI_Comm_size(communicator, &context.size), "MPI_Comm_size"); + return context; + } + + [[nodiscard]] std::string count_range( + const std::uint64_t minimum, + const std::uint64_t maximum + ) { + if (minimum == maximum) { + return std::to_string(minimum); + } + return std::to_string(minimum) + "-" + std::to_string(maximum); + } + + void write_csv_field( + std::ostream &stream, + const std::string_view field + ) { + stream << '"'; + for (const char character : field) { + if (character == '"') { + stream << "\"\""; + } else { + stream << character; + } + } + stream << '"'; + } +} // namespace + +namespace mean_field::profiling { + struct Registry::Impl { + struct Entry { + std::string label; + Statistics statistics; + }; + + mutable std::mutex mutex; + std::map> indices; + std::vector entries; + }; + + Registry &Registry::Get() { + static Registry registry; + return registry; + } + + Registry::Registry() : m_impl(std::make_unique()) { + } + + Registry::~Registry() = default; + + std::size_t Registry::Register( + const std::string_view label, + const std::uint64_t warmup_count + ) { + if (label.empty()) { + throw std::invalid_argument("A profiling region label cannot be empty."); + } + if (label.find('\0') != std::string_view::npos) { + throw std::invalid_argument("A profiling region label cannot contain a null byte."); + } + + std::scoped_lock lock(m_impl->mutex); + if (const auto iterator = m_impl->indices.find(label); iterator != m_impl->indices.end()) { + Impl::Entry &entry = m_impl->entries[iterator->second]; + entry.statistics.warmup_target = std::max(entry.statistics.warmup_target, warmup_count); + return iterator->second; + } + + const std::size_t index = m_impl->entries.size(); + Impl::Entry entry{.label = std::string(label)}; + entry.statistics.warmup_target = warmup_count; + m_impl->entries.push_back(std::move(entry)); + m_impl->indices.emplace(m_impl->entries.back().label, index); + return index; + } + + void Registry::Record( + const std::string_view label, + const double seconds, + const std::uint64_t warmup_count + ) { + if (!std::isfinite(seconds) || seconds < 0.0) { + throw std::invalid_argument("A profiling duration must be finite and nonnegative."); + } + + const std::size_t region = Register(label, warmup_count); + std::scoped_lock lock(m_impl->mutex); + Statistics &statistics = m_impl->entries[region].statistics; + const bool is_warmup = statistics.observations < statistics.warmup_target; + ++statistics.observations; + + if (is_warmup) { + ++statistics.warmups; + return; + } + + ++statistics.samples; + statistics.total_seconds += seconds; + if (statistics.samples == 1) { + statistics.minimum_seconds = seconds; + statistics.maximum_seconds = seconds; + } else { + statistics.minimum_seconds = std::min(statistics.minimum_seconds, seconds); + statistics.maximum_seconds = std::max(statistics.maximum_seconds, seconds); + } + } + + void Registry::AddCount( + const std::string_view label, + const std::uint64_t work_units + ) { + const std::size_t region = Register(label, 0); + std::scoped_lock lock(m_impl->mutex); + Statistics &statistics = m_impl->entries[region].statistics; + if (work_units > std::numeric_limits::max() - statistics.work_units) { + throw std::overflow_error("A profiling work counter overflowed."); + } + statistics.work_units += work_units; + } + + void Registry::Record( + const std::size_t region, + const double seconds + ) noexcept { + if (!std::isfinite(seconds) || seconds < 0.0) { + return; + } + + try { + std::scoped_lock lock(m_impl->mutex); + if (region >= m_impl->entries.size()) { + return; + } + + Statistics &statistics = m_impl->entries[region].statistics; + const bool is_warmup = statistics.observations < statistics.warmup_target; + ++statistics.observations; + + if (is_warmup) { + ++statistics.warmups; + return; + } + + ++statistics.samples; + statistics.total_seconds += seconds; + if (statistics.samples == 1) { + statistics.minimum_seconds = seconds; + statistics.maximum_seconds = seconds; + } else { + statistics.minimum_seconds = std::min(statistics.minimum_seconds, seconds); + statistics.maximum_seconds = std::max(statistics.maximum_seconds, seconds); + } + } catch (...) { + } + } + + void Registry::AddCount( + const std::size_t region, + const std::uint64_t work_units + ) noexcept { + try { + std::scoped_lock lock(m_impl->mutex); + if (region >= m_impl->entries.size()) { + return; + } + + Statistics &statistics = m_impl->entries[region].statistics; + if (work_units > std::numeric_limits::max() - statistics.work_units) { + statistics.work_units = std::numeric_limits::max(); + } else { + statistics.work_units += work_units; + } + } catch (...) { + } + } + + void Registry::Reset() { + std::scoped_lock lock(m_impl->mutex); + for (Impl::Entry &entry : m_impl->entries) { + const std::uint64_t warmup_target = entry.statistics.warmup_target; + entry.statistics = {}; + entry.statistics.warmup_target = warmup_target; + } + } + + std::map< + std::string, + Statistics, + std::less<>> + Registry::Snapshot() const { + std::map> snapshot; + std::scoped_lock lock(m_impl->mutex); + for (const Impl::Entry &entry : m_impl->entries) { + snapshot.emplace(entry.label, entry.statistics); + } + return snapshot; + } + + std::vector Registry::Aggregate(const MPI_Comm communicator) const { + const std::map> local_snapshot = Snapshot(); + const MpiContext mpi_context = get_mpi_context(communicator); + + std::vector labels; + if (!mpi_context.active) { + labels.reserve(local_snapshot.size()); + for (const auto &[label, statistics] : local_snapshot) { + (void)statistics; + labels.push_back(label); + } + } else { + std::string serialized_labels; + for (const auto &[label, statistics] : local_snapshot) { + (void)statistics; + serialized_labels.append(label); + serialized_labels.push_back('\0'); + } + + if (serialized_labels.size() > static_cast(std::numeric_limits::max())) { + throw std::overflow_error("The local profiling label table is too large for MPI_Allgatherv."); + } + + const int local_bytes = static_cast(serialized_labels.size()); + std::vector byte_counts(static_cast(mpi_context.size)); + check_mpi( + MPI_Allgather(&local_bytes, 1, MPI_INT, byte_counts.data(), 1, MPI_INT, communicator), + "MPI_Allgather(profile label sizes)" + ); + + std::vector displacements(static_cast(mpi_context.size)); + int total_bytes = 0; + for (int rank = 0; rank < mpi_context.size; ++rank) { + if (byte_counts[rank] < 0 || byte_counts[rank] > std::numeric_limits::max() - total_bytes) { + throw std::overflow_error("The distributed profiling label table is too large for MPI_Allgatherv."); + } + displacements[rank] = total_bytes; + total_bytes += byte_counts[rank]; + } + + std::vector all_serialized_labels(static_cast(total_bytes)); + check_mpi( + MPI_Allgatherv( + serialized_labels.data(), local_bytes, MPI_CHAR, all_serialized_labels.data(), byte_counts.data(), + displacements.data(), MPI_CHAR, communicator + ), + "MPI_Allgatherv(profile labels)" + ); + + std::set> unique_labels; + for (int rank = 0; rank < mpi_context.size; ++rank) { + const char *position = all_serialized_labels.data() + displacements[rank]; + const char *end = position + byte_counts[rank]; + while (position != end) { + const void *terminator_address = + std::memchr(position, '\0', static_cast(end - position)); + if (terminator_address == nullptr) { + throw std::runtime_error("A distributed profiling label table is malformed."); + } + const auto *terminator = static_cast(terminator_address); + unique_labels.emplace(position, terminator); + position = terminator + 1; + } + } + labels.assign(unique_labels.begin(), unique_labels.end()); + } + + std::vector aggregate(labels.size()); + if (labels.empty()) { + return aggregate; + } + + std::vector local_samples(labels.size(), 0); + std::vector local_warmups(labels.size(), 0); + std::vector local_work_units(labels.size(), 0); + std::vector local_averages(labels.size(), 0.0); + std::vector local_minima(labels.size(), std::numeric_limits::infinity()); + std::vector local_maxima(labels.size(), 0.0); + std::vector local_totals(labels.size(), 0.0); + + for (std::size_t index = 0; index < labels.size(); ++index) { + if (const auto iterator = local_snapshot.find(labels[index]); iterator != local_snapshot.end()) { + const Statistics &statistics = iterator->second; + local_samples[index] = statistics.samples; + local_warmups[index] = statistics.warmups; + local_work_units[index] = statistics.work_units; + local_totals[index] = statistics.total_seconds; + if (statistics.samples != 0) { + local_averages[index] = statistics.total_seconds / static_cast(statistics.samples); + local_minima[index] = statistics.minimum_seconds; + local_maxima[index] = statistics.maximum_seconds; + } + } + } + + std::vector minimum_samples = local_samples; + std::vector maximum_samples = local_samples; + std::vector maximum_warmups = local_warmups; + std::vector minimum_work_units = local_work_units; + std::vector maximum_work_units = local_work_units; + std::vector maximum_rank_averages = local_averages; + std::vector global_minima = local_minima; + std::vector global_maxima = local_maxima; + std::vector maximum_rank_totals = local_totals; + + if (mpi_context.active) { + if (labels.size() > static_cast(std::numeric_limits::max())) { + throw std::overflow_error("There are too many profiling regions for one MPI reduction."); + } + const int count = static_cast(labels.size()); + + check_mpi( + MPI_Allreduce(local_samples.data(), minimum_samples.data(), count, MPI_UINT64_T, MPI_MIN, communicator), + "MPI_Allreduce(minimum profile samples)" + ); + check_mpi( + MPI_Allreduce(local_samples.data(), maximum_samples.data(), count, MPI_UINT64_T, MPI_MAX, communicator), + "MPI_Allreduce(maximum profile samples)" + ); + check_mpi( + MPI_Allreduce(local_warmups.data(), maximum_warmups.data(), count, MPI_UINT64_T, MPI_MAX, communicator), + "MPI_Allreduce(profile warmups)" + ); + check_mpi( + MPI_Allreduce( + local_work_units.data(), minimum_work_units.data(), count, MPI_UINT64_T, MPI_MIN, communicator + ), + "MPI_Allreduce(minimum profile work)" + ); + check_mpi( + MPI_Allreduce( + local_work_units.data(), maximum_work_units.data(), count, MPI_UINT64_T, MPI_MAX, communicator + ), + "MPI_Allreduce(maximum profile work)" + ); + check_mpi( + MPI_Allreduce( + local_averages.data(), maximum_rank_averages.data(), count, MPI_DOUBLE, MPI_MAX, communicator + ), + "MPI_Allreduce(profile averages)" + ); + check_mpi( + MPI_Allreduce(local_minima.data(), global_minima.data(), count, MPI_DOUBLE, MPI_MIN, communicator), + "MPI_Allreduce(profile minima)" + ); + check_mpi( + MPI_Allreduce(local_maxima.data(), global_maxima.data(), count, MPI_DOUBLE, MPI_MAX, communicator), + "MPI_Allreduce(profile maxima)" + ); + check_mpi( + MPI_Allreduce( + local_totals.data(), maximum_rank_totals.data(), count, MPI_DOUBLE, MPI_MAX, communicator + ), + "MPI_Allreduce(profile totals)" + ); + } + + for (std::size_t index = 0; index < labels.size(); ++index) { + aggregate[index] = { + .label = labels[index], + .minimum_samples = minimum_samples[index], + .maximum_samples = maximum_samples[index], + .maximum_warmups = maximum_warmups[index], + .minimum_work_units = minimum_work_units[index], + .maximum_work_units = maximum_work_units[index], + .maximum_rank_average_seconds = maximum_rank_averages[index], + .global_minimum_seconds = std::isfinite(global_minima[index]) ? global_minima[index] : 0.0, + .global_maximum_seconds = global_maxima[index], + .maximum_rank_total_seconds = maximum_rank_totals[index] + }; + } + + return aggregate; + } + + void Registry::Print( + const MPI_Comm communicator, + std::ostream &stream + ) const { + const std::vector aggregate = Aggregate(communicator); + const MpiContext mpi_context = get_mpi_context(communicator); + if (mpi_context.rank != 0) { + return; + } + + std::ios old_state(nullptr); + old_state.copyfmt(stream); + + stream << '\n'; + stream << std::left << std::setw(58) << "Profile Region" << std::right << std::setw(13) << "Samples" + << std::setw(11) << "Warmups" << std::setw(15) << "Work/rank" << std::setw(14) << "Avg max ms" + << std::setw(14) << "Min ms" << std::setw(14) << "Max ms" << std::setw(14) << "Total max s" << '\n'; + stream << std::string(153, '-') << '\n'; + + for (const DistributedStatistics &statistics : aggregate) { + stream << std::left << std::setw(58) << statistics.label << std::right << std::setw(13) + << count_range(statistics.minimum_samples, statistics.maximum_samples) << std::setw(11) + << statistics.maximum_warmups << std::setw(15) + << count_range(statistics.minimum_work_units, statistics.maximum_work_units) << std::setw(14) + << std::fixed << std::setprecision(3) << 1.0e3 * statistics.maximum_rank_average_seconds + << std::setw(14) << 1.0e3 * statistics.global_minimum_seconds << std::setw(14) + << 1.0e3 * statistics.global_maximum_seconds << std::setw(14) << std::setprecision(6) + << statistics.maximum_rank_total_seconds << '\n'; + } + + stream << std::string(153, '=') << '\n'; + stream << "MPI ranks: " << mpi_context.size << "\n\n"; + stream.copyfmt(old_state); + } + + void Registry::Print(const MPI_Comm communicator) const { + Print(communicator, std::cout); + } + + void Registry::PrintCsv( + const MPI_Comm communicator, + std::ostream &stream + ) const { + const std::vector aggregate = Aggregate(communicator); + const MpiContext mpi_context = get_mpi_context(communicator); + if (mpi_context.rank != 0) { + return; + } + + stream << "label,minimum_samples,maximum_samples,maximum_warmups,minimum_work_units,maximum_work_units," + "maximum_rank_average_seconds,global_minimum_seconds,global_maximum_seconds," + "maximum_rank_total_seconds,mpi_ranks\n"; + + for (const DistributedStatistics &statistics : aggregate) { + write_csv_field(stream, statistics.label); + stream << ',' << statistics.minimum_samples << ',' << statistics.maximum_samples << ',' + << statistics.maximum_warmups << ',' << statistics.minimum_work_units << ',' + << statistics.maximum_work_units << ',' << std::setprecision(17) + << statistics.maximum_rank_average_seconds << ',' << statistics.global_minimum_seconds << ',' + << statistics.global_maximum_seconds << ',' << statistics.maximum_rank_total_seconds << ',' + << mpi_context.size << '\n'; + } + } + + Region::Region( + const std::string_view label, + const std::uint64_t warmup_count + ) + : m_region( + Registry::Get().Register( + label, + warmup_count + ) + ) { + } + + void Region::Record(const double seconds) const noexcept { + Registry::Get().Record(m_region, seconds); + } + + void Region::AddCount(const std::uint64_t work_units) const noexcept { + Registry::Get().AddCount(m_region, work_units); + } + + ScopedTimer::ScopedTimer(const Region ®ion) noexcept + : m_region(region), + m_start(std::chrono::steady_clock::now()) { + } + + ScopedTimer::~ScopedTimer() noexcept { + const auto stop = std::chrono::steady_clock::now(); + m_region.Record(std::chrono::duration(stop - m_start).count()); + } +} // namespace mean_field::profiling diff --git a/libmeanfield/include/profile.h b/libmeanfield/include/profile.h index 85d53f6..4dda619 100644 --- a/libmeanfield/include/profile.h +++ b/libmeanfield/include/profile.h @@ -1,194 +1,216 @@ #pragma once -#include #include -#include -#include -#include -#include +#include +#include +#include #include -#include +#include #include +#include #include + #include +#ifndef MEAN_FIELD_ENABLE_PROFILING +#define MEAN_FIELD_ENABLE_PROFILING 0 +#endif + namespace mean_field::profiling { struct Statistics { - unsigned long long observations{0}; - unsigned long long warmups{0}; - unsigned long long samples{0}; - unsigned long long warmup_target{0}; + std::uint64_t observations{0}; + std::uint64_t warmups{0}; + std::uint64_t samples{0}; + std::uint64_t warmup_target{0}; + std::uint64_t work_units{0}; double total_seconds{0.0}; - double minimum_seconds{std::numeric_limits::infinity()}; + double minimum_seconds{0.0}; double maximum_seconds{0.0}; }; + struct DistributedStatistics { + std::string label; + std::uint64_t minimum_samples{0}; + std::uint64_t maximum_samples{0}; + std::uint64_t maximum_warmups{0}; + std::uint64_t minimum_work_units{0}; + std::uint64_t maximum_work_units{0}; + double maximum_rank_average_seconds{0.0}; + double global_minimum_seconds{0.0}; + double global_maximum_seconds{0.0}; + double maximum_rank_total_seconds{0.0}; + }; + class Registry { public: - static Registry& Get() { - static Registry registry; - return registry; - } + static Registry &Get(); - void Record(const std::string& label, const double seconds, const unsigned long long warmup_count) { - std::scoped_lock lock(m_mutex); - Statistics& statistics = m_statistics[label]; + Registry(const Registry &) = delete; + Registry &operator=(const Registry &) = delete; + Registry(Registry &&) = delete; + Registry &operator=(Registry &&) = delete; - statistics.warmup_target = std::max(statistics.warmup_target, warmup_count); - const bool is_warmup = statistics.observations < statistics.warmup_target; - ++statistics.observations; + ~Registry(); - if (is_warmup) { - ++statistics.warmups; - return; - } + void Record( + std::string_view label, + double seconds, + std::uint64_t warmup_count = 0 + ); - ++statistics.samples; - statistics.total_seconds += seconds; - statistics.minimum_seconds = std::min(statistics.minimum_seconds, seconds); - statistics.maximum_seconds = std::max(statistics.maximum_seconds, seconds); - } + void AddCount( + std::string_view label, + std::uint64_t work_units + ); - void Reset() { - std::scoped_lock lock(m_mutex); - m_statistics.clear(); - } + void Reset(); - void Print(MPI_Comm communicator) const { - const std::map snapshot = GetSnapshot(); + [[nodiscard]] std::map< + std::string, + Statistics, + std::less<>> + Snapshot() const; - int mpi_initialized = 0; - int mpi_finalized = 0; - MPI_Initialized(&mpi_initialized); - if (mpi_initialized) MPI_Finalized(&mpi_finalized); + [[nodiscard]] std::vector Aggregate(MPI_Comm communicator) const; - const bool use_mpi = mpi_initialized && !mpi_finalized; - int rank = 0; - int communicator_size = 1; + void Print( + MPI_Comm communicator, + std::ostream &stream + ) const; - if (use_mpi) { - MPI_Comm_rank(communicator, &rank); - MPI_Comm_size(communicator, &communicator_size); - } + void Print(MPI_Comm communicator) const; - if (rank == 0) { - std::cout << '\n'; - std::cout << std::left << std::setw(42) << "Profile Region" - << std::right << std::setw(11) << "Samples" - << std::setw(10) << "Warmups" - << std::setw(14) << "Avg Max ms" - << std::setw(14) << "Min ms" - << std::setw(14) << "Max ms" - << std::setw(14) << "Total Max s" << '\n'; - std::cout << std::string(119, '-') << '\n'; - } - - for (const auto& [label, local_statistics] : snapshot) { - unsigned long long minimum_samples = local_statistics.samples; - unsigned long long maximum_samples = local_statistics.samples; - unsigned long long maximum_warmups = local_statistics.warmups; - - double local_average = local_statistics.samples > 0 ? local_statistics.total_seconds / static_cast(local_statistics.samples) : 0.0; - double local_minimum = local_statistics.samples > 0 ? local_statistics.minimum_seconds : std::numeric_limits::infinity(); - double local_maximum = local_statistics.maximum_seconds; - double local_total = local_statistics.total_seconds; - - double maximum_rank_average = local_average; - double global_minimum = local_minimum; - double global_maximum = local_maximum; - double maximum_rank_total = local_total; - - if (use_mpi) { - MPI_Allreduce(&local_statistics.samples, &minimum_samples, 1, MPI_UNSIGNED_LONG_LONG, MPI_MIN, communicator); - MPI_Allreduce(&local_statistics.samples, &maximum_samples, 1, MPI_UNSIGNED_LONG_LONG, MPI_MAX, communicator); - MPI_Allreduce(&local_statistics.warmups, &maximum_warmups, 1, MPI_UNSIGNED_LONG_LONG, MPI_MAX, communicator); - MPI_Allreduce(&local_average, &maximum_rank_average, 1, MPI_DOUBLE, MPI_MAX, communicator); - MPI_Allreduce(&local_minimum, &global_minimum, 1, MPI_DOUBLE, MPI_MIN, communicator); - MPI_Allreduce(&local_maximum, &global_maximum, 1, MPI_DOUBLE, MPI_MAX, communicator); - MPI_Allreduce(&local_total, &maximum_rank_total, 1, MPI_DOUBLE, MPI_MAX, communicator); - } - - if (!std::isfinite(global_minimum)) global_minimum = 0.0; - - if (rank == 0) { - const std::string sample_string = minimum_samples == maximum_samples - ? std::to_string(minimum_samples) - : std::to_string(minimum_samples) + "-" + std::to_string(maximum_samples); - - std::cout << std::left << std::setw(100) << label - << std::right << std::setw(11) << sample_string - << std::setw(10) << maximum_warmups - << std::setw(14) << std::fixed << std::setprecision(3) << 1.0e3 * maximum_rank_average - << std::setw(14) << 1.0e3 * global_minimum - << std::setw(14) << 1.0e3 * global_maximum - << std::setw(14) << std::setprecision(6) << maximum_rank_total << '\n'; - } - } - - if (rank == 0) { - std::cout << std::string(119, '=') << '\n'; - std::cout << "MPI ranks: " << communicator_size << "\n\n"; - } - } + void PrintCsv( + MPI_Comm communicator, + std::ostream &stream + ) const; private: - [[nodiscard]] std::map GetSnapshot() const { - std::scoped_lock lock(m_mutex); - return m_statistics; - } + friend class Region; + + Registry(); + + [[nodiscard]] std::size_t Register( + std::string_view label, + std::uint64_t warmup_count + ); + + void Record( + std::size_t region, + double seconds + ) noexcept; + + void AddCount( + std::size_t region, + std::uint64_t work_units + ) noexcept; + + struct Impl; + std::unique_ptr m_impl; + }; + + class Region { + public: + explicit Region( + std::string_view label, + std::uint64_t warmup_count = 0 + ); + + void Record(double seconds) const noexcept; + void AddCount(std::uint64_t work_units) const noexcept; private: - mutable std::mutex m_mutex; - std::map m_statistics; + std::size_t m_region; }; class ScopedTimer { public: - ScopedTimer(std::string label, const unsigned long long warmup_count) - : m_label(std::move(label)), - m_warmup_count(warmup_count), - m_start(std::chrono::steady_clock::now()) {} + explicit ScopedTimer(const Region ®ion) noexcept; - ScopedTimer(const ScopedTimer&) = delete; - ScopedTimer& operator=(const ScopedTimer&) = delete; - ScopedTimer(ScopedTimer&&) = delete; - ScopedTimer& operator=(ScopedTimer&&) = delete; + ScopedTimer(const ScopedTimer &) = delete; + ScopedTimer &operator=(const ScopedTimer &) = delete; + ScopedTimer(ScopedTimer &&) = delete; + ScopedTimer &operator=(ScopedTimer &&) = delete; - ~ScopedTimer() { - try { - const auto stop = std::chrono::steady_clock::now(); - const double seconds = std::chrono::duration(stop - m_start).count(); - Registry::Get().Record(m_label, seconds, m_warmup_count); - } catch (...) {} - } + ~ScopedTimer() noexcept; private: - std::string m_label; - unsigned long long m_warmup_count; + const Region &m_region; std::chrono::steady_clock::time_point m_start; }; -} +} // namespace mean_field::profiling #define MEAN_FIELD_PROFILE_JOIN_IMPL(left, right) left##right #define MEAN_FIELD_PROFILE_JOIN(left, right) MEAN_FIELD_PROFILE_JOIN_IMPL(left, right) -#define MEAN_FIELD_PROFILE_SCOPE_WARMUP(label, warmup_count) \ - ::mean_field::profiling::ScopedTimer MEAN_FIELD_PROFILE_JOIN(mean_field_profile_timer_, __COUNTER__)(label, warmup_count) +#if MEAN_FIELD_ENABLE_PROFILING -#define MEAN_FIELD_PROFILE_SCOPE(label) \ - MEAN_FIELD_PROFILE_SCOPE_WARMUP(label, 1) +#define MEAN_FIELD_PROFILE_SCOPE_IMPL(label, warmup_count, identifier) \ + static const ::mean_field::profiling::Region MEAN_FIELD_PROFILE_JOIN(mean_field_profile_region_, identifier)( \ + label, warmup_count \ + ); \ + const ::mean_field::profiling::ScopedTimer MEAN_FIELD_PROFILE_JOIN(mean_field_profile_timer_, identifier)( \ + MEAN_FIELD_PROFILE_JOIN(mean_field_profile_region_, identifier) \ + ) -#define MEAN_FIELD_PROFILE_CALL_WARMUP(label, warmup_count, ...) \ - do { \ - MEAN_FIELD_PROFILE_SCOPE_WARMUP(label, warmup_count); \ - __VA_ARGS__; \ +#define MEAN_FIELD_PROFILE_SCOPE_WARMUP(label, warmup_count) \ + MEAN_FIELD_PROFILE_SCOPE_IMPL(label, warmup_count, __COUNTER__) + +#define MEAN_FIELD_PROFILE_SCOPE(label) MEAN_FIELD_PROFILE_SCOPE_WARMUP(label, 1) + +#define MEAN_FIELD_PROFILE_CALL_WARMUP(label, warmup_count, ...) \ + do { \ + MEAN_FIELD_PROFILE_SCOPE_WARMUP(label, warmup_count); \ + __VA_ARGS__; \ } while (false) -#define MEAN_FIELD_PROFILE_CALL(label, ...) \ - MEAN_FIELD_PROFILE_CALL_WARMUP(label, 1, __VA_ARGS__) +#define MEAN_FIELD_PROFILE_CALL(label, ...) MEAN_FIELD_PROFILE_CALL_WARMUP(label, 1, __VA_ARGS__) -#define MEAN_FIELD_PROFILE_RESET() \ - ::mean_field::profiling::Registry::Get().Reset() +#define MEAN_FIELD_PROFILE_EVALUATE_IMPL(label, warmup_count, identifier, ...) \ + ([&]() -> decltype(auto) { \ + MEAN_FIELD_PROFILE_SCOPE_IMPL(label, warmup_count, identifier); \ + return (__VA_ARGS__); \ + }()) -#define MEAN_FIELD_PROFILE_PRINT(communicator) \ - ::mean_field::profiling::Registry::Get().Print(communicator) \ No newline at end of file +#define MEAN_FIELD_PROFILE_EVALUATE_WARMUP(label, warmup_count, ...) \ + MEAN_FIELD_PROFILE_EVALUATE_IMPL(label, warmup_count, __COUNTER__, __VA_ARGS__) + +#define MEAN_FIELD_PROFILE_EVALUATE(label, ...) MEAN_FIELD_PROFILE_EVALUATE_WARMUP(label, 1, __VA_ARGS__) + +#define MEAN_FIELD_PROFILE_COUNT_IMPL(label, work_units, identifier) \ + do { \ + static const ::mean_field::profiling::Region MEAN_FIELD_PROFILE_JOIN(mean_field_profile_counter_, identifier)( \ + label \ + ); \ + MEAN_FIELD_PROFILE_JOIN(mean_field_profile_counter_, identifier).AddCount(work_units); \ + } while (false) + +#define MEAN_FIELD_PROFILE_COUNT(label, work_units) MEAN_FIELD_PROFILE_COUNT_IMPL(label, work_units, __COUNTER__) + +#define MEAN_FIELD_PROFILE_RESET() ::mean_field::profiling::Registry::Get().Reset() + +#define MEAN_FIELD_PROFILE_PRINT(communicator) ::mean_field::profiling::Registry::Get().Print(communicator) + +#define MEAN_FIELD_PROFILE_PRINT_CSV(communicator, stream) \ + ::mean_field::profiling::Registry::Get().PrintCsv(communicator, stream) + +#else + +#define MEAN_FIELD_PROFILE_SCOPE_WARMUP(label, warmup_count) ((void)0) +#define MEAN_FIELD_PROFILE_SCOPE(label) ((void)0) + +#define MEAN_FIELD_PROFILE_CALL_WARMUP(label, warmup_count, ...) \ + do { \ + __VA_ARGS__; \ + } while (false) + +#define MEAN_FIELD_PROFILE_CALL(label, ...) MEAN_FIELD_PROFILE_CALL_WARMUP(label, 1, __VA_ARGS__) + +#define MEAN_FIELD_PROFILE_EVALUATE_WARMUP(label, warmup_count, ...) (__VA_ARGS__) +#define MEAN_FIELD_PROFILE_EVALUATE(label, ...) (__VA_ARGS__) +#define MEAN_FIELD_PROFILE_COUNT(label, work_units) ((void)0) +#define MEAN_FIELD_PROFILE_RESET() ((void)0) +#define MEAN_FIELD_PROFILE_PRINT(communicator) ((void)0) +#define MEAN_FIELD_PROFILE_PRINT_CSV(communicator, stream) ((void)0) + +#endif diff --git a/libmeanfield/interface/field/field_mfem.cppm b/libmeanfield/interface/field/field_mfem.cppm index 723e9ff..cf5856b 100644 --- a/libmeanfield/interface/field/field_mfem.cppm +++ b/libmeanfield/interface/field/field_mfem.cppm @@ -764,6 +764,11 @@ export namespace mean_field::field { require_reduced_size(reduced); + if (is_identity()) { + reduced = full; + return; + } + for (int reducedDof = 0; reducedDof < reduced_size(); ++reducedDof) { reduced(reducedDof) = full(m_reducedToTrue[reducedDof]); } @@ -798,6 +803,11 @@ export namespace mean_field::field { require_full_size(full); + if (is_identity()) { + full = reduced; + return; + } + full = 0.0; scatter_into(reduced, full); @@ -828,6 +838,11 @@ export namespace mean_field::field { require_full_size(full); + if (is_identity()) { + full = reduced; + return; + } + for (int reducedDof = 0; reducedDof < reduced_size(); ++reducedDof) { full(m_reducedToTrue[reducedDof]) = reduced(reducedDof); } @@ -847,6 +862,11 @@ export namespace mean_field::field { require_full_size(full); + if (is_identity()) { + full.Add(scale, reduced); + return; + } + for (int reducedDof = 0; reducedDof < reduced_size(); ++reducedDof) { full(m_reducedToTrue[reducedDof]) += scale * reduced(reducedDof); } diff --git a/libmeanfield/interface/field/field_registry.cppm b/libmeanfield/interface/field/field_registry.cppm index 1c1de11..d73417b 100644 --- a/libmeanfield/interface/field/field_registry.cppm +++ b/libmeanfield/interface/field/field_registry.cppm @@ -9,6 +9,7 @@ module; export module mean_field:field.registry; +export import :dimensions.quantities; export import :field.base; export import :quadrature.policy; export import :utils.domain; @@ -25,6 +26,7 @@ export namespace mean_field::field { static constexpr std::string_view name = "density"; static constexpr int scalarOrder = 2 + uniformPolynomialOrderIncrement; + using PhysicalQuantity = dimensions::quantity::Density; using Support = DomainSupport; struct Scalar final : ScalarQ> { @@ -261,6 +263,7 @@ export namespace mean_field::field { static constexpr std::string_view name = "specific_enthalpy"; static constexpr int scalarOrder = 3 + uniformPolynomialOrderIncrement; + using PhysicalQuantity = dimensions::quantity::SpecificEnthalpy; using Support = DomainSupport; struct Scalar final : ScalarQ> { diff --git a/libmeanfield/interface/material/thermodynamic_equations.cppm b/libmeanfield/interface/material/thermodynamic_equations.cppm new file mode 100644 index 0000000..c23c7ed --- /dev/null +++ b/libmeanfield/interface/material/thermodynamic_equations.cppm @@ -0,0 +1,271 @@ +module; + +#include +#include + +export module mean_field:material.thermodynamic_equations; + +export import :eos.polytrope; +export import :surface.compiler; +export import :utils.blocks; + +export namespace mean_field::material { + /** + * A thermodynamic field identifies the physical quantity represented by + * its discrete degree of freedom. The quantity belongs to the field, not + * to an EOS-specific aggregate description. + */ + template + concept ThermodynamicField = surface::SurfaceFieldType && requires { + typename std::remove_cvref_t::PhysicalQuantity; + requires eos::ThermodynamicQuantityType::PhysicalQuantity>; + }; + + /** + * Declares the algebraic blocks owned by one thermodynamic governing + * equation. Its physical quantity is inferred from Field. + */ + template + requires std::derived_from && + std::derived_from + struct ThermodynamicEquation final { + using FieldType = Field; + using PhysicalQuantity = typename Field::PhysicalQuantity; + using Correction = CorrectionBlock; + using Residual = ResidualBlock; + }; + + /** + * Registry of thermodynamic governing equations available to a problem. + * The problem form, rather than the registry, selects the active subset. + */ + template struct ThermodynamicEquationCatalog final { + static constexpr int size = sizeof...(Equations); + }; + + namespace detail { + template struct IsThermodynamicEquation : std::false_type { }; + + template + struct IsThermodynamicEquation> : std::true_type { + }; + + template struct IsThermodynamicEquationCatalog : std::false_type { }; + + template + struct IsThermodynamicEquationCatalog> + : std::bool_constant<(sizeof...(Equations) > 0) && (IsThermodynamicEquation::value && ...)> { }; + + template struct CatalogEntriesAreUnique : std::false_type { }; + + template + struct CatalogEntriesAreUnique> + : std::bool_constant< + utils::blocks::types_are_unique_v> && + utils::blocks::types_are_unique_v< + utils::blocks::type_list> && + utils::blocks::types_are_unique_v> && + utils::blocks::types_are_unique_v>> { }; + + template struct EquationForField; + + template + struct EquationForField, Field> + : std::conditional_t< + std::same_as, + std::type_identity, + EquationForField, Field>> { }; + + template struct EquationForField, Field> { + using type = void; + }; + + template struct EquationFieldCount; + + template + struct EquationFieldCount, Field> + : std::integral_constant< + int, + (int{0} + ... + (std::same_as ? 1 : 0))> { }; + + template struct EquationForCorrection; + + template + struct EquationForCorrection, Correction> + : std::conditional_t< + std::same_as, + std::type_identity, + EquationForCorrection, Correction>> { }; + + template struct EquationForCorrection, Correction> { + using type = void; + }; + + template struct PrependEquation; + + template + struct PrependEquation> { + using Type = ThermodynamicEquationCatalog; + }; + + template struct SelectActiveEquations; + + template + struct SelectActiveEquations, AvailableEquations> { + using Type = ThermodynamicEquationCatalog<>; + }; + + template + struct SelectActiveEquations, AvailableEquations> { + private: + using Tail = + typename SelectActiveEquations, AvailableEquations>::Type; + using Match = typename EquationForCorrection::type; + + public: + using Type = + std::conditional_t, Tail, typename PrependEquation::Type>; + }; + + template struct CatalogMatchesForm : std::false_type { }; + + template + struct CatalogMatchesForm< + ThermodynamicEquationCatalog, + utils::blocks::block_form, utils::blocks::type_list>> + : std::bool_constant< + ((utils::blocks:: + contains_type_v> == + utils::blocks:: + contains_type_v>) && + ...)> { }; + + template struct SurfaceBindingsForEquations; + + template + struct SurfaceBindingsForEquations> { + using Type = surface::SurfaceStateBindings< + surface::SurfaceStateBinding...>; + }; + + template + struct SurfaceFieldsBelongToEquations : std::false_type { }; + + template + struct SurfaceFieldsBelongToEquations, Equations> + : std::bool_constant<((EquationFieldCount::value == 1) && ...)> { }; + + template + struct ThermodynamicCompilationIsAvailable : std::false_type { }; + + template + struct ThermodynamicCompilationIsAvailable< + EquationOfState, + Form, + AvailableEquations, + std::enable_if_t< + eos::EquationOfStateModel && utils::blocks::block_form_is_valid_v && + IsThermodynamicEquationCatalog::value && + CatalogEntriesAreUnique::value && + CatalogMatchesForm::value>> { + private: + using ActiveEquations = + typename SelectActiveEquations::Type; + using StateBindings = typename SurfaceBindingsForEquations::Type; + + public: + static constexpr bool value = (ActiveEquations::size > 0) && + surface::PressureSurfaceFormulationCompilable; + }; + + template struct IsCompiledThermodynamicEquations : std::false_type { }; + + template + struct IsCompiledThermodynamicEquations< + Candidate, + std::void_t< + typename Candidate::EquationOfStateType, + typename Candidate::Equations, + typename Candidate::StateBindings, + typename Candidate::PressureSurfaceFormulation>> + : std::bool_constant< + eos::EquationOfStateModel && + IsThermodynamicEquationCatalog::value && + CatalogEntriesAreUnique::value && + surface::ValidSurfaceStateBindings && + std::same_as< + typename Candidate::StateBindings, + typename SurfaceBindingsForEquations::Type> && + surface::SurfaceConstraintFormulationType && + std::same_as< + typename Candidate::PressureSurfaceFormulation::StateBindings, + typename Candidate::StateBindings>> { }; + } // namespace detail + + template + concept ThermodynamicEquationType = detail::IsThermodynamicEquation>::value; + + template + concept ValidThermodynamicEquationCatalog = + detail::IsThermodynamicEquationCatalog>::value && + detail::CatalogEntriesAreUnique>::value; + + template + concept CompiledThermodynamicEquations = + detail::IsCompiledThermodynamicEquations>::value; + + template + requires(detail::EquationFieldCount::value == 1) + using ThermodynamicEquationForFieldT = typename detail::EquationForField::type; + + template + inline constexpr bool fieldsBelongToThermodynamicEquations = + detail::SurfaceFieldsBelongToEquations::value; + + template + concept ThermodynamicEquationsCompilable = detail::ThermodynamicCompilationIsAvailable< + std::remove_cvref_t, + std::remove_cvref_t, + std::remove_cvref_t>::value; + + template + requires surface::PressureSurfaceFormulationCompilable< + typename detail::SurfaceBindingsForEquations::Type, + EquationOfState> + struct ThermodynamicEquationSet final { + using EquationOfStateType = EquationOfState; + using Equations = ActiveEquations; + using StateBindings = typename detail::SurfaceBindingsForEquations::Type; + using PressureSurfaceFormulation = + surface::CompiledPressureSurfaceFormulationT; + }; + + template < + eos::EquationOfStateModel EquationOfState, + typename Form, + ValidThermodynamicEquationCatalog AvailableEquations> + requires ThermodynamicEquationsCompilable + struct CompileThermodynamicEquations final { + using Equations = typename detail::SelectActiveEquations::Type; + using Type = ThermodynamicEquationSet; + }; + + template + requires ThermodynamicEquationsCompilable + using CompiledThermodynamicEquationsT = typename CompileThermodynamicEquations< + std::remove_cvref_t, + std::remove_cvref_t, + std::remove_cvref_t>::Type; + + using StellarEquilibriumThermodynamicEquations = ThermodynamicEquationCatalog< + ThermodynamicEquation< + field::Density, + utils::blocks::density::mass::value, + utils::blocks::density::mass::residual>, + ThermodynamicEquation< + field::Enthalpy, + utils::blocks::enthalpy::specific::value, + utils::blocks::enthalpy::specific::residual>>; + + static_assert(ValidThermodynamicEquationCatalog); +} // namespace mean_field::material diff --git a/libmeanfield/interface/mean_field.cppm b/libmeanfield/interface/mean_field.cppm index 19bb3d1..ae9d288 100644 --- a/libmeanfield/interface/mean_field.cppm +++ b/libmeanfield/interface/mean_field.cppm @@ -26,6 +26,7 @@ export import :quadrature.policy; export import :quadrature.mfem; export import :solver.fields; export import :solver.preconditioning_diagnostics; +export import :preconditioning; export import :utils.blocks; export import :operators.gravity_field; export import :operators.gravity_field_jacobian; @@ -72,6 +73,7 @@ export import :surface.constant; export import :surface.dependencies; export import :surface.compiled; export import :surface.compiler; +export import :material.thermodynamic_equations; export import :deformation.descriptors; export import :deformation.surface_prescription; export import :deformation.nodal_radial_surface; diff --git a/libmeanfield/interface/operators/contexts/gravity_field_context.cppm b/libmeanfield/interface/operators/contexts/gravity_field_context.cppm index d63fb16..cad9261 100644 --- a/libmeanfield/interface/operators/contexts/gravity_field_context.cppm +++ b/libmeanfield/interface/operators/contexts/gravity_field_context.cppm @@ -76,6 +76,12 @@ export namespace mean_field::operators::context::gravity_field { DisplacementRevision displacement_revision ); + GravityFieldGeometryPreparation PreparePrimal( + const mfem::Vector &displacement, + DiscretizationRevision discretization_revision, + DisplacementRevision displacement_revision + ); + [[nodiscard]] const PreparedMappedHDivMassOperator &GetMassOperator() const; [[nodiscard]] const PreparedMappedGravitySourceOperator &GetSourceOperator() const; [[nodiscard]] const mfem::Operator &GetDivergenceOperator() const; @@ -87,12 +93,21 @@ export namespace mean_field::operators::context::gravity_field { [[nodiscard]] bool IsPrepared() const noexcept; private: + enum class PreparationMode : std::uint8_t { primal, linearization }; + + GravityFieldGeometryPreparation PrepareImpl( + const mfem::Vector &displacement, + DiscretizationRevision discretization_revision, + DisplacementRevision displacement_revision, + PreparationMode mode + ); + const fem::FEM &m_fem; const mapping::DomainMapper &m_domain_mapper; std::unique_ptr m_mass_operator; std::unique_ptr m_source_operator; - std::unique_ptr m_divergence_operator; + std::unique_ptr m_divergence_operator; std::unique_ptr m_transpose_divergence_operator; field::FieldDofMap m_displacement_map; @@ -102,6 +117,7 @@ export namespace mean_field::operators::context::gravity_field { DisplacementRevision m_displacement_revision; bool m_is_prepared{false}; + bool m_variation_state_prepared{false}; }; struct GravityFieldPreparationReport { diff --git a/libmeanfield/interface/operators/gravity_field.cppm b/libmeanfield/interface/operators/gravity_field.cppm index 99c87ca..64acb7f 100644 --- a/libmeanfield/interface/operators/gravity_field.cppm +++ b/libmeanfield/interface/operators/gravity_field.cppm @@ -60,6 +60,12 @@ export namespace mean_field::operators { mfem::Array m_state_offsets; mfem::Array m_residual_offsets; GravityFieldJacobianOperator &m_jacobian; + + mutable mfem::Vector m_potential_true; + mutable mfem::Vector m_transpose_divergence_action_true; + mutable mfem::Vector m_transpose_divergence_action; + mutable mfem::Vector m_gradient_true; + mutable mfem::Vector m_divergence_action_true; }; class ReducedGravityFieldOperator final : public mfem::Operator { diff --git a/libmeanfield/interface/operators/prepared_barotropic_closure_operator.cppm b/libmeanfield/interface/operators/prepared_barotropic_closure_operator.cppm index e71ce22..014c133 100644 --- a/libmeanfield/interface/operators/prepared_barotropic_closure_operator.cppm +++ b/libmeanfield/interface/operators/prepared_barotropic_closure_operator.cppm @@ -54,6 +54,10 @@ export namespace mean_field::operators { void BuildResidual(mfem::Vector &residual) const; + // Exact diagonal of the prepared density-to-closure block, expressed + // in the reduced density coordinates used by the root operator. + void AssembleDensityJacobianDiagonal(mfem::Vector &diagonal) const; + [[nodiscard]] bool IsPrepared() const noexcept; [[nodiscard]] std::uint64_t GetPreparationCount() const noexcept; [[nodiscard]] int GetDensitySize() const noexcept; diff --git a/libmeanfield/interface/operators/prepared_gravity_source.cppm b/libmeanfield/interface/operators/prepared_gravity_source.cppm index 0b362a8..ff32c75 100644 --- a/libmeanfield/interface/operators/prepared_gravity_source.cppm +++ b/libmeanfield/interface/operators/prepared_gravity_source.cppm @@ -18,6 +18,7 @@ export namespace mean_field::operators { ); void Prepare(const mfem::Vector &displacement); + void PreparePrimal(const mfem::Vector &displacement); void Mult( const mfem::Vector &density, mfem::Vector &action @@ -29,6 +30,7 @@ export namespace mean_field::operators { ) const; [[nodiscard]] bool IsPrepared() const noexcept; + [[nodiscard]] bool HasVariationData() const noexcept; [[nodiscard]] std::uint64_t GetPreparationCount() const noexcept; [[nodiscard]] const field::FieldDofMap &GetDensityMap() const noexcept; @@ -41,6 +43,8 @@ export namespace mean_field::operators { ) const override; private: + enum class PreparationMode : std::uint8_t { primal, linearization }; + struct ElementPAData { int element_id{-1}; @@ -64,6 +68,11 @@ export namespace mean_field::operators { mfem::Vector quadrature_data; }; + void PrepareImpl( + const mfem::Vector &displacement, + PreparationMode mode + ); + const fem::FEM &m_fem; const mapping::DomainMapper &m_domain_mapper; @@ -77,6 +86,11 @@ export namespace mean_field::operators { mutable mfem::Vector m_density_true; mutable mfem::Vector m_potential_true; mutable mfem::Vector m_action_true; + mutable mfem::Vector m_potential_local; + mutable mfem::Vector m_local_action; + mutable mfem::Vector m_element_input; + mutable mfem::Vector m_quadrature_action; + mutable mfem::Vector m_element_action; mutable mfem::Vector m_density_local; mutable mfem::Vector m_displacement_variation_local; mutable mfem::Vector m_local_variation_action; @@ -90,5 +104,6 @@ export namespace mean_field::operators { std::uint64_t m_preparation_count{0}; bool m_is_prepared{false}; + bool m_has_variation_data{false}; }; } // namespace mean_field::operators diff --git a/libmeanfield/interface/operators/prepared_hdiv_mass.cppm b/libmeanfield/interface/operators/prepared_hdiv_mass.cppm index 46d5df7..c82bfb6 100644 --- a/libmeanfield/interface/operators/prepared_hdiv_mass.cppm +++ b/libmeanfield/interface/operators/prepared_hdiv_mass.cppm @@ -18,6 +18,7 @@ export namespace mean_field::operators { ); void Prepare(const mfem::Vector &displacement); + void PreparePrimal(const mfem::Vector &displacement); void Mult( const mfem::Vector &gravity_gradient, mfem::Vector &action @@ -31,12 +32,15 @@ export namespace mean_field::operators { void AssembleTrueDiagonal(mfem::Vector &diagonal) const; [[nodiscard]] bool IsPrepared() const noexcept; + [[nodiscard]] bool HasVariationData() const noexcept; [[nodiscard]] std::uint64_t GetPreparationCount() const noexcept; [[nodiscard]] const field::FieldDofMap &GetFluxMap() const noexcept; [[nodiscard]] const field::FieldDofMap &GetDisplacementMap() const noexcept; private: + enum class PreparationMode : std::uint8_t { primal, linearization }; + struct ElementVariationData { int elementId{-1}; mfem::Array gravityGradientDofs; @@ -51,6 +55,10 @@ export namespace mean_field::operators { }; void PrepareVariationData(); + void PrepareImpl( + const mfem::Vector &displacement, + PreparationMode mode + ); const fem::FEM &m_fem; const mapping::DomainMapper &m_domain_mapper; @@ -68,6 +76,9 @@ export namespace mean_field::operators { mutable mfem::Vector m_flux_true; mutable mfem::Vector m_action_true; mutable mfem::Vector m_domain_action_true; + mutable mfem::Vector m_flux_local; + mutable mfem::Vector m_action_local; + mutable mfem::Vector m_domain_action_local; mfem::Vector m_displacement_true; std::vector m_variationElements; @@ -86,5 +97,7 @@ export namespace mean_field::operators { mutable mfem::DenseMatrix m_massTensorVariation; std::uint64_t m_preparation_count{0}; bool m_is_prepared{false}; + bool m_has_variation_data{false}; + bool m_single_rank{true}; }; } // namespace mean_field::operators diff --git a/libmeanfield/interface/operators/prepared_hydrostatic_equilibrium_operator.cppm b/libmeanfield/interface/operators/prepared_hydrostatic_equilibrium_operator.cppm index b7b2121..43746ba 100644 --- a/libmeanfield/interface/operators/prepared_hydrostatic_equilibrium_operator.cppm +++ b/libmeanfield/interface/operators/prepared_hydrostatic_equilibrium_operator.cppm @@ -138,6 +138,10 @@ export namespace mean_field::operators { mfem::Vector &action ) const; + // Exact diagonal of the volume enthalpy-to-hydrostatic block. Surface + // boundary-row replacement is deliberately applied by its owner. + void AssembleEnthalpyJacobianDiagonal(mfem::Vector &diagonal) const; + [[nodiscard]] bool IsPrepared() const noexcept; [[nodiscard]] const context::hydrostatic::HydrostaticPreparationStatistics & diff --git a/libmeanfield/interface/operators/stellar_equilibrium_problem.cppm b/libmeanfield/interface/operators/stellar_equilibrium_problem.cppm index 2737d6e..eb2c352 100644 --- a/libmeanfield/interface/operators/stellar_equilibrium_problem.cppm +++ b/libmeanfield/interface/operators/stellar_equilibrium_problem.cppm @@ -11,6 +11,7 @@ export module mean_field:operators.stellar_equilibrium_problem; export import :deformation.domain_deformation; export import :equilibrium.stellar_discretization; +export import :material.thermodynamic_equations; export import :model.typed_stellar; export import :operators.prepared_central_density_stellar_equilibrium; export import :surface.compiler; @@ -39,8 +40,25 @@ export namespace mean_field::equilibrium { hasFixedCentralDensity, operators::PreparedCentralDensityStellarEquilibriumOperator, operators::PreparedStellarEquilibriumOperator>; - using CompiledSurfaceConstraintType = - surface::CompiledPressureSurfaceConstraintT; + using FormType = std::conditional_t< + hasFixedCentralDensity, + operators::CentralDensityStellarEquilibriumForm, + utils::blocks::surface_deformed_stellar_equilibrium_form>; + using JacobianFormType = std::conditional_t< + hasFixedCentralDensity, + operators::CentralDensityStellarEquilibriumJacobianForm, + utils::blocks::surface_deformed_stellar_equilibrium_jacobian_form>; + using ManifestType = std::conditional_t< + hasFixedCentralDensity, + operators::CentralDensityStellarEquilibriumSystemManifest, + operators::StellarEquilibriumSystemManifest>; + using EquationOfStateType = eos::Polytrope; + using AvailableThermodynamicEquations = material::StellarEquilibriumThermodynamicEquations; + using ThermodynamicEquationsType = + material::CompiledThermodynamicEquationsT; + using CompiledSurfaceConstraintType = surface::CompiledPressureSurfaceConstraintT< + typename ThermodynamicEquationsType::PressureSurfaceFormulation, + EquationOfStateType>; StellarEquilibriumProblem( ModelType stellarModel, @@ -113,6 +131,26 @@ export namespace mean_field::equilibrium { return m_preparedOperator.GetRootManifest(); } + [[nodiscard]] bool IsPrepared() const noexcept { + return m_preparedOperator.IsPrepared(); + } + + [[nodiscard]] const operators::StellarEquilibriumDependencies &GetLinearizationDependencies() const { + if constexpr (hasFixedCentralDensity) { + return m_preparedOperator.GetPhysicalOperator().GetDependencies(); + } else { + return m_preparedOperator.GetDependencies(); + } + } + + [[nodiscard]] const operators::StellarEquilibriumDependencyStamp &GetGeometryDependency() const { + if constexpr (hasFixedCentralDensity) { + return m_preparedOperator.GetPhysicalOperator().GetGeneratedDisplacementDependency(); + } else { + return m_preparedOperator.GetGeneratedDisplacementDependency(); + } + } + [[nodiscard]] const field::FieldBoundaryDofMap &GetPressureSurfaceRows() const noexcept { if constexpr (hasFixedCentralDensity) { return m_preparedOperator.GetPhysicalOperator().GetSurfaceConstraintOperator().GetSurfaceRows(); @@ -154,9 +192,10 @@ export namespace mean_field::equilibrium { private: [[nodiscard]] static CompiledSurfaceConstraintType CompileSurfaceConstraint(const ModelType &stellarModel) { - return surface::compilePressureSurfaceConstraint( + return surface::compilePressureSurfaceConstraint< + typename ThermodynamicEquationsType::PressureSurfaceFormulation>( stellarModel.template specification(), - stellarModel.template specification() + stellarModel.template specification() ); } @@ -207,4 +246,12 @@ export namespace mean_field::equilibrium { ) { return discretize(std::forward(stellarModel), StellarDiscretization{finiteElementModel}); } + + template struct IsStellarEquilibriumProblem : std::false_type { }; + + template + struct IsStellarEquilibriumProblem> : std::true_type { }; + + template + concept DiscretizedStellarEquilibriumProblem = IsStellarEquilibriumProblem>::value; } // namespace mean_field::equilibrium diff --git a/libmeanfield/interface/preconditioning/backend.cppm b/libmeanfield/interface/preconditioning/backend.cppm new file mode 100644 index 0000000..5b5eced --- /dev/null +++ b/libmeanfield/interface/preconditioning/backend.cppm @@ -0,0 +1,332 @@ +module; + +#include +#include +#include +#include + +export module mean_field:preconditioning.backend; + +export namespace mean_field::preconditioning { + enum class OperatorCategory : std::uint8_t { + identity, + mass_like, + elliptic_like, + surface_like, + mixed, + dense_border + }; + + enum class OperatorValueStructure : std::uint8_t { scalar, vector, block }; + + enum class OperatorSymmetry : std::uint8_t { symmetric, nonsymmetric }; + + enum class OperatorDefiniteness : std::uint8_t { + positive_definite, + positive_semidefinite, + indefinite, + unspecified + }; + + enum class OperatorRepresentation : std::uint8_t { none, diagonal, matrix_free, assembled_sparse, assembled_dense }; + enum class OperatorDistribution : std::uint8_t { not_applicable, local, distributed_true_dof }; + enum class OperatorFESpace : std::uint8_t { not_applicable, h1, h_curl, h_div, l2, product }; + enum class OperatorNullspace : std::uint8_t { none, constant_mode, supplied_basis }; + enum class ApplicationContract : std::uint8_t { stationary_linear, flexible }; + enum class SymmetryRequirement : std::uint8_t { none, symmetric }; + enum class NullspaceRequirement : std::uint8_t { none, constant_mode_supported, supplied_basis_required }; + enum class SurrogateRequirement : std::uint8_t { none, diagonal, assembled_dense, assembled_sparse }; + + enum class PreparationDependency : std::uint8_t { + discretization = 1U << 0U, + geometry = 1U << 1U, + equation_of_state = 1U << 2U, + linearization = 1U << 3U + }; + + template struct PreparationDependencies final { + static constexpr std::uint8_t mask = (std::uint8_t{0} | ... | static_cast(Dependencies)); + + [[nodiscard]] static consteval bool contains(const PreparationDependency dependency) noexcept { + return (mask & static_cast(dependency)) != 0U; + } + }; + + template struct IsPreparationDependencies : std::false_type { }; + + template + struct IsPreparationDependencies> : std::true_type { }; + + template + concept PreparationDependenciesType = IsPreparationDependencies>::value; + + using NoPreparationDependencies = PreparationDependencies<>; + + template < + OperatorCategory Category, + OperatorValueStructure ValueStructure, + OperatorSymmetry Symmetry, + OperatorDefiniteness Definiteness, + OperatorRepresentation Representation, + OperatorDistribution Distribution, + OperatorFESpace FESpace = OperatorFESpace::not_applicable, + OperatorNullspace Nullspace = OperatorNullspace::none> + struct OperatorCharacteristics final { + static constexpr OperatorCategory category = Category; + static constexpr OperatorValueStructure valueStructure = ValueStructure; + static constexpr OperatorSymmetry symmetry = Symmetry; + static constexpr OperatorDefiniteness definiteness = Definiteness; + static constexpr OperatorRepresentation representation = Representation; + static constexpr OperatorDistribution distribution = Distribution; + static constexpr OperatorFESpace finiteElementSpace = FESpace; + static constexpr OperatorNullspace nullspace = Nullspace; + }; + + template struct IsOperatorCharacteristics : std::false_type { }; + + template < + OperatorCategory Category, + OperatorValueStructure ValueStructure, + OperatorSymmetry Symmetry, + OperatorDefiniteness Definiteness, + OperatorRepresentation Representation, + OperatorDistribution Distribution, + OperatorFESpace FESpace, + OperatorNullspace Nullspace> + struct IsOperatorCharacteristics> : std::true_type { }; + + template + concept OperatorCharacteristicsType = IsOperatorCharacteristics>::value; + + using IdentityOperatorCharacteristics = OperatorCharacteristics< + OperatorCategory::identity, + OperatorValueStructure::block, + OperatorSymmetry::symmetric, + OperatorDefiniteness::positive_definite, + OperatorRepresentation::none, + OperatorDistribution::not_applicable>; + + namespace backend { + struct FixedCycles final { + int cycles{1}; + }; + + struct SolveToTolerance final { + double relativeTolerance{1.0e-8}; + int maximumCycles{100}; + }; + + template struct ApplicationModeTraits { + static constexpr bool registered = false; + }; + + template <> struct ApplicationModeTraits { + static constexpr bool registered = true; + static constexpr ApplicationContract applicationContract = ApplicationContract::stationary_linear; + }; + + template <> struct ApplicationModeTraits { + static constexpr bool registered = true; + static constexpr ApplicationContract applicationContract = ApplicationContract::flexible; + }; + + template + concept ApplicationMode = ApplicationModeTraits>::registered; + + struct Identity final { }; + struct Diagonal final { }; + struct DenseDirect final { }; + + struct MatrixFreeChebyshev final { + int order{2}; + int powerIterations{10}; + double powerTolerance{1.0e-8}; + int powerSeed{12345}; + }; + + template struct HypreBoomerAMG final { + using ApplicationModeType = Mode; + + Mode application{}; + + constexpr HypreBoomerAMG() = default; + + constexpr explicit HypreBoomerAMG(Mode applicationMode) : application(std::move(applicationMode)) { + } + }; + + template HypreBoomerAMG(Mode) -> HypreBoomerAMG; + + template struct Traits { + static constexpr bool registered = false; + + using PreparationDependencies = NoPreparationDependencies; + }; + + template <> struct Traits { + static constexpr bool registered = true; + static constexpr ApplicationContract applicationContract = ApplicationContract::stationary_linear; + static constexpr bool supportsSerialExecution = true; + static constexpr bool supportsDistributedExecution = true; + static constexpr SymmetryRequirement symmetryRequirement = SymmetryRequirement::none; + static constexpr NullspaceRequirement nullspaceRequirement = NullspaceRequirement::none; + static constexpr SurrogateRequirement surrogateRequirement = SurrogateRequirement::none; + static constexpr bool requiresAssembledSparseSurrogate = false; + + using PreparationDependencies = NoPreparationDependencies; + + template + static constexpr bool supports = Characteristics::category == OperatorCategory::identity && + Characteristics::representation == OperatorRepresentation::none; + }; + + template <> struct Traits { + static constexpr bool registered = true; + static constexpr ApplicationContract applicationContract = ApplicationContract::stationary_linear; + static constexpr bool supportsSerialExecution = true; + static constexpr bool supportsDistributedExecution = true; + static constexpr SymmetryRequirement symmetryRequirement = SymmetryRequirement::symmetric; + static constexpr NullspaceRequirement nullspaceRequirement = NullspaceRequirement::none; + static constexpr SurrogateRequirement surrogateRequirement = SurrogateRequirement::diagonal; + static constexpr bool requiresAssembledSparseSurrogate = false; + + using PreparationDependencies = + preconditioning::PreparationDependencies; + + template + static constexpr bool supports = + (Characteristics::category == OperatorCategory::mass_like || + Characteristics::category == OperatorCategory::elliptic_like || + Characteristics::category == OperatorCategory::surface_like) && + Characteristics::symmetry == OperatorSymmetry::symmetric && + (Characteristics::representation == OperatorRepresentation::diagonal || + (Characteristics::category == OperatorCategory::mass_like && + Characteristics::representation == OperatorRepresentation::matrix_free)) && + Characteristics::nullspace == OperatorNullspace::none && + (Characteristics::distribution == OperatorDistribution::local || + Characteristics::distribution == OperatorDistribution::distributed_true_dof); + }; + + template <> struct Traits { + static constexpr bool registered = true; + static constexpr ApplicationContract applicationContract = ApplicationContract::stationary_linear; + static constexpr bool supportsSerialExecution = true; + static constexpr bool supportsDistributedExecution = true; + static constexpr SymmetryRequirement symmetryRequirement = SymmetryRequirement::symmetric; + static constexpr NullspaceRequirement nullspaceRequirement = NullspaceRequirement::none; + static constexpr SurrogateRequirement surrogateRequirement = SurrogateRequirement::diagonal; + static constexpr bool requiresAssembledSparseSurrogate = false; + + using PreparationDependencies = preconditioning::PreparationDependencies< + PreparationDependency::discretization, + PreparationDependency::geometry, + PreparationDependency::linearization>; + + template + static constexpr bool supports = + Characteristics::category == OperatorCategory::mass_like && + Characteristics::symmetry == OperatorSymmetry::symmetric && + Characteristics::definiteness == OperatorDefiniteness::positive_definite && + Characteristics::representation == OperatorRepresentation::matrix_free && + Characteristics::finiteElementSpace == OperatorFESpace::h_div && + Characteristics::nullspace == OperatorNullspace::none && + (Characteristics::distribution == OperatorDistribution::local || + Characteristics::distribution == OperatorDistribution::distributed_true_dof); + }; + + template <> struct Traits { + static constexpr bool registered = true; + static constexpr ApplicationContract applicationContract = ApplicationContract::stationary_linear; + static constexpr bool supportsSerialExecution = true; + static constexpr bool supportsDistributedExecution = false; + static constexpr SymmetryRequirement symmetryRequirement = SymmetryRequirement::none; + static constexpr NullspaceRequirement nullspaceRequirement = NullspaceRequirement::none; + static constexpr SurrogateRequirement surrogateRequirement = SurrogateRequirement::assembled_dense; + static constexpr bool requiresAssembledSparseSurrogate = false; + + using PreparationDependencies = + preconditioning::PreparationDependencies; + + template + static constexpr bool supports = + Characteristics::category == OperatorCategory::dense_border && + Characteristics::representation == OperatorRepresentation::assembled_dense && + Characteristics::distribution == OperatorDistribution::local && + Characteristics::nullspace == OperatorNullspace::none; + }; + + template struct Traits> { + static constexpr bool registered = true; + static constexpr ApplicationContract applicationContract = ApplicationModeTraits::applicationContract; + static constexpr bool supportsSerialExecution = false; + static constexpr bool supportsDistributedExecution = true; + static constexpr SymmetryRequirement symmetryRequirement = SymmetryRequirement::symmetric; + static constexpr NullspaceRequirement nullspaceRequirement = NullspaceRequirement::constant_mode_supported; + static constexpr SurrogateRequirement surrogateRequirement = SurrogateRequirement::assembled_sparse; + static constexpr bool requiresAssembledSparseSurrogate = true; + + using PreparationDependencies = preconditioning::PreparationDependencies< + PreparationDependency::discretization, + PreparationDependency::geometry, + PreparationDependency::equation_of_state, + PreparationDependency::linearization>; + + template + static constexpr bool supports = + Characteristics::category == OperatorCategory::elliptic_like && + Characteristics::valueStructure == OperatorValueStructure::scalar && + Characteristics::symmetry == OperatorSymmetry::symmetric && + (Characteristics::definiteness == OperatorDefiniteness::positive_definite || + Characteristics::definiteness == OperatorDefiniteness::positive_semidefinite) && + Characteristics::representation == OperatorRepresentation::assembled_sparse && + Characteristics::distribution == OperatorDistribution::distributed_true_dof && + (Characteristics::finiteElementSpace == OperatorFESpace::h1 || + Characteristics::finiteElementSpace == OperatorFESpace::l2) && + (Characteristics::nullspace == OperatorNullspace::none || + Characteristics::nullspace == OperatorNullspace::constant_mode); + }; + + template + concept Registered = Traits>::registered; + + namespace detail { + template < + typename Backend, + typename Characteristics, + bool = Registered && OperatorCharacteristicsType> + struct IsCompatible : std::false_type { }; + + template + struct IsCompatible + : std::bool_constant< + Traits>::template supports>> { + }; + } // namespace detail + + template + inline constexpr bool isCompatible = detail::IsCompatible::value; + + template + concept Compatible = isCompatible; + + template + inline constexpr ApplicationContract applicationContract = + Traits>::applicationContract; + + template + concept ArnoldiAdmissible = Registered && applicationContract> == + ApplicationContract::stationary_linear; + + template + inline constexpr bool requiresAssembledSparseSurrogate = + Traits>::requiresAssembledSparseSurrogate; + } // namespace backend +} // namespace mean_field::preconditioning diff --git a/libmeanfield/interface/preconditioning/backend_implementations.cppm b/libmeanfield/interface/preconditioning/backend_implementations.cppm new file mode 100644 index 0000000..9bf7ba7 --- /dev/null +++ b/libmeanfield/interface/preconditioning/backend_implementations.cppm @@ -0,0 +1,422 @@ +module; + +#include +#include +#include +#include +#include +#include +#include + +#include + +export module mean_field:preconditioning.backend_implementations; + +export import :preconditioning.backend; + +export namespace mean_field::preconditioning::backend { + struct BackendStatistics final { + std::uint64_t setups{0}; + std::uint64_t applications{0}; + std::uint64_t innerIterations{0}; + std::uint64_t lastInnerIterations{0}; + }; + + namespace detail { + inline void verifyApplicationDimensions( + const mfem::Solver &solver, + const mfem::Vector &rightHandSide, + const mfem::Vector &action + ) { + if (rightHandSide.Size() != solver.Width() || action.Size() != solver.Height()) { + throw std::invalid_argument( + "A prepared preconditioning backend requires compatible, preallocated input and output vectors." + ); + } + } + + inline void verifySquarePositiveSize( + const int height, + const int width + ) { + if (height <= 0 || height != width) { + throw std::invalid_argument("A preconditioning backend requires a positive square operator."); + } + } + + inline void configure( + mfem::HypreBoomerAMG &solver, + const FixedCycles &mode + ) { + if (mode.cycles <= 0) { + throw std::invalid_argument("Fixed-cycle AMG requires at least one cycle."); + } + solver.SetMaxIter(mode.cycles); + solver.SetTol(0.0); + solver.SetPrintLevel(0); + solver.iterative_mode = false; + } + + inline void configure( + mfem::HypreBoomerAMG &solver, + const SolveToTolerance &mode + ) { + if (!std::isfinite(mode.relativeTolerance) || mode.relativeTolerance <= 0.0 || + mode.relativeTolerance >= 1.0) { + throw std::invalid_argument("Tolerance-driven AMG requires a finite relative tolerance in (0, 1)."); + } + if (mode.maximumCycles <= 0) { + throw std::invalid_argument("Tolerance-driven AMG requires at least one permitted cycle."); + } + solver.SetMaxIter(mode.maximumCycles); + solver.SetTol(mode.relativeTolerance); + solver.SetPrintLevel(0); + solver.iterative_mode = false; + } + } // namespace detail + + class PreparedDiagonal final : public mfem::Solver { + public: + PreparedDiagonal( + Diagonal configuration, + const mfem::Vector &diagonal + ) + : mfem::Solver(diagonal.Size()), + m_configuration(std::move(configuration)) { + Refresh(diagonal); + } + + PreparedDiagonal(const PreparedDiagonal &) = delete; + PreparedDiagonal &operator=(const PreparedDiagonal &) = delete; + PreparedDiagonal(PreparedDiagonal &&) = delete; + PreparedDiagonal &operator=(PreparedDiagonal &&) = delete; + + void SetOperator(const mfem::Operator &operation) override { + if (operation.Height() != Height() || operation.Width() != Width()) { + throw std::invalid_argument("The diagonal backend received an operator with incompatible dimensions."); + } + } + + void Mult( + const mfem::Vector &rightHandSide, + mfem::Vector &action + ) const override { + detail::verifyApplicationDimensions(*this, rightHandSide, action); + for (int index = 0; index < Height(); ++index) { + action(index) = m_inverseDiagonal(index) * rightHandSide(index); + } + ++m_statistics.applications; + } + + void Refresh(const mfem::Vector &diagonal) { + if (diagonal.Size() <= 0 || diagonal.Size() != Height()) { + throw std::invalid_argument("The diagonal backend requires a positive diagonal of unchanged size."); + } + + m_inverseDiagonal.SetSize(diagonal.Size()); + for (int index = 0; index < diagonal.Size(); ++index) { + const double entry = diagonal(index); + if (!std::isfinite(entry) || entry == 0.0) { + throw std::invalid_argument("The diagonal backend cannot invert a zero or non-finite entry."); + } + m_inverseDiagonal(index) = 1.0 / entry; + } + ++m_statistics.setups; + } + + [[nodiscard]] const Diagonal &GetConfiguration() const noexcept { + return m_configuration; + } + + [[nodiscard]] const mfem::Vector &GetInverseDiagonal() const noexcept { + return m_inverseDiagonal; + } + + [[nodiscard]] const BackendStatistics &GetStatistics() const noexcept { + return m_statistics; + } + + private: + Diagonal m_configuration; + mfem::Vector m_inverseDiagonal; + mutable BackendStatistics m_statistics; + }; + + class PreparedMatrixFreeChebyshev final : public mfem::Solver { + public: + PreparedMatrixFreeChebyshev( + MatrixFreeChebyshev configuration, + const mfem::Operator &operation, + const MPI_Comm communicator + ) + : mfem::Solver(operation.Height()), + m_configuration(std::move(configuration)), + m_communicator(communicator) { + ValidateConfiguration(); + Refresh(operation); + } + + PreparedMatrixFreeChebyshev(const PreparedMatrixFreeChebyshev &) = delete; + PreparedMatrixFreeChebyshev &operator=(const PreparedMatrixFreeChebyshev &) = delete; + PreparedMatrixFreeChebyshev(PreparedMatrixFreeChebyshev &&) = delete; + PreparedMatrixFreeChebyshev &operator=(PreparedMatrixFreeChebyshev &&) = delete; + + void SetOperator(const mfem::Operator &operation) override { + Refresh(operation); + } + + void Mult( + const mfem::Vector &rightHandSide, + mfem::Vector &action + ) const override { + detail::verifyApplicationDimensions(*this, rightHandSide, action); + m_smoother->Mult(rightHandSide, action); + ++m_statistics.applications; + m_statistics.lastInnerIterations = static_cast(m_configuration.order); + m_statistics.innerIterations += static_cast(m_configuration.order); + } + + void Refresh(const mfem::Operator &operation) { + detail::verifySquarePositiveSize(operation.Height(), operation.Width()); + if (operation.Height() != Height()) { + throw std::invalid_argument("The matrix-free Chebyshev backend cannot change size during refresh."); + } + + operation.AssembleDiagonal(m_diagonal); + if (m_diagonal.Size() != Height()) { + throw std::invalid_argument( + "The matrix-free Chebyshev backend received an incompatible assembled diagonal." + ); + } + for (int index = 0; index < m_diagonal.Size(); ++index) { + if (!std::isfinite(m_diagonal(index)) || m_diagonal(index) <= 0.0) { + throw std::invalid_argument( + "The matrix-free Chebyshev backend requires a finite, strictly positive diagonal." + ); + } + } + + m_operation = std::addressof(operation); + m_essentialTrueDofs.SetSize(0); + m_smoother = std::make_unique( + operation, m_diagonal, m_essentialTrueDofs, m_configuration.order, m_communicator, + m_configuration.powerIterations, m_configuration.powerTolerance, m_configuration.powerSeed + ); + m_smoother->iterative_mode = false; + ++m_statistics.setups; + } + + [[nodiscard]] const MatrixFreeChebyshev &GetConfiguration() const noexcept { + return m_configuration; + } + + [[nodiscard]] const mfem::Operator &GetOperator() const noexcept { + return *m_operation; + } + + [[nodiscard]] const mfem::Vector &GetDiagonal() const noexcept { + return m_diagonal; + } + + [[nodiscard]] const BackendStatistics &GetStatistics() const noexcept { + return m_statistics; + } + + private: + void ValidateConfiguration() const { + if (m_configuration.order <= 0 || m_configuration.order > 5) { + throw std::invalid_argument("Matrix-free Chebyshev requires a polynomial order in [1, 5]."); + } + if (m_configuration.powerIterations <= 0) { + throw std::invalid_argument("Matrix-free Chebyshev requires at least one power iteration."); + } + if (!std::isfinite(m_configuration.powerTolerance) || m_configuration.powerTolerance <= 0.0 || + m_configuration.powerTolerance >= 1.0) { + throw std::invalid_argument( + "Matrix-free Chebyshev requires a finite power-method tolerance strictly between zero and one." + ); + } + if (m_configuration.powerSeed <= 0) { + throw std::invalid_argument("Matrix-free Chebyshev requires a strictly positive power-method seed."); + } + } + + MatrixFreeChebyshev m_configuration; + MPI_Comm m_communicator; + const mfem::Operator *m_operation{nullptr}; + mfem::Vector m_diagonal; + mfem::Array m_essentialTrueDofs; + std::unique_ptr m_smoother; + mutable BackendStatistics m_statistics; + }; + + class PreparedDenseDirect final : public mfem::Solver { + public: + PreparedDenseDirect( + DenseDirect configuration, + const mfem::DenseMatrix &matrix + ) + : mfem::Solver(matrix.Height()), + m_configuration(std::move(configuration)) { + Refresh(matrix); + } + + PreparedDenseDirect(const PreparedDenseDirect &) = delete; + PreparedDenseDirect &operator=(const PreparedDenseDirect &) = delete; + PreparedDenseDirect(PreparedDenseDirect &&) = delete; + PreparedDenseDirect &operator=(PreparedDenseDirect &&) = delete; + + void SetOperator(const mfem::Operator &operation) override { + const auto *matrix = dynamic_cast(&operation); + if (matrix == nullptr) { + throw std::invalid_argument("The dense-direct backend requires an mfem::DenseMatrix."); + } + Refresh(*matrix); + } + + void Mult( + const mfem::Vector &rightHandSide, + mfem::Vector &action + ) const override { + detail::verifyApplicationDimensions(*this, rightHandSide, action); + m_inverse->Mult(rightHandSide, action); + ++m_statistics.applications; + } + + void Refresh(const mfem::DenseMatrix &matrix) { + detail::verifySquarePositiveSize(matrix.Height(), matrix.Width()); + if (matrix.Height() != Height()) { + throw std::invalid_argument("The dense-direct backend cannot change size during refresh."); + } + + m_matrix = matrix; + m_inverse = std::make_unique(m_matrix); + ++m_statistics.setups; + } + + [[nodiscard]] const DenseDirect &GetConfiguration() const noexcept { + return m_configuration; + } + + [[nodiscard]] const mfem::DenseMatrix &GetDenseSurrogate() const noexcept { + return m_matrix; + } + + [[nodiscard]] const BackendStatistics &GetStatistics() const noexcept { + return m_statistics; + } + + private: + DenseDirect m_configuration; + mfem::DenseMatrix m_matrix; + std::unique_ptr m_inverse; + mutable BackendStatistics m_statistics; + }; + + template class PreparedHypreBoomerAMG final : public mfem::Solver { + public: + using Configuration = HypreBoomerAMG; + + PreparedHypreBoomerAMG( + Configuration configuration, + const mfem::HypreParMatrix &matrix + ) + : mfem::Solver(matrix.Height()), + m_configuration(std::move(configuration)) { + Refresh(matrix); + } + + PreparedHypreBoomerAMG(const PreparedHypreBoomerAMG &) = delete; + PreparedHypreBoomerAMG &operator=(const PreparedHypreBoomerAMG &) = delete; + PreparedHypreBoomerAMG(PreparedHypreBoomerAMG &&) = delete; + PreparedHypreBoomerAMG &operator=(PreparedHypreBoomerAMG &&) = delete; + + void SetOperator(const mfem::Operator &operation) override { + const auto *matrix = dynamic_cast(&operation); + if (matrix == nullptr) { + throw std::invalid_argument("The BoomerAMG backend requires an mfem::HypreParMatrix surrogate."); + } + Refresh(*matrix); + } + + void Mult( + const mfem::Vector &rightHandSide, + mfem::Vector &action + ) const override { + detail::verifyApplicationDimensions(*this, rightHandSide, action); + m_solver->Mult(rightHandSide, action); + + int iterations = 0; + m_solver->GetNumIterations(iterations); + ++m_statistics.applications; + m_statistics.lastInnerIterations = static_cast(iterations); + m_statistics.innerIterations += static_cast(iterations); + } + + void Refresh(const mfem::HypreParMatrix &matrix) { + detail::verifySquarePositiveSize(matrix.Height(), matrix.Width()); + if (matrix.Height() != Height()) { + throw std::invalid_argument("The BoomerAMG backend cannot change size during refresh."); + } + + m_sparseSurrogate = std::addressof(matrix); + m_solver = std::make_unique(matrix); + detail::configure(*m_solver, m_configuration.application); + + mfem::Vector setupRightHandSide(Width()); + mfem::Vector setupAction(Height()); + setupRightHandSide = 0.0; + setupAction = 0.0; + m_solver->Setup(setupRightHandSide, setupAction); + ++m_statistics.setups; + } + + [[nodiscard]] const Configuration &GetConfiguration() const noexcept { + return m_configuration; + } + + [[nodiscard]] const mfem::HypreParMatrix &GetSparseSurrogate() const noexcept { + return *m_sparseSurrogate; + } + + [[nodiscard]] const BackendStatistics &GetStatistics() const noexcept { + return m_statistics; + } + + private: + Configuration m_configuration; + const mfem::HypreParMatrix *m_sparseSurrogate{nullptr}; + std::unique_ptr m_solver; + mutable BackendStatistics m_statistics; + }; + + [[nodiscard]] inline PreparedDiagonal prepare( + Diagonal configuration, + const mfem::Vector &diagonal + ) { + return PreparedDiagonal{std::move(configuration), diagonal}; + } + + [[nodiscard]] inline PreparedMatrixFreeChebyshev prepare( + MatrixFreeChebyshev configuration, + const mfem::Operator &operation, + const MPI_Comm communicator + ) { + return PreparedMatrixFreeChebyshev{std::move(configuration), operation, communicator}; + } + + [[nodiscard]] inline PreparedDenseDirect prepare( + DenseDirect configuration, + const mfem::DenseMatrix &matrix + ) { + return PreparedDenseDirect{std::move(configuration), matrix}; + } + + template + [[nodiscard]] PreparedHypreBoomerAMG prepare( + HypreBoomerAMG configuration, + const mfem::HypreParMatrix &matrix + ) { + return PreparedHypreBoomerAMG{std::move(configuration), matrix}; + } +} // namespace mean_field::preconditioning::backend diff --git a/libmeanfield/interface/preconditioning/equilibrium_coordinates.cppm b/libmeanfield/interface/preconditioning/equilibrium_coordinates.cppm new file mode 100644 index 0000000..2fabd7b --- /dev/null +++ b/libmeanfield/interface/preconditioning/equilibrium_coordinates.cppm @@ -0,0 +1,402 @@ +module; + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +export module mean_field:preconditioning.equilibrium_coordinates; + +export import :preconditioning.specification_border; + +export namespace mean_field::preconditioning { + namespace detail { + template struct EquilibriumCoordinateListIsSubset; + + template + struct EquilibriumCoordinateListIsSubset, Universe> + : std::bool_constant<(utils::blocks::contains_type_v && ...)> { }; + } // namespace detail + + template + concept EquilibriumCoordinateComponentFor = + PreconditionerComponent && utils::blocks::block_form_is_valid_v && + std::remove_cvref_t::CorrectionBlocks::size == Form::value_block_count && + std::remove_cvref_t::ResidualBlocks::size == Form::residual_block_count && + detail::EquilibriumCoordinateListIsSubset< + typename std::remove_cvref_t::CorrectionBlocks, + typename Form::value_blocks>::value && + detail::EquilibriumCoordinateListIsSubset< + typename std::remove_cvref_t::ResidualBlocks, + typename Form::residual_blocks>::value; + + struct EquilibriumCoordinateRange final { + int equilibriumOffset{0}; + int preconditionerOffset{0}; + int size{0}; + + constexpr bool operator==(const EquilibriumCoordinateRange &) const = default; + }; + + struct EquilibriumCoordinateMapStatistics final { + std::uint64_t residualPacks{0}; + std::uint64_t residualUnpacks{0}; + std::uint64_t correctionPacks{0}; + std::uint64_t correctionUnpacks{0}; + }; + + template + requires EquilibriumCoordinateComponentFor + class EquilibriumPreconditionerCoordinateMap final { + private: + using ComponentType = std::remove_cvref_t; + using Layout = utils::blocks::form_layout; + + static constexpr std::size_t correctionBlockCount = ComponentType::CorrectionBlocks::size; + static constexpr std::size_t residualBlockCount = ComponentType::ResidualBlocks::size; + + public: + explicit EquilibriumPreconditionerCoordinateMap(const Layout &layout) + : m_correctionRanges(MakeCorrectionRanges( + layout, + typename ComponentType::CorrectionBlocks{} + )), + m_residualRanges(MakeResidualRanges( + layout, + typename ComponentType::ResidualBlocks{} + )), + m_equilibriumStateSize(layout.value_offsets().Last()), + m_equilibriumResidualSize(layout.residual_offsets().Last()), + m_preconditionerCorrectionSize(TotalSize(m_correctionRanges)), + m_preconditionerResidualSize(TotalSize(m_residualRanges)) { + if (m_preconditionerCorrectionSize != m_equilibriumStateSize || + m_preconditionerResidualSize != m_equilibriumResidualSize) { + throw std::logic_error( + "The typed preconditioner coordinate map does not span the complete equilibrium operator." + ); + } + } + + void PackResidual( + const mfem::Vector &equilibriumResidual, + mfem::Vector &preconditionerResidual + ) const { + VerifySizes( + equilibriumResidual, m_equilibriumResidualSize, preconditionerResidual, m_preconditionerResidualSize, + "residual pack" + ); + EquilibriumToPreconditioner(equilibriumResidual, preconditionerResidual, m_residualRanges); + ++m_statistics.residualPacks; + } + + void UnpackResidual( + const mfem::Vector &preconditionerResidual, + mfem::Vector &equilibriumResidual + ) const { + VerifySizes( + preconditionerResidual, m_preconditionerResidualSize, equilibriumResidual, m_equilibriumResidualSize, + "residual unpack" + ); + PreconditionerToEquilibrium(preconditionerResidual, equilibriumResidual, m_residualRanges); + ++m_statistics.residualUnpacks; + } + + void PackCorrection( + const mfem::Vector &equilibriumCorrection, + mfem::Vector &preconditionerCorrection + ) const { + VerifySizes( + equilibriumCorrection, m_equilibriumStateSize, preconditionerCorrection, m_preconditionerCorrectionSize, + "correction pack" + ); + EquilibriumToPreconditioner(equilibriumCorrection, preconditionerCorrection, m_correctionRanges); + ++m_statistics.correctionPacks; + } + + void UnpackCorrection( + const mfem::Vector &preconditionerCorrection, + mfem::Vector &equilibriumCorrection + ) const { + VerifySizes( + preconditionerCorrection, m_preconditionerCorrectionSize, equilibriumCorrection, m_equilibriumStateSize, + "correction unpack" + ); + PreconditionerToEquilibrium(preconditionerCorrection, equilibriumCorrection, m_correctionRanges); + ++m_statistics.correctionUnpacks; + } + + [[nodiscard]] int EquilibriumStateSize() const noexcept { + return m_equilibriumStateSize; + } + + [[nodiscard]] int EquilibriumResidualSize() const noexcept { + return m_equilibriumResidualSize; + } + + [[nodiscard]] int PreconditionerCorrectionSize() const noexcept { + return m_preconditionerCorrectionSize; + } + + [[nodiscard]] int PreconditionerResidualSize() const noexcept { + return m_preconditionerResidualSize; + } + + [[nodiscard]] const std::array< + EquilibriumCoordinateRange, + correctionBlockCount> & + GetCorrectionRanges() const noexcept { + return m_correctionRanges; + } + + [[nodiscard]] const std::array< + EquilibriumCoordinateRange, + residualBlockCount> & + GetResidualRanges() const noexcept { + return m_residualRanges; + } + + [[nodiscard]] const EquilibriumCoordinateMapStatistics &GetStatistics() const noexcept { + return m_statistics; + } + + private: + template + [[nodiscard]] static std::array< + EquilibriumCoordinateRange, + sizeof...(Blocks)> + MakeCorrectionRanges( + const Layout &layout, + utils::blocks::type_list + ) { + std::array ranges{}; + int preconditionerOffset = 0; + std::size_t range = 0; + ( + [&] { + constexpr int equilibriumBlock = utils::blocks::type_index_v; + const int size = layout.size(utils::blocks::value_block{}); + ranges[range++] = { + .equilibriumOffset = layout.offset(utils::blocks::value_block{}), + .preconditionerOffset = preconditionerOffset, + .size = size + }; + preconditionerOffset += size; + }(), + ...); + return ranges; + } + + template + [[nodiscard]] static std::array< + EquilibriumCoordinateRange, + sizeof...(Blocks)> + MakeResidualRanges( + const Layout &layout, + utils::blocks::type_list + ) { + std::array ranges{}; + int preconditionerOffset = 0; + std::size_t range = 0; + ( + [&] { + constexpr int equilibriumBlock = + utils::blocks::type_index_v; + const int size = layout.size(utils::blocks::residual_block{}); + ranges[range++] = { + .equilibriumOffset = layout.offset(utils::blocks::residual_block{}), + .preconditionerOffset = preconditionerOffset, + .size = size + }; + preconditionerOffset += size; + }(), + ...); + return ranges; + } + + template + [[nodiscard]] static int TotalSize( + const std::array< + EquilibriumCoordinateRange, + Size> &ranges + ) noexcept { + int size = 0; + for (const auto &range : ranges) { + size += range.size; + } + return size; + } + + template + static void EquilibriumToPreconditioner( + const mfem::Vector &equilibrium, + mfem::Vector &preconditioner, + const std::array< + EquilibriumCoordinateRange, + Size> &ranges + ) { + for (const auto &range : ranges) { + for (int index = 0; index < range.size; ++index) { + preconditioner(range.preconditionerOffset + index) = equilibrium(range.equilibriumOffset + index); + } + } + } + + template + static void PreconditionerToEquilibrium( + const mfem::Vector &preconditioner, + mfem::Vector &equilibrium, + const std::array< + EquilibriumCoordinateRange, + Size> &ranges + ) { + for (const auto &range : ranges) { + for (int index = 0; index < range.size; ++index) { + equilibrium(range.equilibriumOffset + index) = preconditioner(range.preconditionerOffset + index); + } + } + } + + static void VerifySizes( + const mfem::Vector &source, + const int expectedSourceSize, + const mfem::Vector &destination, + const int expectedDestinationSize, + const char *operation + ) { + if (source.Size() != expectedSourceSize || destination.Size() != expectedDestinationSize) { + throw std::invalid_argument( + std::string("The equilibrium preconditioner ") + operation + " received an incompatible vector." + ); + } + } + + std::array m_correctionRanges; + std::array m_residualRanges; + int m_equilibriumStateSize; + int m_equilibriumResidualSize; + int m_preconditionerCorrectionSize; + int m_preconditionerResidualSize; + mutable EquilibriumCoordinateMapStatistics m_statistics; + }; + + struct PreparedStellarPreconditionerStatistics final { + std::uint64_t applications{0}; + std::uint64_t residualCoordinateMappings{0}; + std::uint64_t correctionCoordinateMappings{0}; + }; + + template + requires EquilibriumCoordinateComponentFor::FormType> + class PreparedStellarPreconditioner final : public mfem::Solver { + private: + using ProblemType = std::remove_cvref_t; + using BlockType = std::remove_cvref_t; + + public: + using Form = typename ProblemType::FormType; + using BackendType = typename BlockType::BackendType; + using GroupedPreconditioner = PreparedSpecificationBorderBlock; + using CoordinateMap = EquilibriumPreconditionerCoordinateMap; + + PreparedStellarPreconditioner( + const ProblemType &problem, + BlockType block + ) + : mfem::Solver(problem.StateSize()), + m_grouped( + problem, + std::move(block) + ), + m_coordinates(problem.GetManifest().layout()), + m_groupedResidual(m_coordinates.PreconditionerResidualSize()), + m_groupedCorrection(m_coordinates.PreconditionerCorrectionSize()) { + if (problem.StateSize() != problem.EquationSize() || m_grouped.Height() != problem.StateSize() || + m_grouped.Width() != problem.EquationSize()) { + throw std::logic_error( + "The prepared stellar preconditioner is incompatible with the complete equilibrium operator." + ); + } + } + + PreparedStellarPreconditioner(const PreparedStellarPreconditioner &) = delete; + PreparedStellarPreconditioner &operator=(const PreparedStellarPreconditioner &) = delete; + PreparedStellarPreconditioner(PreparedStellarPreconditioner &&) = delete; + PreparedStellarPreconditioner &operator=(PreparedStellarPreconditioner &&) = delete; + + void SetOperator(const mfem::Operator &operation) override { + if (operation.Height() != Height() || operation.Width() != Width()) { + throw std::invalid_argument( + "The prepared stellar preconditioner received an incompatible equilibrium operator." + ); + } + m_grouped.SetOperator(operation); + } + + void Mult( + const mfem::Vector &equilibriumResidual, + mfem::Vector &equilibriumCorrection + ) const override { + if (equilibriumResidual.Size() != Width() || equilibriumCorrection.Size() != Height()) { + throw std::invalid_argument( + "The prepared stellar preconditioner requires compatible, preallocated equilibrium vectors." + ); + } + m_coordinates.PackResidual(equilibriumResidual, m_groupedResidual); + ++m_statistics.residualCoordinateMappings; + m_grouped.Mult(m_groupedResidual, m_groupedCorrection); + m_coordinates.UnpackCorrection(m_groupedCorrection, equilibriumCorrection); + ++m_statistics.correctionCoordinateMappings; + ++m_statistics.applications; + } + + [[nodiscard]] SpecificationBorderBlockPreparationReport Refresh() { + return m_grouped.Refresh(); + } + + [[nodiscard]] bool IsCurrent() const { + return m_grouped.IsCurrent(); + } + + [[nodiscard]] const BlockType &GetBlock() const noexcept { + return m_grouped.GetBlock(); + } + + [[nodiscard]] const GroupedPreconditioner &GetGroupedPreconditioner() const noexcept { + return m_grouped; + } + + [[nodiscard]] const CoordinateMap &GetCoordinateMap() const noexcept { + return m_coordinates; + } + + [[nodiscard]] const PreparedStellarPreconditionerStatistics &GetStatistics() const noexcept { + return m_statistics; + } + + private: + GroupedPreconditioner m_grouped; + CoordinateMap m_coordinates; + mutable mfem::Vector m_groupedResidual; + mutable mfem::Vector m_groupedCorrection; + mutable PreparedStellarPreconditionerStatistics m_statistics; + }; + + template < + equilibrium::DiscretizedStellarEquilibriumProblem Problem, + SpecificationBorderBlockType Block> + requires EquilibriumCoordinateComponentFor< + Block, + typename std::remove_cvref_t::FormType> + [[nodiscard]] auto prepare( + const Problem &problem, + Block block + ) { + return PreparedStellarPreconditioner{problem, std::move(block)}; + } +} // namespace mean_field::preconditioning diff --git a/libmeanfield/interface/preconditioning/gravity_field.cppm b/libmeanfield/interface/preconditioning/gravity_field.cppm new file mode 100644 index 0000000..35c7555 --- /dev/null +++ b/libmeanfield/interface/preconditioning/gravity_field.cppm @@ -0,0 +1,676 @@ +module; + +#include +#include +#include +#include +#include +#include +#include + +#include + +export module mean_field:preconditioning.gravity_field; + +export import :fem; +export import :operators.context.gravity_field; +export import :preconditioning.backend_implementations; +export import :preconditioning.plan; + +export namespace mean_field::preconditioning { + struct GravityBlockDiagonal final { }; + struct GravityLowerTriangular final { }; + struct GravityUpperTriangular final { }; + struct GravityApproximateLDU final { }; + + template struct IsGravityFactorizationPolicy : std::false_type { }; + + template <> struct IsGravityFactorizationPolicy : std::true_type { }; + template <> struct IsGravityFactorizationPolicy : std::true_type { }; + template <> struct IsGravityFactorizationPolicy : std::true_type { }; + template <> struct IsGravityFactorizationPolicy : std::true_type { }; + + template + concept GravityFactorizationPolicy = IsGravityFactorizationPolicy>::value; + + using GravityMassInverseCharacteristics = OperatorCharacteristics< + OperatorCategory::mass_like, + OperatorValueStructure::vector, + OperatorSymmetry::symmetric, + OperatorDefiniteness::positive_definite, + OperatorRepresentation::matrix_free, + OperatorDistribution::distributed_true_dof, + OperatorFESpace::h_div>; + + using GravityPotentialSchurCharacteristics = OperatorCharacteristics< + OperatorCategory::elliptic_like, + OperatorValueStructure::scalar, + OperatorSymmetry::symmetric, + OperatorDefiniteness::positive_semidefinite, + OperatorRepresentation::assembled_sparse, + OperatorDistribution::distributed_true_dof, + OperatorFESpace::l2, + OperatorNullspace::constant_mode>; + + using CoupledGravityCharacteristics = OperatorCharacteristics< + OperatorCategory::mixed, + OperatorValueStructure::block, + OperatorSymmetry::symmetric, + OperatorDefiniteness::indefinite, + OperatorRepresentation::matrix_free, + OperatorDistribution::distributed_true_dof, + OperatorFESpace::product>; + + namespace backend { + template + requires Compatible && + Compatible + struct CoupledGravity final { + using MassBackendType = MassInverseBackend; + using PotentialSchurBackendType = PotentialSchurBackend; + using FactorizationPolicyType = Policy; + }; + + template + requires Compatible && + Compatible + struct Traits> { + static constexpr bool registered = true; + static constexpr ApplicationContract applicationContract = + ::mean_field::preconditioning::backend::applicationContract == + ApplicationContract::stationary_linear && + ::mean_field::preconditioning::backend::applicationContract == + ApplicationContract::stationary_linear + ? ApplicationContract::stationary_linear + : ApplicationContract::flexible; + static constexpr bool supportsSerialExecution = false; + static constexpr bool supportsDistributedExecution = true; + static constexpr SymmetryRequirement symmetryRequirement = SymmetryRequirement::symmetric; + static constexpr NullspaceRequirement nullspaceRequirement = NullspaceRequirement::constant_mode_supported; + static constexpr SurrogateRequirement surrogateRequirement = SurrogateRequirement::assembled_sparse; + static constexpr bool requiresAssembledSparseSurrogate = true; + + using PreparationDependencies = preconditioning::PreparationDependencies< + PreparationDependency::discretization, + PreparationDependency::geometry, + PreparationDependency::equation_of_state, + PreparationDependency::linearization>; + + template + static constexpr bool supports = + Characteristics::category == OperatorCategory::mixed && + Characteristics::valueStructure == OperatorValueStructure::block && + Characteristics::symmetry == OperatorSymmetry::symmetric && + Characteristics::definiteness == OperatorDefiniteness::indefinite && + Characteristics::representation == OperatorRepresentation::matrix_free && + Characteristics::distribution == OperatorDistribution::distributed_true_dof && + Characteristics::finiteElementSpace == OperatorFESpace::product; + }; + } // namespace backend + + template < + backend::Registered MassBackendT, + backend::Registered PotentialSchurBackendT, + GravityFactorizationPolicy FactorizationPolicyT> + requires backend::Compatible && + backend::Compatible + class GravityFieldBlock final { + public: + using CorrectionBlocks = + utils::blocks::type_list; + using ResidualBlocks = utils::blocks:: + type_list; + using RequiredCouplings = utils::blocks::type_list< + Coupling, + Coupling, + Coupling>; + using OperatorDescription = CoupledGravityCharacteristics; + using BackendType = backend::CoupledGravity; + using PreparationDependencies = typename backend::Traits::PreparationDependencies; + using MassBackend = MassBackendT; + using PotentialSchurBackend = PotentialSchurBackendT; + using Factorization = FactorizationPolicyT; + + constexpr GravityFieldBlock( + MassBackendT massInverseBackend = {}, + PotentialSchurBackendT potentialSchurBackend = {}, + FactorizationPolicyT factorizationPolicy = {} + ) + : m_massInverseBackend(std::move(massInverseBackend)), + m_potentialSchurBackend(std::move(potentialSchurBackend)), + m_factorizationPolicy(std::move(factorizationPolicy)) { + } + + [[nodiscard]] constexpr const MassBackendT &massInverseBackend() const noexcept { + return m_massInverseBackend; + } + + [[nodiscard]] constexpr const PotentialSchurBackendT &potentialSchurBackend() const noexcept { + return m_potentialSchurBackend; + } + + [[nodiscard]] constexpr const FactorizationPolicyT &factorizationPolicy() const noexcept { + return m_factorizationPolicy; + } + + private: + MassBackendT m_massInverseBackend; + PotentialSchurBackendT m_potentialSchurBackend; + FactorizationPolicyT m_factorizationPolicy; + }; + + template < + typename MassInverseBackend, + typename PotentialSchurBackend, + typename FactorizationPolicy> + GravityFieldBlock( + MassInverseBackend, + PotentialSchurBackend, + FactorizationPolicy + ) + -> GravityFieldBlock< + MassInverseBackend, + PotentialSchurBackend, + FactorizationPolicy>; + + struct GravityFactorizationStatistics final { + std::uint64_t applications{0}; + std::uint64_t massInverseApplications{0}; + std::uint64_t potentialSchurApplications{0}; + std::uint64_t divergenceApplications{0}; + std::uint64_t transposeDivergenceApplications{0}; + }; + + template class GravityFactorizationOperator final : public mfem::Solver { + public: + GravityFactorizationOperator( + Policy policy, + const mfem::Solver &massInverse, + const mfem::Solver &potentialSchurInverse, + const mfem::Operator &divergence + ) + : mfem::Solver(massInverse.Height() + potentialSchurInverse.Height()), + m_policy(std::move(policy)), + m_massInverse(std::addressof(massInverse)), + m_potentialSchurInverse(std::addressof(potentialSchurInverse)), + m_divergence(std::addressof(divergence)), + m_offsets(3), + m_potentialWorkspace(potentialSchurInverse.Height()), + m_gradientWorkspace(massInverse.Height()), + m_massCorrection(massInverse.Height()) { + if (massInverse.Height() <= 0 || massInverse.Height() != massInverse.Width()) { + throw std::invalid_argument("The gravity factorization requires a square gradient-mass inverse."); + } + if (potentialSchurInverse.Height() <= 0 || + potentialSchurInverse.Height() != potentialSchurInverse.Width()) { + throw std::invalid_argument("The gravity factorization requires a square potential-Schur inverse."); + } + if (divergence.Width() != massInverse.Width() || divergence.Height() != potentialSchurInverse.Width()) { + throw std::invalid_argument("The gravity divergence does not connect the supplied inverse blocks."); + } + + m_offsets[0] = 0; + m_offsets[1] = massInverse.Height(); + m_offsets[2] = Height(); + } + + GravityFactorizationOperator(const GravityFactorizationOperator &) = delete; + GravityFactorizationOperator &operator=(const GravityFactorizationOperator &) = delete; + GravityFactorizationOperator(GravityFactorizationOperator &&) = delete; + GravityFactorizationOperator &operator=(GravityFactorizationOperator &&) = delete; + + void SetOperator(const mfem::Operator &operation) override { + if (operation.Height() != Height() || operation.Width() != Width()) { + throw std::invalid_argument("The gravity factorization received an operator of incompatible size."); + } + } + + void Mult( + const mfem::Vector &rightHandSide, + mfem::Vector &action + ) const override { + if (rightHandSide.Size() != Width() || action.Size() != Height()) { + throw std::invalid_argument( + "The gravity factorization requires compatible, preallocated input and output vectors." + ); + } + + const mfem::Vector gradientRightHandSide( + const_cast(rightHandSide.GetData()) + m_offsets[0], m_offsets[1] - m_offsets[0] + ); + const mfem::Vector potentialRightHandSide( + const_cast(rightHandSide.GetData()) + m_offsets[1], m_offsets[2] - m_offsets[1] + ); + mfem::Vector gradientAction(action.GetData() + m_offsets[0], m_offsets[1] - m_offsets[0]); + mfem::Vector potentialAction(action.GetData() + m_offsets[1], m_offsets[2] - m_offsets[1]); + + if constexpr (std::same_as) { + m_massInverse->Mult(gradientRightHandSide, gradientAction); + m_potentialSchurInverse->Mult(potentialRightHandSide, potentialAction); + ++m_statistics.massInverseApplications; + ++m_statistics.potentialSchurApplications; + } else if constexpr (std::same_as) { + m_massInverse->Mult(gradientRightHandSide, gradientAction); + m_divergence->Mult(gradientAction, m_potentialWorkspace); + m_potentialWorkspace -= potentialRightHandSide; + m_potentialSchurInverse->Mult(m_potentialWorkspace, potentialAction); + ++m_statistics.massInverseApplications; + ++m_statistics.divergenceApplications; + ++m_statistics.potentialSchurApplications; + } else if constexpr (std::same_as) { + m_potentialWorkspace = potentialRightHandSide; + m_potentialWorkspace *= -1.0; + m_potentialSchurInverse->Mult(m_potentialWorkspace, potentialAction); + m_divergence->MultTranspose(potentialAction, m_gradientWorkspace); + m_gradientWorkspace *= -1.0; + m_gradientWorkspace += gradientRightHandSide; + m_massInverse->Mult(m_gradientWorkspace, gradientAction); + ++m_statistics.potentialSchurApplications; + ++m_statistics.transposeDivergenceApplications; + ++m_statistics.massInverseApplications; + } else { + static_assert(std::same_as); + m_massInverse->Mult(gradientRightHandSide, gradientAction); + m_divergence->Mult(gradientAction, m_potentialWorkspace); + m_potentialWorkspace -= potentialRightHandSide; + m_potentialSchurInverse->Mult(m_potentialWorkspace, potentialAction); + m_divergence->MultTranspose(potentialAction, m_gradientWorkspace); + m_massInverse->Mult(m_gradientWorkspace, m_massCorrection); + gradientAction -= m_massCorrection; + m_statistics.massInverseApplications += 2; + ++m_statistics.divergenceApplications; + ++m_statistics.potentialSchurApplications; + ++m_statistics.transposeDivergenceApplications; + } + + ++m_statistics.applications; + } + + [[nodiscard]] const mfem::Array &GetOffsets() const noexcept { + return m_offsets; + } + + [[nodiscard]] const GravityFactorizationStatistics &GetStatistics() const noexcept { + return m_statistics; + } + + private: + Policy m_policy; + const mfem::Solver *m_massInverse; + const mfem::Solver *m_potentialSchurInverse; + const mfem::Operator *m_divergence; + mfem::Array m_offsets; + mutable mfem::Vector m_potentialWorkspace; + mutable mfem::Vector m_gradientWorkspace; + mutable mfem::Vector m_massCorrection; + mutable GravityFactorizationStatistics m_statistics; + }; + + class ReducedGravityDivergenceOperator final : public mfem::Operator { + public: + ReducedGravityDivergenceOperator( + const mfem::Operator &trueDofDivergence, + field::FieldDofMap gradientMap, + field::FieldDofMap potentialMap + ) + : mfem::Operator( + potentialMap.reduced_size(), + gradientMap.reduced_size() + ), + m_trueDofDivergence(std::addressof(trueDofDivergence)), + m_gradientMap(std::move(gradientMap)), + m_potentialMap(std::move(potentialMap)), + m_gradientTrue(m_gradientMap.full_size()), + m_potentialTrue(m_potentialMap.full_size()) { + VerifyOperator(trueDofDivergence); + } + + void Rebind(const mfem::Operator &trueDofDivergence) { + VerifyOperator(trueDofDivergence); + m_trueDofDivergence = std::addressof(trueDofDivergence); + } + + void Mult( + const mfem::Vector &gradient, + mfem::Vector &potentialAction + ) const override { + if (gradient.Size() != Width() || potentialAction.Size() != Height()) { + throw std::invalid_argument("The reduced gravity divergence received incompatible vectors."); + } + m_gradientMap.scatter(gradient, m_gradientTrue); + m_trueDofDivergence->Mult(m_gradientTrue, m_potentialTrue); + m_potentialMap.gather(m_potentialTrue, potentialAction); + } + + void MultTranspose( + const mfem::Vector &potential, + mfem::Vector &gradientAction + ) const override { + if (potential.Size() != Height() || gradientAction.Size() != Width()) { + throw std::invalid_argument("The reduced transpose divergence received incompatible vectors."); + } + m_potentialMap.scatter(potential, m_potentialTrue); + m_trueDofDivergence->MultTranspose(m_potentialTrue, m_gradientTrue); + m_gradientMap.gather(m_gradientTrue, gradientAction); + } + + private: + void VerifyOperator(const mfem::Operator &operation) const { + if (operation.Width() != m_gradientMap.full_size() || operation.Height() != m_potentialMap.full_size()) { + throw std::invalid_argument("The true-DOF divergence is incompatible with the gravity field maps."); + } + } + + const mfem::Operator *m_trueDofDivergence; + field::FieldDofMap m_gradientMap; + field::FieldDofMap m_potentialMap; + mutable mfem::Vector m_gradientTrue; + mutable mfem::Vector m_potentialTrue; + }; + + class ReducedFieldSolverAdapter final : public mfem::Solver { + public: + ReducedFieldSolverAdapter( + const mfem::Solver &trueDofSolver, + field::FieldDofMap map + ) + : mfem::Solver(map.reduced_size()), + m_trueDofSolver(std::addressof(trueDofSolver)), + m_map(std::move(map)), + m_rightHandSideTrue(m_map.full_size()), + m_actionTrue(m_map.full_size()) { + if (trueDofSolver.Height() != m_map.full_size() || trueDofSolver.Width() != m_map.full_size()) { + throw std::invalid_argument("The true-DOF solver is incompatible with the reduced field map."); + } + } + + void SetOperator(const mfem::Operator &operation) override { + if (operation.Height() != Height() || operation.Width() != Width()) { + throw std::invalid_argument("The reduced field solver received an operator of incompatible size."); + } + } + + void Mult( + const mfem::Vector &rightHandSide, + mfem::Vector &action + ) const override { + if (rightHandSide.Size() != Width() || action.Size() != Height()) { + throw std::invalid_argument("The reduced field solver received incompatible vectors."); + } + m_map.scatter(rightHandSide, m_rightHandSideTrue); + m_trueDofSolver->Mult(m_rightHandSideTrue, m_actionTrue); + m_map.gather(m_actionTrue, action); + } + + private: + const mfem::Solver *m_trueDofSolver; + field::FieldDofMap m_map; + mutable mfem::Vector m_rightHandSideTrue; + mutable mfem::Vector m_actionTrue; + }; + + [[nodiscard]] std::unique_ptr assembleGravityDivergenceSurrogate(const fem::FEM &f); + + [[nodiscard]] std::unique_ptr assembleGravityPotentialSchurSurrogate( + const fem::FEM &f, + const mfem::Vector &trueMassDiagonal + ); + + struct GravityFieldBlockPreparationReport final { + bool discretizationChanged{false}; + bool geometryChanged{false}; + bool rebuiltMassInverse{false}; + bool rebuiltDivergenceBinding{false}; + bool rebuiltPotentialSchur{false}; + + [[nodiscard]] bool DidAnyWork() const noexcept { + return rebuiltMassInverse || rebuiltDivergenceBinding || rebuiltPotentialSchur; + } + }; + + struct PreparedGravityFieldBlockStatistics final { + std::uint64_t setups{0}; + std::uint64_t refreshChecks{0}; + std::uint64_t refreshes{0}; + std::uint64_t noOpRefreshes{0}; + }; + + template + concept ImplementedGravityMassBackend = std::same_as, backend::Diagonal> || + std::same_as, backend::MatrixFreeChebyshev>; + + template + requires ImplementedGravityMassBackend && + backend::Compatible + class PreparedGravityFieldBlock final : public mfem::Solver { + public: + using Block = GravityFieldBlock, Policy>; + using PreparedMassInverse = std::conditional_t< + std::same_as, + backend::PreparedDiagonal, + backend::PreparedMatrixFreeChebyshev>; + + PreparedGravityFieldBlock( + const fem::FEM &f, + const operators::context::gravity_field::GravityFieldGeometryContext &geometryContext, + Block block + ) + : mfem::Solver(GravitySize(geometryContext)), + m_block(std::move(block)), + m_geometryContext(std::addressof(geometryContext)), + m_gradientMap(geometryContext.GetMassOperator().GetFluxMap()), + m_potentialMap(geometryContext.GetSourceOperator().GetPotentialMap()), + m_divergence( + geometryContext.GetDivergenceOperator(), + m_gradientMap, + m_potentialMap + ), + m_massInverse(MakeMassInverse( + f, + geometryContext, + m_block.massInverseBackend() + )), + m_potentialSchurSurrogate(AssemblePotentialSchur( + f, + geometryContext + )), + m_potentialSchurInverse( + m_block.potentialSchurBackend(), + *m_potentialSchurSurrogate + ), + m_reducedPotentialSchurInverse( + m_potentialSchurInverse, + m_potentialMap + ), + m_factorization( + m_block.factorizationPolicy(), + m_massInverse, + m_reducedPotentialSchurInverse, + m_divergence + ), + m_discretizationRevision(geometryContext.GetDiscretizationRevision()), + m_displacementRevision(geometryContext.GetDisplacementRevision()) { + if (!geometryContext.IsPrepared()) { + throw std::logic_error("The gravity field block requires a prepared gravity geometry context."); + } + m_statistics.setups = 1; + } + + PreparedGravityFieldBlock(const PreparedGravityFieldBlock &) = delete; + PreparedGravityFieldBlock &operator=(const PreparedGravityFieldBlock &) = delete; + PreparedGravityFieldBlock(PreparedGravityFieldBlock &&) = delete; + PreparedGravityFieldBlock &operator=(PreparedGravityFieldBlock &&) = delete; + + void SetOperator(const mfem::Operator &operation) override { + m_factorization.SetOperator(operation); + } + + void Mult( + const mfem::Vector &rightHandSide, + mfem::Vector &action + ) const override { + if (!IsCurrent()) { + throw std::logic_error("The gravity field block is stale; refresh it before application."); + } + m_factorization.Mult(rightHandSide, action); + } + + [[nodiscard]] bool IsCurrent() const noexcept { + return m_geometryContext->IsPrepared() && + m_geometryContext->GetDiscretizationRevision() == m_discretizationRevision && + m_geometryContext->GetDisplacementRevision() == m_displacementRevision; + } + + [[nodiscard]] GravityFieldBlockPreparationReport Refresh( + const fem::FEM &f, + const operators::context::gravity_field::GravityFieldGeometryContext &geometryContext + ) { + if (!geometryContext.IsPrepared()) { + throw std::logic_error("The gravity field block cannot refresh from unprepared geometry."); + } + if (std::addressof(geometryContext) != m_geometryContext) { + throw std::invalid_argument("A prepared gravity field block cannot change geometry-context identity."); + } + + ++m_statistics.refreshChecks; + GravityFieldBlockPreparationReport report{ + .discretizationChanged = geometryContext.GetDiscretizationRevision() != m_discretizationRevision, + .geometryChanged = geometryContext.GetDisplacementRevision() != m_displacementRevision + }; + if (!report.discretizationChanged && !report.geometryChanged) { + ++m_statistics.noOpRefreshes; + return report; + } + + m_divergence.Rebind(geometryContext.GetDivergenceOperator()); + report.rebuiltDivergenceBinding = report.discretizationChanged; + + RefreshMassInverse(geometryContext); + report.rebuiltMassInverse = true; + + auto potentialSchur = AssemblePotentialSchur(f, geometryContext); + m_potentialSchurInverse.Refresh(*potentialSchur); + m_potentialSchurSurrogate = std::move(potentialSchur); + report.rebuiltPotentialSchur = true; + + m_discretizationRevision = geometryContext.GetDiscretizationRevision(); + m_displacementRevision = geometryContext.GetDisplacementRevision(); + ++m_statistics.refreshes; + return report; + } + + [[nodiscard]] const Block &GetBlock() const noexcept { + return m_block; + } + + [[nodiscard]] const mfem::Array &GetOffsets() const noexcept { + return m_factorization.GetOffsets(); + } + + [[nodiscard]] const PreparedMassInverse &GetMassInverse() const { + if (!IsCurrent()) { + throw std::logic_error("The gravity mass inverse is stale; refresh its owning gravity block first."); + } + return m_massInverse; + } + + [[nodiscard]] const backend::PreparedHypreBoomerAMG &GetPotentialSchurInverse() const noexcept { + return m_potentialSchurInverse; + } + + [[nodiscard]] const mfem::HypreParMatrix &GetPotentialSchurSurrogate() const noexcept { + return *m_potentialSchurSurrogate; + } + + [[nodiscard]] const GravityFactorizationOperator &GetFactorization() const noexcept { + return m_factorization; + } + + [[nodiscard]] const PreparedGravityFieldBlockStatistics &GetStatistics() const noexcept { + return m_statistics; + } + + private: + [[nodiscard]] static int + GravitySize(const operators::context::gravity_field::GravityFieldGeometryContext &geometryContext) { + if (!geometryContext.IsPrepared()) { + throw std::logic_error("The gravity field block requires a prepared gravity geometry context."); + } + return geometryContext.GetMassOperator().GetFluxMap().reduced_size() + + geometryContext.GetSourceOperator().GetPotentialMap().reduced_size(); + } + + [[nodiscard]] static mfem::Vector AssembleReducedMassDiagonal( + const operators::context::gravity_field::GravityFieldGeometryContext &geometryContext + ) { + mfem::Vector diagonal; + geometryContext.GetMassOperator().AssembleDiagonal(diagonal); + return diagonal; + } + + [[nodiscard]] static PreparedMassInverse MakeMassInverse( + const fem::FEM &f, + const operators::context::gravity_field::GravityFieldGeometryContext &geometryContext, + const MassBackend &backendConfiguration + ) { + if constexpr (std::same_as) { + return PreparedMassInverse{backendConfiguration, AssembleReducedMassDiagonal(geometryContext)}; + } else { + static_assert(std::same_as); + return PreparedMassInverse{ + backendConfiguration, geometryContext.GetMassOperator(), f.gravityFluxFes->GetComm() + }; + } + } + + void RefreshMassInverse(const operators::context::gravity_field::GravityFieldGeometryContext &geometryContext) { + if constexpr (std::same_as) { + m_massInverse.Refresh(AssembleReducedMassDiagonal(geometryContext)); + } else { + static_assert(std::same_as); + m_massInverse.Refresh(geometryContext.GetMassOperator()); + } + } + + [[nodiscard]] static std::unique_ptr AssemblePotentialSchur( + const fem::FEM &f, + const operators::context::gravity_field::GravityFieldGeometryContext &geometryContext + ) { + mfem::Vector trueMassDiagonal; + geometryContext.GetMassOperator().AssembleTrueDiagonal(trueMassDiagonal); + return assembleGravityPotentialSchurSurrogate(f, trueMassDiagonal); + } + + Block m_block; + const operators::context::gravity_field::GravityFieldGeometryContext *m_geometryContext; + field::FieldDofMap m_gradientMap; + field::FieldDofMap m_potentialMap; + ReducedGravityDivergenceOperator m_divergence; + PreparedMassInverse m_massInverse; + std::unique_ptr m_potentialSchurSurrogate; + backend::PreparedHypreBoomerAMG m_potentialSchurInverse; + ReducedFieldSolverAdapter m_reducedPotentialSchurInverse; + GravityFactorizationOperator m_factorization; + operators::context::gravity_field::DiscretizationRevision m_discretizationRevision; + operators::context::gravity_field::DisplacementRevision m_displacementRevision; + PreparedGravityFieldBlockStatistics m_statistics; + }; + + template < + backend::Registered MassBackend, + backend::ApplicationMode Mode, + GravityFactorizationPolicy Policy> + requires ImplementedGravityMassBackend && backend::Compatible< + MassBackend, + GravityMassInverseCharacteristics> + [[nodiscard]] auto prepare( + const fem::FEM &f, + const operators::context::gravity_field::GravityFieldGeometryContext &geometryContext, + GravityFieldBlock< + MassBackend, + backend::HypreBoomerAMG, + Policy> block + ) { + return PreparedGravityFieldBlock{f, geometryContext, std::move(block)}; + } +} // namespace mean_field::preconditioning diff --git a/libmeanfield/interface/preconditioning/material_surface.cppm b/libmeanfield/interface/preconditioning/material_surface.cppm new file mode 100644 index 0000000..509c885 --- /dev/null +++ b/libmeanfield/interface/preconditioning/material_surface.cppm @@ -0,0 +1,2096 @@ +module; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +export module mean_field:preconditioning.material_surface; + +export import :operators.prepared_stellar_equilibrium; +export import :operators.stellar_equilibrium_problem; +export import :preconditioning.backend_implementations; +export import :preconditioning.plan; + +export namespace mean_field::preconditioning { + struct MaterialSurfaceBlockDiagonal final { }; + struct CoupledMaterialIndependentSurface final { }; + struct MaterialThenSurfaceTriangular final { }; + struct SurfaceThenMaterialTriangular final { }; + struct ApproximateMaterialSurfaceLDU final { }; + + template struct IsMaterialSurfaceFactorizationPolicy : std::false_type { }; + template <> struct IsMaterialSurfaceFactorizationPolicy : std::true_type { }; + template <> struct IsMaterialSurfaceFactorizationPolicy : std::true_type { }; + template <> struct IsMaterialSurfaceFactorizationPolicy : std::true_type { }; + template <> struct IsMaterialSurfaceFactorizationPolicy : std::true_type { }; + template <> struct IsMaterialSurfaceFactorizationPolicy : std::true_type { }; + + template + concept MaterialSurfaceFactorizationPolicy = + IsMaterialSurfaceFactorizationPolicy>::value; + + namespace detail { + template struct MaterialSurfaceConcatenate; + + template <> struct MaterialSurfaceConcatenate<> { + using Type = utils::blocks::type_list<>; + }; + + template struct MaterialSurfaceConcatenate> { + using Type = utils::blocks::type_list; + }; + + template + struct MaterialSurfaceConcatenate< + utils::blocks::type_list, + utils::blocks::type_list, + Remaining...> { + using Type = + typename MaterialSurfaceConcatenate, Remaining...>::Type; + }; + + template + using MaterialSurfaceConcatenateT = typename MaterialSurfaceConcatenate::Type; + + template struct NonCarrierEquations; + + template + struct NonCarrierEquations, CarrierField> { + using Type = material::ThermodynamicEquationCatalog<>; + }; + + template + struct NonCarrierEquations, CarrierField> { + private: + using Tail = + typename NonCarrierEquations, CarrierField>::Type; + + template struct Prepend; + + template + struct Prepend> { + using Type = material::ThermodynamicEquationCatalog; + }; + + public: + using Type = std::conditional_t< + std::same_as, + Tail, + typename Prepend::Type>; + }; + + template struct EquationCorrectionBlocks; + + template + struct EquationCorrectionBlocks> { + using Type = utils::blocks::type_list; + }; + + template struct EquationResidualBlocks; + + template + struct EquationResidualBlocks> { + using Type = utils::blocks::type_list; + }; + + template struct CouplingsForResidual; + + template + struct CouplingsForResidual, JacobianForm> { + using Type = utils::blocks::type_list<>; + }; + + template + struct CouplingsForResidual, JacobianForm> { + private: + using Tail = + typename CouplingsForResidual, JacobianForm>::Type; + + public: + using Type = std::conditional_t< + utils::blocks::has_jacobian_coupling_v, + MaterialSurfaceConcatenateT>, Tail>, + Tail>; + }; + + template struct InducedCouplings; + + template + struct InducedCouplings, Corrections, JacobianForm> { + using Type = utils::blocks::type_list<>; + }; + + template + struct InducedCouplings, Corrections, JacobianForm> { + using Type = MaterialSurfaceConcatenateT< + typename CouplingsForResidual::Type, + typename InducedCouplings, Corrections, JacobianForm>::Type>; + }; + + template struct IsMaterialSurfaceDescriptor : std::false_type { }; + + template struct MaterialSurfaceListIsSubset; + + template + struct MaterialSurfaceListIsSubset, Universe> + : std::bool_constant<(utils::blocks::contains_type_v && ...)> { }; + } // namespace detail + + template < + material::CompiledThermodynamicEquations ThermodynamicEquationsT, + typename SurfaceConstraintT, + typename FormT, + typename JacobianFormT> + requires requires { + typename SurfaceConstraintT::CarrierField; + typename SurfaceConstraintT::SurfaceDependencies; + typename SurfaceConstraintT::SurfaceDependencies::StateFieldTypes; + } && utils::blocks::valid_jacobian_form + struct CompiledMaterialSurfaceDescriptor final { + using ThermodynamicEquations = ThermodynamicEquationsT; + using SurfaceConstraint = SurfaceConstraintT; + using Form = FormT; + using JacobianForm = JacobianFormT; + using CarrierField = typename SurfaceConstraint::CarrierField; + using CarrierEquation = + material::ThermodynamicEquationForFieldT; + using NonCarrierMaterialEquations = + typename detail::NonCarrierEquations::Type; + using NonCarrierCorrectionBlocks = typename detail::EquationCorrectionBlocks::Type; + using NonCarrierResidualBlocks = typename detail::EquationResidualBlocks::Type; + using SurfaceStateFields = typename SurfaceConstraint::SurfaceDependencies::StateFieldTypes; + using CorrectionBlocks = detail::MaterialSurfaceConcatenateT< + NonCarrierCorrectionBlocks, + utils::blocks::type_list, + utils::blocks::type_list>; + using ResidualBlocks = detail::MaterialSurfaceConcatenateT< + NonCarrierResidualBlocks, + utils::blocks::type_list, + utils::blocks::type_list>; + using RequiredCouplings = + typename detail::InducedCouplings::Type; + + static constexpr bool symbolicallySquare = CorrectionBlocks::size == ResidualBlocks::size; + static constexpr bool surfaceDependenciesBelongToMaterial = material:: + fieldsBelongToThermodynamicEquations; + static constexpr bool correctionBlocksBelongToForm = + detail::MaterialSurfaceListIsSubset::value; + static constexpr bool residualBlocksBelongToForm = + detail::MaterialSurfaceListIsSubset::value; + }; + + template + requires equilibrium::DiscretizedStellarEquilibriumProblem> + using MaterialSurfaceDescriptorFor = CompiledMaterialSurfaceDescriptor< + typename std::remove_cvref_t::ThermodynamicEquationsType, + typename std::remove_cvref_t::CompiledSurfaceConstraintType, + typename std::remove_cvref_t::FormType, + typename std::remove_cvref_t::JacobianFormType>; + + namespace detail { + template + struct IsMaterialSurfaceDescriptor< + CompiledMaterialSurfaceDescriptor> + : std::bool_constant< + material::CompiledThermodynamicEquations::ThermodynamicEquations> && + CompiledMaterialSurfaceDescriptor:: + symbolicallySquare && + CompiledMaterialSurfaceDescriptor:: + surfaceDependenciesBelongToMaterial && + CompiledMaterialSurfaceDescriptor:: + correctionBlocksBelongToForm && + CompiledMaterialSurfaceDescriptor:: + residualBlocksBelongToForm && + utils::blocks::types_are_unique_v::CorrectionBlocks> && + utils::blocks::types_are_unique_v::ResidualBlocks>> { }; + } // namespace detail + + template + concept MaterialSurfaceDescriptor = detail::IsMaterialSurfaceDescriptor>::value; + + template + concept ImplementedMaterialSurfaceDescriptor = + MaterialSurfaceDescriptor && + std::same_as; + + using DensityMassDiagonalCharacteristics = OperatorCharacteristics< + OperatorCategory::mass_like, + OperatorValueStructure::scalar, + OperatorSymmetry::symmetric, + OperatorDefiniteness::positive_definite, + OperatorRepresentation::diagonal, + OperatorDistribution::distributed_true_dof, + OperatorFESpace::l2>; + + using EnthalpyMassDiagonalCharacteristics = OperatorCharacteristics< + OperatorCategory::mass_like, + OperatorValueStructure::scalar, + OperatorSymmetry::symmetric, + OperatorDefiniteness::positive_definite, + OperatorRepresentation::diagonal, + OperatorDistribution::distributed_true_dof, + OperatorFESpace::h1>; + + // This describes the assembled diagonal surrogate, not the generally + // nonsymmetric pulled-back q-to-R_q operator that it approximates. A + // calibrated scalar multiple may carry either sign, so definiteness is not + // claimed by this legacy path. + using SurfaceDiagonalCharacteristics = OperatorCharacteristics< + OperatorCategory::surface_like, + OperatorValueStructure::scalar, + OperatorSymmetry::symmetric, + OperatorDefiniteness::unspecified, + OperatorRepresentation::diagonal, + OperatorDistribution::distributed_true_dof, + OperatorFESpace::h1>; + + // The frequency-aware surface surrogate is an elliptic scalar operator on + // the ambient H1 space whose trace supplies the deformation parameters. + // A positive mass coefficient removes the constant-mode nullspace of the + // tangential stiffness contribution. + using SurfaceH1MassStiffnessCharacteristics = OperatorCharacteristics< + OperatorCategory::elliptic_like, + OperatorValueStructure::scalar, + OperatorSymmetry::symmetric, + OperatorDefiniteness::positive_definite, + OperatorRepresentation::assembled_sparse, + OperatorDistribution::distributed_true_dof, + OperatorFESpace::h1>; + + enum class SurfaceRieszCalibrationTarget { none, surface_jacobian, approximate_material_schur }; + + // operator_action fits the surrogate s M directly to the requested target + // T. right_preconditioned_action instead fits T (alpha M^{-1}) to the + // identity and stores s = 1 / alpha. Both produce one fixed linear + // diagonal operator after setup; the distinction is solely the calibration + // objective. + enum class SurfaceRieszCalibrationObjective { operator_action, right_preconditioned_action }; + + struct SurfaceRieszCalibrationOptions final { + SurfaceRieszCalibrationTarget target{SurfaceRieszCalibrationTarget::none}; + int probeCount{0}; + SurfaceRieszCalibrationObjective objective{SurfaceRieszCalibrationObjective::operator_action}; + }; + + struct MaterialSurfaceDiagonalOptions final { + double relativeFloor{1.0e-12}; + double absoluteFloor{1.0e-14}; + SurfaceRieszCalibrationOptions surfaceCalibration{}; + }; + + struct SurfaceMassDiagonal final { }; + + struct SurfaceH1MassStiffness final { + SurfaceRieszCalibrationOptions calibration{ + .target = SurfaceRieszCalibrationTarget::approximate_material_schur, + .probeCount = 6 + }; + double relativeMassCoefficientFloor{1.0e-10}; + double gramRelativeTolerance{1.0e-12}; + }; + + template struct SurfaceSurrogateTraits { + static constexpr bool registered = false; + }; + + template <> struct SurfaceSurrogateTraits { + static constexpr bool registered = true; + using OperatorDescription = SurfaceDiagonalCharacteristics; + }; + + template <> struct SurfaceSurrogateTraits { + static constexpr bool registered = true; + using OperatorDescription = SurfaceH1MassStiffnessCharacteristics; + }; + + template + concept MaterialSurfaceSurrogate = SurfaceSurrogateTraits>::registered; + + struct SurfaceH1NormalEquations final { + double massMass{0.0}; + double massStiffness{0.0}; + double stiffnessStiffness{0.0}; + double massTarget{0.0}; + double stiffnessTarget{0.0}; + double targetTarget{0.0}; + }; + + struct SurfaceH1FitReport final { + SurfaceRieszCalibrationTarget target{SurfaceRieszCalibrationTarget::none}; + int probeCount{0}; + double sign{1.0}; + double massCoefficient{1.0}; + double stiffnessCoefficient{0.0}; + double relativeResidual{0.0}; + double relativeGramDeterminant{0.0}; + SurfaceH1NormalEquations normalEquations{}; + + [[nodiscard]] bool WasCalibrated() const noexcept { + return target != SurfaceRieszCalibrationTarget::none; + } + }; + + namespace detail { + [[nodiscard]] inline SurfaceH1FitReport fitSurfaceH1Coefficients( + const SurfaceH1NormalEquations &equations, + const SurfaceH1MassStiffness &configuration + ) { + const std::array values{ + equations.massMass, + equations.massStiffness, + equations.stiffnessStiffness, + equations.massTarget, + equations.stiffnessTarget, + equations.targetTarget, + configuration.relativeMassCoefficientFloor, + configuration.gramRelativeTolerance + }; + if (!std::all_of(values.begin(), values.end(), [](const double value) { return std::isfinite(value); }) || + equations.massMass <= 0.0 || equations.stiffnessStiffness <= 0.0 || equations.targetTarget <= 0.0 || + configuration.relativeMassCoefficientFloor <= 0.0 || configuration.gramRelativeTolerance <= 0.0 || + configuration.gramRelativeTolerance >= 1.0 || + configuration.calibration.target == SurfaceRieszCalibrationTarget::none || + configuration.calibration.probeCount <= 0) { + throw std::invalid_argument( + "The surface H1 fit requires a calibration target, probes, and finite positive Gram data." + ); + } + + const double massNorm = std::sqrt(equations.massMass); + const double stiffnessNorm = std::sqrt(equations.stiffnessStiffness); + const double correlation = equations.massStiffness / massNorm / stiffnessNorm; + const double relativeDeterminant = 1.0 - correlation * correlation; + const double normalizedMassTarget = equations.massTarget / massNorm; + const double normalizedStiffnessTarget = equations.stiffnessTarget / stiffnessNorm; + if (!std::isfinite(massNorm) || !std::isfinite(stiffnessNorm) || !std::isfinite(correlation) || + !std::isfinite(relativeDeterminant) || !std::isfinite(normalizedMassTarget) || + !std::isfinite(normalizedStiffnessTarget) || + relativeDeterminant <= configuration.gramRelativeTolerance) { + throw std::runtime_error("The surface H1 calibration probes do not distinguish mass and stiffness."); + } + + const double amplitude = std::sqrt(equations.targetTarget) / massNorm; + const double minimumMassCoefficient = configuration.relativeMassCoefficientFloor * amplitude; + if (!std::isfinite(amplitude) || !std::isfinite(minimumMassCoefficient) || minimumMassCoefficient <= 0.0) { + throw std::runtime_error("The surface H1 calibration produced an invalid positive mass floor."); + } + struct Candidate final { + double sign{1.0}; + double mass{0.0}; + double stiffness{0.0}; + double residual{std::numeric_limits::infinity()}; + }; + const auto evaluate = [&](const double sign, const double mass, const double stiffness) { + Candidate candidate{.sign = sign, .mass = mass, .stiffness = stiffness}; + if (!std::isfinite(mass) || !std::isfinite(stiffness) || mass < minimumMassCoefficient || + stiffness < 0.0) { + return candidate; + } + const double normalizedMass = mass * massNorm; + const double normalizedStiffness = stiffness * stiffnessNorm; + candidate.residual = + equations.targetTarget + normalizedMass * normalizedMass + + 2.0 * correlation * normalizedMass * normalizedStiffness + + normalizedStiffness * normalizedStiffness - + 2.0 * sign * + (normalizedMass * normalizedMassTarget + normalizedStiffness * normalizedStiffnessTarget); + candidate.residual = std::max(0.0, candidate.residual); + return candidate; + }; + + Candidate best; + for (const double sign : {-1.0, 1.0}) { + const double unconstrainedMass = sign * + (normalizedMassTarget - correlation * normalizedStiffnessTarget) / + relativeDeterminant / massNorm; + const double unconstrainedStiffness = sign * + (normalizedStiffnessTarget - correlation * normalizedMassTarget) / + relativeDeterminant / stiffnessNorm; + if (unconstrainedMass >= minimumMassCoefficient && unconstrainedStiffness >= 0.0) { + const Candidate candidate = evaluate(sign, unconstrainedMass, unconstrainedStiffness); + if (candidate.residual < best.residual) { + best = candidate; + } + } + + const Candidate massOnly = + evaluate(sign, std::max(minimumMassCoefficient, sign * normalizedMassTarget / massNorm), 0.0); + if (massOnly.residual < best.residual) { + best = massOnly; + } + + const double boundaryStiffness = std::max( + 0.0, + (sign * normalizedStiffnessTarget - minimumMassCoefficient * massNorm * correlation) / stiffnessNorm + ); + const Candidate massFloor = evaluate(sign, minimumMassCoefficient, boundaryStiffness); + if (massFloor.residual < best.residual) { + best = massFloor; + } + } + if (!std::isfinite(best.residual) || !std::isfinite(best.mass) || !std::isfinite(best.stiffness)) { + throw std::runtime_error("The surface H1 calibration failed to produce a finite constrained fit."); + } + return { + .target = configuration.calibration.target, + .probeCount = configuration.calibration.probeCount, + .sign = best.sign, + .massCoefficient = best.mass, + .stiffnessCoefficient = best.stiffness, + .relativeResidual = std::sqrt(best.residual / equations.targetTarget), + .relativeGramDeterminant = relativeDeterminant, + .normalEquations = equations + }; + } + } // namespace detail + + using CoupledMaterialSurfaceCharacteristics = OperatorCharacteristics< + OperatorCategory::mixed, + OperatorValueStructure::block, + OperatorSymmetry::nonsymmetric, + OperatorDefiniteness::unspecified, + OperatorRepresentation::matrix_free, + OperatorDistribution::distributed_true_dof, + OperatorFESpace::product>; + + namespace backend { + template < + MaterialSurfaceDescriptor Descriptor, + Registered MaterialBackend, + Registered SurfaceBackend, + MaterialSurfaceFactorizationPolicy Policy, + MaterialSurfaceSurrogate SurfaceSurrogate = SurfaceMassDiagonal> + requires Compatible && + Compatible && + Compatible::OperatorDescription> + struct MaterialSurface final { + using DescriptorType = Descriptor; + using MaterialBackendType = MaterialBackend; + using SurfaceBackendType = SurfaceBackend; + using FactorizationPolicyType = Policy; + using SurfaceSurrogateType = SurfaceSurrogate; + }; + + template < + MaterialSurfaceDescriptor Descriptor, + Registered MaterialBackend, + Registered SurfaceBackend, + MaterialSurfaceFactorizationPolicy Policy, + MaterialSurfaceSurrogate SurfaceSurrogate> + requires Compatible && + Compatible && + Compatible::OperatorDescription> + struct Traits> { + static constexpr bool registered = true; + static constexpr ApplicationContract applicationContract = + ::mean_field::preconditioning::backend::applicationContract == + ApplicationContract::stationary_linear && + ::mean_field::preconditioning::backend::applicationContract == + ApplicationContract::stationary_linear + ? ApplicationContract::stationary_linear + : ApplicationContract::flexible; + static constexpr bool supportsSerialExecution = + Traits::supportsSerialExecution && Traits::supportsSerialExecution; + static constexpr bool supportsDistributedExecution = + Traits::supportsDistributedExecution && + Traits::supportsDistributedExecution; + static constexpr SymmetryRequirement symmetryRequirement = SymmetryRequirement::none; + static constexpr NullspaceRequirement nullspaceRequirement = NullspaceRequirement::none; + static constexpr SurrogateRequirement surrogateRequirement = Traits::surrogateRequirement; + static constexpr bool requiresAssembledSparseSurrogate = + Traits::requiresAssembledSparseSurrogate; + + using PreparationDependencies = preconditioning::PreparationDependencies< + PreparationDependency::discretization, + PreparationDependency::geometry, + PreparationDependency::equation_of_state, + PreparationDependency::linearization>; + + template + static constexpr bool supports = + Characteristics::category == OperatorCategory::mixed && + Characteristics::valueStructure == OperatorValueStructure::block && + Characteristics::symmetry == OperatorSymmetry::nonsymmetric && + Characteristics::representation == OperatorRepresentation::matrix_free && + Characteristics::distribution == OperatorDistribution::distributed_true_dof && + Characteristics::finiteElementSpace == OperatorFESpace::product; + }; + } // namespace backend + + template < + ImplementedMaterialSurfaceDescriptor DescriptorT, + backend::Registered MaterialBackendT, + backend::Registered SurfaceBackendT, + MaterialSurfaceFactorizationPolicy FactorizationPolicyT, + MaterialSurfaceSurrogate SurfaceSurrogateT = SurfaceMassDiagonal> + requires backend::Compatible && + backend::Compatible && + backend::Compatible< + SurfaceBackendT, + typename SurfaceSurrogateTraits::OperatorDescription> + class MaterialSurfaceBlock final { + public: + using Descriptor = DescriptorT; + using CorrectionBlocks = typename Descriptor::CorrectionBlocks; + using ResidualBlocks = typename Descriptor::ResidualBlocks; + using RequiredCouplings = typename Descriptor::RequiredCouplings; + using OperatorDescription = CoupledMaterialSurfaceCharacteristics; + using BackendType = backend:: + MaterialSurface; + using PreparationDependencies = typename backend::Traits::PreparationDependencies; + using MaterialBackend = MaterialBackendT; + using SurfaceBackend = SurfaceBackendT; + using Factorization = FactorizationPolicyT; + using SurfaceSurrogate = SurfaceSurrogateT; + + constexpr MaterialSurfaceBlock( + MaterialBackendT materialBackend = {}, + SurfaceBackendT surfaceBackend = {}, + FactorizationPolicyT factorizationPolicy = {}, + MaterialSurfaceDiagonalOptions diagonalOptions = {}, + SurfaceSurrogateT surfaceSurrogate = {} + ) + : m_materialBackend(std::move(materialBackend)), + m_surfaceBackend(std::move(surfaceBackend)), + m_factorizationPolicy(std::move(factorizationPolicy)), + m_diagonalOptions(diagonalOptions), + m_surfaceSurrogate(std::move(surfaceSurrogate)) { + } + + [[nodiscard]] constexpr const MaterialBackendT &materialBackend() const noexcept { + return m_materialBackend; + } + [[nodiscard]] constexpr const SurfaceBackendT &surfaceBackend() const noexcept { + return m_surfaceBackend; + } + [[nodiscard]] constexpr const FactorizationPolicyT &factorizationPolicy() const noexcept { + return m_factorizationPolicy; + } + [[nodiscard]] constexpr const MaterialSurfaceDiagonalOptions &diagonalOptions() const noexcept { + return m_diagonalOptions; + } + [[nodiscard]] constexpr const SurfaceSurrogateT &surfaceSurrogate() const noexcept { + return m_surfaceSurrogate; + } + + private: + MaterialBackendT m_materialBackend; + SurfaceBackendT m_surfaceBackend; + FactorizationPolicyT m_factorizationPolicy; + MaterialSurfaceDiagonalOptions m_diagonalOptions; + SurfaceSurrogateT m_surfaceSurrogate; + }; + + template < + equilibrium::DiscretizedStellarEquilibriumProblem Problem, + backend::Registered MaterialBackend = backend::Diagonal, + backend::Registered SurfaceBackend = backend::Diagonal, + MaterialSurfaceFactorizationPolicy Policy = SurfaceThenMaterialTriangular> + [[nodiscard]] constexpr auto materialSurfaceBlock( + const Problem &, + MaterialBackend materialBackend = {}, + SurfaceBackend surfaceBackend = {}, + Policy policy = {}, + const MaterialSurfaceDiagonalOptions diagonalOptions = {} + ) { + using Descriptor = MaterialSurfaceDescriptorFor>; + return MaterialSurfaceBlock{ + std::move(materialBackend), std::move(surfaceBackend), std::move(policy), diagonalOptions + }; + } + + template < + equilibrium::DiscretizedStellarEquilibriumProblem Problem, + backend::Registered MaterialBackend, + backend::Registered SurfaceBackend, + MaterialSurfaceFactorizationPolicy Policy, + MaterialSurfaceSurrogate SurfaceSurrogate> + [[nodiscard]] constexpr auto materialSurfaceBlock( + const Problem &, + MaterialBackend materialBackend, + SurfaceBackend surfaceBackend, + Policy policy, + SurfaceSurrogate surfaceSurrogate, + const MaterialSurfaceDiagonalOptions diagonalOptions = {} + ) { + using Descriptor = MaterialSurfaceDescriptorFor>; + return MaterialSurfaceBlock{ + std::move(materialBackend), std::move(surfaceBackend), std::move(policy), diagonalOptions, + std::move(surfaceSurrogate) + }; + } + + class MaterialSurfaceJacobianOperator final : public mfem::Operator { + public: + explicit MaterialSurfaceJacobianOperator(const operators::PreparedStellarEquilibriumOperator &operation) + : mfem::Operator(TotalSize(operation)), + m_operation(std::addressof(operation)), + m_offsets(4), + m_zeroDensity(operation.GetBarotropicClosureOperator().GetDensitySize()), + m_zeroGravity( + operation.GetDisplacementOperator().GetGravityContext().GetGravityGradientMap().reduced_size() + ), + m_zeroPotential(operation.GetHydrostaticOperator().GetGravityPotentialMap().reduced_size()), + m_zeroEnthalpy(operation.GetBarotropicClosureOperator().GetEnthalpySize()), + m_volumeDisplacement(operation.GetDomainDeformation().volumeDisplacementSize()), + m_mechanicalAction(operation.GetDomainDeformation().volumeDisplacementSize()), + m_pullbackAction(operation.GetDomainDeformation().parameterCount()), + m_fullDirection(operation.Width()), + m_fullAction(operation.Height()) { + m_offsets[0] = 0; + m_offsets[1] = m_zeroDensity.Size(); + m_offsets[2] = m_offsets[1] + operation.GetDomainDeformation().parameterCount(); + m_offsets[3] = Height(); + m_zeroDensity = 0.0; + m_zeroGravity = 0.0; + m_zeroPotential = 0.0; + m_zeroEnthalpy = 0.0; + } + + void Mult( + const mfem::Vector &direction, + mfem::Vector &action + ) const override { + VerifyVectors(direction, action); + const auto densityDirection = ConstBlock(direction, 0); + const auto surfaceDirection = ConstBlock(direction, 1); + const auto enthalpyDirection = ConstBlock(direction, 2); + auto densityAction = MutableBlock(action, 0); + auto surfaceAction = MutableBlock(action, 1); + auto enthalpyAction = MutableBlock(action, 2); + + // This operator is the diagnostic restriction R_material J + // P_material, so construct it through the authoritative stellar + // Jacobian. The block factorization below continues to use the + // direct coupling actions and does not pay for a full Jacobian + // application. + m_fullDirection = 0.0; + const auto fullDirectionView = m_operation->GetRootManifest().directionView(m_fullDirection); + mfem::Vector fullDensityDirection = fullDirectionView.block(utils::blocks::density_field.mass_term); + mfem::Vector fullSurfaceDirection = + fullDirectionView.block(utils::blocks::surface_deformation_field.parameters_term); + mfem::Vector fullEnthalpyDirection = fullDirectionView.block(utils::blocks::enthalpy_field.specific_term); + fullDensityDirection = densityDirection; + fullSurfaceDirection = surfaceDirection; + fullEnthalpyDirection = enthalpyDirection; + + m_operation->Mult(m_fullDirection, m_fullAction); + const auto fullActionView = m_operation->GetRootManifest().residualView(m_fullAction); + const mfem::Vector fullDensityAction = fullActionView.block(utils::blocks::density_field.mass_term); + const mfem::Vector fullSurfaceAction = + fullActionView.block(utils::blocks::surface_deformation_field.shape_equilibrium_term); + const mfem::Vector fullEnthalpyAction = fullActionView.block(utils::blocks::enthalpy_field.specific_term); + densityAction = fullDensityAction; + surfaceAction = fullSurfaceAction; + enthalpyAction = fullEnthalpyAction; + } + + void ApplyEnthalpyToDensity( + const mfem::Vector &enthalpy, + mfem::Vector &densityAction + ) const { + VerifyBlock(enthalpy, 2, "enthalpy direction"); + VerifyBlock(densityAction, 0, "density action"); + m_operation->GetBarotropicClosureOperator().Mult( + m_zeroDensity, enthalpy, ZeroVolumeDisplacement(), densityAction + ); + } + + void ApplySurfaceToMaterial( + const mfem::Vector &surface, + mfem::Vector &densityAction, + mfem::Vector &enthalpyAction + ) const { + VerifyBlock(surface, 1, "surface direction"); + VerifyBlock(densityAction, 0, "density action"); + VerifyBlock(enthalpyAction, 2, "enthalpy action"); + GenerateDisplacement(surface); + m_operation->GetBarotropicClosureOperator().Mult( + m_zeroDensity, m_zeroEnthalpy, m_volumeDisplacement, densityAction + ); + m_operation->GetHydrostaticOperator().ApplyDisplacementJacobianAction(m_volumeDisplacement, enthalpyAction); + m_operation->GetSurfaceConstraintOperator().ApplyJacobianRows(m_zeroEnthalpy, enthalpyAction); + } + + void ApplyMaterialToSurface( + const mfem::Vector &density, + const mfem::Vector &enthalpy, + mfem::Vector &surfaceAction + ) const { + VerifyBlock(density, 0, "density direction"); + VerifyBlock(enthalpy, 2, "enthalpy direction"); + VerifyBlock(surfaceAction, 1, "surface action"); + m_operation->GetDisplacementOperator().ApplyCompleteJacobianAction( + density, ZeroVolumeDisplacement(), m_zeroGravity, enthalpy, m_mechanicalAction + ); + m_operation->GetDomainDeformation().applyJacobianTranspose( + m_operation->GetSurfaceDeformationParameters(), m_mechanicalAction, surfaceAction + ); + } + + void ApplySurfaceToSurface( + const mfem::Vector &surface, + mfem::Vector &surfaceAction + ) const { + VerifyBlock(surface, 1, "surface direction"); + VerifyBlock(surfaceAction, 1, "surface action"); + GenerateDisplacement(surface); + m_operation->GetDisplacementOperator().ApplyDisplacementJacobianAction( + m_volumeDisplacement, m_mechanicalAction + ); + m_operation->GetDomainDeformation().applyJacobianTranspose( + m_operation->GetSurfaceDeformationParameters(), m_mechanicalAction, surfaceAction + ); + m_operation->GetDomainDeformation().applyPullbackDerivative( + m_operation->GetSurfaceDeformationParameters(), surface, m_operation->GetFullMechanicalResidual(), + m_pullbackAction + ); + surfaceAction += m_pullbackAction; + } + + [[nodiscard]] const mfem::Array &GetOffsets() const noexcept { + return m_offsets; + } + + private: + [[nodiscard]] static int TotalSize(const operators::PreparedStellarEquilibriumOperator &operation) { + if (!operation.IsPrepared()) { + throw std::logic_error("The material-surface Jacobian requires a prepared stellar operator."); + } + return operation.GetBarotropicClosureOperator().GetDensitySize() + + operation.GetDomainDeformation().parameterCount() + + operation.GetBarotropicClosureOperator().GetEnthalpySize(); + } + + [[nodiscard]] mfem::Vector ConstBlock( + const mfem::Vector &vector, + int block + ) const { + return mfem::Vector( + const_cast(vector.GetData()) + m_offsets[block], m_offsets[block + 1] - m_offsets[block] + ); + } + [[nodiscard]] mfem::Vector MutableBlock( + mfem::Vector &vector, + int block + ) const { + return mfem::Vector(vector.GetData() + m_offsets[block], m_offsets[block + 1] - m_offsets[block]); + } + void VerifyBlock( + const mfem::Vector &vector, + int block, + const char *name + ) const { + if (vector.Size() != m_offsets[block + 1] - m_offsets[block]) { + throw std::invalid_argument(std::string("The material-surface ") + name + " has the wrong size."); + } + } + void VerifyVectors( + const mfem::Vector &direction, + const mfem::Vector &action + ) const { + if (direction.Size() != Width() || action.Size() != Height()) { + throw std::invalid_argument("The material-surface Jacobian requires compatible, preallocated vectors."); + } + } + void GenerateDisplacement(const mfem::Vector &surface) const { + m_operation->GetDomainDeformation().applyJacobian( + m_operation->GetSurfaceDeformationParameters(), surface, m_volumeDisplacement + ); + } + [[nodiscard]] const mfem::Vector &ZeroVolumeDisplacement() const { + m_volumeDisplacement = 0.0; + return m_volumeDisplacement; + } + + const operators::PreparedStellarEquilibriumOperator *m_operation; + mfem::Array m_offsets; + mfem::Vector m_zeroDensity; + mfem::Vector m_zeroGravity; + mfem::Vector m_zeroPotential; + mfem::Vector m_zeroEnthalpy; + mutable mfem::Vector m_volumeDisplacement; + mutable mfem::Vector m_mechanicalAction; + mutable mfem::Vector m_pullbackAction; + mutable mfem::Vector m_fullDirection; + mutable mfem::Vector m_fullAction; + }; + + struct MaterialSurfaceFactorizationStatistics final { + std::uint64_t applications{0}; + std::uint64_t densityInverseApplications{0}; + std::uint64_t surfaceInverseApplications{0}; + std::uint64_t enthalpyInverseApplications{0}; + std::uint64_t enthalpyToDensityApplications{0}; + std::uint64_t surfaceToMaterialApplications{0}; + std::uint64_t materialToSurfaceApplications{0}; + }; + + template + concept MaterialSurfaceCouplingOperator = requires( + const Candidate &couplings, + const mfem::Vector &density, + const mfem::Vector &surface, + const mfem::Vector &enthalpy, + mfem::Vector &densityAction, + mfem::Vector &surfaceAction, + mfem::Vector &enthalpyAction + ) { + { couplings.Height() } -> std::same_as; + { couplings.GetOffsets() } -> std::same_as &>; + couplings.ApplyEnthalpyToDensity(enthalpy, densityAction); + couplings.ApplySurfaceToMaterial(surface, densityAction, enthalpyAction); + couplings.ApplyMaterialToSurface(density, enthalpy, surfaceAction); + }; + + template < + MaterialSurfaceFactorizationPolicy Policy, + MaterialSurfaceCouplingOperator CouplingOperator = MaterialSurfaceJacobianOperator> + class MaterialSurfaceFactorizationOperator final : public mfem::Solver { + public: + MaterialSurfaceFactorizationOperator( + Policy policy, + const mfem::Solver &densityInverse, + const mfem::Solver &surfaceInverse, + const mfem::Solver &enthalpyInverse, + const CouplingOperator &couplings + ) + : mfem::Solver(couplings.Height()), + m_policy(std::move(policy)), + m_densityInverse(std::addressof(densityInverse)), + m_surfaceInverse(std::addressof(surfaceInverse)), + m_enthalpyInverse(std::addressof(enthalpyInverse)), + m_couplings(std::addressof(couplings)), + m_offsets(couplings.GetOffsets()), + m_densityWorkspace(densityInverse.Height()), + m_densityCoupling(densityInverse.Height()), + m_surfaceWorkspace(surfaceInverse.Height()), + m_enthalpyWorkspace(enthalpyInverse.Height()) { + if (densityInverse.Height() != densityInverse.Width() || + surfaceInverse.Height() != surfaceInverse.Width() || + enthalpyInverse.Height() != enthalpyInverse.Width() || densityInverse.Height() != m_offsets[1] || + surfaceInverse.Height() != m_offsets[2] - m_offsets[1] || + enthalpyInverse.Height() != m_offsets[3] - m_offsets[2]) { + throw std::invalid_argument("Material-surface inverse blocks do not match the coupled operator."); + } + } + + void SetOperator(const mfem::Operator &operation) override { + if (operation.Height() != Height() || operation.Width() != Width()) { + throw std::invalid_argument("The material-surface factorization received an incompatible operator."); + } + } + + void Mult( + const mfem::Vector &rightHandSide, + mfem::Vector &action + ) const override { + if (rightHandSide.Size() != Width() || action.Size() != Height()) { + throw std::invalid_argument( + "The material-surface factorization requires compatible, preallocated vectors." + ); + } + const auto densityRightHandSide = ConstBlock(rightHandSide, 0); + const auto surfaceRightHandSide = ConstBlock(rightHandSide, 1); + const auto enthalpyRightHandSide = ConstBlock(rightHandSide, 2); + auto densityAction = MutableBlock(action, 0); + auto surfaceAction = MutableBlock(action, 1); + auto enthalpyAction = MutableBlock(action, 2); + + if constexpr (std::same_as) { + m_densityInverse->Mult(densityRightHandSide, densityAction); + m_surfaceInverse->Mult(surfaceRightHandSide, surfaceAction); + m_enthalpyInverse->Mult(enthalpyRightHandSide, enthalpyAction); + } else if constexpr (std::same_as) { + m_surfaceInverse->Mult(surfaceRightHandSide, surfaceAction); + m_couplings->ApplySurfaceToMaterial(surfaceAction, m_densityWorkspace, m_enthalpyWorkspace); + m_enthalpyWorkspace *= -1.0; + m_enthalpyWorkspace += enthalpyRightHandSide; + m_enthalpyInverse->Mult(m_enthalpyWorkspace, enthalpyAction); + m_couplings->ApplyEnthalpyToDensity(enthalpyAction, m_densityCoupling); + m_densityWorkspace += m_densityCoupling; + m_densityWorkspace *= -1.0; + m_densityWorkspace += densityRightHandSide; + m_densityInverse->Mult(m_densityWorkspace, densityAction); + ++m_statistics.surfaceToMaterialApplications; + ++m_statistics.enthalpyToDensityApplications; + } else if constexpr (std::same_as) { + // Form m_0 = M^{-1} b_m with the upper-triangular material + // inverse, where m = (rho, h). + m_enthalpyInverse->Mult(enthalpyRightHandSide, enthalpyAction); + m_couplings->ApplyEnthalpyToDensity(enthalpyAction, m_densityWorkspace); + m_densityWorkspace *= -1.0; + m_densityWorkspace += densityRightHandSide; + m_densityInverse->Mult(m_densityWorkspace, densityAction); + + // Apply the surface inverse to b_q - A_qm m_0. The surface + // surrogate is an approximation to the resulting Schur + // complement rather than merely to A_qq. + m_couplings->ApplyMaterialToSurface(densityAction, enthalpyAction, m_surfaceWorkspace); + m_surfaceWorkspace *= -1.0; + m_surfaceWorkspace += surfaceRightHandSide; + m_surfaceInverse->Mult(m_surfaceWorkspace, surfaceAction); + + // Recover m = M^{-1}(b_m - A_mq q). Recomputing the material + // solve is algebraically equivalent to the conventional LDU + // correction m_0 - M^{-1} A_mq q. + m_couplings->ApplySurfaceToMaterial(surfaceAction, m_densityWorkspace, m_enthalpyWorkspace); + m_enthalpyWorkspace *= -1.0; + m_enthalpyWorkspace += enthalpyRightHandSide; + m_enthalpyInverse->Mult(m_enthalpyWorkspace, enthalpyAction); + m_couplings->ApplyEnthalpyToDensity(enthalpyAction, m_densityCoupling); + m_densityWorkspace += m_densityCoupling; + m_densityWorkspace *= -1.0; + m_densityWorkspace += densityRightHandSide; + m_densityInverse->Mult(m_densityWorkspace, densityAction); + + ++m_statistics.materialToSurfaceApplications; + ++m_statistics.surfaceToMaterialApplications; + m_statistics.enthalpyToDensityApplications += 2; + ++m_statistics.densityInverseApplications; + ++m_statistics.enthalpyInverseApplications; + } else { + m_enthalpyInverse->Mult(enthalpyRightHandSide, enthalpyAction); + m_couplings->ApplyEnthalpyToDensity(enthalpyAction, m_densityWorkspace); + m_densityWorkspace *= -1.0; + m_densityWorkspace += densityRightHandSide; + m_densityInverse->Mult(m_densityWorkspace, densityAction); + ++m_statistics.enthalpyToDensityApplications; + + if constexpr (std::same_as) { + m_couplings->ApplyMaterialToSurface(densityAction, enthalpyAction, m_surfaceWorkspace); + m_surfaceWorkspace *= -1.0; + m_surfaceWorkspace += surfaceRightHandSide; + m_surfaceInverse->Mult(m_surfaceWorkspace, surfaceAction); + ++m_statistics.materialToSurfaceApplications; + } else { + static_assert(std::same_as); + m_surfaceInverse->Mult(surfaceRightHandSide, surfaceAction); + } + } + + ++m_statistics.densityInverseApplications; + ++m_statistics.surfaceInverseApplications; + ++m_statistics.enthalpyInverseApplications; + ++m_statistics.applications; + } + + [[nodiscard]] const mfem::Array &GetOffsets() const noexcept { + return m_offsets; + } + [[nodiscard]] const MaterialSurfaceFactorizationStatistics &GetStatistics() const noexcept { + return m_statistics; + } + + private: + [[nodiscard]] mfem::Vector ConstBlock( + const mfem::Vector &vector, + int block + ) const { + return mfem::Vector( + const_cast(vector.GetData()) + m_offsets[block], m_offsets[block + 1] - m_offsets[block] + ); + } + [[nodiscard]] mfem::Vector MutableBlock( + mfem::Vector &vector, + int block + ) const { + return mfem::Vector(vector.GetData() + m_offsets[block], m_offsets[block + 1] - m_offsets[block]); + } + + Policy m_policy; + const mfem::Solver *m_densityInverse; + const mfem::Solver *m_surfaceInverse; + const mfem::Solver *m_enthalpyInverse; + const CouplingOperator *m_couplings; + mfem::Array m_offsets; + mutable mfem::Vector m_densityWorkspace; + mutable mfem::Vector m_densityCoupling; + mutable mfem::Vector m_surfaceWorkspace; + mutable mfem::Vector m_enthalpyWorkspace; + mutable MaterialSurfaceFactorizationStatistics m_statistics; + }; + + struct DiagonalPreparationQuality final { + double minimumAbsoluteEntryBeforeRegularization{0.0}; + double maximumAbsoluteEntryBeforeRegularization{0.0}; + double appliedFloor{0.0}; + std::uint64_t regularizedEntries{0}; + }; + + struct SurfaceRieszCalibrationReport final { + SurfaceRieszCalibrationTarget target{SurfaceRieszCalibrationTarget::none}; + int probeCount{0}; + SurfaceRieszCalibrationObjective objective{SurfaceRieszCalibrationObjective::operator_action}; + double leastSquaresNumerator{0.0}; + double leastSquaresDenominator{0.0}; + double scale{1.0}; + double inverseMultiplier{1.0}; + + [[nodiscard]] bool WasCalibrated() const noexcept { + return target != SurfaceRieszCalibrationTarget::none; + } + }; + + namespace detail { + struct SurfaceRieszScalarFit final { + double surrogateScale{1.0}; + double inverseMultiplier{1.0}; + }; + + [[nodiscard]] inline SurfaceRieszScalarFit fitSurfaceRieszScalar( + const double leastSquaresNumerator, + const double leastSquaresDenominator, + const SurfaceRieszCalibrationObjective objective + ) { + if (!std::isfinite(leastSquaresNumerator) || !std::isfinite(leastSquaresDenominator) || + leastSquaresDenominator <= 0.0) { + throw std::invalid_argument("Surface Riesz scalar calibration requires finite, nondegenerate data."); + } + const double fittedMultiplier = leastSquaresNumerator / leastSquaresDenominator; + if (!std::isfinite(fittedMultiplier) || fittedMultiplier == 0.0) { + throw std::runtime_error("Surface Riesz scalar calibration produced a zero or non-finite multiplier."); + } + const double surrogateScale = objective == SurfaceRieszCalibrationObjective::operator_action + ? fittedMultiplier + : 1.0 / fittedMultiplier; + if (!std::isfinite(surrogateScale) || surrogateScale == 0.0) { + throw std::runtime_error("Surface Riesz scalar calibration produced a zero or non-finite scale."); + } + return {.surrogateScale = surrogateScale, .inverseMultiplier = 1.0 / surrogateScale}; + } + } // namespace detail + + struct MaterialSurfaceBlockPreparationReport final { + bool linearizationChanged{false}; + bool rebuiltDensityInverse{false}; + bool rebuiltSurfaceInverse{false}; + bool rebuiltEnthalpyInverse{false}; + DiagonalPreparationQuality densityDiagonal; + DiagonalPreparationQuality surfaceDiagonal; + DiagonalPreparationQuality enthalpyDiagonal; + + [[nodiscard]] bool DidAnyWork() const noexcept { + return rebuiltDensityInverse || rebuiltSurfaceInverse || rebuiltEnthalpyInverse; + } + }; + + struct PreparedMaterialSurfaceBlockStatistics final { + std::uint64_t setups{0}; + std::uint64_t refreshChecks{0}; + std::uint64_t refreshes{0}; + std::uint64_t noOpRefreshes{0}; + std::uint64_t surfaceJacobianProbes{0}; + std::uint64_t surfaceRieszAssemblies{0}; + std::uint64_t surfaceH1Assemblies{0}; + }; + + template + class PreparedMaterialSurfaceBlock final : public mfem::Solver { + public: + using Block = MaterialSurfaceBlock; + + PreparedMaterialSurfaceBlock( + const operators::PreparedStellarEquilibriumOperator &operation, + Block block + ) + : mfem::Solver(MaterialSurfaceJacobianOperator(operation).Height()), + m_block(std::move(block)), + m_operation(std::addressof(operation)), + m_couplings(operation), + m_densityDiagonal(AssembleDensityDiagonal(operation)), + m_surfaceDiagonal(AssembleSurfaceRieszDiagonal(operation)), + m_enthalpyDiagonal(AssembleEnthalpyDiagonal(operation)), + m_densityQuality(Regularize( + m_densityDiagonal, + m_block.diagonalOptions(), + Communicator(operation) + )), + m_surfaceQuality(Regularize( + m_surfaceDiagonal, + m_block.diagonalOptions(), + Communicator(operation) + )), + m_enthalpyQuality(Regularize( + m_enthalpyDiagonal, + m_block.diagonalOptions(), + Communicator(operation) + )), + m_densityInverse( + m_block.materialBackend(), + m_densityDiagonal + ), + m_surfaceInverse( + m_block.surfaceBackend(), + m_surfaceDiagonal + ), + m_enthalpyInverse( + m_block.materialBackend(), + m_enthalpyDiagonal + ), + m_factorization( + m_block.factorizationPolicy(), + m_densityInverse, + m_surfaceInverse, + m_enthalpyInverse, + m_couplings + ), + m_dependencies(operation.GetDependencies()) { + m_statistics.setups = 1; + m_statistics.surfaceRieszAssemblies = 1; + m_surfaceCalibration = CalibrateSurfaceRiesz(); + } + + PreparedMaterialSurfaceBlock(const PreparedMaterialSurfaceBlock &) = delete; + PreparedMaterialSurfaceBlock &operator=(const PreparedMaterialSurfaceBlock &) = delete; + PreparedMaterialSurfaceBlock(PreparedMaterialSurfaceBlock &&) = delete; + PreparedMaterialSurfaceBlock &operator=(PreparedMaterialSurfaceBlock &&) = delete; + + void SetOperator(const mfem::Operator &operation) override { + m_factorization.SetOperator(operation); + } + + void Mult( + const mfem::Vector &rightHandSide, + mfem::Vector &action + ) const override { + if (!IsCurrent()) { + throw std::logic_error("The material-surface block is stale; refresh it before application."); + } + m_factorization.Mult(rightHandSide, action); + } + + [[nodiscard]] bool IsCurrent() const noexcept { + return m_operation->IsPrepared() && m_operation->GetDependencies() == m_dependencies; + } + + [[nodiscard]] MaterialSurfaceBlockPreparationReport + Refresh(const operators::PreparedStellarEquilibriumOperator &operation) { + if (std::addressof(operation) != m_operation) { + throw std::invalid_argument("A material-surface block cannot change stellar-operator identity."); + } + if (!operation.IsPrepared()) { + throw std::logic_error("A material-surface block cannot refresh from an unprepared operator."); + } + ++m_statistics.refreshChecks; + MaterialSurfaceBlockPreparationReport report{ + .linearizationChanged = operation.GetDependencies() != m_dependencies + }; + if (!report.linearizationChanged) { + ++m_statistics.noOpRefreshes; + return report; + } + + const operators::StellarEquilibriumDependencies currentDependencies = operation.GetDependencies(); + const bool geometryChanged = currentDependencies.discretization != m_dependencies.discretization || + currentDependencies.surfaceDeformation != m_dependencies.surfaceDeformation; + const bool calibratedSurface = + m_block.diagonalOptions().surfaceCalibration.target != SurfaceRieszCalibrationTarget::none; + + if (geometryChanged || calibratedSurface) { + m_densityDiagonal = AssembleDensityDiagonal(operation); + m_surfaceDiagonal = AssembleSurfaceRieszDiagonal(operation); + ++m_statistics.surfaceRieszAssemblies; + m_enthalpyDiagonal = AssembleEnthalpyDiagonal(operation); + m_densityQuality = Regularize(m_densityDiagonal, m_block.diagonalOptions(), Communicator(operation)); + m_surfaceQuality = Regularize(m_surfaceDiagonal, m_block.diagonalOptions(), Communicator(operation)); + m_enthalpyQuality = Regularize(m_enthalpyDiagonal, m_block.diagonalOptions(), Communicator(operation)); + m_densityInverse.Refresh(m_densityDiagonal); + // Calibration always starts from the newly assembled, + // unscaled surface mass inverse. This is required by the + // right-preconditioned objective and also prevents repeated + // refreshes from compounding the previous calibration scale. + m_surfaceInverse.Refresh(m_surfaceDiagonal); + m_enthalpyInverse.Refresh(m_enthalpyDiagonal); + if (calibratedSurface) { + m_surfaceCalibration = CalibrateSurfaceRiesz(); + } + report.rebuiltDensityInverse = true; + report.rebuiltSurfaceInverse = true; + report.rebuiltEnthalpyInverse = true; + report.densityDiagonal = m_densityQuality; + report.surfaceDiagonal = m_surfaceQuality; + report.enthalpyDiagonal = m_enthalpyQuality; + ++m_statistics.refreshes; + } else { + ++m_statistics.noOpRefreshes; + } + + m_dependencies = currentDependencies; + return report; + } + + [[nodiscard]] const MaterialSurfaceJacobianOperator &GetCoupledOperator() const noexcept { + return m_couplings; + } + [[nodiscard]] const MaterialSurfaceFactorizationOperator &GetFactorization() const noexcept { + return m_factorization; + } + [[nodiscard]] const mfem::Vector &GetDensityDiagonal() const noexcept { + return m_densityDiagonal; + } + [[nodiscard]] const mfem::Vector &GetSurfaceDiagonal() const noexcept { + return m_surfaceDiagonal; + } + [[nodiscard]] const mfem::Vector &GetEnthalpyDiagonal() const noexcept { + return m_enthalpyDiagonal; + } + [[nodiscard]] const DiagonalPreparationQuality &GetDensityDiagonalQuality() const noexcept { + return m_densityQuality; + } + [[nodiscard]] const DiagonalPreparationQuality &GetSurfaceDiagonalQuality() const noexcept { + return m_surfaceQuality; + } + [[nodiscard]] const SurfaceRieszCalibrationReport &GetSurfaceCalibration() const noexcept { + return m_surfaceCalibration; + } + [[nodiscard]] const DiagonalPreparationQuality &GetEnthalpyDiagonalQuality() const noexcept { + return m_enthalpyQuality; + } + [[nodiscard]] const PreparedMaterialSurfaceBlockStatistics &GetStatistics() const noexcept { + return m_statistics; + } + + private: + [[nodiscard]] static MPI_Comm Communicator(const operators::PreparedStellarEquilibriumOperator &operation) { + return operation.GetHydrostaticOperator().GetFEM().mesh->GetComm(); + } + [[nodiscard]] static mfem::Vector + AssembleDensityDiagonal(const operators::PreparedStellarEquilibriumOperator &operation) { + mfem::Vector diagonal; + operation.GetBarotropicClosureOperator().AssembleDensityJacobianDiagonal(diagonal); + return diagonal; + } + [[nodiscard]] static mfem::Vector + AssembleEnthalpyDiagonal(const operators::PreparedStellarEquilibriumOperator &operation) { + mfem::Vector diagonal; + operation.GetHydrostaticOperator().AssembleEnthalpyJacobianDiagonal(diagonal); + mfem::Vector ones(diagonal.Size()); + ones = 1.0; + operation.GetSurfaceConstraintOperator().ApplyJacobianRows(ones, diagonal); + return diagonal; + } + [[nodiscard]] static mfem::Vector + AssembleSurfaceRieszDiagonal(const operators::PreparedStellarEquilibriumOperator &operation) { + using DomainSchema = utils::domain::CoreEnvelopeVacuumDomainSchema; + + const fem::FEM &finiteElements = operation.GetHydrostaticOperator().GetFEM(); + MFEM_VERIFY( + finiteElements.mesh != nullptr && finiteElements.surfaceDeformationFes != nullptr, + "The material/free-surface Riesz surrogate requires the surface finite-element space." + ); + + mfem::Array stellarSurfaceMarker(finiteElements.mesh->bdr_attributes.Max()); + stellarSurfaceMarker = 0; + constexpr int stellarSurfaceAttribute = + DomainSchema::template boundary_attribute(); + MFEM_VERIFY( + stellarSurfaceAttribute > 0 && stellarSurfaceAttribute <= stellarSurfaceMarker.Size(), + "The stellar-surface boundary attribute is absent from the finite-element mesh." + ); + stellarSurfaceMarker[stellarSurfaceAttribute - 1] = 1; + + mfem::ParBilinearForm surfaceRiesz(finiteElements.surfaceDeformationFes.get()); + surfaceRiesz.AddBoundaryIntegrator(new mfem::MassIntegrator(), stellarSurfaceMarker); + surfaceRiesz.Assemble(); + surfaceRiesz.Finalize(); + + std::unique_ptr surfaceRieszMatrix(surfaceRiesz.ParallelAssemble()); + MFEM_VERIFY(surfaceRieszMatrix != nullptr, "The stellar-surface Riesz surrogate failed to assemble."); + mfem::Vector ambientDiagonal; + surfaceRieszMatrix->GetDiag(ambientDiagonal); + + const field::ScalarBoundaryDofMap surfaceMap = + field::make_stellar_surface_scalar_dof_map(*finiteElements.surfaceDeformationFes); + mfem::Vector diagonal = surfaceMap.gather(ambientDiagonal); + MFEM_VERIFY( + diagonal.Size() == operation.GetDomainDeformation().parameterCount(), + "The stellar-surface Riesz diagonal does not match the deformation parameter space." + ); + return diagonal; + } + + [[nodiscard]] SurfaceRieszCalibrationReport CalibrateSurfaceRiesz() { + const SurfaceRieszCalibrationOptions calibration = m_block.diagonalOptions().surfaceCalibration; + if (calibration.target == SurfaceRieszCalibrationTarget::none) { + if (calibration.probeCount != 0 || + calibration.objective != SurfaceRieszCalibrationObjective::operator_action) { + throw std::invalid_argument( + "An uncalibrated surface Riesz surrogate must request zero probes and the default objective." + ); + } + return {}; + } + if (calibration.probeCount <= 0) { + throw std::invalid_argument("Surface Riesz calibration requires at least one deterministic probe."); + } + + using DomainSchema = utils::domain::CoreEnvelopeVacuumDomainSchema; + const fem::FEM &finiteElements = m_operation->GetHydrostaticOperator().GetFEM(); + const field::ScalarBoundaryDofMap surfaceMap = + field::make_stellar_surface_scalar_dof_map(*finiteElements.surfaceDeformationFes); + if (surfaceMap.local_size() != m_surfaceDiagonal.Size()) { + throw std::logic_error("Surface Riesz calibration received an incompatible boundary coordinate map."); + } + + mfem::Vector probe(m_surfaceDiagonal.Size()); + mfem::Vector rieszAction(m_surfaceDiagonal.Size()); + mfem::Vector targetAction(m_surfaceDiagonal.Size()); + mfem::Vector baseInverseAction(m_surfaceDiagonal.Size()); + mfem::Vector materialFeedback(m_surfaceDiagonal.Size()); + mfem::Vector densityRightHandSide(m_densityDiagonal.Size()); + mfem::Vector densityCorrection(m_densityDiagonal.Size()); + mfem::Vector densityCoupling(m_densityDiagonal.Size()); + mfem::Vector enthalpyRightHandSide(m_enthalpyDiagonal.Size()); + mfem::Vector enthalpyCorrection(m_enthalpyDiagonal.Size()); + + double localNumerator = 0.0; + double localDenominator = 0.0; + for (int sample = 0; sample < calibration.probeCount; ++sample) { + for (int index = 0; index < probe.Size(); ++index) { + std::uint64_t value = static_cast(surfaceMap.global_boundary_dof(index)); + value += 0x9e3779b97f4a7c15ULL * static_cast(sample + 1); + value = (value ^ (value >> 30U)) * 0xbf58476d1ce4e5b9ULL; + value = (value ^ (value >> 27U)) * 0x94d049bb133111ebULL; + value ^= value >> 31U; + probe(index) = (value & 1ULL) == 0ULL ? -1.0 : 1.0; + rieszAction(index) = m_surfaceDiagonal(index) * probe(index); + } + + if (calibration.objective == SurfaceRieszCalibrationObjective::right_preconditioned_action) { + m_surfaceInverse.Mult(probe, baseInverseAction); + } + const mfem::Vector &targetDirection = + calibration.objective == SurfaceRieszCalibrationObjective::operator_action ? probe + : baseInverseAction; + m_couplings.ApplySurfaceToSurface(targetDirection, targetAction); + if (calibration.target == SurfaceRieszCalibrationTarget::approximate_material_schur) { + m_couplings.ApplySurfaceToMaterial(targetDirection, densityRightHandSide, enthalpyRightHandSide); + m_enthalpyInverse.Mult(enthalpyRightHandSide, enthalpyCorrection); + m_couplings.ApplyEnthalpyToDensity(enthalpyCorrection, densityCoupling); + densityRightHandSide -= densityCoupling; + m_densityInverse.Mult(densityRightHandSide, densityCorrection); + m_couplings.ApplyMaterialToSurface(densityCorrection, enthalpyCorrection, materialFeedback); + targetAction -= materialFeedback; + } + + if (calibration.objective == SurfaceRieszCalibrationObjective::operator_action) { + // Fit s M q ~= T q. The stored surface surrogate is s M. + localNumerator += rieszAction * targetAction; + localDenominator += rieszAction * rieszAction; + } else { + // Fit alpha T M^{-1} q ~= q. Since the stored surrogate is + // s M, its inverse multiplier is alpha = 1 / s. This is the + // surface factor in the right-preconditioned product. + localNumerator += targetAction * probe; + localDenominator += targetAction * targetAction; + } + } + + const MPI_Comm communicator = Communicator(*m_operation); + double globalNumerator = 0.0; + double globalDenominator = 0.0; + MPI_Allreduce(&localNumerator, &globalNumerator, 1, MPI_DOUBLE, MPI_SUM, communicator); + MPI_Allreduce(&localDenominator, &globalDenominator, 1, MPI_DOUBLE, MPI_SUM, communicator); + if (!std::isfinite(globalNumerator) || !std::isfinite(globalDenominator) || globalDenominator <= 0.0) { + throw std::runtime_error("Surface Riesz calibration produced an invalid least-squares problem."); + } + const detail::SurfaceRieszScalarFit fit = + detail::fitSurfaceRieszScalar(globalNumerator, globalDenominator, calibration.objective); + + m_surfaceDiagonal *= fit.surrogateScale; + m_surfaceQuality = Regularize(m_surfaceDiagonal, m_block.diagonalOptions(), communicator); + m_surfaceInverse.Refresh(m_surfaceDiagonal); + m_statistics.surfaceJacobianProbes += static_cast(calibration.probeCount); + return { + .target = calibration.target, + .probeCount = calibration.probeCount, + .objective = calibration.objective, + .leastSquaresNumerator = globalNumerator, + .leastSquaresDenominator = globalDenominator, + .scale = fit.surrogateScale, + .inverseMultiplier = fit.inverseMultiplier + }; + } + + [[nodiscard]] static DiagonalPreparationQuality Regularize( + mfem::Vector &diagonal, + const MaterialSurfaceDiagonalOptions options, + const MPI_Comm communicator + ) { + if (!std::isfinite(options.relativeFloor) || options.relativeFloor < 0.0 || + !std::isfinite(options.absoluteFloor) || options.absoluteFloor <= 0.0) { + throw std::invalid_argument("Material-surface diagonal floors must be finite and nonnegative."); + } + double localMaximum = 0.0; + double localMinimum = std::numeric_limits::infinity(); + for (int index = 0; index < diagonal.Size(); ++index) { + if (!std::isfinite(diagonal(index))) { + throw std::invalid_argument("A material-surface diagonal contains a non-finite entry."); + } + const double magnitude = std::abs(diagonal(index)); + localMaximum = std::max(localMaximum, magnitude); + localMinimum = std::min(localMinimum, magnitude); + } + double globalMaximum = 0.0; + double globalMinimum = 0.0; + MPI_Allreduce(&localMaximum, &globalMaximum, 1, MPI_DOUBLE, MPI_MAX, communicator); + MPI_Allreduce(&localMinimum, &globalMinimum, 1, MPI_DOUBLE, MPI_MIN, communicator); + const double floor = std::max(options.absoluteFloor, options.relativeFloor * globalMaximum); + std::uint64_t localRegularized = 0; + for (int index = 0; index < diagonal.Size(); ++index) { + if (std::abs(diagonal(index)) < floor) { + diagonal(index) = std::copysign(floor, diagonal(index) == 0.0 ? 1.0 : diagonal(index)); + ++localRegularized; + } + } + std::uint64_t globalRegularized = 0; + MPI_Allreduce(&localRegularized, &globalRegularized, 1, MPI_UINT64_T, MPI_SUM, communicator); + return { + .minimumAbsoluteEntryBeforeRegularization = globalMinimum, + .maximumAbsoluteEntryBeforeRegularization = globalMaximum, + .appliedFloor = floor, + .regularizedEntries = globalRegularized + }; + } + + Block m_block; + const operators::PreparedStellarEquilibriumOperator *m_operation; + MaterialSurfaceJacobianOperator m_couplings; + mfem::Vector m_densityDiagonal; + mfem::Vector m_surfaceDiagonal; + mfem::Vector m_enthalpyDiagonal; + DiagonalPreparationQuality m_densityQuality; + DiagonalPreparationQuality m_surfaceQuality; + DiagonalPreparationQuality m_enthalpyQuality; + SurfaceRieszCalibrationReport m_surfaceCalibration; + backend::PreparedDiagonal m_densityInverse; + backend::PreparedDiagonal m_surfaceInverse; + backend::PreparedDiagonal m_enthalpyInverse; + MaterialSurfaceFactorizationOperator m_factorization; + operators::StellarEquilibriumDependencies m_dependencies; + PreparedMaterialSurfaceBlockStatistics m_statistics; + }; + + // The AMG backend acts on the ambient scalar H1 true-DOF space. Surface + // deformation parameters, however, contain only the locally owned trace + // DOFs. This adapter is the sole conversion point between those two + // coordinate systems. The sign is deliberately applied outside AMG so the + // sparse surrogate supplied to BoomerAMG remains positive definite. + class SignedScalarBoundarySolverAdapter final : public mfem::Solver { + public: + SignedScalarBoundarySolverAdapter( + const mfem::Solver &ambientSolver, + field::ScalarBoundaryDofMap surfaceMap, + const double sign + ) + : mfem::Solver(surfaceMap.local_size()), + m_ambientSolver(std::addressof(ambientSolver)), + m_surfaceMap(std::move(surfaceMap)), + m_ambientRightHandSide(m_surfaceMap.volume_true_dof_size()), + m_ambientAction(m_surfaceMap.volume_true_dof_size()) { + if (ambientSolver.Height() != m_surfaceMap.volume_true_dof_size() || + ambientSolver.Width() != m_surfaceMap.volume_true_dof_size()) { + throw std::invalid_argument("The surface AMG solver is incompatible with its ambient H1 space."); + } + SetSign(sign); + } + + void SetOperator(const mfem::Operator &operation) override { + if (operation.Height() != Height() || operation.Width() != Width()) { + throw std::invalid_argument("The surface trace solver received an operator of incompatible size."); + } + } + + void SetSign(const double sign) { + if (sign != -1.0 && sign != 1.0) { + throw std::invalid_argument("The fitted surface-operator sign must be exactly -1 or +1."); + } + m_sign = sign; + } + + void Mult( + const mfem::Vector &rightHandSide, + mfem::Vector &action + ) const override { + if (rightHandSide.Size() != Width() || action.Size() != Height()) { + throw std::invalid_argument("The surface trace solver received incompatible vectors."); + } + m_surfaceMap.scatter(rightHandSide, m_ambientRightHandSide); + m_ambientSolver->Mult(m_ambientRightHandSide, m_ambientAction); + m_surfaceMap.gather(m_ambientAction, action); + action *= m_sign; + } + + [[nodiscard]] double GetSign() const noexcept { + return m_sign; + } + + [[nodiscard]] const field::ScalarBoundaryDofMap &GetSurfaceMap() const noexcept { + return m_surfaceMap; + } + + private: + const mfem::Solver *m_ambientSolver; + field::ScalarBoundaryDofMap m_surfaceMap; + double m_sign{1.0}; + mutable mfem::Vector m_ambientRightHandSide; + mutable mfem::Vector m_ambientAction; + }; + + template < + ImplementedMaterialSurfaceDescriptor Descriptor, + MaterialSurfaceFactorizationPolicy Policy, + backend::ApplicationMode Mode> + class PreparedH1MaterialSurfaceBlock final : public mfem::Solver { + public: + using SurfaceBackend = backend::HypreBoomerAMG; + using Block = + MaterialSurfaceBlock; + + PreparedH1MaterialSurfaceBlock( + const operators::PreparedStellarEquilibriumOperator &operation, + Block block + ) + : mfem::Solver(MaterialSurfaceJacobianOperator(operation).Height()), + m_block(std::move(block)), + m_operation(std::addressof(operation)), + m_couplings(operation), + m_surfaceMap(SurfaceMap(operation)), + m_densityDiagonal(AssembleDensityDiagonal(operation)), + m_enthalpyDiagonal(AssembleEnthalpyDiagonal(operation)), + m_dependencies(operation.GetDependencies()) { + ValidateConfiguration(m_block); + m_densityQuality = Regularize(m_densityDiagonal, m_block.diagonalOptions(), Communicator(operation)); + m_enthalpyQuality = Regularize(m_enthalpyDiagonal, m_block.diagonalOptions(), Communicator(operation)); + m_densityInverse = + std::make_unique(m_block.materialBackend(), m_densityDiagonal); + m_enthalpyInverse = + std::make_unique(m_block.materialBackend(), m_enthalpyDiagonal); + + m_surfaceMass = AssembleSurfaceOperator(operation, 1.0, 0.0, false); + m_surfaceStiffness = AssembleSurfaceOperator(operation, 0.0, 1.0, false); + m_surfaceFit = FitSurfaceOperator(); + m_surfaceSurrogate = AssembleSurfaceOperator( + operation, m_surfaceFit.massCoefficient, m_surfaceFit.stiffnessCoefficient, true + ); + m_surfaceInverse = + std::make_unique>(m_block.surfaceBackend(), *m_surfaceSurrogate); + RebindFactorization(); + + m_statistics.setups = 1; + m_statistics.surfaceJacobianProbes = + static_cast(m_block.surfaceSurrogate().calibration.probeCount); + m_statistics.surfaceH1Assemblies = 3; + } + + PreparedH1MaterialSurfaceBlock(const PreparedH1MaterialSurfaceBlock &) = delete; + PreparedH1MaterialSurfaceBlock &operator=(const PreparedH1MaterialSurfaceBlock &) = delete; + PreparedH1MaterialSurfaceBlock(PreparedH1MaterialSurfaceBlock &&) = delete; + PreparedH1MaterialSurfaceBlock &operator=(PreparedH1MaterialSurfaceBlock &&) = delete; + + void SetOperator(const mfem::Operator &operation) override { + m_factorization->SetOperator(operation); + } + + void Mult( + const mfem::Vector &rightHandSide, + mfem::Vector &action + ) const override { + if (!IsCurrent()) { + throw std::logic_error("The H1 material-surface block is stale; refresh it before application."); + } + m_factorization->Mult(rightHandSide, action); + } + + [[nodiscard]] bool IsCurrent() const noexcept { + return m_operation->IsPrepared() && m_operation->GetDependencies() == m_dependencies; + } + + [[nodiscard]] MaterialSurfaceBlockPreparationReport + Refresh(const operators::PreparedStellarEquilibriumOperator &operation) { + if (std::addressof(operation) != m_operation) { + throw std::invalid_argument("An H1 material-surface block cannot change stellar-operator identity."); + } + if (!operation.IsPrepared()) { + throw std::logic_error("An H1 material-surface block cannot refresh from an unprepared operator."); + } + ++m_statistics.refreshChecks; + MaterialSurfaceBlockPreparationReport report{ + .linearizationChanged = operation.GetDependencies() != m_dependencies + }; + if (!report.linearizationChanged) { + ++m_statistics.noOpRefreshes; + return report; + } + + const operators::StellarEquilibriumDependencies currentDependencies = operation.GetDependencies(); + const bool geometryChanged = currentDependencies.discretization != m_dependencies.discretization || + currentDependencies.surfaceDeformation != m_dependencies.surfaceDeformation; + + m_densityDiagonal = AssembleDensityDiagonal(operation); + m_enthalpyDiagonal = AssembleEnthalpyDiagonal(operation); + m_densityQuality = Regularize(m_densityDiagonal, m_block.diagonalOptions(), Communicator(operation)); + m_enthalpyQuality = Regularize(m_enthalpyDiagonal, m_block.diagonalOptions(), Communicator(operation)); + m_densityInverse->Refresh(m_densityDiagonal); + m_enthalpyInverse->Refresh(m_enthalpyDiagonal); + + if (geometryChanged) { + field::ScalarBoundaryDofMap currentSurfaceMap = SurfaceMap(operation); + if (currentSurfaceMap.local_size() != m_surfaceMap.local_size() || + currentSurfaceMap.volume_true_dof_size() != m_surfaceMap.volume_true_dof_size()) { + throw std::invalid_argument( + "An H1 material-surface block cannot change discretization size during refresh." + ); + } + m_surfaceMap = std::move(currentSurfaceMap); + m_surfaceMass = AssembleSurfaceOperator(operation, 1.0, 0.0, false); + m_surfaceStiffness = AssembleSurfaceOperator(operation, 0.0, 1.0, false); + m_statistics.surfaceH1Assemblies += 2; + } + + m_surfaceFit = FitSurfaceOperator(); + auto refreshedSurrogate = AssembleSurfaceOperator( + operation, m_surfaceFit.massCoefficient, m_surfaceFit.stiffnessCoefficient, true + ); + m_surfaceInverse->Refresh(*refreshedSurrogate); + m_surfaceSurrogate = std::move(refreshedSurrogate); + RebindFactorization(); + + m_statistics.surfaceJacobianProbes += + static_cast(m_block.surfaceSurrogate().calibration.probeCount); + ++m_statistics.surfaceH1Assemblies; + ++m_statistics.refreshes; + report.rebuiltDensityInverse = true; + report.rebuiltSurfaceInverse = true; + report.rebuiltEnthalpyInverse = true; + report.densityDiagonal = m_densityQuality; + report.enthalpyDiagonal = m_enthalpyQuality; + m_dependencies = currentDependencies; + return report; + } + + [[nodiscard]] const MaterialSurfaceJacobianOperator &GetCoupledOperator() const noexcept { + return m_couplings; + } + [[nodiscard]] const MaterialSurfaceFactorizationOperator &GetFactorization() const noexcept { + return *m_factorization; + } + [[nodiscard]] const SurfaceH1FitReport &GetSurfaceFit() const noexcept { + return m_surfaceFit; + } + [[nodiscard]] const mfem::HypreParMatrix &GetSurfaceMassMatrix() const noexcept { + return *m_surfaceMass; + } + [[nodiscard]] const mfem::HypreParMatrix &GetSurfaceStiffnessMatrix() const noexcept { + return *m_surfaceStiffness; + } + [[nodiscard]] const mfem::HypreParMatrix &GetSurfaceSurrogateMatrix() const noexcept { + return *m_surfaceSurrogate; + } + [[nodiscard]] const SignedScalarBoundarySolverAdapter &GetSurfaceInverse() const noexcept { + return *m_surfaceBoundaryInverse; + } + [[nodiscard]] const backend::PreparedHypreBoomerAMG &GetSurfaceBackend() const noexcept { + return *m_surfaceInverse; + } + [[nodiscard]] const mfem::Vector &GetDensityDiagonal() const noexcept { + return m_densityDiagonal; + } + [[nodiscard]] const mfem::Vector &GetEnthalpyDiagonal() const noexcept { + return m_enthalpyDiagonal; + } + [[nodiscard]] const DiagonalPreparationQuality &GetDensityDiagonalQuality() const noexcept { + return m_densityQuality; + } + [[nodiscard]] const DiagonalPreparationQuality &GetEnthalpyDiagonalQuality() const noexcept { + return m_enthalpyQuality; + } + [[nodiscard]] const PreparedMaterialSurfaceBlockStatistics &GetStatistics() const noexcept { + return m_statistics; + } + + private: + static void ValidateConfiguration(const Block &block) { + const SurfaceRieszCalibrationOptions legacyCalibration = block.diagonalOptions().surfaceCalibration; + if (legacyCalibration.target != SurfaceRieszCalibrationTarget::none || legacyCalibration.probeCount != 0) { + throw std::invalid_argument( + "Surface H1 calibration must be configured on SurfaceH1MassStiffness, not diagonal options." + ); + } + const SurfaceH1MassStiffness &surface = block.surfaceSurrogate(); + if (surface.calibration.target == SurfaceRieszCalibrationTarget::none || + surface.calibration.probeCount < 3 || surface.calibration.probeCount > 64 || + surface.calibration.objective != SurfaceRieszCalibrationObjective::operator_action || + !std::isfinite(surface.relativeMassCoefficientFloor) || surface.relativeMassCoefficientFloor <= 0.0 || + !std::isfinite(surface.gramRelativeTolerance) || surface.gramRelativeTolerance <= 0.0 || + surface.gramRelativeTolerance >= 1.0) { + throw std::invalid_argument( + "SurfaceH1MassStiffness requires an operator-action target, 3--64 probes, and finite positive fit " + "tolerances." + ); + } + } + + [[nodiscard]] static MPI_Comm Communicator(const operators::PreparedStellarEquilibriumOperator &operation) { + return operation.GetHydrostaticOperator().GetFEM().mesh->GetComm(); + } + + [[nodiscard]] static field::ScalarBoundaryDofMap + SurfaceMap(const operators::PreparedStellarEquilibriumOperator &operation) { + using DomainSchema = utils::domain::CoreEnvelopeVacuumDomainSchema; + const fem::FEM &finiteElements = operation.GetHydrostaticOperator().GetFEM(); + return field::make_stellar_surface_scalar_dof_map(*finiteElements.surfaceDeformationFes); + } + + [[nodiscard]] static mfem::Vector + AssembleDensityDiagonal(const operators::PreparedStellarEquilibriumOperator &operation) { + mfem::Vector diagonal; + operation.GetBarotropicClosureOperator().AssembleDensityJacobianDiagonal(diagonal); + return diagonal; + } + + [[nodiscard]] static mfem::Vector + AssembleEnthalpyDiagonal(const operators::PreparedStellarEquilibriumOperator &operation) { + mfem::Vector diagonal; + operation.GetHydrostaticOperator().AssembleEnthalpyJacobianDiagonal(diagonal); + mfem::Vector ones(diagonal.Size()); + ones = 1.0; + operation.GetSurfaceConstraintOperator().ApplyJacobianRows(ones, diagonal); + return diagonal; + } + + [[nodiscard]] static std::unique_ptr AssembleSurfaceOperator( + const operators::PreparedStellarEquilibriumOperator &operation, + const double massCoefficient, + const double stiffnessCoefficient, + const bool eliminateZeroRows + ) { + if (!std::isfinite(massCoefficient) || !std::isfinite(stiffnessCoefficient) || massCoefficient < 0.0 || + stiffnessCoefficient < 0.0 || massCoefficient + stiffnessCoefficient <= 0.0) { + throw std::invalid_argument("Surface H1 coefficients must be finite, nonnegative, and nonzero."); + } + using DomainSchema = utils::domain::CoreEnvelopeVacuumDomainSchema; + const fem::FEM &finiteElements = operation.GetHydrostaticOperator().GetFEM(); + MFEM_VERIFY( + finiteElements.mesh != nullptr && finiteElements.surfaceDeformationFes != nullptr, + "The surface H1 surrogate requires the ambient scalar finite-element space." + ); + + mfem::Array stellarSurfaceMarker(finiteElements.mesh->bdr_attributes.Max()); + stellarSurfaceMarker = 0; + constexpr int stellarSurfaceAttribute = + DomainSchema::template boundary_attribute(); + MFEM_VERIFY( + stellarSurfaceAttribute > 0 && stellarSurfaceAttribute <= stellarSurfaceMarker.Size(), + "The stellar-surface boundary attribute is absent from the finite-element mesh." + ); + stellarSurfaceMarker[stellarSurfaceAttribute - 1] = 1; + + mfem::ConstantCoefficient massWeight(massCoefficient); + mfem::ConstantCoefficient stiffnessWeight(stiffnessCoefficient); + mfem::ParBilinearForm surfaceOperator(finiteElements.surfaceDeformationFes.get()); + if (massCoefficient > 0.0) { + surfaceOperator.AddBoundaryIntegrator(new mfem::MassIntegrator(massWeight), stellarSurfaceMarker); + } + if (stiffnessCoefficient > 0.0) { + surfaceOperator.AddBoundaryIntegrator( + new mfem::DiffusionIntegrator(stiffnessWeight), stellarSurfaceMarker + ); + } + surfaceOperator.Assemble(); + surfaceOperator.Finalize(); + std::unique_ptr matrix(surfaceOperator.ParallelAssemble()); + MFEM_VERIFY(matrix != nullptr, "The surface H1 surrogate failed to assemble."); + if (eliminateZeroRows) { + matrix->EliminateZeroRows(); + } + return matrix; + } + + [[nodiscard]] static double Legendre12(const double coordinate) { + double previous = 1.0; + double current = coordinate; + for (int degree = 2; degree <= 12; ++degree) { + const double next = ((2.0 * static_cast(degree) - 1.0) * coordinate * current - + (static_cast(degree) - 1.0) * previous) / + static_cast(degree); + previous = current; + current = next; + } + return current; + } + + void FillProbe( + const int sample, + mfem::Vector &probe + ) const { + const bool useOrderedMode = sample < 3; + for (int index = 0; index < probe.Size(); ++index) { + const long long globalDof = m_surfaceMap.global_boundary_dof(index); + if (useOrderedMode) { + const double coordinate = -1.0 + 2.0 * (static_cast(globalDof) + 0.5) / + static_cast(m_surfaceMap.global_size()); + const double mode = sample == 0 ? 1.0 + : (sample == 1 ? 0.5 * (3.0 * coordinate * coordinate - 1.0) + : Legendre12(coordinate)); + probe(index) = mode; + } else { + std::uint64_t value = static_cast(globalDof); + value += 0x9e3779b97f4a7c15ULL * static_cast(sample + 1); + value = (value ^ (value >> 30U)) * 0xbf58476d1ce4e5b9ULL; + value = (value ^ (value >> 27U)) * 0x94d049bb133111ebULL; + value ^= value >> 31U; + probe(index) = (value & 1ULL) == 0ULL ? -1.0 : 1.0; + } + } + const double localNormSquared = probe * probe; + double globalNormSquared = 0.0; + MPI_Allreduce(&localNormSquared, &globalNormSquared, 1, MPI_DOUBLE, MPI_SUM, Communicator(*m_operation)); + if (!std::isfinite(globalNormSquared) || globalNormSquared <= 0.0) { + throw std::runtime_error("A deterministic surface H1 calibration probe has zero norm."); + } + probe /= std::sqrt(globalNormSquared); + } + + void ApplyAmbientSurfaceOperator( + const mfem::HypreParMatrix &surfaceOperator, + const mfem::Vector &probe, + mfem::Vector &action, + mfem::Vector &ambientProbe, + mfem::Vector &ambientAction + ) const { + m_surfaceMap.scatter(probe, ambientProbe); + surfaceOperator.Mult(ambientProbe, ambientAction); + m_surfaceMap.gather(ambientAction, action); + } + + [[nodiscard]] SurfaceH1FitReport FitSurfaceOperator() { + const SurfaceH1MassStiffness &configuration = m_block.surfaceSurrogate(); + if (configuration.calibration.target == SurfaceRieszCalibrationTarget::none || + configuration.calibration.probeCount < 3) { + throw std::invalid_argument( + "The surface H1 mass-plus-stiffness surrogate requires at least three calibration probes." + ); + } + + mfem::Vector probe(m_surfaceMap.local_size()); + mfem::Vector massAction(m_surfaceMap.local_size()); + mfem::Vector stiffnessAction(m_surfaceMap.local_size()); + mfem::Vector targetAction(m_surfaceMap.local_size()); + mfem::Vector materialFeedback(m_surfaceMap.local_size()); + mfem::Vector ambientProbe(m_surfaceMap.volume_true_dof_size()); + mfem::Vector ambientAction(m_surfaceMap.volume_true_dof_size()); + mfem::Vector densityRightHandSide(m_densityDiagonal.Size()); + mfem::Vector densityCorrection(m_densityDiagonal.Size()); + mfem::Vector densityCoupling(m_densityDiagonal.Size()); + mfem::Vector enthalpyRightHandSide(m_enthalpyDiagonal.Size()); + mfem::Vector enthalpyCorrection(m_enthalpyDiagonal.Size()); + + SurfaceH1NormalEquations local; + for (int sample = 0; sample < configuration.calibration.probeCount; ++sample) { + FillProbe(sample, probe); + ApplyAmbientSurfaceOperator(*m_surfaceMass, probe, massAction, ambientProbe, ambientAction); + ApplyAmbientSurfaceOperator(*m_surfaceStiffness, probe, stiffnessAction, ambientProbe, ambientAction); + + m_couplings.ApplySurfaceToSurface(probe, targetAction); + if (configuration.calibration.target == SurfaceRieszCalibrationTarget::approximate_material_schur) { + m_couplings.ApplySurfaceToMaterial(probe, densityRightHandSide, enthalpyRightHandSide); + m_enthalpyInverse->Mult(enthalpyRightHandSide, enthalpyCorrection); + m_couplings.ApplyEnthalpyToDensity(enthalpyCorrection, densityCoupling); + densityRightHandSide -= densityCoupling; + m_densityInverse->Mult(densityRightHandSide, densityCorrection); + m_couplings.ApplyMaterialToSurface(densityCorrection, enthalpyCorrection, materialFeedback); + targetAction -= materialFeedback; + } + + local.massMass += massAction * massAction; + local.massStiffness += massAction * stiffnessAction; + local.stiffnessStiffness += stiffnessAction * stiffnessAction; + local.massTarget += massAction * targetAction; + local.stiffnessTarget += stiffnessAction * targetAction; + local.targetTarget += targetAction * targetAction; + } + + std::array localValues{local.massMass, local.massStiffness, local.stiffnessStiffness, + local.massTarget, local.stiffnessTarget, local.targetTarget}; + std::array globalValues{}; + MPI_Allreduce( + localValues.data(), globalValues.data(), static_cast(globalValues.size()), MPI_DOUBLE, MPI_SUM, + Communicator(*m_operation) + ); + const SurfaceH1NormalEquations global{ + .massMass = globalValues[0], + .massStiffness = globalValues[1], + .stiffnessStiffness = globalValues[2], + .massTarget = globalValues[3], + .stiffnessTarget = globalValues[4], + .targetTarget = globalValues[5] + }; + return detail::fitSurfaceH1Coefficients(global, configuration); + } + + void RebindFactorization() { + auto surfaceBoundaryInverse = + std::make_unique(*m_surfaceInverse, m_surfaceMap, m_surfaceFit.sign); + auto factorization = std::make_unique>( + m_block.factorizationPolicy(), *m_densityInverse, *surfaceBoundaryInverse, *m_enthalpyInverse, + m_couplings + ); + m_factorization = std::move(factorization); + m_surfaceBoundaryInverse = std::move(surfaceBoundaryInverse); + } + + [[nodiscard]] static DiagonalPreparationQuality Regularize( + mfem::Vector &diagonal, + const MaterialSurfaceDiagonalOptions options, + const MPI_Comm communicator + ) { + if (!std::isfinite(options.relativeFloor) || options.relativeFloor < 0.0 || + !std::isfinite(options.absoluteFloor) || options.absoluteFloor <= 0.0) { + throw std::invalid_argument("Material-surface diagonal floors must be finite and nonnegative."); + } + double localMaximum = 0.0; + double localMinimum = std::numeric_limits::infinity(); + for (int index = 0; index < diagonal.Size(); ++index) { + if (!std::isfinite(diagonal(index))) { + throw std::invalid_argument("A material-surface diagonal contains a non-finite entry."); + } + const double magnitude = std::abs(diagonal(index)); + localMaximum = std::max(localMaximum, magnitude); + localMinimum = std::min(localMinimum, magnitude); + } + double globalMaximum = 0.0; + double globalMinimum = 0.0; + MPI_Allreduce(&localMaximum, &globalMaximum, 1, MPI_DOUBLE, MPI_MAX, communicator); + MPI_Allreduce(&localMinimum, &globalMinimum, 1, MPI_DOUBLE, MPI_MIN, communicator); + const double floor = std::max(options.absoluteFloor, options.relativeFloor * globalMaximum); + std::uint64_t localRegularized = 0; + for (int index = 0; index < diagonal.Size(); ++index) { + if (std::abs(diagonal(index)) < floor) { + diagonal(index) = std::copysign(floor, diagonal(index) == 0.0 ? 1.0 : diagonal(index)); + ++localRegularized; + } + } + std::uint64_t globalRegularized = 0; + MPI_Allreduce(&localRegularized, &globalRegularized, 1, MPI_UINT64_T, MPI_SUM, communicator); + return { + .minimumAbsoluteEntryBeforeRegularization = globalMinimum, + .maximumAbsoluteEntryBeforeRegularization = globalMaximum, + .appliedFloor = floor, + .regularizedEntries = globalRegularized + }; + } + + Block m_block; + const operators::PreparedStellarEquilibriumOperator *m_operation; + MaterialSurfaceJacobianOperator m_couplings; + field::ScalarBoundaryDofMap m_surfaceMap; + mfem::Vector m_densityDiagonal; + mfem::Vector m_enthalpyDiagonal; + DiagonalPreparationQuality m_densityQuality; + DiagonalPreparationQuality m_enthalpyQuality; + std::unique_ptr m_densityInverse; + std::unique_ptr m_enthalpyInverse; + std::unique_ptr m_surfaceMass; + std::unique_ptr m_surfaceStiffness; + std::unique_ptr m_surfaceSurrogate; + SurfaceH1FitReport m_surfaceFit; + std::unique_ptr> m_surfaceInverse; + std::unique_ptr m_surfaceBoundaryInverse; + std::unique_ptr> m_factorization; + operators::StellarEquilibriumDependencies m_dependencies; + PreparedMaterialSurfaceBlockStatistics m_statistics; + }; + + template < + ImplementedMaterialSurfaceDescriptor Descriptor, + MaterialSurfaceFactorizationPolicy Policy> + [[nodiscard]] auto prepare( + const operators::PreparedStellarEquilibriumOperator &operation, + MaterialSurfaceBlock< + Descriptor, + backend::Diagonal, + backend::Diagonal, + Policy> block + ) { + return PreparedMaterialSurfaceBlock{operation, std::move(block)}; + } + + template < + equilibrium::StellarEquilibriumModel Model, + MaterialSurfaceFactorizationPolicy Policy> + [[nodiscard]] auto prepare( + const equilibrium::StellarEquilibriumProblem &problem, + MaterialSurfaceBlock< + MaterialSurfaceDescriptorFor>, + backend::Diagonal, + backend::Diagonal, + Policy> block + ) { + if constexpr (equilibrium::StellarEquilibriumProblem::hasFixedCentralDensity) { + return prepare(problem.GetPreparedOperator().GetPhysicalOperator(), std::move(block)); + } else { + return prepare(problem.GetPreparedOperator(), std::move(block)); + } + } + + template < + ImplementedMaterialSurfaceDescriptor Descriptor, + MaterialSurfaceFactorizationPolicy Policy, + backend::ApplicationMode Mode> + [[nodiscard]] auto prepare( + const operators::PreparedStellarEquilibriumOperator &operation, + MaterialSurfaceBlock< + Descriptor, + backend::Diagonal, + backend::HypreBoomerAMG, + Policy, + SurfaceH1MassStiffness> block + ) { + return PreparedH1MaterialSurfaceBlock{operation, std::move(block)}; + } + + template < + equilibrium::StellarEquilibriumModel Model, + MaterialSurfaceFactorizationPolicy Policy, + backend::ApplicationMode Mode> + [[nodiscard]] auto prepare( + const equilibrium::StellarEquilibriumProblem &problem, + MaterialSurfaceBlock< + MaterialSurfaceDescriptorFor>, + backend::Diagonal, + backend::HypreBoomerAMG, + Policy, + SurfaceH1MassStiffness> block + ) { + if constexpr (equilibrium::StellarEquilibriumProblem::hasFixedCentralDensity) { + return prepare(problem.GetPreparedOperator().GetPhysicalOperator(), std::move(block)); + } else { + return prepare(problem.GetPreparedOperator(), std::move(block)); + } + } +} // namespace mean_field::preconditioning diff --git a/libmeanfield/interface/preconditioning/plan.cppm b/libmeanfield/interface/preconditioning/plan.cppm new file mode 100644 index 0000000..4dcae73 --- /dev/null +++ b/libmeanfield/interface/preconditioning/plan.cppm @@ -0,0 +1,381 @@ +module; + +#include +#include +#include +#include + +export module mean_field:preconditioning.plan; + +export import :preconditioning.backend; +export import :utils.blocks; + +export namespace mean_field::preconditioning { + template struct Coupling final { + using Residual = ResidualBlock; + using Correction = CorrectionBlock; + }; + + template < + typename CorrectionBlockList, + typename ResidualBlockList, + typename RequiredCouplingList, + typename Characteristics, + typename Backend, + typename PreparationRequirements = + typename backend::Traits>::PreparationDependencies> + struct ComponentDeclaration { + using CorrectionBlocks = CorrectionBlockList; + using ResidualBlocks = ResidualBlockList; + using RequiredCouplings = RequiredCouplingList; + using OperatorDescription = Characteristics; + using BackendType = Backend; + using PreparationDependencies = PreparationRequirements; + }; + + template + requires std::derived_from && + std::derived_from + struct IdentityBlock final { + using CorrectionBlocks = utils::blocks::type_list; + using ResidualBlocks = utils::blocks::type_list; + using RequiredCouplings = utils::blocks::type_list<>; + using OperatorDescription = IdentityOperatorCharacteristics; + using BackendType = backend::Identity; + using PreparationDependencies = NoPreparationDependencies; + }; + + namespace detail { + template struct IsTypeList : std::false_type { }; + + template struct IsTypeList> : std::true_type { }; + + template + inline constexpr bool isTypeList = IsTypeList>::value; + + template struct IsUniqueDerivedBlockList : std::false_type { }; + + template + struct IsUniqueDerivedBlockList, Base> + : std::bool_constant< + (std::derived_from && ...) && + utils::blocks::types_are_unique_v>> { }; + + template struct IsCoupling : std::false_type { }; + + template + struct IsCoupling> + : std::bool_constant< + std::derived_from && + std::derived_from> { }; + + template struct IsCouplingList : std::false_type { }; + + template + struct IsCouplingList> + : std::bool_constant< + (IsCoupling::value && ...) && + utils::blocks::types_are_unique_v>> { }; + + template struct ComponentTraits { + static constexpr bool valid = false; + }; + + template + struct ComponentTraits< + Candidate, + std::void_t< + typename Candidate::CorrectionBlocks, + typename Candidate::ResidualBlocks, + typename Candidate::RequiredCouplings, + typename Candidate::OperatorDescription, + typename Candidate::BackendType, + typename Candidate::PreparationDependencies>> { + using CorrectionBlocks = typename Candidate::CorrectionBlocks; + using ResidualBlocks = typename Candidate::ResidualBlocks; + using RequiredCouplings = typename Candidate::RequiredCouplings; + using OperatorDescription = typename Candidate::OperatorDescription; + using BackendType = typename Candidate::BackendType; + using PreparationDependencies = typename Candidate::PreparationDependencies; + using BackendPreparationDependencies = typename backend::Traits::PreparationDependencies; + + static constexpr bool valid = + IsUniqueDerivedBlockList::value && + IsUniqueDerivedBlockList::value && + IsCouplingList::value && OperatorCharacteristicsType && + backend::Registered && backend::isCompatible && + PreparationDependenciesType && + ((PreparationDependencies::mask & BackendPreparationDependencies::mask) == + BackendPreparationDependencies::mask); + }; + + template struct Concatenate; + + template <> struct Concatenate<> { + using Type = utils::blocks::type_list<>; + }; + + template struct Concatenate> { + using Type = utils::blocks::type_list; + }; + + template + struct Concatenate, utils::blocks::type_list, Remaining...> { + using Type = typename Concatenate, Remaining...>::Type; + }; + + template using ConcatenateT = typename Concatenate::Type; + + template struct Append; + + template struct Append, Type> { + using Result = utils::blocks::type_list; + }; + + template using AppendT = typename Append::Result; + + template + using AppendUniqueT = std::conditional_t, List, AppendT>; + + template struct ListDifference; + + template struct ListDifference, Excluded> { + using Type = utils::blocks::type_list<>; + }; + + template + struct ListDifference, Excluded> { + private: + using Remaining = typename ListDifference, Excluded>::Type; + + public: + using Type = std::conditional_t< + utils::blocks::contains_type_v, + Remaining, + ConcatenateT, Remaining>>; + }; + + template + using ListDifferenceT = typename ListDifference::Type; + + template struct CollectRepeatedTypes; + + template + struct CollectRepeatedTypes, Original, Repeated> { + using Type = Repeated; + }; + + template + struct CollectRepeatedTypes, Original, Repeated> { + private: + using Next = std::conditional_t< + (utils::blocks::type_count_v > 1), + AppendUniqueT, + Repeated>; + + public: + using Type = typename CollectRepeatedTypes, Original, Next>::Type; + }; + + template + using RepeatedTypesT = typename CollectRepeatedTypes>::Type; + + template class PlanStorage { + public: + using ComponentTypes = utils::blocks::type_list; + using CorrectionBlocks = ConcatenateT::CorrectionBlocks...>; + using ResidualBlocks = ConcatenateT::ResidualBlocks...>; + using RequiredCouplings = ConcatenateT::RequiredCouplings...>; + + static constexpr bool allowsOverlappingOwnership = AllowsOverlap; + static constexpr bool stationaryLinear = + ((backend::applicationContract::BackendType> == + ApplicationContract::stationary_linear) && + ...); + + constexpr explicit PlanStorage(Components... components) : m_components(std::move(components)...) { + } + + template [[nodiscard]] constexpr const Component &component() const noexcept { + return std::get(m_components); + } + + [[nodiscard]] constexpr const std::tuple &components() const noexcept { + return m_components; + } + + private: + std::tuple m_components; + }; + + template < + bool ComponentsAreValid, + typename DeclaredCorrectionBlocks, + typename DeclaredResidualBlocks, + typename DeclaredCouplings, + typename... Components> + struct CoherentPlanDeclaration : std::false_type { }; + + template < + typename DeclaredCorrectionBlocks, + typename DeclaredResidualBlocks, + typename DeclaredCouplings, + typename... Components> + struct CoherentPlanDeclaration< + true, + DeclaredCorrectionBlocks, + DeclaredResidualBlocks, + DeclaredCouplings, + Components...> + : std::bool_constant< + std::same_as< + DeclaredCorrectionBlocks, + ConcatenateT::CorrectionBlocks...>> && + std::same_as< + DeclaredResidualBlocks, + ConcatenateT::ResidualBlocks...>> && + std::same_as< + DeclaredCouplings, + ConcatenateT::RequiredCouplings...>>> { }; + + template < + typename ComponentList, + typename DeclaredCorrectionBlocks, + typename DeclaredResidualBlocks, + typename DeclaredCouplings> + struct PlanDeclarationIsCoherent : std::false_type { }; + + template < + typename... Components, + typename DeclaredCorrectionBlocks, + typename DeclaredResidualBlocks, + typename DeclaredCouplings> + struct PlanDeclarationIsCoherent< + utils::blocks::type_list, + DeclaredCorrectionBlocks, + DeclaredResidualBlocks, + DeclaredCouplings> + : CoherentPlanDeclaration< + (ComponentTraits::valid && ...), + DeclaredCorrectionBlocks, + DeclaredResidualBlocks, + DeclaredCouplings, + Components...> { }; + + template struct PlanTraits { + static constexpr bool valid = false; + }; + + template + struct PlanTraits< + Candidate, + std::void_t< + typename Candidate::ComponentTypes, + typename Candidate::CorrectionBlocks, + typename Candidate::ResidualBlocks, + typename Candidate::RequiredCouplings>> { + static constexpr bool valid = + isTypeList && isTypeList && + isTypeList && isTypeList && + PlanDeclarationIsCoherent< + typename Candidate::ComponentTypes, + typename Candidate::CorrectionBlocks, + typename Candidate::ResidualBlocks, + typename Candidate::RequiredCouplings>::value; + }; + + template struct CouplingsExistInJacobian; + + template + struct CouplingsExistInJacobian, JacobianForm> : std::true_type { }; + + template + struct CouplingsExistInJacobian< + utils::blocks::type_list, Remaining...>, + JacobianForm> + : std::bool_constant< + utils::blocks::has_jacobian_coupling_v && + CouplingsExistInJacobian, JacobianForm>::value> { }; + + template >::valid> + struct RequiredCouplingsExist : std::false_type { }; + + template + struct RequiredCouplingsExist + : CouplingsExistInJacobian::RequiredCouplings, JacobianForm> { }; + } // namespace detail + + template + concept PreconditionerComponent = detail::ComponentTraits>::valid; + + template + class PreconditionerPlan final : public detail::PlanStorage { + using Base = detail::PlanStorage; + + public: + using Base::Base; + }; + + template PreconditionerPlan(Components...) -> PreconditionerPlan; + + template + class OverlappingPreconditionerPlan final : public detail::PlanStorage { + using Base = detail::PlanStorage; + + public: + using Base::Base; + }; + + template + OverlappingPreconditionerPlan(Components...) -> OverlappingPreconditionerPlan; + + template + concept PreconditionerPlanType = detail::PlanTraits>::valid; + + template + requires utils::blocks::block_form_is_valid_v && PreconditionerPlanType + struct PreconditionerCoverage final { + using DeclaredCorrectionBlocks = typename Plan::CorrectionBlocks; + using DeclaredResidualBlocks = typename Plan::ResidualBlocks; + + using MissingCorrectionBlocks = detail::ListDifferenceT; + using UnexpectedCorrectionBlocks = + detail::ListDifferenceT; + using RepeatedCorrectionBlocks = detail::RepeatedTypesT; + + using MissingResidualBlocks = detail::ListDifferenceT; + using UnexpectedResidualBlocks = + detail::ListDifferenceT; + using RepeatedResidualBlocks = detail::RepeatedTypesT; + + static constexpr bool hasEveryCorrectionBlock = MissingCorrectionBlocks::size == 0; + static constexpr bool hasOnlyCorrectionBlocks = UnexpectedCorrectionBlocks::size == 0; + static constexpr bool hasUniqueCorrectionOwners = + Plan::allowsOverlappingOwnership || RepeatedCorrectionBlocks::size == 0; + + static constexpr bool hasEveryResidualBlock = MissingResidualBlocks::size == 0; + static constexpr bool hasOnlyResidualBlocks = UnexpectedResidualBlocks::size == 0; + static constexpr bool hasUniqueResidualOwners = + Plan::allowsOverlappingOwnership || RepeatedResidualBlocks::size == 0; + + static constexpr bool complete = hasEveryCorrectionBlock && hasOnlyCorrectionBlocks && + hasUniqueCorrectionOwners && hasEveryResidualBlock && hasOnlyResidualBlocks && + hasUniqueResidualOwners; + }; + + template + concept CompletePreconditionerFor = utils::blocks::block_form_is_valid_v && PreconditionerPlanType && + PreconditionerCoverage>::complete; + + template + inline constexpr bool requiredCouplingsExist = detail::RequiredCouplingsExist::value; + + template + concept CompatiblePreconditionerFor = + CompletePreconditionerFor && utils::blocks::valid_jacobian_form && + requiredCouplingsExist; + + template + concept StationaryLinearPreconditionerPlan = + PreconditionerPlanType && std::remove_cvref_t::stationaryLinear; +} // namespace mean_field::preconditioning diff --git a/libmeanfield/interface/preconditioning/preconditioning.cppm b/libmeanfield/interface/preconditioning/preconditioning.cppm new file mode 100644 index 0000000..49ec461 --- /dev/null +++ b/libmeanfield/interface/preconditioning/preconditioning.cppm @@ -0,0 +1,11 @@ +export module mean_field:preconditioning; + +export import :preconditioning.backend; +export import :preconditioning.backend_implementations; +export import :preconditioning.gravity_field; +export import :preconditioning.material_surface; +export import :preconditioning.plan; +export import :preconditioning.stellar_equilibrium; +export import :preconditioning.stellar_structure; +export import :preconditioning.specification_border; +export import :preconditioning.equilibrium_coordinates; diff --git a/libmeanfield/interface/preconditioning/specification_border.cppm b/libmeanfield/interface/preconditioning/specification_border.cppm new file mode 100644 index 0000000..c8c6ef0 --- /dev/null +++ b/libmeanfield/interface/preconditioning/specification_border.cppm @@ -0,0 +1,1077 @@ +module; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +export module mean_field:preconditioning.specification_border; + +export import :preconditioning.stellar_equilibrium; +export import :preconditioning.stellar_structure; + +export namespace mean_field::preconditioning { + template struct SpecificationBorderContribution { + using CorrectionBlocks = utils::blocks::type_list<>; + using ResidualBlocks = utils::blocks::type_list<>; + using RequiredCouplings = utils::blocks::type_list<>; + + static constexpr bool registered = false; + }; + + template <> struct SpecificationBorderContribution { + using LayoutRequest = models::FixedMassLayoutRequest; + using CorrectionBlock = typename LayoutRequest::ValueBlockType; + using ResidualBlock = typename LayoutRequest::ResidualBlockType; + using CorrectionBlocks = utils::blocks::type_list; + using ResidualBlocks = utils::blocks::type_list; + using RequiredCouplings = utils::blocks::type_list< + Coupling, + Coupling, + Coupling>; + + static constexpr bool registered = true; + }; + + template <> struct SpecificationBorderContribution { + using LayoutRequest = models::CentralDensityLayoutRequest; + using CorrectionBlock = typename LayoutRequest::ValueBlockType; + using ResidualBlock = typename LayoutRequest::ResidualBlockType; + using CorrectionBlocks = utils::blocks::type_list; + using ResidualBlocks = utils::blocks::type_list; + using RequiredCouplings = utils::blocks::type_list< + Coupling, + Coupling>; + + static constexpr bool registered = true; + }; + + namespace detail { + template + inline constexpr std::size_t generatedBorderValueArity = + models::specificationDescriptor().generatedValueArity; + + template + inline constexpr std::size_t generatedBorderResidualArity = + models::specificationDescriptor().generatedResidualArity; + + template + inline constexpr bool specificationGeneratesBorder = + generatedBorderValueArity != 0 || generatedBorderResidualArity != 0; + + template + inline constexpr bool specificationBorderContributionIsComplete = + !specificationGeneratesBorder || + (SpecificationBorderContribution::registered && + generatedBorderValueArity == generatedBorderResidualArity && + SpecificationBorderContribution::CorrectionBlocks::size == 1 && + SpecificationBorderContribution::ResidualBlocks::size == 1); + + template struct CompiledSpecificationBorder; + + template + struct CompiledSpecificationBorder> { + static_assert( + (specificationBorderContributionIsComplete && ...), + "Every specification-generated border requires a registered preconditioning contribution with " + "balanced value and residual arity." + ); + + using SpecificationTypes = models::detail::SpecificationSetStorage; + using CorrectionBlocks = preconditioning::detail::ConcatenateT< + typename SpecificationBorderContribution::CorrectionBlocks...>; + using ResidualBlocks = preconditioning::detail::ConcatenateT< + typename SpecificationBorderContribution::ResidualBlocks...>; + using RequiredCouplings = preconditioning::detail::ConcatenateT< + typename SpecificationBorderContribution::RequiredCouplings...>; + + static constexpr std::size_t valueArity = + (std::size_t{0} + ... + generatedBorderValueArity); + static constexpr std::size_t residualArity = + (std::size_t{0} + ... + generatedBorderResidualArity); + static constexpr std::size_t specificationCount = + (std::size_t{0} + ... + (SpecificationBorderContribution::registered ? 1U : 0U)); + static constexpr bool symbolicallySquare = valueArity == residualArity; + }; + + template struct SpecificationBorderValueOffset; + + template + struct SpecificationBorderValueOffset> { + static constexpr std::size_t value = [] { + if constexpr (std::same_as) { + return std::size_t{0}; + } else { + static_assert(sizeof...(Tail) > 0, "The requested border specification is not in the model."); + return generatedBorderValueArity + + SpecificationBorderValueOffset< + Query, models::detail::SpecificationSetStorage>::value; + } + }(); + }; + + template struct SpecificationBorderResidualOffset; + + template + struct SpecificationBorderResidualOffset> { + static constexpr std::size_t value = [] { + if constexpr (std::same_as) { + return std::size_t{0}; + } else { + static_assert(sizeof...(Tail) > 0, "The requested border specification is not in the model."); + return generatedBorderResidualArity + + SpecificationBorderResidualOffset< + Query, models::detail::SpecificationSetStorage>::value; + } + }(); + }; + } // namespace detail + + template + using CompiledSpecificationBorderFor = + detail::CompiledSpecificationBorder::SpecificationTypes>; + + template + inline constexpr std::size_t specificationBorderValueOffset = detail:: + SpecificationBorderValueOffset::SpecificationTypes>::value; + + template + inline constexpr std::size_t specificationBorderResidualOffset = detail::SpecificationBorderResidualOffset< + Specification, + typename std::remove_cvref_t::SpecificationTypes>::value; + + using SpecificationBorderCharacteristics = OperatorCharacteristics< + OperatorCategory::dense_border, + OperatorValueStructure::block, + OperatorSymmetry::nonsymmetric, + OperatorDefiniteness::indefinite, + OperatorRepresentation::assembled_dense, + OperatorDistribution::local>; + + using BorderedStellarStructureCharacteristics = OperatorCharacteristics< + OperatorCategory::mixed, + OperatorValueStructure::block, + OperatorSymmetry::nonsymmetric, + OperatorDefiniteness::unspecified, + OperatorRepresentation::matrix_free, + OperatorDistribution::distributed_true_dof, + OperatorFESpace::product>; + + namespace backend { + template + struct BorderedStellarStructure final { + using StructureBackendType = StructureBackend; + using BorderBackendType = BorderBackend; + }; + + template + struct Traits> { + static constexpr bool registered = true; + static constexpr ApplicationContract applicationContract = + ::mean_field::preconditioning::backend::applicationContract == + ApplicationContract::stationary_linear && + ::mean_field::preconditioning::backend::applicationContract == + ApplicationContract::stationary_linear + ? ApplicationContract::stationary_linear + : ApplicationContract::flexible; + static constexpr bool supportsSerialExecution = Traits::supportsSerialExecution; + static constexpr bool supportsDistributedExecution = + Traits::supportsDistributedExecution && + Traits::supportsSerialExecution; + static constexpr SymmetryRequirement symmetryRequirement = SymmetryRequirement::none; + static constexpr NullspaceRequirement nullspaceRequirement = NullspaceRequirement::constant_mode_supported; + static constexpr SurrogateRequirement surrogateRequirement = SurrogateRequirement::assembled_sparse; + static constexpr bool requiresAssembledSparseSurrogate = + Traits::requiresAssembledSparseSurrogate; + + using PreparationDependencies = preconditioning::PreparationDependencies< + PreparationDependency::discretization, + PreparationDependency::geometry, + PreparationDependency::equation_of_state, + PreparationDependency::linearization>; + + template + static constexpr bool supports = + Characteristics::category == OperatorCategory::mixed && + Characteristics::valueStructure == OperatorValueStructure::block && + Characteristics::symmetry == OperatorSymmetry::nonsymmetric && + Characteristics::representation == OperatorRepresentation::matrix_free && + Characteristics::distribution == OperatorDistribution::distributed_true_dof && + Characteristics::finiteElementSpace == OperatorFESpace::product; + }; + } // namespace backend + + template < + PreconditionerComponent StructureComponentT, + model::StellarModelType ModelT, + typename FormT, + typename JacobianFormT> + requires utils::blocks::valid_jacobian_form + class SpecificationBorderBlock final { + private: + using CompiledBorder = CompiledSpecificationBorderFor; + + public: + using StructureComponent = StructureComponentT; + using Model = ModelT; + using Form = FormT; + using JacobianForm = JacobianFormT; + using CorrectionBlocks = preconditioning::detail:: + ConcatenateT; + using ResidualBlocks = preconditioning::detail:: + ConcatenateT; + using RequiredCouplings = preconditioning::detail:: + ConcatenateT; + using OperatorDescription = BorderedStellarStructureCharacteristics; + using BackendType = + backend::BorderedStellarStructure; + using PreparationDependencies = typename backend::Traits::PreparationDependencies; + + static constexpr std::size_t borderValueArity = CompiledBorder::valueArity; + static constexpr std::size_t borderResidualArity = CompiledBorder::residualArity; + + constexpr explicit SpecificationBorderBlock( + StructureComponent structureComponent, + backend::DenseDirect borderBackend = {} + ) + : m_structureComponent(std::move(structureComponent)), + m_borderBackend(std::move(borderBackend)) { + static_assert(CompiledBorder::symbolicallySquare); + } + + [[nodiscard]] constexpr const StructureComponent &structureComponent() const noexcept { + return m_structureComponent; + } + + [[nodiscard]] constexpr const backend::DenseDirect &borderBackend() const noexcept { + return m_borderBackend; + } + + private: + StructureComponent m_structureComponent; + backend::DenseDirect m_borderBackend; + }; + + template struct IsSpecificationBorderBlock : std::false_type { }; + + template < + PreconditionerComponent StructureComponent, + model::StellarModelType Model, + typename Form, + typename JacobianForm> + struct IsSpecificationBorderBlock> + : std::true_type { }; + + template + concept SpecificationBorderBlockType = IsSpecificationBorderBlock>::value; + + struct StellarStructureDirectionView final { + const mfem::Vector &density; + const mfem::Vector &surface; + const mfem::Vector &enthalpy; + const mfem::Vector &gravityGradient; + const mfem::Vector &gravityPotential; + }; + + struct StellarStructureActionView final { + mfem::Vector &density; + mfem::Vector &surface; + mfem::Vector &enthalpy; + mfem::Vector &gravityGradient; + mfem::Vector &gravityPotential; + }; + + namespace detail { + template + [[nodiscard]] const operators::PreparedStellarEquilibriumOperator & + specificationBorderPhysicalOperator(const Problem &problem) { + if constexpr (std::remove_cvref_t::hasFixedCentralDensity) { + return problem.GetPreparedOperator().GetPhysicalOperator(); + } else { + return problem.GetPreparedOperator(); + } + } + + template + class PreparedSpecificationBorderAction { + static_assert( + !specificationGeneratesBorder, + "A generated model specification requires a prepared specification-border action specialization." + ); + + public: + explicit PreparedSpecificationBorderAction(const Problem &) noexcept { + } + + void ApplyStructureToBorder( + const StellarStructureDirectionView &, + mfem::Vector & + ) const noexcept { + } + + void ApplyBorderToStructure( + const mfem::Vector &, + StellarStructureActionView + ) const noexcept { + } + + void ApplyBorderToBorder( + const mfem::Vector &, + mfem::Vector & + ) const noexcept { + } + }; + + template + class PreparedSpecificationBorderAction { + private: + using Model = typename std::remove_cvref_t::ModelType; + + public: + explicit PreparedSpecificationBorderAction(const Problem &problem) + : m_physical(std::addressof(specificationBorderPhysicalOperator(problem))), + m_volumeDisplacement(m_physical->GetDomainDeformation().volumeDisplacementSize()), + m_enthalpyWorkspace(m_physical->GetBarotropicClosureOperator().GetEnthalpySize()), + m_zeroEnthalpy(m_physical->GetBarotropicClosureOperator().GetEnthalpySize()) { + m_zeroEnthalpy = 0.0; + } + + void ApplyStructureToBorder( + const StellarStructureDirectionView &structure, + mfem::Vector &borderAction + ) const { + constexpr int residualOffset = + static_cast(specificationBorderResidualOffset); + mfem::Vector massAction(borderAction, residualOffset, 1); + m_physical->GetDomainDeformation().applyJacobian( + m_physical->GetSurfaceDeformationParameters(), structure.surface, m_volumeDisplacement + ); + m_physical->GetMassNormalizationOperator().ApplyCompleteJacobianAction( + structure.density, m_volumeDisplacement, massAction + ); + massAction.SyncAliasMemory(borderAction); + } + + void ApplyBorderToStructure( + const mfem::Vector &borderDirection, + StellarStructureActionView structureAction + ) const { + constexpr int valueOffset = + static_cast(specificationBorderValueOffset); + m_physical->GetHydrostaticOperator().ApplyBernoulliConstantJacobianAction( + borderDirection(valueOffset), m_enthalpyWorkspace + ); + m_physical->GetSurfaceConstraintOperator().ApplyJacobianRows(m_zeroEnthalpy, m_enthalpyWorkspace); + structureAction.enthalpy += m_enthalpyWorkspace; + } + + void ApplyBorderToBorder( + const mfem::Vector &, + mfem::Vector & + ) const noexcept { + } + + private: + const operators::PreparedStellarEquilibriumOperator *m_physical; + mutable mfem::Vector m_volumeDisplacement; + mutable mfem::Vector m_enthalpyWorkspace; + mfem::Vector m_zeroEnthalpy; + }; + + template + class PreparedSpecificationBorderAction { + private: + using ProblemType = std::remove_cvref_t; + using Model = typename ProblemType::ModelType; + static_assert(ProblemType::hasFixedCentralDensity); + + public: + explicit PreparedSpecificationBorderAction(const Problem &problem) + : m_constraint(std::addressof(problem.GetPreparedOperator().GetCentralDensityConstraint())), + m_zeroEnthalpy( + specificationBorderPhysicalOperator(problem).GetBarotropicClosureOperator().GetEnthalpySize() + ), + m_enthalpyWorkspace(m_zeroEnthalpy.Size()), + m_phaseWorkspace(1) { + m_zeroEnthalpy = 0.0; + } + + void ApplyStructureToBorder( + const StellarStructureDirectionView &structure, + mfem::Vector &borderAction + ) const { + constexpr int residualOffset = + static_cast(specificationBorderResidualOffset); + mfem::Vector phaseAction(borderAction, residualOffset, 1); + m_enthalpyWorkspace = 0.0; + m_constraint->ApplyJacobian( + {.enthalpyVariation = structure.enthalpy, .borderVariation = 0.0}, + {.enthalpyAction = m_enthalpyWorkspace, .phaseAction = phaseAction} + ); + phaseAction.SyncAliasMemory(borderAction); + } + + void ApplyBorderToStructure( + const mfem::Vector &borderDirection, + StellarStructureActionView structureAction + ) const { + constexpr int valueOffset = + static_cast(specificationBorderValueOffset); + m_enthalpyWorkspace = 0.0; + m_phaseWorkspace = 0.0; + m_constraint->ApplyJacobian( + {.enthalpyVariation = m_zeroEnthalpy, .borderVariation = borderDirection(valueOffset)}, + {.enthalpyAction = m_enthalpyWorkspace, .phaseAction = m_phaseWorkspace} + ); + structureAction.enthalpy += m_enthalpyWorkspace; + } + + void ApplyBorderToBorder( + const mfem::Vector &, + mfem::Vector & + ) const noexcept { + } + + private: + const operators::PreparedCentralDensityConstraint *m_constraint; + mfem::Vector m_zeroEnthalpy; + mutable mfem::Vector m_enthalpyWorkspace; + mutable mfem::Vector m_phaseWorkspace; + }; + + template + class PreparedSpecificationBorderActions; + + template < + models::ModelSpecification... Specifications, + equilibrium::DiscretizedStellarEquilibriumProblem Problem> + class PreparedSpecificationBorderActions, Problem> { + public: + explicit PreparedSpecificationBorderActions(const Problem &problem) + : m_actions( + PreparedSpecificationBorderAction< + Specifications, + Problem>{problem}... + ) { + } + + void ApplyStructureToBorder( + const StellarStructureDirectionView &structure, + mfem::Vector &borderAction + ) const { + std::apply( + [&](const auto &...actions) { (actions.ApplyStructureToBorder(structure, borderAction), ...); }, + m_actions + ); + } + + void ApplyBorderToStructure( + const mfem::Vector &borderDirection, + StellarStructureActionView structureAction + ) const { + std::apply( + [&](const auto &...actions) { + (actions.ApplyBorderToStructure(borderDirection, structureAction), ...); + }, + m_actions + ); + } + + void ApplyBorderToBorder( + const mfem::Vector &borderDirection, + mfem::Vector &borderAction + ) const { + std::apply( + [&](const auto &...actions) { (actions.ApplyBorderToBorder(borderDirection, borderAction), ...); }, + m_actions + ); + } + + private: + std::tuple...> m_actions; + }; + } // namespace detail + + template + class SpecificationBorderJacobianOperator final : public mfem::Operator { + private: + using ProblemType = std::remove_cvref_t; + using Model = typename ProblemType::ModelType; + using CompiledBorder = CompiledSpecificationBorderFor; + using Actions = detail::PreparedSpecificationBorderActions; + + public: + explicit SpecificationBorderJacobianOperator(const ProblemType &problem) + : mfem::Operator(StructureSizeOf(problem) + BorderSizeOf(problem)), + m_structureOffsets(6), + m_actions(problem) { + const auto &physical = detail::specificationBorderPhysicalOperator(problem); + m_structureOffsets[0] = 0; + m_structureOffsets[1] = physical.GetGravityContext().GetDensityMap().reduced_size(); + m_structureOffsets[2] = m_structureOffsets[1] + physical.GetDomainDeformation().parameterCount(); + m_structureOffsets[3] = m_structureOffsets[2] + physical.GetBarotropicClosureOperator().GetEnthalpySize(); + m_structureOffsets[4] = + m_structureOffsets[3] + physical.GetGravityContext().GetGravityGradientMap().reduced_size(); + m_structureOffsets[5] = StructureSizeOf(problem); + + if (StructureSize() + BorderSize() != problem.StateSize() || + StructureSize() + BorderSize() != problem.EquationSize()) { + throw std::logic_error( + "The compiled specification border does not complete the stellar-equilibrium problem." + ); + } + } + + void Mult( + const mfem::Vector &direction, + mfem::Vector &action + ) const override { + VerifyCombined(direction, action); + action = 0.0; + const mfem::Vector structureDirection(const_cast(direction.GetData()), StructureSize()); + const mfem::Vector borderDirection( + const_cast(direction.GetData()) + StructureSize(), BorderSize() + ); + mfem::Vector structureAction(action, 0, StructureSize()); + mfem::Vector borderAction(action, StructureSize(), BorderSize()); + + ApplyBorderToStructure(borderDirection, structureAction); + ApplyStructureToBorder(structureDirection, borderAction); + mfem::Vector borderDiagonalAction(BorderSize()); + ApplyBorderToBorder(borderDirection, borderDiagonalAction); + borderAction += borderDiagonalAction; + + structureAction.SyncAliasMemory(action); + borderAction.SyncAliasMemory(action); + } + + void ApplyStructureToBorder( + const mfem::Vector &structureDirection, + mfem::Vector &borderAction + ) const { + VerifyStructure(structureDirection, "direction"); + VerifyBorder(borderAction, "action"); + borderAction = 0.0; + const auto directionView = StructureDirection(structureDirection); + m_actions.ApplyStructureToBorder(directionView, borderAction); + } + + void ApplyBorderToStructure( + const mfem::Vector &borderDirection, + mfem::Vector &structureAction + ) const { + VerifyBorder(borderDirection, "direction"); + VerifyStructure(structureAction, "action"); + structureAction = 0.0; + auto densityAction = MutableStructureBlock(structureAction, 0); + auto surfaceAction = MutableStructureBlock(structureAction, 1); + auto enthalpyAction = MutableStructureBlock(structureAction, 2); + auto gravityGradientAction = MutableStructureBlock(structureAction, 3); + auto gravityPotentialAction = MutableStructureBlock(structureAction, 4); + m_actions.ApplyBorderToStructure( + borderDirection, {.density = densityAction, + .surface = surfaceAction, + .enthalpy = enthalpyAction, + .gravityGradient = gravityGradientAction, + .gravityPotential = gravityPotentialAction} + ); + densityAction.SyncAliasMemory(structureAction); + surfaceAction.SyncAliasMemory(structureAction); + enthalpyAction.SyncAliasMemory(structureAction); + gravityGradientAction.SyncAliasMemory(structureAction); + gravityPotentialAction.SyncAliasMemory(structureAction); + } + + void ApplyBorderToBorder( + const mfem::Vector &borderDirection, + mfem::Vector &borderAction + ) const { + VerifyBorder(borderDirection, "direction"); + VerifyBorder(borderAction, "action"); + borderAction = 0.0; + m_actions.ApplyBorderToBorder(borderDirection, borderAction); + } + + [[nodiscard]] int StructureSize() const noexcept { + return m_structureOffsets.Last(); + } + + [[nodiscard]] static constexpr int BorderSize() noexcept { + return static_cast(CompiledBorder::valueArity); + } + + [[nodiscard]] const mfem::Array &GetStructureOffsets() const noexcept { + return m_structureOffsets; + } + + private: + [[nodiscard]] static int StructureSizeOf(const ProblemType &problem) { + const auto &physical = detail::specificationBorderPhysicalOperator(problem); + return physical.GetGravityContext().GetDensityMap().reduced_size() + + physical.GetDomainDeformation().parameterCount() + + physical.GetBarotropicClosureOperator().GetEnthalpySize() + + physical.GetGravityContext().GetGravityGradientMap().reduced_size() + + physical.GetGravityContext().GetGravityPotentialMap().reduced_size(); + } + + [[nodiscard]] static constexpr int BorderSizeOf(const ProblemType &) noexcept { + return BorderSize(); + } + + [[nodiscard]] mfem::Vector ConstStructureBlock( + const mfem::Vector &vector, + const int block + ) const { + return mfem::Vector( + const_cast(vector.GetData()) + m_structureOffsets[block], + m_structureOffsets[block + 1] - m_structureOffsets[block] + ); + } + + [[nodiscard]] mfem::Vector MutableStructureBlock( + mfem::Vector &vector, + const int block + ) const { + return mfem::Vector( + vector, m_structureOffsets[block], m_structureOffsets[block + 1] - m_structureOffsets[block] + ); + } + + [[nodiscard]] StellarStructureDirectionView StructureDirection(const mfem::Vector &direction) const { + m_directionDensity = ConstStructureBlock(direction, 0); + m_directionSurface = ConstStructureBlock(direction, 1); + m_directionEnthalpy = ConstStructureBlock(direction, 2); + m_directionGravityGradient = ConstStructureBlock(direction, 3); + m_directionGravityPotential = ConstStructureBlock(direction, 4); + return { + .density = m_directionDensity, + .surface = m_directionSurface, + .enthalpy = m_directionEnthalpy, + .gravityGradient = m_directionGravityGradient, + .gravityPotential = m_directionGravityPotential + }; + } + + void VerifyCombined( + const mfem::Vector &direction, + const mfem::Vector &action + ) const { + if (direction.Size() != Width() || action.Size() != Height()) { + throw std::invalid_argument( + "The specification-border Jacobian requires compatible, preallocated vectors." + ); + } + } + + void VerifyStructure( + const mfem::Vector &vector, + const char *role + ) const { + if (vector.Size() != StructureSize()) { + throw std::invalid_argument( + std::string("The specification-border structure ") + role + " has the wrong size." + ); + } + } + + void VerifyBorder( + const mfem::Vector &vector, + const char *role + ) const { + if (vector.Size() != BorderSize()) { + throw std::invalid_argument(std::string("The specification border ") + role + " has the wrong size."); + } + } + + mfem::Array m_structureOffsets; + Actions m_actions; + mutable mfem::Vector m_directionDensity; + mutable mfem::Vector m_directionSurface; + mutable mfem::Vector m_directionEnthalpy; + mutable mfem::Vector m_directionGravityGradient; + mutable mfem::Vector m_directionGravityPotential; + }; + + template + SpecificationBorderJacobianOperator(const Problem &) + -> SpecificationBorderJacobianOperator>; + + template + concept SpecificationBorderCouplingOperator = requires( + const Candidate &couplings, + const mfem::Vector &structureDirection, + const mfem::Vector &borderDirection, + mfem::Vector &structureAction, + mfem::Vector &borderAction + ) { + { couplings.StructureSize() } -> std::same_as; + { couplings.BorderSize() } -> std::same_as; + couplings.ApplyStructureToBorder(structureDirection, borderAction); + couplings.ApplyBorderToStructure(borderDirection, structureAction); + couplings.ApplyBorderToBorder(borderDirection, borderAction); + }; + + struct SpecificationBorderFactorizationStatistics final { + std::uint64_t setups{0}; + std::uint64_t applications{0}; + std::uint64_t structureInverseApplications{0}; + std::uint64_t cachedStructureInverseBorderApplications{0}; + std::uint64_t structureToBorderApplications{0}; + std::uint64_t borderToStructureApplications{0}; + std::uint64_t borderToBorderApplications{0}; + std::uint64_t schurProbes{0}; + }; + + template < + SpecificationBorderCouplingOperator CouplingOperator, + ApplicationContract StructureInverseContract = ApplicationContract::stationary_linear> + class SpecificationBorderFactorizationOperator final : public mfem::Solver { + public: + static constexpr bool cachesStructureInverseBorderCoupling = + StructureInverseContract == ApplicationContract::stationary_linear; + + SpecificationBorderFactorizationOperator( + const mfem::Solver &structureInverse, + const CouplingOperator &couplings, + backend::DenseDirect borderBackend = {} + ) + : mfem::Solver(couplings.StructureSize() + couplings.BorderSize()), + m_structureInverse(std::addressof(structureInverse)), + m_couplings(std::addressof(couplings)), + m_borderBackend(std::move(borderBackend)), + m_schurComplement(couplings.BorderSize()), + m_structureInverseBorderCoupling( + cachesStructureInverseBorderCoupling ? couplings.StructureSize() : 0, + cachesStructureInverseBorderCoupling ? couplings.BorderSize() : 0 + ), + m_structureWorkspace(couplings.StructureSize()), + m_structureCoupling(couplings.StructureSize()), + m_borderWorkspace(couplings.BorderSize()), + m_borderCoupling(couplings.BorderSize()), + m_borderDiagonal(couplings.BorderSize()), + m_borderBasis(couplings.BorderSize()) { + if (structureInverse.Height() <= 0 || structureInverse.Height() != structureInverse.Width() || + structureInverse.Height() != couplings.StructureSize() || couplings.BorderSize() < 0) { + throw std::invalid_argument( + "The structure inverse and specification-border couplings have incompatible dimensions." + ); + } + AssembleSchurComplement(); + } + + SpecificationBorderFactorizationOperator(const SpecificationBorderFactorizationOperator &) = delete; + SpecificationBorderFactorizationOperator &operator=(const SpecificationBorderFactorizationOperator &) = delete; + SpecificationBorderFactorizationOperator(SpecificationBorderFactorizationOperator &&) = delete; + SpecificationBorderFactorizationOperator &operator=(SpecificationBorderFactorizationOperator &&) = delete; + + void SetOperator(const mfem::Operator &operation) override { + if (operation.Height() != Height() || operation.Width() != Width()) { + throw std::invalid_argument( + "The specification-border factorization received an incompatible operator." + ); + } + } + + void Mult( + const mfem::Vector &rightHandSide, + mfem::Vector &action + ) const override { + if (rightHandSide.Size() != Width() || action.Size() != Height()) { + throw std::invalid_argument( + "The specification-border factorization requires compatible, preallocated vectors." + ); + } + + const mfem::Vector structureRightHandSide( + const_cast(rightHandSide.GetData()), m_couplings->StructureSize() + ); + const mfem::Vector borderRightHandSide( + const_cast(rightHandSide.GetData()) + m_couplings->StructureSize(), + m_couplings->BorderSize() + ); + action = 0.0; + mfem::Vector structureAction(action, 0, m_couplings->StructureSize()); + mfem::Vector borderAction(action, m_couplings->StructureSize(), m_couplings->BorderSize()); + + if (m_couplings->BorderSize() == 0) { + m_structureInverse->Mult(structureRightHandSide, structureAction); + ++m_statistics.structureInverseApplications; + } else { + m_structureInverse->Mult(structureRightHandSide, m_structureWorkspace); + m_couplings->ApplyStructureToBorder(m_structureWorkspace, m_borderCoupling); + m_borderWorkspace = borderRightHandSide; + m_borderWorkspace -= m_borderCoupling; + m_borderInverse->Mult(m_borderWorkspace, borderAction); + + if constexpr (cachesStructureInverseBorderCoupling) { + // W = A^{-1} B was assembled with the border Schur complement, so the + // stationary-linear structure correction is A^{-1} f - W y. + m_structureInverseBorderCoupling.Mult(borderAction, m_structureCoupling); + structureAction = m_structureWorkspace; + structureAction -= m_structureCoupling; + ++m_statistics.structureInverseApplications; + ++m_statistics.cachedStructureInverseBorderApplications; + } else { + m_couplings->ApplyBorderToStructure(borderAction, m_structureCoupling); + m_structureCoupling *= -1.0; + m_structureCoupling += structureRightHandSide; + m_structureInverse->Mult(m_structureCoupling, structureAction); + m_statistics.structureInverseApplications += 2; + ++m_statistics.borderToStructureApplications; + } + ++m_statistics.structureToBorderApplications; + } + + structureAction.SyncAliasMemory(action); + borderAction.SyncAliasMemory(action); + ++m_statistics.applications; + } + + void RefreshSchurComplement() { + AssembleSchurComplement(); + } + + [[nodiscard]] const mfem::DenseMatrix &GetSchurComplement() const noexcept { + return m_schurComplement; + } + + [[nodiscard]] const backend::PreparedDenseDirect *GetBorderInverse() const noexcept { + return m_borderInverse.get(); + } + + [[nodiscard]] const SpecificationBorderFactorizationStatistics &GetStatistics() const noexcept { + return m_statistics; + } + + private: + void AssembleSchurComplement() { + const int borderSize = m_couplings->BorderSize(); + if (borderSize == 0) { + m_schurComplement.SetSize(0, 0); + m_borderInverse.reset(); + ++m_statistics.setups; + return; + } + + mfem::Vector schurColumn(borderSize); + for (int column = 0; column < borderSize; ++column) { + m_borderBasis = 0.0; + m_borderBasis(column) = 1.0; + m_couplings->ApplyBorderToStructure(m_borderBasis, m_structureCoupling); + ++m_statistics.borderToStructureApplications; + m_structureInverse->Mult(m_structureCoupling, m_structureWorkspace); + ++m_statistics.structureInverseApplications; + if constexpr (cachesStructureInverseBorderCoupling) { + m_structureInverseBorderCoupling.SetCol(column, m_structureWorkspace); + } + m_couplings->ApplyStructureToBorder(m_structureWorkspace, m_borderCoupling); + ++m_statistics.structureToBorderApplications; + m_couplings->ApplyBorderToBorder(m_borderBasis, m_borderDiagonal); + ++m_statistics.borderToBorderApplications; + schurColumn = m_borderDiagonal; + schurColumn -= m_borderCoupling; + for (int row = 0; row < borderSize; ++row) { + m_schurComplement(row, column) = schurColumn(row); + } + ++m_statistics.schurProbes; + } + + if (m_borderInverse == nullptr) { + m_borderInverse = std::make_unique(m_borderBackend, m_schurComplement); + } else { + m_borderInverse->Refresh(m_schurComplement); + } + ++m_statistics.setups; + } + + const mfem::Solver *m_structureInverse; + const CouplingOperator *m_couplings; + backend::DenseDirect m_borderBackend; + mfem::DenseMatrix m_schurComplement; + mfem::DenseMatrix m_structureInverseBorderCoupling; + std::unique_ptr m_borderInverse; + mutable mfem::Vector m_structureWorkspace; + mutable mfem::Vector m_structureCoupling; + mutable mfem::Vector m_borderWorkspace; + mutable mfem::Vector m_borderCoupling; + mutable mfem::Vector m_borderDiagonal; + mutable mfem::Vector m_borderBasis; + mutable SpecificationBorderFactorizationStatistics m_statistics; + }; + + struct SpecificationBorderBlockPreparationReport final { + bool structureRefreshed{false}; + bool rebuiltSchurComplement{false}; + + [[nodiscard]] bool DidAnyWork() const noexcept { + return structureRefreshed || rebuiltSchurComplement; + } + }; + + struct PreparedSpecificationBorderBlockStatistics final { + std::uint64_t setups{0}; + std::uint64_t refreshChecks{0}; + std::uint64_t refreshes{0}; + std::uint64_t noOpRefreshes{0}; + }; + + template + class PreparedSpecificationBorderBlock final : public mfem::Solver { + private: + using ProblemType = std::remove_cvref_t; + using BlockType = std::remove_cvref_t; + using PreparedStructure = decltype(preconditioning::prepare( + std::declval(), + std::declval() + )); + static constexpr ApplicationContract structureInverseContract = + backend::applicationContract; + + public: + using Factorization = SpecificationBorderFactorizationOperator< + SpecificationBorderJacobianOperator, + structureInverseContract>; + + PreparedSpecificationBorderBlock( + const ProblemType &problem, + BlockType block + ) + : mfem::Solver(problem.StateSize()), + m_problem(std::addressof(problem)), + m_block(std::move(block)), + m_structure( + preconditioning::prepare( + problem, + m_block.structureComponent() + ) + ), + m_couplings(problem), + m_factorization( + m_structure, + m_couplings, + m_block.borderBackend() + ), + m_snapshot(StellarEquilibriumProblemTraits::Snapshot(problem)) { + if (m_factorization.Height() != Height() || m_factorization.Width() != Width()) { + throw std::logic_error( + "The prepared specification border does not span the grouped equilibrium coordinates." + ); + } + m_statistics.setups = 1; + } + + PreparedSpecificationBorderBlock(const PreparedSpecificationBorderBlock &) = delete; + PreparedSpecificationBorderBlock &operator=(const PreparedSpecificationBorderBlock &) = delete; + PreparedSpecificationBorderBlock(PreparedSpecificationBorderBlock &&) = delete; + PreparedSpecificationBorderBlock &operator=(PreparedSpecificationBorderBlock &&) = delete; + + void SetOperator(const mfem::Operator &operation) override { + m_factorization.SetOperator(operation); + } + + void Mult( + const mfem::Vector &rightHandSide, + mfem::Vector &action + ) const override { + if (!IsCurrent()) { + throw std::logic_error("The specification-border block is stale; refresh it before application."); + } + m_factorization.Mult(rightHandSide, action); + } + + [[nodiscard]] SpecificationBorderBlockPreparationReport Refresh() { + const auto current = StellarEquilibriumProblemTraits::Snapshot(*m_problem); + ++m_statistics.refreshChecks; + SpecificationBorderBlockPreparationReport report; + const auto structureReport = m_structure.Refresh(); + report.structureRefreshed = structureReport.DidAnyWork(); + + if (current != m_snapshot) { + m_factorization.RefreshSchurComplement(); + report.rebuiltSchurComplement = true; + m_snapshot = current; + ++m_statistics.refreshes; + } else { + ++m_statistics.noOpRefreshes; + } + return report; + } + + [[nodiscard]] bool IsCurrent() const { + return m_structure.IsCurrent() && m_problem->IsPrepared() && + StellarEquilibriumProblemTraits::Snapshot(*m_problem) == m_snapshot; + } + + [[nodiscard]] const BlockType &GetBlock() const noexcept { + return m_block; + } + + [[nodiscard]] const PreparedStructure &GetStructurePreconditioner() const noexcept { + return m_structure; + } + + [[nodiscard]] const SpecificationBorderJacobianOperator &GetCouplings() const noexcept { + return m_couplings; + } + + [[nodiscard]] const Factorization &GetFactorization() const noexcept { + return m_factorization; + } + + [[nodiscard]] const PreparedSpecificationBorderBlockStatistics &GetStatistics() const noexcept { + return m_statistics; + } + + private: + const ProblemType *m_problem; + BlockType m_block; + PreparedStructure m_structure; + SpecificationBorderJacobianOperator m_couplings; + Factorization m_factorization; + StellarPreconditionerLifecycleSnapshot m_snapshot; + PreparedSpecificationBorderBlockStatistics m_statistics; + }; + + template < + equilibrium::DiscretizedStellarEquilibriumProblem Problem, + PreconditionerComponent StructureComponent> + [[nodiscard]] constexpr auto specificationBorderBlock( + const Problem &, + StructureComponent structureComponent, + backend::DenseDirect borderBackend = {} + ) { + using ProblemType = std::remove_cvref_t; + using Block = SpecificationBorderBlock< + StructureComponent, typename ProblemType::ModelType, typename ProblemType::FormType, + typename ProblemType::JacobianFormType>; + using Plan = PreconditionerPlan; + static_assert( + CompletePreconditionerFor, + "The model-compiled preconditioner must own every correction and residual block exactly once." + ); + static_assert( + CompatiblePreconditionerFor, + "Every coupling required by the model-compiled preconditioner must exist in the compiled Jacobian." + ); + return Block{std::move(structureComponent), std::move(borderBackend)}; + } + + template + [[nodiscard]] constexpr auto specificationBorderBlock(const Problem &problem) { + return specificationBorderBlock(problem, stellarStructureBlock(problem), backend::DenseDirect{}); + } + + template + [[nodiscard]] constexpr auto makePreconditioner(const Problem &problem) { + return specificationBorderBlock(problem); + } + +} // namespace mean_field::preconditioning diff --git a/libmeanfield/interface/preconditioning/stellar_equilibrium.cppm b/libmeanfield/interface/preconditioning/stellar_equilibrium.cppm new file mode 100644 index 0000000..c152212 --- /dev/null +++ b/libmeanfield/interface/preconditioning/stellar_equilibrium.cppm @@ -0,0 +1,513 @@ +module; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +export module mean_field:preconditioning.stellar_equilibrium; + +export import :operators.stellar_equilibrium_problem; +export import :preconditioning.plan; + +export namespace mean_field::preconditioning { + struct StellarPreconditionerLifecycleSnapshot final { + operators::StellarEquilibriumDependencyStamp discretization; + operators::StellarEquilibriumDependencyStamp geometry; + const void *equationOfStateIdentity{nullptr}; + operators::StellarEquilibriumDependencies linearization; + + constexpr bool operator==(const StellarPreconditionerLifecycleSnapshot &) const = default; + }; + + struct StellarPreconditionerPreparationChanges final { + bool discretization{false}; + bool geometry{false}; + bool equationOfState{false}; + bool linearization{false}; + + [[nodiscard]] constexpr bool Any() const noexcept { + return discretization || geometry || equationOfState || linearization; + } + + [[nodiscard]] constexpr bool Contains(const PreparationDependency dependency) const noexcept { + switch (dependency) { + case PreparationDependency::discretization: + return discretization; + case PreparationDependency::geometry: + return geometry; + case PreparationDependency::equation_of_state: + return equationOfState; + case PreparationDependency::linearization: + return linearization; + } + return false; + } + }; + + [[nodiscard]] constexpr StellarPreconditionerPreparationChanges preparationChanges( + const StellarPreconditionerLifecycleSnapshot &prepared, + const StellarPreconditionerLifecycleSnapshot ¤t + ) noexcept { + return { + .discretization = prepared.discretization != current.discretization, + .geometry = prepared.geometry != current.geometry, + .equationOfState = prepared.equationOfStateIdentity != current.equationOfStateIdentity, + .linearization = prepared.linearization != current.linearization + }; + } + + struct StellarPreconditionerPreparationReport final { + StellarPreconditionerPreparationChanges changes; + std::uint64_t refreshedComponents{0}; + + [[nodiscard]] constexpr bool DidAnyWork() const noexcept { + return refreshedComponents != 0; + } + }; + + struct StellarPreconditionerStatistics final { + std::uint64_t setups{0}; + std::uint64_t refreshChecks{0}; + std::uint64_t refreshes{0}; + std::uint64_t noOpRefreshes{0}; + std::uint64_t componentSetups{0}; + std::uint64_t componentRefreshes{0}; + std::uint64_t operatorBindings{0}; + std::uint64_t applications{0}; + std::uint64_t backendApplications{0}; + std::uint64_t innerIterations{0}; + double setupSeconds{0.0}; + double refreshSeconds{0.0}; + double applicationSeconds{0.0}; + double maximumApplicationSeconds{0.0}; + }; + + template struct StellarEquilibriumProblemTraits { + static constexpr bool registered = false; + }; + + template + struct StellarEquilibriumProblemTraits> { + using Problem = equilibrium::StellarEquilibriumProblem; + using Form = typename Problem::FormType; + using JacobianForm = typename Problem::JacobianFormType; + using Manifest = typename Problem::ManifestType; + + static constexpr bool registered = true; + + [[nodiscard]] static bool IsPrepared(const Problem &problem) noexcept { + return problem.IsPrepared(); + } + + [[nodiscard]] static int StateSize(const Problem &problem) noexcept { + return problem.StateSize(); + } + + [[nodiscard]] static int EquationSize(const Problem &problem) noexcept { + return problem.EquationSize(); + } + + [[nodiscard]] static const Manifest &ManifestOf(const Problem &problem) noexcept { + return problem.GetManifest(); + } + + [[nodiscard]] static const mfem::Operator &LinearizationOperator(const Problem &problem) noexcept { + return problem.GetLinearizationOperator(); + } + + [[nodiscard]] static StellarPreconditionerLifecycleSnapshot Snapshot(const Problem &problem) { + const operators::StellarEquilibriumDependencies &dependencies = problem.GetLinearizationDependencies(); + return { + .discretization = dependencies.discretization, + .geometry = problem.GetGeometryDependency(), + .equationOfStateIdentity = + std::addressof(problem.GetStellarModel().template specification()), + .linearization = dependencies + }; + } + }; + + template + concept StellarPreconditionerProblem = StellarEquilibriumProblemTraits>::registered; + + namespace backend { + template + class PreparedComponent; + + template + class PreparedComponent final { + public: + void Setup( + const Problem &, + const Component & + ) noexcept { + } + + [[nodiscard]] bool Refresh( + const Problem &, + const Component &, + const StellarPreconditionerPreparationChanges & + ) noexcept { + return true; + } + }; + + template + concept PreparedComponentFor = requires( + PreparedComponent &prepared, + const Problem &problem, + const Component &component, + const StellarPreconditionerPreparationChanges &changes + ) { + prepared.Setup(problem, component); + { prepared.Refresh(problem, component, changes) } -> std::same_as; + }; + } // namespace backend + + namespace detail { + using DensityIdentity = + IdentityBlock; + using SurfaceIdentity = IdentityBlock< + utils::blocks::surface_deformation::parameters::value, + utils::blocks::surface_deformation::shape_equilibrium::residual>; + using GravityGradientIdentity = + IdentityBlock; + using GravityPotentialIdentity = + IdentityBlock; + using EnthalpyIdentity = + IdentityBlock; + using FixedMassIdentity = IdentityBlock< + utils::blocks::fixed_total_mass::mass_normalization::value, + utils::blocks::fixed_total_mass::mass_normalization::residual>; + using FixedCentralDensityIdentity = IdentityBlock< + utils::blocks::fixed_central_density::central_value::value, + utils::blocks::fixed_central_density::central_value::residual>; + + template struct IdentityPlanForForm; + + template <> struct IdentityPlanForForm { + using Type = PreconditionerPlan< + DensityIdentity, + SurfaceIdentity, + GravityGradientIdentity, + GravityPotentialIdentity, + EnthalpyIdentity, + FixedMassIdentity>; + + [[nodiscard]] static constexpr Type Make() { + return Type{DensityIdentity{}, SurfaceIdentity{}, GravityGradientIdentity{}, + GravityPotentialIdentity{}, EnthalpyIdentity{}, FixedMassIdentity{}}; + } + }; + + template <> struct IdentityPlanForForm { + using Type = PreconditionerPlan< + DensityIdentity, + SurfaceIdentity, + GravityGradientIdentity, + GravityPotentialIdentity, + EnthalpyIdentity, + FixedMassIdentity, + FixedCentralDensityIdentity>; + + [[nodiscard]] static constexpr Type Make() { + return Type{ + DensityIdentity{}, SurfaceIdentity{}, GravityGradientIdentity{}, GravityPotentialIdentity{}, + EnthalpyIdentity{}, FixedMassIdentity{}, FixedCentralDensityIdentity{} + }; + } + }; + + template struct UsesOnlyIdentityBackends : std::false_type { }; + + template + struct UsesOnlyIdentityBackends> + : std::bool_constant<(std::same_as && ...)> { }; + + template struct PreparedComponentTuple; + + template + struct PreparedComponentTuple, Problem> { + using Type = std::tuple...>; + + static constexpr bool available = (backend::PreparedComponentFor && ...); + }; + + template + [[nodiscard]] constexpr bool + componentRequiresRefresh(const StellarPreconditionerPreparationChanges &changes) noexcept { + return (Requirements::contains(PreparationDependency::discretization) && changes.discretization) || + (Requirements::contains(PreparationDependency::geometry) && changes.geometry) || + (Requirements::contains(PreparationDependency::equation_of_state) && changes.equationOfState) || + (Requirements::contains(PreparationDependency::linearization) && changes.linearization); + } + } // namespace detail + + template + concept PreparedPreconditionerPlanFor = + StellarPreconditionerProblem && PreconditionerPlanType && + CompletePreconditionerFor::Form> && + CompatiblePreconditionerFor< + Plan, + typename StellarEquilibriumProblemTraits::Form, + typename StellarEquilibriumProblemTraits::JacobianForm> && + (!std::remove_cvref_t::allowsOverlappingOwnership) && + detail::UsesOnlyIdentityBackends::ComponentTypes>::value && + detail::PreparedComponentTuple< + typename std::remove_cvref_t::ComponentTypes, + std::remove_cvref_t>::available; + + template + using IdentityPreconditionerPlanFor = typename detail::IdentityPlanForForm< + typename StellarEquilibriumProblemTraits>::Form>::Type; + + template + [[nodiscard]] constexpr IdentityPreconditionerPlanFor makeIdentityPlan(const Problem &) { + using Form = typename StellarEquilibriumProblemTraits>::Form; + return detail::IdentityPlanForForm::Make(); + } + + template + requires PreparedPreconditionerPlanFor + class StellarEquilibriumPreconditioner final : public mfem::Solver { + private: + using ProblemType = std::remove_cvref_t; + using PlanType = std::remove_cvref_t; + using Traits = StellarEquilibriumProblemTraits; + using Components = typename PlanType::ComponentTypes; + using PreparedComponents = typename detail::PreparedComponentTuple::Type; + using Clock = std::chrono::steady_clock; + + public: + using FormType = typename Traits::Form; + using JacobianFormType = typename Traits::JacobianForm; + + StellarEquilibriumPreconditioner( + ProblemType &problem, + PlanType plan + ) + : mfem::Solver(Traits::StateSize(problem)), + m_problem(std::addressof(problem)), + m_manifest(std::addressof(Traits::ManifestOf(problem))), + m_linearization(std::addressof(Traits::LinearizationOperator(problem))), + m_plan(std::move(plan)) { + const Clock::time_point start = Clock::now(); + VerifyPreparedProblem(); + SetupComponents(std::make_index_sequence>{}); + m_snapshot = Traits::Snapshot(*m_problem); + m_statistics.setups = 1; + m_statistics.setupSeconds = std::chrono::duration(Clock::now() - start).count(); + } + + StellarEquilibriumPreconditioner(const StellarEquilibriumPreconditioner &) = delete; + StellarEquilibriumPreconditioner &operator=(const StellarEquilibriumPreconditioner &) = delete; + StellarEquilibriumPreconditioner(StellarEquilibriumPreconditioner &&) = delete; + StellarEquilibriumPreconditioner &operator=(StellarEquilibriumPreconditioner &&) = delete; + + void SetOperator(const mfem::Operator &operation) override { + if (operation.Height() != Height() || operation.Width() != Width()) { + throw std::invalid_argument( + "The stellar-equilibrium preconditioner received an operator with incompatible dimensions." + ); + } + ++m_statistics.operatorBindings; + } + + void Mult( + const mfem::Vector &residual, + mfem::Vector &correction + ) const override { + VerifyCurrent(); + if (residual.Size() != Width()) { + throw std::invalid_argument( + "The stellar-equilibrium preconditioner received a residual with the wrong size." + ); + } + if (correction.Size() != Height()) { + throw std::invalid_argument( + "The stellar-equilibrium preconditioner requires a preallocated correction of the correct size." + ); + } + + const Clock::time_point start = Clock::now(); + correction = residual; + const double elapsed = std::chrono::duration(Clock::now() - start).count(); + + ++m_statistics.applications; + ++m_statistics.backendApplications; + m_statistics.applicationSeconds += elapsed; + m_statistics.maximumApplicationSeconds = std::max(m_statistics.maximumApplicationSeconds, elapsed); + } + + [[nodiscard]] StellarPreconditionerPreparationReport Refresh() { + const Clock::time_point start = Clock::now(); + VerifyPreparedProblem(); + + const StellarPreconditionerLifecycleSnapshot current = Traits::Snapshot(*m_problem); + const StellarPreconditionerPreparationChanges changes = preparationChanges(m_snapshot, current); + ++m_statistics.refreshChecks; + + StellarPreconditionerPreparationReport report{.changes = changes}; + if (!changes.Any()) { + ++m_statistics.noOpRefreshes; + } else { + report.refreshedComponents = + RefreshComponents(changes, std::make_index_sequence>{}); + ++m_statistics.refreshes; + m_statistics.componentRefreshes += report.refreshedComponents; + m_snapshot = current; + } + + m_statistics.refreshSeconds += std::chrono::duration(Clock::now() - start).count(); + return report; + } + + [[nodiscard]] bool IsCurrent() const { + return Traits::IsPrepared(*m_problem) && Traits::Snapshot(*m_problem) == m_snapshot; + } + + [[nodiscard]] const ProblemType &GetProblem() const noexcept { + return *m_problem; + } + + [[nodiscard]] const typename Traits::Manifest &GetManifest() const noexcept { + return *m_manifest; + } + + [[nodiscard]] const mfem::Operator &GetLinearizationOperator() const noexcept { + return *m_linearization; + } + + template + requires utils::blocks::contains_type_v< + CorrectionBlock, + typename FormType::value_blocks> + [[nodiscard]] mfem::Vector GetCorrectionBlock(mfem::Vector &correction) const { + if (correction.Size() != Height()) { + throw std::invalid_argument("A correction block view requires a complete correction vector."); + } + constexpr int index = utils::blocks::type_index_v; + return mfem::Vector( + correction.GetData() + m_manifest->layout().offset(utils::blocks::value_block{}), + m_manifest->layout().size(utils::blocks::value_block{}) + ); + } + + template + requires utils::blocks::contains_type_v< + ResidualBlock, + typename FormType::residual_blocks> + [[nodiscard]] mfem::Vector GetResidualBlock(const mfem::Vector &residual) const { + if (residual.Size() != Width()) { + throw std::invalid_argument("A residual block view requires a complete residual vector."); + } + constexpr int index = utils::blocks::type_index_v; + return mfem::Vector( + const_cast(residual.GetData()) + + m_manifest->layout().offset(utils::blocks::residual_block{}), + m_manifest->layout().size(utils::blocks::residual_block{}) + ); + } + + [[nodiscard]] const PlanType &GetPlan() const noexcept { + return m_plan; + } + + [[nodiscard]] const StellarPreconditionerLifecycleSnapshot &GetLifecycleSnapshot() const noexcept { + return m_snapshot; + } + + [[nodiscard]] const StellarPreconditionerStatistics &GetStatistics() const noexcept { + return m_statistics; + } + + private: + void VerifyPreparedProblem() const { + if (!Traits::IsPrepared(*m_problem)) { + throw std::logic_error( + "The stellar-equilibrium problem must be prepared before its preconditioner is prepared or " + "refreshed." + ); + } + if (Traits::StateSize(*m_problem) <= 0 || + Traits::StateSize(*m_problem) != Traits::EquationSize(*m_problem)) { + throw std::logic_error("A stellar-equilibrium preconditioner requires a positive square problem."); + } + + const auto &layout = Traits::ManifestOf(*m_problem).layout(); + if (layout.value_offsets().Last() != Traits::StateSize(*m_problem) || + layout.residual_offsets().Last() != Traits::EquationSize(*m_problem)) { + throw std::logic_error( + "The stellar-equilibrium manifest and discrete problem dimensions are inconsistent." + ); + } + } + + void VerifyCurrent() const { + if (!IsCurrent()) { + throw std::logic_error( + "The stellar-equilibrium preconditioner is stale; call Refresh after preparing a new linearization." + ); + } + } + + template void SetupComponents(std::index_sequence) { + ((std::get(m_preparedComponents).Setup(*m_problem, std::get(m_plan.components())), + ++m_statistics.componentSetups), + ...); + } + + template + [[nodiscard]] std::uint64_t RefreshComponent(const StellarPreconditionerPreparationChanges &changes) { + using ComponentTuple = std::remove_cvref_t; + using Component = std::tuple_element_t; + if (!detail::componentRequiresRefresh(changes)) { + return 0; + } + return std::get(m_preparedComponents) + .Refresh(*m_problem, std::get(m_plan.components()), changes) + ? 1U + : 0U; + } + + template + [[nodiscard]] std::uint64_t RefreshComponents( + const StellarPreconditionerPreparationChanges &changes, + std::index_sequence + ) { + return (std::uint64_t{0} + ... + RefreshComponent(changes)); + } + + ProblemType *m_problem; + const typename Traits::Manifest *m_manifest; + const mfem::Operator *m_linearization; + PlanType m_plan; + PreparedComponents m_preparedComponents; + StellarPreconditionerLifecycleSnapshot m_snapshot; + mutable StellarPreconditionerStatistics m_statistics; + }; + + template < + StellarPreconditionerProblem Problem, + typename Plan> + requires PreparedPreconditionerPlanFor< + std::remove_cvref_t, + std::remove_cvref_t> + [[nodiscard]] auto prepare( + Problem &problem, + Plan &&plan + ) { + using ProblemType = std::remove_cvref_t; + using PlanType = std::remove_cvref_t; + return StellarEquilibriumPreconditioner{problem, std::forward(plan)}; + } +} // namespace mean_field::preconditioning diff --git a/libmeanfield/interface/preconditioning/stellar_structure.cppm b/libmeanfield/interface/preconditioning/stellar_structure.cppm new file mode 100644 index 0000000..dba31f2 --- /dev/null +++ b/libmeanfield/interface/preconditioning/stellar_structure.cppm @@ -0,0 +1,830 @@ +module; + +#include +#include +#include +#include +#include +#include +#include + +#include + +export module mean_field:preconditioning.stellar_structure; + +export import :preconditioning.gravity_field; +export import :preconditioning.material_surface; + +export namespace mean_field::preconditioning { + struct IndependentStellarSubsystems final { }; + struct MaterialThenGravityTriangular final { }; + struct GravityThenMaterialTriangular final { }; + struct ApproximateStellarBlockLDU final { }; + + template struct IsStellarStructureFactorizationPolicy : std::false_type { }; + template <> struct IsStellarStructureFactorizationPolicy : std::true_type { }; + template <> struct IsStellarStructureFactorizationPolicy : std::true_type { }; + template <> struct IsStellarStructureFactorizationPolicy : std::true_type { }; + template <> struct IsStellarStructureFactorizationPolicy : std::true_type { }; + + template + concept StellarStructureFactorizationPolicy = + IsStellarStructureFactorizationPolicy>::value; + + namespace detail { + template struct StellarStructureConcatenate; + + template <> struct StellarStructureConcatenate<> { + using Type = utils::blocks::type_list<>; + }; + + template struct StellarStructureConcatenate> { + using Type = utils::blocks::type_list; + }; + + template + struct StellarStructureConcatenate< + utils::blocks::type_list, + utils::blocks::type_list, + Remaining...> { + using Type = + typename StellarStructureConcatenate, Remaining...>::Type; + }; + + template + using StellarStructureConcatenateT = typename StellarStructureConcatenate::Type; + + template + struct StellarStructureCouplingsForResidual; + + template + struct StellarStructureCouplingsForResidual, JacobianForm> { + using Type = utils::blocks::type_list<>; + }; + + template + struct StellarStructureCouplingsForResidual< + Residual, + utils::blocks::type_list, + JacobianForm> { + private: + using Tail = typename StellarStructureCouplingsForResidual< + Residual, + utils::blocks::type_list, + JacobianForm>::Type; + + public: + using Type = std::conditional_t< + utils::blocks::has_jacobian_coupling_v, + StellarStructureConcatenateT>, Tail>, + Tail>; + }; + + template + struct StellarStructureInducedCouplings; + + template + struct StellarStructureInducedCouplings, Corrections, JacobianForm> { + using Type = utils::blocks::type_list<>; + }; + + template + struct StellarStructureInducedCouplings< + utils::blocks::type_list, + Corrections, + JacobianForm> { + using Type = StellarStructureConcatenateT< + typename StellarStructureCouplingsForResidual::Type, + typename StellarStructureInducedCouplings< + utils::blocks::type_list, + Corrections, + JacobianForm>::Type>; + }; + + template struct StellarStructureListsAreDisjoint; + + template + struct StellarStructureListsAreDisjoint, Right> + : std::bool_constant<(!utils::blocks::contains_type_v && ...)> { }; + + template struct StellarStructureListIsSubset; + + template + struct StellarStructureListIsSubset, Universe> + : std::bool_constant<(utils::blocks::contains_type_v && ...)> { }; + } // namespace detail + + using CoupledStellarStructureCharacteristics = OperatorCharacteristics< + OperatorCategory::mixed, + OperatorValueStructure::block, + OperatorSymmetry::nonsymmetric, + OperatorDefiniteness::unspecified, + OperatorRepresentation::matrix_free, + OperatorDistribution::distributed_true_dof, + OperatorFESpace::product>; + + namespace backend { + template < + Registered MaterialSurfaceBackend, + Registered GravityBackend, + StellarStructureFactorizationPolicy Policy> + struct CoupledStellarStructure final { + using MaterialSurfaceBackendType = MaterialSurfaceBackend; + using GravityBackendType = GravityBackend; + using FactorizationPolicyType = Policy; + }; + + template < + Registered MaterialSurfaceBackend, + Registered GravityBackend, + StellarStructureFactorizationPolicy Policy> + struct Traits> { + static constexpr bool registered = true; + static constexpr ApplicationContract applicationContract = + ::mean_field::preconditioning::backend::applicationContract == + ApplicationContract::stationary_linear && + ::mean_field::preconditioning::backend::applicationContract == + ApplicationContract::stationary_linear + ? ApplicationContract::stationary_linear + : ApplicationContract::flexible; + static constexpr bool supportsSerialExecution = Traits::supportsSerialExecution && + Traits::supportsSerialExecution; + static constexpr bool supportsDistributedExecution = + Traits::supportsDistributedExecution && + Traits::supportsDistributedExecution; + static constexpr SymmetryRequirement symmetryRequirement = SymmetryRequirement::none; + static constexpr NullspaceRequirement nullspaceRequirement = NullspaceRequirement::constant_mode_supported; + static constexpr SurrogateRequirement surrogateRequirement = SurrogateRequirement::assembled_sparse; + static constexpr bool requiresAssembledSparseSurrogate = + Traits::requiresAssembledSparseSurrogate || + Traits::requiresAssembledSparseSurrogate; + + using PreparationDependencies = preconditioning::PreparationDependencies< + PreparationDependency::discretization, + PreparationDependency::geometry, + PreparationDependency::equation_of_state, + PreparationDependency::linearization>; + + template + static constexpr bool supports = + Characteristics::category == OperatorCategory::mixed && + Characteristics::valueStructure == OperatorValueStructure::block && + Characteristics::symmetry == OperatorSymmetry::nonsymmetric && + Characteristics::representation == OperatorRepresentation::matrix_free && + Characteristics::distribution == OperatorDistribution::distributed_true_dof && + Characteristics::finiteElementSpace == OperatorFESpace::product; + }; + } // namespace backend + + template < + PreconditionerComponent MaterialSurfaceComponentT, + PreconditionerComponent GravityComponentT, + typename FormT, + typename JacobianFormT, + StellarStructureFactorizationPolicy PolicyT> + requires utils::blocks::valid_jacobian_form && + detail::StellarStructureListsAreDisjoint< + typename MaterialSurfaceComponentT::CorrectionBlocks, + typename GravityComponentT::CorrectionBlocks>::value && + detail::StellarStructureListsAreDisjoint< + typename MaterialSurfaceComponentT::ResidualBlocks, + typename GravityComponentT::ResidualBlocks>::value && + detail::StellarStructureListIsSubset< + typename MaterialSurfaceComponentT::CorrectionBlocks, + typename FormT::value_blocks>::value && + detail::StellarStructureListIsSubset< + typename GravityComponentT::CorrectionBlocks, + typename FormT::value_blocks>::value && + detail::StellarStructureListIsSubset< + typename MaterialSurfaceComponentT::ResidualBlocks, + typename FormT::residual_blocks>::value && + detail::StellarStructureListIsSubset< + typename GravityComponentT::ResidualBlocks, + typename FormT::residual_blocks>::value + class StellarStructureBlock final { + public: + using MaterialSurfaceComponent = MaterialSurfaceComponentT; + using GravityComponent = GravityComponentT; + using Form = FormT; + using JacobianForm = JacobianFormT; + using Factorization = PolicyT; + using CorrectionBlocks = detail::StellarStructureConcatenateT< + typename MaterialSurfaceComponent::CorrectionBlocks, + typename GravityComponent::CorrectionBlocks>; + using ResidualBlocks = detail::StellarStructureConcatenateT< + typename MaterialSurfaceComponent::ResidualBlocks, + typename GravityComponent::ResidualBlocks>; + using MaterialToGravityCouplings = typename detail::StellarStructureInducedCouplings< + typename GravityComponent::ResidualBlocks, + typename MaterialSurfaceComponent::CorrectionBlocks, + JacobianForm>::Type; + using GravityToMaterialCouplings = typename detail::StellarStructureInducedCouplings< + typename MaterialSurfaceComponent::ResidualBlocks, + typename GravityComponent::CorrectionBlocks, + JacobianForm>::Type; + using RequiredCouplings = detail::StellarStructureConcatenateT< + typename MaterialSurfaceComponent::RequiredCouplings, + typename GravityComponent::RequiredCouplings, + MaterialToGravityCouplings, + GravityToMaterialCouplings>; + using OperatorDescription = CoupledStellarStructureCharacteristics; + using BackendType = backend::CoupledStellarStructure< + typename MaterialSurfaceComponent::BackendType, + typename GravityComponent::BackendType, + Factorization>; + using PreparationDependencies = typename backend::Traits::PreparationDependencies; + + constexpr StellarStructureBlock( + MaterialSurfaceComponent materialSurfaceComponent, + GravityComponent gravityComponent, + Factorization factorization = {} + ) + : m_materialSurfaceComponent(std::move(materialSurfaceComponent)), + m_gravityComponent(std::move(gravityComponent)), + m_factorization(std::move(factorization)) { + } + + [[nodiscard]] constexpr const MaterialSurfaceComponent &materialSurfaceComponent() const noexcept { + return m_materialSurfaceComponent; + } + + [[nodiscard]] constexpr const GravityComponent &gravityComponent() const noexcept { + return m_gravityComponent; + } + + [[nodiscard]] constexpr const Factorization &factorizationPolicy() const noexcept { + return m_factorization; + } + + private: + MaterialSurfaceComponent m_materialSurfaceComponent; + GravityComponent m_gravityComponent; + Factorization m_factorization; + }; + + template + concept StellarStructureCrossCouplingOperator = requires( + const Candidate &couplings, + const mfem::Vector &materialDirection, + const mfem::Vector &gravityDirection, + mfem::Vector &materialAction, + mfem::Vector &gravityAction + ) { + { couplings.MaterialSize() } -> std::same_as; + { couplings.GravitySize() } -> std::same_as; + couplings.ApplyMaterialToGravity(materialDirection, gravityAction); + couplings.ApplyGravityToMaterial(gravityDirection, materialAction); + }; + + struct StellarStructureFactorizationStatistics final { + std::uint64_t applications{0}; + std::uint64_t materialSurfaceInverseApplications{0}; + std::uint64_t gravityInverseApplications{0}; + std::uint64_t materialToGravityApplications{0}; + std::uint64_t gravityToMaterialApplications{0}; + }; + + template + class StellarStructureFactorizationOperator final : public mfem::Solver { + public: + StellarStructureFactorizationOperator( + Policy policy, + const mfem::Solver &materialSurfaceInverse, + const mfem::Solver &gravityInverse, + const CouplingOperator &couplings + ) + : mfem::Solver(materialSurfaceInverse.Height() + gravityInverse.Height()), + m_policy(std::move(policy)), + m_materialSurfaceInverse(std::addressof(materialSurfaceInverse)), + m_gravityInverse(std::addressof(gravityInverse)), + m_couplings(std::addressof(couplings)), + m_materialWorkspace(materialSurfaceInverse.Height()), + m_gravityWorkspace(gravityInverse.Height()) { + if (materialSurfaceInverse.Height() <= 0 || + materialSurfaceInverse.Height() != materialSurfaceInverse.Width() || gravityInverse.Height() <= 0 || + gravityInverse.Height() != gravityInverse.Width() || + materialSurfaceInverse.Height() != couplings.MaterialSize() || + gravityInverse.Height() != couplings.GravitySize()) { + throw std::invalid_argument( + "The stellar-structure inverse blocks do not match the cross-coupling operator." + ); + } + } + + StellarStructureFactorizationOperator(const StellarStructureFactorizationOperator &) = delete; + StellarStructureFactorizationOperator &operator=(const StellarStructureFactorizationOperator &) = delete; + StellarStructureFactorizationOperator(StellarStructureFactorizationOperator &&) = delete; + StellarStructureFactorizationOperator &operator=(StellarStructureFactorizationOperator &&) = delete; + + void SetOperator(const mfem::Operator &operation) override { + if (operation.Height() != Height() || operation.Width() != Width()) { + throw std::invalid_argument("The stellar-structure factorization received an incompatible operator."); + } + } + + void Mult( + const mfem::Vector &rightHandSide, + mfem::Vector &action + ) const override { + if (rightHandSide.Size() != Width() || action.Size() != Height()) { + throw std::invalid_argument( + "The stellar-structure factorization requires compatible, preallocated vectors." + ); + } + + action = 0.0; + const mfem::Vector materialRightHandSide( + const_cast(rightHandSide.GetData()), m_materialSurfaceInverse->Width() + ); + const mfem::Vector gravityRightHandSide( + const_cast(rightHandSide.GetData()) + m_materialSurfaceInverse->Width(), + m_gravityInverse->Width() + ); + mfem::Vector materialAction(action, 0, m_materialSurfaceInverse->Height()); + mfem::Vector gravityAction(action, m_materialSurfaceInverse->Height(), m_gravityInverse->Height()); + + if constexpr (std::same_as) { + m_materialSurfaceInverse->Mult(materialRightHandSide, materialAction); + m_gravityInverse->Mult(gravityRightHandSide, gravityAction); + ++m_statistics.materialSurfaceInverseApplications; + ++m_statistics.gravityInverseApplications; + } else if constexpr (std::same_as) { + m_materialSurfaceInverse->Mult(materialRightHandSide, materialAction); + m_couplings->ApplyMaterialToGravity(materialAction, m_gravityWorkspace); + m_gravityWorkspace *= -1.0; + m_gravityWorkspace += gravityRightHandSide; + m_gravityInverse->Mult(m_gravityWorkspace, gravityAction); + ++m_statistics.materialSurfaceInverseApplications; + ++m_statistics.materialToGravityApplications; + ++m_statistics.gravityInverseApplications; + } else if constexpr (std::same_as) { + m_gravityInverse->Mult(gravityRightHandSide, gravityAction); + m_couplings->ApplyGravityToMaterial(gravityAction, m_materialWorkspace); + m_materialWorkspace *= -1.0; + m_materialWorkspace += materialRightHandSide; + m_materialSurfaceInverse->Mult(m_materialWorkspace, materialAction); + ++m_statistics.gravityInverseApplications; + ++m_statistics.gravityToMaterialApplications; + ++m_statistics.materialSurfaceInverseApplications; + } else { + static_assert(std::same_as); + m_materialSurfaceInverse->Mult(materialRightHandSide, materialAction); + m_couplings->ApplyMaterialToGravity(materialAction, m_gravityWorkspace); + m_gravityWorkspace *= -1.0; + m_gravityWorkspace += gravityRightHandSide; + m_gravityInverse->Mult(m_gravityWorkspace, gravityAction); + m_couplings->ApplyGravityToMaterial(gravityAction, m_materialWorkspace); + m_materialWorkspace *= -1.0; + m_materialWorkspace += materialRightHandSide; + m_materialSurfaceInverse->Mult(m_materialWorkspace, materialAction); + m_statistics.materialSurfaceInverseApplications += 2; + ++m_statistics.materialToGravityApplications; + ++m_statistics.gravityInverseApplications; + ++m_statistics.gravityToMaterialApplications; + } + materialAction.SyncAliasMemory(action); + gravityAction.SyncAliasMemory(action); + ++m_statistics.applications; + } + + [[nodiscard]] const StellarStructureFactorizationStatistics &GetStatistics() const noexcept { + return m_statistics; + } + + private: + Policy m_policy; + const mfem::Solver *m_materialSurfaceInverse; + const mfem::Solver *m_gravityInverse; + const CouplingOperator *m_couplings; + mutable mfem::Vector m_materialWorkspace; + mutable mfem::Vector m_gravityWorkspace; + mutable StellarStructureFactorizationStatistics m_statistics; + }; + + class StellarStructureCrossJacobianOperator final : public mfem::Operator { + public: + explicit StellarStructureCrossJacobianOperator(const operators::PreparedStellarEquilibriumOperator &operation) + : mfem::Operator(MaterialSizeOf(operation) + GravitySizeOf(operation)), + m_operation(std::addressof(operation)), + m_materialOffsets(4), + m_gravityOffsets(3), + m_combinedOffsets(3), + m_gravityDirection(operation.GetGravityJacobianOperator().Width()), + m_volumeDisplacement(operation.GetDomainDeformation().volumeDisplacementSize()), + m_mechanicalAction(operation.GetDomainDeformation().volumeDisplacementSize()), + m_zeroEnthalpy(operation.GetBarotropicClosureOperator().GetEnthalpySize()) { + const auto &context = operation.GetGravityContext(); + m_materialOffsets[0] = 0; + m_materialOffsets[1] = context.GetDensityMap().reduced_size(); + m_materialOffsets[2] = m_materialOffsets[1] + operation.GetDomainDeformation().parameterCount(); + m_materialOffsets[3] = MaterialSizeOf(operation); + m_gravityOffsets[0] = 0; + m_gravityOffsets[1] = context.GetGravityGradientMap().reduced_size(); + m_gravityOffsets[2] = GravitySizeOf(operation); + m_combinedOffsets[0] = 0; + m_combinedOffsets[1] = MaterialSize(); + m_combinedOffsets[2] = Height(); + m_zeroEnthalpy = 0.0; + } + + void Mult( + const mfem::Vector &direction, + mfem::Vector &action + ) const override { + VerifyCombined(direction, action); + action = 0.0; + const mfem::Vector materialDirection(const_cast(direction.GetData()), MaterialSize()); + const mfem::Vector gravityDirection( + const_cast(direction.GetData()) + MaterialSize(), GravitySize() + ); + mfem::Vector materialAction(action, 0, MaterialSize()); + mfem::Vector gravityAction(action, MaterialSize(), GravitySize()); + ApplyMaterialToGravity(materialDirection, gravityAction); + ApplyGravityToMaterial(gravityDirection, materialAction); + materialAction.SyncAliasMemory(action); + gravityAction.SyncAliasMemory(action); + } + + void ApplyMaterialToGravity( + const mfem::Vector &materialDirection, + mfem::Vector &gravityAction + ) const { + VerifyMaterial(materialDirection, "direction"); + VerifyGravity(gravityAction, "action"); + const auto densityDirection = MaterialBlock(materialDirection, 0); + const auto surfaceDirection = MaterialBlock(materialDirection, 1); + const auto &gravityOffsets = m_operation->GetGravityOperator().GetStateOffsets(); + using GravityForm = utils::blocks::gravity_field_form; + constexpr auto densityBlock = + utils::blocks::get_value_block(utils::blocks::density_field.mass_term); + constexpr auto displacementBlock = + utils::blocks::get_value_block(utils::blocks::displacement_field.geometry_term); + + m_gravityDirection = 0.0; + auto packedDensityDirection = MutableBlock(m_gravityDirection, gravityOffsets, densityBlock.index); + packedDensityDirection = densityDirection; + packedDensityDirection.SyncAliasMemory(m_gravityDirection); + m_operation->GetDomainDeformation().applyJacobian( + m_operation->GetSurfaceDeformationParameters(), surfaceDirection, m_volumeDisplacement + ); + auto packedDisplacementDirection = + MutableBlock(m_gravityDirection, gravityOffsets, displacementBlock.index); + packedDisplacementDirection = m_volumeDisplacement; + packedDisplacementDirection.SyncAliasMemory(m_gravityDirection); + m_operation->GetGravityJacobianOperator().Mult(m_gravityDirection, gravityAction); + } + + void ApplyGravityToMaterial( + const mfem::Vector &gravityDirection, + mfem::Vector &materialAction + ) const { + VerifyGravity(gravityDirection, "direction"); + VerifyMaterial(materialAction, "action"); + const auto gravityGradientDirection = GravityBlock(gravityDirection, 0); + const auto gravityPotentialDirection = GravityBlock(gravityDirection, 1); + auto densityAction = MaterialBlock(materialAction, 0); + auto surfaceAction = MaterialBlock(materialAction, 1); + auto enthalpyAction = MaterialBlock(materialAction, 2); + + densityAction = 0.0; + m_operation->GetDisplacementOperator().ApplyGravityGradientJacobianAction( + gravityGradientDirection, m_mechanicalAction + ); + m_operation->GetDomainDeformation().applyJacobianTranspose( + m_operation->GetSurfaceDeformationParameters(), m_mechanicalAction, surfaceAction + ); + m_operation->GetHydrostaticOperator().ApplyGravityPotentialJacobianAction( + gravityPotentialDirection, enthalpyAction + ); + m_operation->GetSurfaceConstraintOperator().ApplyJacobianRows(m_zeroEnthalpy, enthalpyAction); + densityAction.SyncAliasMemory(materialAction); + surfaceAction.SyncAliasMemory(materialAction); + enthalpyAction.SyncAliasMemory(materialAction); + } + + [[nodiscard]] int MaterialSize() const noexcept { + return m_materialOffsets.Last(); + } + + [[nodiscard]] int GravitySize() const noexcept { + return m_gravityOffsets.Last(); + } + + [[nodiscard]] const mfem::Array &GetMaterialOffsets() const noexcept { + return m_materialOffsets; + } + + [[nodiscard]] const mfem::Array &GetGravityOffsets() const noexcept { + return m_gravityOffsets; + } + + [[nodiscard]] const mfem::Array &GetCombinedOffsets() const noexcept { + return m_combinedOffsets; + } + + private: + [[nodiscard]] static int MaterialSizeOf(const operators::PreparedStellarEquilibriumOperator &operation) { + if (!operation.IsPrepared()) { + throw std::logic_error("The stellar-structure cross Jacobian requires a prepared operator."); + } + return operation.GetGravityContext().GetDensityMap().reduced_size() + + operation.GetDomainDeformation().parameterCount() + + operation.GetBarotropicClosureOperator().GetEnthalpySize(); + } + + [[nodiscard]] static int GravitySizeOf(const operators::PreparedStellarEquilibriumOperator &operation) { + return operation.GetGravityContext().GetGravityGradientMap().reduced_size() + + operation.GetGravityContext().GetGravityPotentialMap().reduced_size(); + } + + [[nodiscard]] static mfem::Vector MutableBlock( + mfem::Vector &vector, + const mfem::Array &offsets, + const int block + ) { + return mfem::Vector(vector, offsets[block], offsets[block + 1] - offsets[block]); + } + + [[nodiscard]] mfem::Vector MaterialBlock( + const mfem::Vector &vector, + const int block + ) const { + return mfem::Vector( + const_cast(vector.GetData()) + m_materialOffsets[block], + m_materialOffsets[block + 1] - m_materialOffsets[block] + ); + } + + [[nodiscard]] mfem::Vector MaterialBlock( + mfem::Vector &vector, + const int block + ) const { + return mfem::Vector( + vector, m_materialOffsets[block], m_materialOffsets[block + 1] - m_materialOffsets[block] + ); + } + + [[nodiscard]] mfem::Vector GravityBlock( + const mfem::Vector &vector, + const int block + ) const { + return mfem::Vector( + const_cast(vector.GetData()) + m_gravityOffsets[block], + m_gravityOffsets[block + 1] - m_gravityOffsets[block] + ); + } + + void VerifyCombined( + const mfem::Vector &direction, + const mfem::Vector &action + ) const { + if (direction.Size() != Width() || action.Size() != Height()) { + throw std::invalid_argument( + "The stellar-structure cross Jacobian requires compatible, preallocated vectors." + ); + } + } + + void VerifyMaterial( + const mfem::Vector &vector, + const char *role + ) const { + if (vector.Size() != MaterialSize()) { + throw std::invalid_argument( + std::string("The stellar-structure material ") + role + " has the wrong size." + ); + } + } + + void VerifyGravity( + const mfem::Vector &vector, + const char *role + ) const { + if (vector.Size() != GravitySize()) { + throw std::invalid_argument( + std::string("The stellar-structure gravity ") + role + " has the wrong size." + ); + } + } + + const operators::PreparedStellarEquilibriumOperator *m_operation; + mfem::Array m_materialOffsets; + mfem::Array m_gravityOffsets; + mfem::Array m_combinedOffsets; + mutable mfem::Vector m_gravityDirection; + mutable mfem::Vector m_volumeDisplacement; + mutable mfem::Vector m_mechanicalAction; + mfem::Vector m_zeroEnthalpy; + }; + + struct StellarStructureBlockPreparationReport final { + MaterialSurfaceBlockPreparationReport materialSurface; + GravityFieldBlockPreparationReport gravity; + + [[nodiscard]] bool DidAnyWork() const noexcept { + return materialSurface.DidAnyWork() || gravity.DidAnyWork(); + } + }; + + namespace detail { + template + [[nodiscard]] const operators::PreparedStellarEquilibriumOperator & + physicalOperator(const equilibrium::StellarEquilibriumProblem &problem) { + if constexpr (equilibrium::StellarEquilibriumProblem::hasFixedCentralDensity) { + return problem.GetPreparedOperator().GetPhysicalOperator(); + } else { + return problem.GetPreparedOperator(); + } + } + } // namespace detail + + template < + equilibrium::StellarEquilibriumModel Model, + typename MaterialComponent, + backend::Registered GravityMassBackend, + backend::ApplicationMode Mode, + GravityFactorizationPolicy GravityPolicy, + StellarStructureFactorizationPolicy StructurePolicy> + class PreparedStellarStructureBlock final : public mfem::Solver { + private: + using Problem = equilibrium::StellarEquilibriumProblem; + using GravityComponent = GravityFieldBlock, GravityPolicy>; + using Structure = StellarStructureBlock< + MaterialComponent, + GravityComponent, + typename Problem::FormType, + typename Problem::JacobianFormType, + StructurePolicy>; + using MaterialPrepared = + decltype(preconditioning::prepare(std::declval(), std::declval())); + using GravityPrepared = decltype(preconditioning::prepare( + std::declval(), + std::declval(), + std::declval() + )); + + public: + PreparedStellarStructureBlock( + const Problem &problem, + Structure structure + ) + : mfem::Solver(StructureSize(problem)), + m_problem(std::addressof(problem)), + m_structure(std::move(structure)), + m_materialSurface( + preconditioning::prepare( + problem, + m_structure.materialSurfaceComponent() + ) + ), + m_gravity( + preconditioning::prepare( + detail::physicalOperator(problem).GetHydrostaticOperator().GetFEM(), + detail::physicalOperator(problem).GetGravityContext().GetGeometryContext(), + m_structure.gravityComponent() + ) + ), + m_crossCouplings(detail::physicalOperator(problem)), + m_factorization( + m_structure.factorizationPolicy(), + m_materialSurface, + m_gravity, + m_crossCouplings + ) { + } + + PreparedStellarStructureBlock(const PreparedStellarStructureBlock &) = delete; + PreparedStellarStructureBlock &operator=(const PreparedStellarStructureBlock &) = delete; + PreparedStellarStructureBlock(PreparedStellarStructureBlock &&) = delete; + PreparedStellarStructureBlock &operator=(PreparedStellarStructureBlock &&) = delete; + + void SetOperator(const mfem::Operator &operation) override { + m_factorization.SetOperator(operation); + } + + void Mult( + const mfem::Vector &rightHandSide, + mfem::Vector &action + ) const override { + if (!IsCurrent()) { + throw std::logic_error("The stellar-structure block is stale; refresh it before application."); + } + m_factorization.Mult(rightHandSide, action); + } + + [[nodiscard]] bool IsCurrent() const noexcept { + return m_materialSurface.IsCurrent() && m_gravity.IsCurrent() && + detail::physicalOperator(*m_problem).IsPrepared(); + } + + [[nodiscard]] StellarStructureBlockPreparationReport Refresh() { + const auto &physical = detail::physicalOperator(*m_problem); + return { + .materialSurface = m_materialSurface.Refresh(physical), + .gravity = m_gravity.Refresh( + physical.GetHydrostaticOperator().GetFEM(), physical.GetGravityContext().GetGeometryContext() + ) + }; + } + + [[nodiscard]] const Structure &GetBlock() const noexcept { + return m_structure; + } + + [[nodiscard]] const MaterialPrepared &GetMaterialSurfacePreconditioner() const noexcept { + return m_materialSurface; + } + + [[nodiscard]] const GravityPrepared &GetGravityPreconditioner() const noexcept { + return m_gravity; + } + + [[nodiscard]] const StellarStructureCrossJacobianOperator &GetCrossCouplings() const noexcept { + return m_crossCouplings; + } + + [[nodiscard]] const StellarStructureFactorizationOperator< + StructurePolicy, + StellarStructureCrossJacobianOperator> & + GetFactorization() const noexcept { + return m_factorization; + } + + private: + [[nodiscard]] static int StructureSize(const Problem &problem) { + const auto &physical = detail::physicalOperator(problem); + return physical.GetGravityContext().GetDensityMap().reduced_size() + + physical.GetDomainDeformation().parameterCount() + + physical.GetBarotropicClosureOperator().GetEnthalpySize() + + physical.GetGravityContext().GetGravityGradientMap().reduced_size() + + physical.GetGravityContext().GetGravityPotentialMap().reduced_size(); + } + + const Problem *m_problem; + Structure m_structure; + MaterialPrepared m_materialSurface; + GravityPrepared m_gravity; + StellarStructureCrossJacobianOperator m_crossCouplings; + StellarStructureFactorizationOperator m_factorization; + }; + + template < + equilibrium::DiscretizedStellarEquilibriumProblem Problem, + typename MaterialComponent, + backend::Registered GravityMassBackend, + backend::ApplicationMode Mode, + GravityFactorizationPolicy GravityPolicy, + StellarStructureFactorizationPolicy StructurePolicy> + [[nodiscard]] constexpr auto stellarStructureBlock( + const Problem &, + MaterialComponent materialComponent, + GravityFieldBlock< + GravityMassBackend, + backend::HypreBoomerAMG, + GravityPolicy> gravityComponent, + StructurePolicy policy + ) { + using ProblemType = std::remove_cvref_t; + return StellarStructureBlock< + MaterialComponent, GravityFieldBlock, GravityPolicy>, + typename ProblemType::FormType, typename ProblemType::JacobianFormType, StructurePolicy>{ + std::move(materialComponent), std::move(gravityComponent), std::move(policy) + }; + } + + template + [[nodiscard]] constexpr auto stellarStructureBlock(const Problem &problem) { + using FixedAMG = backend::HypreBoomerAMG; + auto material = materialSurfaceBlock(problem); + auto gravity = GravityFieldBlock( + backend::MatrixFreeChebyshev{.order = 5, .powerIterations = 20}, + FixedAMG{backend::FixedCycles{.cycles = 3}}, GravityApproximateLDU{} + ); + return stellarStructureBlock(problem, std::move(material), std::move(gravity), IndependentStellarSubsystems{}); + } + + template < + equilibrium::StellarEquilibriumModel Model, + typename MaterialComponent, + backend::Registered GravityMassBackend, + backend::ApplicationMode Mode, + GravityFactorizationPolicy GravityPolicy, + StellarStructureFactorizationPolicy StructurePolicy> + [[nodiscard]] auto prepare( + const equilibrium::StellarEquilibriumProblem &problem, + StellarStructureBlock< + MaterialComponent, + GravityFieldBlock< + GravityMassBackend, + backend::HypreBoomerAMG, + GravityPolicy>, + typename equilibrium::StellarEquilibriumProblem::FormType, + typename equilibrium::StellarEquilibriumProblem::JacobianFormType, + StructurePolicy> structure + ) { + return PreparedStellarStructureBlock< + Model, MaterialComponent, GravityMassBackend, Mode, GravityPolicy, StructurePolicy>{ + problem, std::move(structure) + }; + } +} // namespace mean_field::preconditioning diff --git a/libmeanfield/interface/surface/compiler.cppm b/libmeanfield/interface/surface/compiler.cppm index d7c1153..29a37c3 100644 --- a/libmeanfield/interface/surface/compiler.cppm +++ b/libmeanfield/interface/surface/compiler.cppm @@ -29,6 +29,25 @@ export namespace mean_field::surface { InputQuantities>)) && ...)> { }; + template + struct PressureSurfaceRelationMatchesBindings : std::false_type { }; + + template + struct PressureSurfaceRelationMatchesBindings< + eos::Relation, + Bindings, + EquationOfState> + : std::bool_constant< + (surfaceBindingCount == 1) && + (std::same_as || ...) && + ((std::same_as || + (surfaceBindingCount == 1 && + eos::SupportsPartialDerivative< + EquationOfState, + eos::Relation, + InputQuantities>)) && + ...)> { }; + template struct MatchingPressureSurfaceRelations; @@ -44,6 +63,24 @@ export namespace mean_field::surface { static constexpr std::size_t count = std::tuple_size_v; }; + template + struct MatchingPressureSurfaceRelationsForBindings; + + template + struct MatchingPressureSurfaceRelationsForBindings< + eos::RelationCatalog, + Bindings, + EquationOfState> { + using Tuple = decltype(std::tuple_cat( + std::conditional_t< + PressureSurfaceRelationMatchesBindings::value, + std::tuple, + std::tuple<>>{}... + )); + + static constexpr std::size_t count = std::tuple_size_v; + }; + template struct UniquePressureSurfaceRelation { using Type = void; }; @@ -52,6 +89,17 @@ export namespace mean_field::surface { using Type = std::tuple_element_t<0, Tuple>; }; + template struct UniquePressureSurfaceFormulation { + using Type = void; + }; + + template struct UniquePressureSurfaceFormulation<1, Tuple, Bindings> { + using Relation = std::tuple_element_t<0, Tuple>; + using CarrierQuantity = eos::RelationOutputT; + using CarrierField = SurfaceFieldForQuantityT; + using Type = SurfaceConstraintFormulation; + }; + template struct AppendSurfaceDependency; template @@ -103,6 +151,16 @@ export namespace mean_field::surface { using Relation = typename UniquePressureSurfaceRelation::Type; }; + template + struct PressureSurfaceFormulationCompilation { + using Matches = MatchingPressureSurfaceRelationsForBindings< + typename EquationOfState::Relations, + Bindings, + EquationOfState>; + using Formulation = + typename UniquePressureSurfaceFormulation::Type; + }; + template requires(PressureSurfaceCompilation::Matches::count == 1) struct CompiledPressureSurfaceConstraintType { @@ -119,6 +177,19 @@ export namespace mean_field::surface { (detail::PressureSurfaceCompilation, std::remove_cvref_t>:: Matches::count == 1); + template + concept PressureSurfaceFormulationCompilable = + ValidSurfaceStateBindings && eos::EquationOfStateModel && + (detail::PressureSurfaceFormulationCompilation< + std::remove_cvref_t, + std::remove_cvref_t>::Matches::count == 1); + + template + requires PressureSurfaceFormulationCompilable + using CompiledPressureSurfaceFormulationT = typename detail::PressureSurfaceFormulationCompilation< + std::remove_cvref_t, + std::remove_cvref_t>::Formulation; + template requires PressureSurfaceCompilable using CompiledPressureSurfaceConstraintT = typename detail::CompiledPressureSurfaceConstraintType< diff --git a/tests/integrators/centrifugal.cpp b/tests/integrators/centrifugal.cpp index b070afa..93d3a18 100644 --- a/tests/integrators/centrifugal.cpp +++ b/tests/integrators/centrifugal.cpp @@ -1,3 +1,4 @@ +#include "profile.h" #include #include @@ -576,7 +577,8 @@ TEST_CASE( integrator.AssembleElementVector(elements, *transformation, element_state, element_residual); mfem::Vector position_test_dofs(velocity_size); - mfem::Vector x_physical(dim); + mapping::MappingPointContext point_context; + mapping::VolumeMappingContext volume_context; position_test_dofs = 0.0; const mfem::IntegrationRule &velocity_nodes = velocity_element->GetNodes(); @@ -584,10 +586,14 @@ TEST_CASE( for (int i = 0; i < velocity_dofs_count; ++i) { const mfem::IntegrationPoint &node = velocity_nodes.IntPoint(i); transformation->SetIntPoint(&node); - mapping_evaluator.GetPhysicalPoint(*transformation, node, x_physical); + MFEM_VERIFY( + mapping_evaluator.EvaluatePoint(*transformation, node, point_context) == + mapping::MappingStatus::valid, + "Centrifugal residual reference encountered an invalid nodal mapping." + ); for (int d = 0; d < dim; ++d) { - position_test_dofs(i + d * velocity_dofs_count) = x_physical(d); + position_test_dofs(i + d * velocity_dofs_count) = point_context.physical_position(d); } } @@ -605,14 +611,17 @@ TEST_CASE( const mfem::IntegrationPoint &integration_point = reference_rule.IntPoint(q); transformation->SetIntPoint(&integration_point); - const mapping::VolumeQuadratureContext context = - mapping_evaluator.GetQuadratureContext(*transformation, integration_point); - const double signed_map_determinant = context.detJ; + MFEM_VERIFY( + mapping_evaluator.EvaluateVolume(*transformation, integration_point, volume_context) == + mapping::MappingStatus::valid, + "Centrifugal residual reference encountered an invalid volume mapping." + ); + const double signed_map_determinant = volume_context.quadrature.detJ; local_minimum_map_determinant = std::min(local_minimum_map_determinant, signed_map_determinant); local_maximum_map_determinant = std::max(local_maximum_map_determinant, signed_map_determinant); - mapping_evaluator.GetPhysicalPoint(*transformation, integration_point, x_physical); + const mfem::Vector &x_physical = volume_context.mapping.physical_position; velocity_element->CalcShape(integration_point, velocity_shape); position_test_value = 0.0; @@ -634,9 +643,9 @@ TEST_CASE( const double density_value = density.GetValue(elem_id, integration_point); local_discrete_reference_action += - density_value * (position_test_value * centrifugal_acceleration) * context.weight; + density_value * (position_test_value * centrifugal_acceleration) * volume_context.quadrature.weight; local_continuous_reference_action += - density_value * (x_physical * centrifugal_acceleration) * context.weight; + density_value * (x_physical * centrifugal_acceleration) * volume_context.quadrature.weight; } } @@ -780,7 +789,8 @@ TEST_CASE( const int velocity_size = dim * velocity_dofs_count; mfem::Vector position_test_dofs(velocity_size); - mfem::Vector x_physical(dim); + mapping::MappingPointContext point_context; + mapping::VolumeMappingContext volume_context; position_test_dofs = 0.0; const mfem::IntegrationRule &velocity_nodes = velocity_element->GetNodes(); @@ -788,10 +798,14 @@ TEST_CASE( for (int i = 0; i < velocity_dofs_count; ++i) { const mfem::IntegrationPoint &node = velocity_nodes.IntPoint(i); transformation->SetIntPoint(&node); - mapping_evaluator.GetPhysicalPoint(*transformation, node, x_physical); + MFEM_VERIFY( + mapping_evaluator.EvaluatePoint(*transformation, node, point_context) == + mapping::MappingStatus::valid, + "Centrifugal p-refinement reference encountered an invalid nodal mapping." + ); for (int d = 0; d < dim; ++d) { - position_test_dofs(i + d * velocity_dofs_count) = x_physical(d); + position_test_dofs(i + d * velocity_dofs_count) = point_context.physical_position(d); } } @@ -807,13 +821,16 @@ TEST_CASE( const mfem::IntegrationPoint &integration_point = reference_rule.IntPoint(q); transformation->SetIntPoint(&integration_point); - const mapping::VolumeQuadratureContext context = - mapping_evaluator.GetQuadratureContext(*transformation, integration_point); - const double signed_map_determinant = context.detJ; + MFEM_VERIFY( + mapping_evaluator.EvaluateVolume(*transformation, integration_point, volume_context) == + mapping::MappingStatus::valid, + "Centrifugal p-refinement reference encountered an invalid volume mapping." + ); + const double signed_map_determinant = volume_context.quadrature.detJ; local_minimum_determinant = std::min(local_minimum_determinant, signed_map_determinant); - mapping_evaluator.GetPhysicalPoint(*transformation, integration_point, x_physical); + const mfem::Vector &x_physical = volume_context.mapping.physical_position; velocity_element->CalcShape(integration_point, velocity_shape); position_test_value = 0.0; @@ -838,9 +855,10 @@ TEST_CASE( const double density_value = density.GetValue(elem_id, integration_point); - local_discrete_action += - density_value * (position_test_value * centrifugal_acceleration) * context.weight; - local_continuous_action += density_value * (x_physical * centrifugal_acceleration) * context.weight; + local_discrete_action += density_value * (position_test_value * centrifugal_acceleration) * + volume_context.quadrature.weight; + local_continuous_action += + density_value * (x_physical * centrifugal_acceleration) * volume_context.quadrature.weight; } } @@ -882,6 +900,8 @@ TEST_CASE( "Centrifugal Virial Position Representation Converges Under H Refinement", tags::rotation_integrator_convergence ) { + MEAN_FIELD_PROFILE_RESET(); + constexpr int dim = 3; constexpr double concentration = 4.0; constexpr double minimum_rate = 1.5; @@ -893,9 +913,12 @@ TEST_CASE( std::array, rotation_fractions.size()> minimum_determinants{}; for (std::size_t refinement_index = 0; refinement_index < refinement_levels.size(); ++refinement_index) { - auto args = test_utils::setup_args(); + auto args = test_utils::setup_args(); - fem::FEM f = fem::setup_fem(args.mesh_file, args, refinement_levels[refinement_index]); + fem::FEM f = MEAN_FIELD_PROFILE_EVALUATE_WARMUP( + "centrifugal virial: FEM setup", 0, + fem::setup_fem(args.mesh_file, args, refinement_levels[refinement_index]) + ); const double radius = utils::RADIUS; @@ -942,8 +965,11 @@ TEST_CASE( }; mfem::VectorFunctionCoefficient displacement_coefficient(dim, rotation_displacement); - displacement.ProjectCoefficient(displacement_coefficient); - *f.displacement = displacement; + MEAN_FIELD_PROFILE_CALL_WARMUP( + "centrifugal virial: displacement projection", 0, + displacement.ProjectCoefficient(displacement_coefficient); + *f.displacement = displacement + ); mapping::GridFunctionMappingEvaluator mapping_evaluator( *f.domainMapperStateless, *f.displacement, *f.compactificationCoordinate ); @@ -955,9 +981,13 @@ TEST_CASE( const int reference_order = 2 * std::max(f.displacementFes->GetMaxElementOrder(), f.densityFes->GetMaxElementOrder()) + 16; - double local_discrete_action = 0.0; - double local_continuous_action = 0.0; - double local_minimum_determinant = std::numeric_limits::infinity(); + double local_discrete_action = 0.0; + double local_continuous_action = 0.0; + double local_minimum_determinant = std::numeric_limits::infinity(); + std::uint64_t nodal_mapping_evaluations = 0; + std::uint64_t quadrature_mapping_evaluations = 0; + + MEAN_FIELD_PROFILE_SCOPE_WARMUP("centrifugal virial: integration traversal", 0); for (int elem_id = 0; elem_id < f.mesh->GetNE(); ++elem_id) { if (f.mesh->GetAttribute(elem_id) == 3) { @@ -971,7 +1001,8 @@ TEST_CASE( const int velocity_size = dim * velocity_dofs_count; mfem::Vector position_test_dofs(velocity_size); - mfem::Vector x_physical(dim); + mapping::MappingPointContext point_context; + mapping::VolumeMappingContext volume_context; position_test_dofs = 0.0; const mfem::IntegrationRule &velocity_nodes = velocity_element->GetNodes(); @@ -979,10 +1010,15 @@ TEST_CASE( for (int i = 0; i < velocity_dofs_count; ++i) { const mfem::IntegrationPoint &node = velocity_nodes.IntPoint(i); transformation->SetIntPoint(&node); - mapping_evaluator.GetPhysicalPoint(*transformation, node, x_physical); + MFEM_VERIFY( + mapping_evaluator.EvaluatePoint(*transformation, node, point_context) == + mapping::MappingStatus::valid, + "Centrifugal h-refinement reference encountered an invalid nodal mapping." + ); + ++nodal_mapping_evaluations; for (int d = 0; d < dim; ++d) { - position_test_dofs(i + d * velocity_dofs_count) = x_physical(d); + position_test_dofs(i + d * velocity_dofs_count) = point_context.physical_position(d); } } @@ -998,13 +1034,17 @@ TEST_CASE( const mfem::IntegrationPoint &integration_point = reference_rule.IntPoint(q); transformation->SetIntPoint(&integration_point); - const mapping::VolumeQuadratureContext context = - mapping_evaluator.GetQuadratureContext(*transformation, integration_point); - const double signed_map_determinant = context.detJ; + MFEM_VERIFY( + mapping_evaluator.EvaluateVolume(*transformation, integration_point, volume_context) == + mapping::MappingStatus::valid, + "Centrifugal h-refinement reference encountered an invalid volume mapping." + ); + ++quadrature_mapping_evaluations; + const double signed_map_determinant = volume_context.quadrature.detJ; local_minimum_determinant = std::min(local_minimum_determinant, signed_map_determinant); - mapping_evaluator.GetPhysicalPoint(*transformation, integration_point, x_physical); + const mfem::Vector &x_physical = volume_context.mapping.physical_position; velocity_element->CalcShape(integration_point, velocity_shape); position_test_value = 0.0; @@ -1029,12 +1069,18 @@ TEST_CASE( const double density_value = density.GetValue(elem_id, integration_point); - local_discrete_action += - density_value * (position_test_value * centrifugal_acceleration) * context.weight; - local_continuous_action += density_value * (x_physical * centrifugal_acceleration) * context.weight; + local_discrete_action += density_value * (position_test_value * centrifugal_acceleration) * + volume_context.quadrature.weight; + local_continuous_action += + density_value * (x_physical * centrifugal_acceleration) * volume_context.quadrature.weight; } } + MEAN_FIELD_PROFILE_COUNT("centrifugal virial: nodal mapping evaluations", nodal_mapping_evaluations); + MEAN_FIELD_PROFILE_COUNT( + "centrifugal virial: quadrature mapping evaluations", quadrature_mapping_evaluations + ); + double global_discrete_action = 0.0; double global_continuous_action = 0.0; double global_minimum_determinant = 0.0; @@ -1054,6 +1100,8 @@ TEST_CASE( *f.displacement = 0.0; } + MEAN_FIELD_PROFILE_PRINT(MPI_COMM_WORLD); + for (std::size_t rotation_index = 0; rotation_index < rotation_fractions.size(); ++rotation_index) { const double error_h = position_errors[rotation_index][0]; const double error_h2 = position_errors[rotation_index][1]; diff --git a/tests/material/thermodynamic_equation_compilation.cpp b/tests/material/thermodynamic_equation_compilation.cpp new file mode 100644 index 0000000..1123a11 --- /dev/null +++ b/tests/material/thermodynamic_equation_compilation.cpp @@ -0,0 +1,345 @@ +#include +#include +#include + +#include + +import mean_field; + +namespace { + namespace blocks = mean_field::utils::blocks; + namespace eos = mean_field::eos; + namespace field = mean_field::field; + namespace material = mean_field::material; + namespace surface = mean_field::surface; + + struct Entropy final : eos::ThermodynamicQuantity { + static constexpr std::string_view identifier = "mock_entropy"; + }; + + struct Composition final : eos::ThermodynamicQuantity { + static constexpr std::string_view identifier = "mock_composition"; + }; + + struct Temperature final : eos::ThermodynamicQuantity { + static constexpr std::string_view identifier = "mock_temperature"; + }; + + struct EntropyField final { + static constexpr std::string_view name = "mock_entropy"; + using PhysicalQuantity = Entropy; + }; + + struct CompositionField final { + static constexpr std::string_view name = "mock_composition"; + using PhysicalQuantity = Composition; + }; + + struct TemperatureField final { + static constexpr std::string_view name = "mock_temperature"; + using PhysicalQuantity = Temperature; + }; + + struct AlternateEntropyField final { + static constexpr std::string_view name = "alternate_mock_entropy"; + using PhysicalQuantity = Entropy; + }; + + struct EntropyValue final : blocks::value_block_base { }; + struct EntropyResidual final : blocks::residual_block_base { }; + struct CompositionValue final : blocks::value_block_base { }; + struct CompositionResidual final : blocks::residual_block_base { }; + struct TemperatureValue final : blocks::value_block_base { }; + struct TemperatureResidual final : blocks::residual_block_base { }; + struct DensityValue final : blocks::value_block_base { }; + struct DensityResidual final : blocks::residual_block_base { }; + struct EnthalpyValue final : blocks::value_block_base { }; + struct EnthalpyResidual final : blocks::residual_block_base { }; + struct UnrelatedValue final : blocks::value_block_base { }; + struct UnrelatedResidual final : blocks::residual_block_base { }; + + using EntropyEquation = material::ThermodynamicEquation; + using CompositionEquation = + material::ThermodynamicEquation; + using TemperatureEquation = + material::ThermodynamicEquation; + using DensityEquation = material::ThermodynamicEquation; + using EnthalpyEquation = material::ThermodynamicEquation; + + using EnthalpyFromPressureEntropyComposition = + eos::Relation; + + class GeneralEquationOfState final { + public: + using Relations = eos::RelationCatalog; + + [[nodiscard]] constexpr eos::SpecificEnthalpyValue evaluate( + EnthalpyFromPressureEntropyComposition, + const eos::PressureValue pressure, + const eos::QuantityValue entropy, + const eos::QuantityValue composition + ) const noexcept { + return eos::SpecificEnthalpyValue{ + 2.0 * pressure.value() + 3.0 * entropy.value() + 5.0 * composition.value() + }; + } + + [[nodiscard]] constexpr eos::PartialDerivative< + eos::quantity::SpecificEnthalpy, + Entropy> + partialDerivative( + EnthalpyFromPressureEntropyComposition, + eos::WithRespectTo, + eos::PressureValue, + eos::QuantityValue, + eos::QuantityValue + ) const noexcept { + return eos::PartialDerivative{3.0}; + } + + [[nodiscard]] constexpr eos::PartialDerivative< + eos::quantity::SpecificEnthalpy, + Composition> + partialDerivative( + EnthalpyFromPressureEntropyComposition, + eos::WithRespectTo, + eos::PressureValue, + eos::QuantityValue, + eos::QuantityValue + ) const noexcept { + return eos::PartialDerivative{5.0}; + } + }; + + class MissingPressureSurfaceRelationEquationOfState final { + public: + using Relations = eos::RelationCatalog; + + [[nodiscard]] constexpr eos::PressureValue evaluate( + eos::PressureFromDensity, + const eos::DensityValue density + ) const noexcept { + return eos::PressureValue{density.value()}; + } + }; + + using EnthalpyFromPressureEntropy = + eos::Relation; + + class AmbiguousEquationOfState final { + public: + using Relations = eos::RelationCatalog; + + [[nodiscard]] constexpr eos::SpecificEnthalpyValue evaluate( + EnthalpyFromPressureEntropyComposition, + const eos::PressureValue pressure, + const eos::QuantityValue entropy, + const eos::QuantityValue composition + ) const noexcept { + return eos::SpecificEnthalpyValue{pressure.value() + entropy.value() + composition.value()}; + } + + [[nodiscard]] constexpr eos::SpecificEnthalpyValue evaluate( + EnthalpyFromPressureEntropy, + const eos::PressureValue pressure, + const eos::QuantityValue entropy + ) const noexcept { + return eos::SpecificEnthalpyValue{pressure.value() + entropy.value()}; + } + + [[nodiscard]] constexpr eos::PartialDerivative< + eos::quantity::SpecificEnthalpy, + Entropy> + partialDerivative( + EnthalpyFromPressureEntropyComposition, + eos::WithRespectTo, + eos::PressureValue, + eos::QuantityValue, + eos::QuantityValue + ) const noexcept { + return eos::PartialDerivative{1.0}; + } + + [[nodiscard]] constexpr eos::PartialDerivative< + eos::quantity::SpecificEnthalpy, + Composition> + partialDerivative( + EnthalpyFromPressureEntropyComposition, + eos::WithRespectTo, + eos::PressureValue, + eos::QuantityValue, + eos::QuantityValue + ) const noexcept { + return eos::PartialDerivative{1.0}; + } + + [[nodiscard]] constexpr eos::PartialDerivative< + eos::quantity::SpecificEnthalpy, + Entropy> + partialDerivative( + EnthalpyFromPressureEntropy, + eos::WithRespectTo, + eos::PressureValue, + eos::QuantityValue + ) const noexcept { + return eos::PartialDerivative{1.0}; + } + }; + + using GeneralForm = blocks::block_form< + blocks::type_list, + blocks::type_list>; + + using ReorderedAvailableEquations = material:: + ThermodynamicEquationCatalog; + + using DifferentlyReorderedAvailableEquations = material:: + ThermodynamicEquationCatalog; + + using ExpectedGeneralEquations = + material::ThermodynamicEquationCatalog; + + using GeneralCompilation = + material::CompiledThermodynamicEquationsT; + + using GeneralCompilationFromOtherOrder = material:: + CompiledThermodynamicEquationsT; + + using HalfPresentEquationForm = blocks::block_form< + blocks::type_list, + blocks::type_list>; + + using NoThermodynamicEquationForm = + blocks::block_form, blocks::type_list>; + + using DuplicateFieldCatalog = material::ThermodynamicEquationCatalog< + EntropyEquation, + material::ThermodynamicEquation>; + + using DuplicateQuantityCatalog = material::ThermodynamicEquationCatalog< + EntropyEquation, + material::ThermodynamicEquation>; + + using DensityFromPressureEntropy = eos::Relation; + + class DensityCarrierEquationOfState final { + public: + using Relations = eos::RelationCatalog; + + [[nodiscard]] constexpr eos::DensityValue evaluate( + DensityFromPressureEntropy, + const eos::PressureValue pressure, + const eos::QuantityValue entropy + ) const noexcept { + return eos::DensityValue{4.0 * pressure.value() + 2.0 * entropy.value()}; + } + + [[nodiscard]] constexpr eos::PartialDerivative< + eos::quantity::Density, + Entropy> + partialDerivative( + DensityFromPressureEntropy, + eos::WithRespectTo, + eos::PressureValue, + eos::QuantityValue + ) const noexcept { + return eos::PartialDerivative{2.0}; + } + }; + + using DensityCarrierForm = blocks:: + block_form, blocks::type_list>; + using DensityCarrierAvailableEquations = + material::ThermodynamicEquationCatalog; + using DensityCarrierCompilation = material::CompiledThermodynamicEquationsT< + DensityCarrierEquationOfState, + DensityCarrierForm, + DensityCarrierAvailableEquations>; + + struct DensityCarrierState final { + double density; + double entropy; + + [[nodiscard]] constexpr eos::DensityValue value(eos::quantity::Density) const noexcept { + return eos::DensityValue{density}; + } + + [[nodiscard]] constexpr eos::QuantityValue value(Entropy) const noexcept { + return eos::QuantityValue{entropy}; + } + }; +} // namespace + +TEST_CASE( + "Thermodynamic Equation Compilation Is Inferred From Fields And Canonical Problem Blocks", + "[material][thermodynamic_equations][compilation][unit][type_contract]" +) { + STATIC_CHECK(material::ThermodynamicField); + STATIC_CHECK(material::ThermodynamicField); + STATIC_CHECK(material::ThermodynamicField); + STATIC_CHECK(material::ThermodynamicEquationType); + STATIC_CHECK(material::ValidThermodynamicEquationCatalog); + STATIC_CHECK(material::CompiledThermodynamicEquations); + STATIC_CHECK(std::same_as); + STATIC_CHECK(std::same_as); + STATIC_CHECK(GeneralCompilation::Equations::size == 3); + STATIC_CHECK( + std::same_as< + typename GeneralCompilation::StateBindings, + surface::SurfaceStateBindings< + surface::SurfaceStateBinding, + surface::SurfaceStateBinding, + surface::SurfaceStateBinding>> + ); + STATIC_CHECK(std::same_as); +} + +TEST_CASE( + "Thermodynamic Equation Compilation Rejects Incomplete Ambiguous And Missing Physics", + "[material][thermodynamic_equations][compilation][unit][negative]" +) { + STATIC_CHECK_FALSE( + material::ThermodynamicEquationsCompilable< + GeneralEquationOfState, HalfPresentEquationForm, ReorderedAvailableEquations> + ); + STATIC_CHECK_FALSE( + material::ThermodynamicEquationsCompilable< + GeneralEquationOfState, NoThermodynamicEquationForm, ReorderedAvailableEquations> + ); + STATIC_CHECK_FALSE( + material::ThermodynamicEquationsCompilable< + MissingPressureSurfaceRelationEquationOfState, GeneralForm, ReorderedAvailableEquations> + ); + STATIC_CHECK_FALSE( + material::ThermodynamicEquationsCompilable + ); + STATIC_CHECK_FALSE(material::ValidThermodynamicEquationCatalog); + STATIC_CHECK_FALSE(material::ValidThermodynamicEquationCatalog); +} + +TEST_CASE( + "Pressure Surface Carrier Is Selected By The EOS Relation Rather Than A Preferred Field", + "[material][thermodynamic_equations][surface][unit]" +) { + using Formulation = DensityCarrierCompilation::PressureSurfaceFormulation; + using Constraint = surface::CompiledPressureSurfaceConstraintT; + + STATIC_CHECK(material::CompiledThermodynamicEquations); + STATIC_CHECK(std::same_as); + STATIC_CHECK(std::same_as); + STATIC_CHECK(std::same_as); + STATIC_CHECK( + std::same_as< + typename Constraint::SurfaceDependencies::StateFieldTypes, field::TypeList> + ); + + constexpr DensityCarrierEquationOfState equationOfState; + const surface::ConstantPressureSurface condition{eos::PressureValue{0.25}}; + const auto constraint = surface::compilePressureSurfaceConstraint(condition, equationOfState); + + constexpr DensityCarrierState state{.density = 1.6, .entropy = 0.3}; + constexpr DensityCarrierState direction{.density = -0.2, .entropy = 0.4}; + + CHECK(constraint.residual(state) == 0.0); + CHECK(constraint.jacobianAction(state, direction) == -1.0); +} diff --git a/tests/mpi/distributed_execution.cpp b/tests/mpi/distributed_execution.cpp new file mode 100644 index 0000000..8c2b955 --- /dev/null +++ b/tests/mpi/distributed_execution.cpp @@ -0,0 +1,292 @@ +#include +#include +#include +#include +#include +#include + +import mean_field; +import test_helpers; + +namespace { + bool vector_is_finite(const mfem::Vector &vector) { + for (int index = 0; index < vector.Size(); ++index) { + if (!std::isfinite(vector(index))) { + return false; + } + } + return true; + } + + mfem::Vector make_deterministic_vector( + const int size, + const double phase + ) { + mfem::Vector vector(size); + for (int index = 0; index < size; ++index) { + const double coordinate = static_cast(index + 1); + vector(index) = std::sin(phase + 0.017 * coordinate) + 0.25 * std::cos(0.031 * coordinate); + } + return vector; + } + + double global_dot( + const mfem::Vector &left, + const mfem::Vector &right, + const MPI_Comm communicator + ) { + REQUIRE(left.Size() == right.Size()); + const double local = left * right; + double global = 0.0; + REQUIRE(MPI_Allreduce(&local, &global, 1, MPI_DOUBLE, MPI_SUM, communicator) == MPI_SUCCESS); + return global; + } + + double global_norm( + const mfem::Vector &vector, + const MPI_Comm communicator + ) { + return std::sqrt(global_dot(vector, vector, communicator)); + } +} // namespace + +TEST_CASE( + "MPI Runtime Preserves World And Split Communicator Membership", + "[mpi][distributed][unit]" +) { + int rank = 0; + int size = 1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + std::vector ranks(static_cast(size), -1); + MPI_Allgather(&rank, 1, MPI_INT, ranks.data(), 1, MPI_INT, MPI_COMM_WORLD); + + CHECK(size >= 2); + for (int expected = 0; expected < size; ++expected) { + CHECK(ranks[expected] == expected); + } + + MPI_Comm parity_communicator = MPI_COMM_NULL; + MPI_Comm_split(MPI_COMM_WORLD, rank % 2, rank, &parity_communicator); + + int parity_size = 0; + MPI_Comm_size(parity_communicator, &parity_size); + const int expected_parity_size = (size + 1 - rank % 2) / 2; + CHECK(parity_size == expected_parity_size); + + MPI_Comm_free(&parity_communicator); +} + +TEST_CASE( + "MPI FEM Setup Partitions Every Element Exactly Once", + "[mpi][distributed][mesh][integration]" +) { + const mean_field::utils::Args args = test_utils::setup_args(); + const mean_field::fem::FEM f = mean_field::fem::setup_fem(args.mesh_file, args, 0); + + const long long local_elements = f.mesh->GetNE(); + long long global_elements = 0; + long long minimum_elements = 0; + + MPI_Allreduce(&local_elements, &global_elements, 1, MPI_LONG_LONG, MPI_SUM, f.mesh->GetComm()); + MPI_Allreduce(&local_elements, &minimum_elements, 1, MPI_LONG_LONG, MPI_MIN, f.mesh->GetComm()); + + CHECK(global_elements == f.smesh.mesh->GetNE()); + CHECK(minimum_elements > 0); + CHECK(f.logicalReferenceMesh->GetNE() == f.mesh->GetNE()); +} + +TEST_CASE( + "MPI Prepared Gravity Operators Preserve Global Algebraic Identities", + "[mpi][distributed][gravity][operators][unit]" +) { + const auto args = test_utils::setup_args(); + mean_field::fem::FEM f = mean_field::fem::setup_fem(args.mesh_file, args, 0); + + using GeometryContext = mean_field::operators::context::gravity_field::GravityFieldGeometryContext; + GeometryContext geometry_context(f, *f.domainMapperStateless); + + mfem::Vector displacement_true(f.displacementFes->GetTrueVSize()); + displacement_true = 0.0; + const mfem::Vector displacement = geometry_context.GetDisplacementMap().gather(displacement_true); + geometry_context.PreparePrimal(displacement, {0}, {0}); + + const mfem::Operator &mass = geometry_context.GetMassOperator(); + const mfem::Vector first = make_deterministic_vector(mass.Width(), 0.17); + const mfem::Vector second = make_deterministic_vector(mass.Width(), 0.83); + mfem::Vector combination(first); + combination *= 1.7; + combination.Add(-0.4, second); + + mfem::Vector first_action; + mfem::Vector second_action; + mfem::Vector combination_action; + mass.Mult(first, first_action); + mass.Mult(second, second_action); + mass.Mult(combination, combination_action); + + mfem::Vector expected_combination(first_action); + expected_combination *= 1.7; + expected_combination.Add(-0.4, second_action); + mfem::Vector linearity_difference(combination_action); + linearity_difference -= expected_combination; + + const MPI_Comm communicator = f.mesh->GetComm(); + const double symmetry_scale = std::max( + {std::abs(global_dot(first, second_action, communicator)), + std::abs(global_dot(second, first_action, communicator)), std::numeric_limits::epsilon()} + ); + const double symmetry_error = + std::abs(global_dot(first, second_action, communicator) - global_dot(second, first_action, communicator)) / + symmetry_scale; + const double linearity_error = + global_norm(linearity_difference, communicator) / + std::max(global_norm(expected_combination, communicator), std::numeric_limits::epsilon()); + + CHECK(symmetry_error <= 2.0e-12); + CHECK(linearity_error <= 2.0e-12); + + const mfem::Operator &divergence = geometry_context.GetDivergenceOperator(); + const mfem::Operator &transpose_divergence = geometry_context.GetTransposeDivergenceOperator(); + const mfem::Vector flux = make_deterministic_vector(divergence.Width(), 0.41); + const mfem::Vector potential = make_deterministic_vector(divergence.Height(), 0.67); + mfem::Vector divergence_action; + mfem::Vector transpose_action; + divergence.Mult(flux, divergence_action); + transpose_divergence.Mult(potential, transpose_action); + + const double forward_product = global_dot(potential, divergence_action, communicator); + const double transpose_product = global_dot(flux, transpose_action, communicator); + const double adjoint_scale = + std::max({std::abs(forward_product), std::abs(transpose_product), std::numeric_limits::epsilon()}); + const double adjoint_error = std::abs(forward_product - transpose_product) / adjoint_scale; + + CHECK(adjoint_error <= 2.0e-12); +} + +TEST_CASE( + "MPI Coupled Gravity LDU Is Stationary Linear And Does Not Reprepare Geometry", + "[mpi][distributed][gravity][preconditioning][integration]" +) { + const auto args = test_utils::setup_args(); + mean_field::fem::FEM f = mean_field::fem::setup_fem(args.mesh_file, args, 0); + + using GeometryContext = mean_field::operators::context::gravity_field::GravityFieldGeometryContext; + GeometryContext geometryContext(f, *f.domainMapperStateless); + + mfem::Vector displacementTrue(f.displacementFes->GetTrueVSize()); + displacementTrue = 0.0; + const mfem::Vector displacement = geometryContext.GetDisplacementMap().gather(displacementTrue); + geometryContext.PreparePrimal(displacement, {.value = 1}, {.value = 1}); + + namespace backend = mean_field::preconditioning::backend; + namespace preconditioning = mean_field::preconditioning; + const auto block = preconditioning::GravityFieldBlock( + backend::Diagonal{}, backend::HypreBoomerAMG{backend::FixedCycles{.cycles = 1}}, + preconditioning::GravityApproximateLDU{} + ); + auto prepared = preconditioning::prepare(f, geometryContext, block); + + const mfem::Vector first = make_deterministic_vector(prepared.Width(), 0.23); + const mfem::Vector second = make_deterministic_vector(prepared.Width(), 0.79); + mfem::Vector combined(first); + combined *= 1.3; + combined.Add(-0.45, second); + + mfem::Vector firstAction(prepared.Height()); + mfem::Vector secondAction(prepared.Height()); + mfem::Vector combinedAction(prepared.Height()); + mfem::Vector repeatedAction(prepared.Height()); + firstAction = 0.0; + secondAction = 0.0; + combinedAction = 0.0; + repeatedAction = 0.0; + + const std::uint64_t massPreparations = geometryContext.GetMassOperator().GetPreparationCount(); + const std::uint64_t sourcePreparations = geometryContext.GetSourceOperator().GetPreparationCount(); + double *const combinedStorage = combinedAction.GetData(); + + prepared.Mult(first, firstAction); + prepared.Mult(second, secondAction); + prepared.Mult(combined, combinedAction); + prepared.Mult(first, repeatedAction); + + mfem::Vector expectedCombined(firstAction); + expectedCombined *= 1.3; + expectedCombined.Add(-0.45, secondAction); + + const MPI_Comm communicator = f.mesh->GetComm(); + mfem::Vector linearityDifference(combinedAction); + linearityDifference -= expectedCombined; + mfem::Vector determinismDifference(repeatedAction); + determinismDifference -= firstAction; + const double linearityError = + global_norm(linearityDifference, communicator) / + std::max(global_norm(expectedCombined, communicator), std::numeric_limits::epsilon()); + + CHECK(vector_is_finite(combinedAction)); + CHECK(linearityError <= 5.0e-12); + CHECK(global_norm(determinismDifference, communicator) <= 5.0e-14); + CHECK(combinedAction.GetData() == combinedStorage); + CHECK(geometryContext.GetMassOperator().GetPreparationCount() == massPreparations); + CHECK(geometryContext.GetSourceOperator().GetPreparationCount() == sourcePreparations); + CHECK(prepared.GetFactorization().GetStatistics().applications == 4); +} + +TEST_CASE( + "MPI Gravity Analysis And Solve Produce Finite Distributed Fields", + "[mpi][distributed][gravity][integration]" +) { + auto args = test_utils::setup_args(); + mean_field::fem::FEM f = mean_field::fem::setup_fem(args.mesh_file, args, 0); + + *f.displacement = 0.0; + + using DomainSchema = mean_field::utils::domain::CoreEnvelopeVacuumDomainSchema; + mfem::Vector attribute_density(f.smesh.mesh->attributes.Max()); + attribute_density = 0.0; + for (int index = 0; index < f.smesh.mesh->attributes.Size(); ++index) { + const int attribute = f.smesh.mesh->attributes[index]; + if (DomainSchema::template attribute_belongs_to(attribute)) { + attribute_density(attribute - 1) = 1.0; + } + } + + mfem::PWConstCoefficient density_coefficient(attribute_density); + mfem::ParGridFunction density(f.densityFes.get()); + density.ProjectCoefficient(density_coefficient); + mean_field::analysis::conserve_mass(f, density, mean_field::utils::MASS); + + const double integrated_mass = mean_field::analysis::domain_integrate_grid_function( + f, density, mean_field::utils::DOMAINS::STELLAR, mean_field::mapping::COORDINATE_SPACE::PHYSICAL + ); + f.com = mean_field::analysis::get_com(f, density); + f.Q = mean_field::physics::compute_quadrupole_moment_tensor(f, density, f.com); + + const mean_field::physics::GravitySolution solution = mean_field::physics::solve_gravity_field( + f, + mean_field::physics::GravitySolveOptions{ + .relativeTolerance = 1.0e-12, .absoluteTolerance = 1.0e-15, .maximumIterations = 1000 + }, + density, *f.displacement + ); + + mfem::Vector flux_true; + mfem::Vector potential_true; + solution.gradPhi.GetTrueDofs(flux_true); + solution.phi.GetTrueDofs(potential_true); + + const int local_finite = vector_is_finite(flux_true) && vector_is_finite(potential_true) ? 1 : 0; + int globally_finite = 0; + MPI_Allreduce(&local_finite, &globally_finite, 1, MPI_INT, MPI_MIN, f.mesh->GetComm()); + + const double local_norms[2]{flux_true * flux_true, potential_true * potential_true}; + double global_norms[2]{}; + MPI_Allreduce(local_norms, global_norms, 2, MPI_DOUBLE, MPI_SUM, f.mesh->GetComm()); + + CHECK(globally_finite == 1); + CHECK(std::abs(integrated_mass - mean_field::utils::MASS) <= 1.0e-12 * mean_field::utils::MASS); + CHECK(global_norms[0] > std::numeric_limits::min()); + CHECK(global_norms[1] > std::numeric_limits::min()); +} diff --git a/tests/mpi/mpi_test_main.cpp b/tests/mpi/mpi_test_main.cpp new file mode 100644 index 0000000..39515ec --- /dev/null +++ b/tests/mpi/mpi_test_main.cpp @@ -0,0 +1,58 @@ +#include +#include +#include +#include +#include +#include + +int main( + int argc, + char *argv[] +) { + mfem::Mpi::Init(argc, argv); + + const int rank = mfem::Mpi::WorldRank(); + std::vector arguments; + arguments.reserve(static_cast(argc) + 6); + for (int index = 0; index < argc; ++index) { + arguments.emplace_back(argv[index]); + } + + arguments.emplace_back("--order"); + arguments.emplace_back("lex"); + arguments.emplace_back("--rng-seed"); + arguments.emplace_back("184467"); + + if (rank != 0) { + arguments.emplace_back("--out"); + arguments.emplace_back("/dev/null"); + } + + std::vector catch_arguments; + catch_arguments.reserve(arguments.size()); + for (const std::string &argument : arguments) { + catch_arguments.push_back(argument.c_str()); + } + + int local_result = 0; + { + Catch::Session session; + if (const int parse_result = + session.applyCommandLine(static_cast(catch_arguments.size()), catch_arguments.data()); + parse_result != 0) { + local_result = parse_result; + } else { + local_result = session.run(); + } + } + + int global_result = 0; + MPI_Allreduce(&local_result, &global_result, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + + if (rank == 0 && global_result != 0 && local_result == 0) { + std::cerr << "At least one non-root MPI rank reported a test failure.\n"; + } + + mfem::Mpi::Finalize(); + return global_result; +} diff --git a/tests/mpi/profiling.cpp b/tests/mpi/profiling.cpp new file mode 100644 index 0000000..f7bcb21 --- /dev/null +++ b/tests/mpi/profiling.cpp @@ -0,0 +1,66 @@ +#include "profile.h" + +#include +#include +#include +#include +#include + +namespace { + const mean_field::profiling::DistributedStatistics *find_region( + const std::vector &statistics, + const std::string &label + ) { + const auto iterator = + std::ranges::find(statistics, label, &mean_field::profiling::DistributedStatistics::label); + return iterator != statistics.end() ? &*iterator : nullptr; + } +} // namespace + +TEST_CASE( + "MPI Profiling Aggregates Rank-Local Label Sets Without Collective Divergence", + "[mpi][profiling][distributed]" +) { + int rank = 0; + int size = 1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + mean_field::profiling::Registry ®istry = mean_field::profiling::Registry::Get(); + registry.Reset(); + registry.Record("common-region", static_cast(rank + 1)); + registry.AddCount("common-region", static_cast(10 + rank)); + + const std::string local_label = "rank-" + std::to_string(rank) + "-only"; + registry.Record(local_label, 0.125 * static_cast(rank + 1)); + + const auto aggregate = registry.Aggregate(MPI_COMM_WORLD); + const auto *common = find_region(aggregate, "common-region"); + CHECK(common != nullptr); + if (common != nullptr) { + CHECK(common->minimum_samples == 1); + CHECK(common->maximum_samples == 1); + CHECK(common->minimum_work_units == 10); + CHECK(common->maximum_work_units == static_cast(9 + size)); + CHECK(common->global_minimum_seconds == 1.0); + CHECK(common->global_maximum_seconds == static_cast(size)); + } + + for (int owner = 0; owner < size; ++owner) { + const auto *local = find_region(aggregate, "rank-" + std::to_string(owner) + "-only"); + CHECK(local != nullptr); + if (local != nullptr) { + CHECK(local->minimum_samples == 0); + CHECK(local->maximum_samples == 1); + } + } + + std::ostringstream csv; + registry.PrintCsv(MPI_COMM_WORLD, csv); + if (rank == 0) { + CHECK(csv.str().find("common-region") != std::string::npos); + CHECK(csv.str().find("," + std::to_string(size) + "\n") != std::string::npos); + } else { + CHECK(csv.str().empty()); + } +} diff --git a/tests/operators/contexts/gravity_field_context.cpp b/tests/operators/contexts/gravity_field_context.cpp index 4908390..755fb05 100644 --- a/tests/operators/contexts/gravity_field_context.cpp +++ b/tests/operators/contexts/gravity_field_context.cpp @@ -1,4 +1,5 @@ #include +#include #include import mean_field; @@ -50,6 +51,8 @@ TEST_CASE( CHECK(initial_report.DidAnyWork()); const auto &geometry_context = context.GetGeometryContext(); + CHECK(geometry_context.GetMassOperator().HasVariationData()); + CHECK(geometry_context.GetSourceOperator().HasVariationData()); CHECK(geometry_context.GetDivergenceOperator().Width() == f.gravityFluxFes->GetTrueVSize()); CHECK(geometry_context.GetDivergenceOperator().Height() == f.gravityPotentialFes->GetTrueVSize()); CHECK(geometry_context.GetTransposeDivergenceOperator().Width() == f.gravityPotentialFes->GetTrueVSize()); @@ -122,6 +125,51 @@ TEST_CASE( CHECK(context.GetGeometryContext().GetSourceOperator().GetPreparationCount() == 1); } +TEST_CASE( + "Gravity Field Geometry Context Distinguishes Primal And Linearization Preparation", + tags::gravity_context +) { + auto args = test_utils::setup_args(); + fem::FEM f = fem::setup_fem(args.mesh_file, args, 0); + + gravity_context::GravityFieldGeometryContext context(f, *f.domainMapperStateless); + const mfem::Vector displacement = context.GetDisplacementMap().gather(prepared_test::make_displacement(f, 0.25)); + + const gravity_context::GravityFieldGeometryPreparation primal_report = + context.PreparePrimal(displacement, {.value = 0}, {.value = 0}); + + REQUIRE(context.IsPrepared()); + CHECK(primal_report.reconstructed_operators); + CHECK(primal_report.rebuilt_mass_operator); + CHECK(primal_report.rebuilt_source_operator); + CHECK(primal_report.rebuilt_divergence_operator); + CHECK_FALSE(primal_report.refreshed_variation_state); + CHECK_FALSE(context.GetMassOperator().HasVariationData()); + CHECK_FALSE(context.GetSourceOperator().HasVariationData()); + + const std::uint64_t primal_mass_preparations = context.GetMassOperator().GetPreparationCount(); + const std::uint64_t primal_source_preparations = context.GetSourceOperator().GetPreparationCount(); + const gravity_context::GravityFieldGeometryPreparation repeated_primal_report = + context.PreparePrimal(displacement, {.value = 0}, {.value = 0}); + + CHECK_FALSE(repeated_primal_report.DidAnyWork()); + CHECK(context.GetMassOperator().GetPreparationCount() == primal_mass_preparations); + CHECK(context.GetSourceOperator().GetPreparationCount() == primal_source_preparations); + + const gravity_context::GravityFieldGeometryPreparation upgrade_report = + context.Prepare(displacement, {.value = 0}, {.value = 0}); + + CHECK_FALSE(upgrade_report.reconstructed_operators); + CHECK(upgrade_report.rebuilt_mass_operator); + CHECK(upgrade_report.rebuilt_source_operator); + CHECK_FALSE(upgrade_report.rebuilt_divergence_operator); + CHECK(upgrade_report.refreshed_variation_state); + CHECK(context.GetMassOperator().HasVariationData()); + CHECK(context.GetSourceOperator().HasVariationData()); + CHECK(context.GetMassOperator().GetPreparationCount() == primal_mass_preparations + 1); + CHECK(context.GetSourceOperator().GetPreparationCount() == primal_source_preparations + 1); +} + TEST_CASE( "Gravity Field Linearization Context Owns Frozen Base Fields", tags::gravity_context diff --git a/tests/operators/gravity_field.cpp b/tests/operators/gravity_field.cpp index 15c1857..a9ba8d1 100644 --- a/tests/operators/gravity_field.cpp +++ b/tests/operators/gravity_field.cpp @@ -2889,6 +2889,8 @@ TEST_CASE( operators::ReducedGravityFieldPreconditioner reduced_preconditioner(f, reduced_geometry_context); REQUIRE(reduced_geometry_context.IsPrepared()); + REQUIRE_FALSE(reduced_geometry_context.GetMassOperator().HasVariationData()); + REQUIRE_FALSE(reduced_geometry_context.GetSourceOperator().HasVariationData()); REQUIRE(reduced_operator.Width() == layout.residual_offsets().Last()); REQUIRE(reduced_operator.Height() == layout.residual_offsets().Last()); REQUIRE(reduced_operator.Width() == reduced_operator.Height()); diff --git a/tests/operators/stellar_equilibrium_system.cpp b/tests/operators/stellar_equilibrium_system.cpp index 8f5420f..2e91f39 100644 --- a/tests/operators/stellar_equilibrium_system.cpp +++ b/tests/operators/stellar_equilibrium_system.cpp @@ -91,8 +91,10 @@ TEST_CASE( STATIC_CHECK( std::same_as< typename BaseProblem::CompiledSurfaceConstraintType, - surface::CompiledPressureSurfaceConstraintT> + surface::CompiledPressureSurfaceConstraintT< + typename BaseProblem::ThermodynamicEquationsType::PressureSurfaceFormulation, eos::Polytrope>> ); + STATIC_CHECK(material::CompiledThermodynamicEquations); } TEST_CASE( diff --git a/tests/physics/gravity_monopole_accuracy.cpp b/tests/physics/gravity_monopole_accuracy.cpp index 685f428..7a28723 100644 --- a/tests/physics/gravity_monopole_accuracy.cpp +++ b/tests/physics/gravity_monopole_accuracy.cpp @@ -1,3 +1,4 @@ +#include "profile.h" #include #include @@ -716,6 +717,9 @@ namespace { const mean_field::physics::GravitySolution &solution, const mfem::GridFunction &displacement ) { + MEAN_FIELD_PROFILE_SCOPE_WARMUP("gravity virial: energy integration", 0); + + std::uint64_t mapping_evaluations = 0; mean_field::mapping::DomainMapper::Workspace workspace(f.mesh->Dimension()); double local_binding = 0.0; @@ -729,6 +733,13 @@ namespace { const int order = 2 * std::max(f.gravityPotentialFes->GetMaxElementOrder(), f.gravityFluxFes->GetMaxElementOrder()) + 8; std::array local_status_counts{}; + mean_field::mapping::VolumeMappingContext context; + mfem::Vector reference_field(3); + mfem::Vector physical_field(3); + mfem::Array displacement_dofs; + mfem::Array compactification_dofs; + mfem::Vector element_displacement; + mfem::Vector element_compactification; for (int element_id = 0; element_id < f.mesh->GetNE(); ++element_id) { mfem::ElementTransformation *transformation = f.mesh->GetElementTransformation(element_id); @@ -739,17 +750,11 @@ namespace { const mfem::FiniteElement &displacement_element = *f.displacementFes->GetFE(element_id); const mfem::FiniteElement &compactification_element = *f.compactificationFes->GetFE(element_id); - mfem::Array displacement_dofs; - mfem::Array compactification_dofs; - mfem::DofTransformation *displacement_transform = f.displacementFes->GetElementVDofs(element_id, displacement_dofs); mfem::DofTransformation *compactification_transform = f.compactificationFes->GetElementDofs(element_id, compactification_dofs); - mfem::Vector element_displacement; - mfem::Vector element_compactification; - displacement.GetSubVector(displacement_dofs, element_displacement); f.compactificationCoordinate->GetSubVector(compactification_dofs, element_compactification); @@ -778,11 +783,11 @@ namespace { for (int q = 0; q < rule.GetNPoints(); ++q) { const mfem::IntegrationPoint &point = rule.IntPoint(q); - - mean_field::mapping::VolumeMappingContext context; + context.mapping.mapping_determinant = 0.0; const mean_field::mapping::MappingStatus status = f.domainMapperStateless->EvaluateVolume(mapping_data, *transformation, point, workspace, context); + ++mapping_evaluations; const double mapping_determinant = context.mapping.mapping_determinant; if (std::isfinite(mapping_determinant)) { @@ -799,14 +804,6 @@ namespace { continue; } - if (status != mean_field::mapping::MappingStatus::valid) { - ++local_invalid_points; - continue; - } - - mfem::Vector reference_field(3); - mfem::Vector physical_field(3); - solution.gradPhi.GetVectorValue(element_id, point, reference_field); mean_field::mapping::MapHDivFluxToPhysical(context.mapping, reference_field, physical_field); @@ -819,23 +816,33 @@ namespace { } } + MEAN_FIELD_PROFILE_COUNT("gravity virial: energy mapping evaluations", mapping_evaluations); + GravitationalEnergies energies; MPI_Comm communicator = f.densityFes->GetComm(); - MPI_Allreduce(&local_binding, &energies.binding, 1, MPI_DOUBLE, MPI_SUM, communicator); - MPI_Allreduce(&local_virial, &energies.virial, 1, MPI_DOUBLE, MPI_SUM, communicator); - MPI_Allreduce(&local_invalid_points, &energies.invalid_points, 1, MPI_LONG_LONG, MPI_SUM, communicator); + const std::array local_energy{local_binding, local_virial}; + std::array global_energy{}; + MPI_Allreduce(local_energy.data(), global_energy.data(), 2, MPI_DOUBLE, MPI_SUM, communicator); + energies.binding = global_energy[0]; + energies.virial = global_energy[1]; + + std::array local_counts{}; + std::array global_counts{}; + local_counts[0] = local_invalid_points; + std::copy(local_status_counts.begin(), local_status_counts.end(), local_counts.begin() + 1); MPI_Allreduce( - local_status_counts.data(), energies.mapping_status_counts.data(), mapping_status_count, MPI_LONG_LONG, - MPI_SUM, communicator - ); - MPI_Allreduce( - &local_minimum_determinant, &energies.minimum_mapping_determinant, 1, MPI_DOUBLE, MPI_MIN, communicator - ); - MPI_Allreduce( - &local_maximum_determinant, &energies.maximum_mapping_determinant, 1, MPI_DOUBLE, MPI_MAX, communicator + local_counts.data(), global_counts.data(), mapping_status_count + 1, MPI_LONG_LONG, MPI_SUM, communicator ); + energies.invalid_points = global_counts[0]; + std::copy(global_counts.begin() + 1, global_counts.end(), energies.mapping_status_counts.begin()); + + const std::array local_extrema{local_minimum_determinant, -local_maximum_determinant}; + std::array global_extrema{}; + MPI_Allreduce(local_extrema.data(), global_extrema.data(), 2, MPI_DOUBLE, MPI_MIN, communicator); + energies.minimum_mapping_determinant = global_extrema[0]; + energies.maximum_mapping_determinant = -global_extrema[1]; return energies; } @@ -999,6 +1006,8 @@ TEST_CASE( "Gravity Field Virial Consistency Across Volume Preserving Deformation", tags::gravity_consistency_accuracy ) { + MEAN_FIELD_PROFILE_RESET(); + auto args = test_utils::setup_args(); args.p.rtol = 1.0e-13; args.p.max_iters = std::max(args.p.max_iters, 1000); @@ -1054,12 +1063,13 @@ TEST_CASE( mfem::VectorFunctionCoefficient displacement_coefficient(3, displacement_function); mfem::ParGridFunction displacement(f.displacementFes.get()); - displacement.ProjectCoefficient(displacement_coefficient); + MEAN_FIELD_PROFILE_CALL_WARMUP( + "gravity virial: displacement projection", 0, displacement.ProjectCoefficient(displacement_coefficient); + *f.displacement = displacement + ); - *f.displacement = displacement; - - f.com = mean_field::analysis::get_com(f, density); - f.Q = mean_field::physics::compute_quadrupole_moment_tensor(f, density, f.com); + f.com = mean_field::analysis::get_com(f, density); + f.Q = mean_field::physics::compute_quadrupole_moment_tensor(f, density, f.com); const mfem::FiniteElementSpace *nodal_space = f.mesh->GetNodalFESpace(); const mean_field::physics::GravitySolution solution = @@ -1131,6 +1141,8 @@ TEST_CASE( << ", consistency error=" << consistency_errors[index] << '\n'; } + MEAN_FIELD_PROFILE_PRINT(f.mesh->GetComm()); + INFO(report.str()); for (std::size_t index = 1; index < amplitudes.size(); ++index) { diff --git a/tests/preconditioning/backends.cpp b/tests/preconditioning/backends.cpp new file mode 100644 index 0000000..438cda4 --- /dev/null +++ b/tests/preconditioning/backends.cpp @@ -0,0 +1,392 @@ +#include +#include +#include +#include + +#include +#include +#include + +import mean_field; +import test_helpers; + +namespace { + namespace blocks = mean_field::utils::blocks; + namespace preconditioning = mean_field::preconditioning; + namespace backend = mean_field::preconditioning::backend; + + using DiagonalMass = preconditioning::OperatorCharacteristics< + preconditioning::OperatorCategory::mass_like, + preconditioning::OperatorValueStructure::scalar, + preconditioning::OperatorSymmetry::symmetric, + preconditioning::OperatorDefiniteness::positive_definite, + preconditioning::OperatorRepresentation::diagonal, + preconditioning::OperatorDistribution::local, + preconditioning::OperatorFESpace::h1>; + using MatrixFreeHdivMass = preconditioning::OperatorCharacteristics< + preconditioning::OperatorCategory::mass_like, + preconditioning::OperatorValueStructure::vector, + preconditioning::OperatorSymmetry::symmetric, + preconditioning::OperatorDefiniteness::positive_definite, + preconditioning::OperatorRepresentation::matrix_free, + preconditioning::OperatorDistribution::distributed_true_dof, + preconditioning::OperatorFESpace::h_div>; + using DenseBorder = preconditioning::OperatorCharacteristics< + preconditioning::OperatorCategory::dense_border, + preconditioning::OperatorValueStructure::block, + preconditioning::OperatorSymmetry::nonsymmetric, + preconditioning::OperatorDefiniteness::indefinite, + preconditioning::OperatorRepresentation::assembled_dense, + preconditioning::OperatorDistribution::local>; + using ScalarH1Elliptic = preconditioning::OperatorCharacteristics< + preconditioning::OperatorCategory::elliptic_like, + preconditioning::OperatorValueStructure::scalar, + preconditioning::OperatorSymmetry::symmetric, + preconditioning::OperatorDefiniteness::positive_definite, + preconditioning::OperatorRepresentation::assembled_sparse, + preconditioning::OperatorDistribution::distributed_true_dof, + preconditioning::OperatorFESpace::h1>; + using ConstantNullspaceH1Elliptic = preconditioning::OperatorCharacteristics< + preconditioning::OperatorCategory::elliptic_like, + preconditioning::OperatorValueStructure::scalar, + preconditioning::OperatorSymmetry::symmetric, + preconditioning::OperatorDefiniteness::positive_semidefinite, + preconditioning::OperatorRepresentation::assembled_sparse, + preconditioning::OperatorDistribution::distributed_true_dof, + preconditioning::OperatorFESpace::h1, + preconditioning::OperatorNullspace::constant_mode>; + using HdivElliptic = preconditioning::OperatorCharacteristics< + preconditioning::OperatorCategory::elliptic_like, + preconditioning::OperatorValueStructure::vector, + preconditioning::OperatorSymmetry::symmetric, + preconditioning::OperatorDefiniteness::positive_definite, + preconditioning::OperatorRepresentation::assembled_sparse, + preconditioning::OperatorDistribution::distributed_true_dof, + preconditioning::OperatorFESpace::h_div>; + using NonsymmetricH1Elliptic = preconditioning::OperatorCharacteristics< + preconditioning::OperatorCategory::elliptic_like, + preconditioning::OperatorValueStructure::scalar, + preconditioning::OperatorSymmetry::nonsymmetric, + preconditioning::OperatorDefiniteness::indefinite, + preconditioning::OperatorRepresentation::assembled_sparse, + preconditioning::OperatorDistribution::distributed_true_dof, + preconditioning::OperatorFESpace::h1>; + using SuppliedNullspaceH1Elliptic = preconditioning::OperatorCharacteristics< + preconditioning::OperatorCategory::elliptic_like, + preconditioning::OperatorValueStructure::scalar, + preconditioning::OperatorSymmetry::symmetric, + preconditioning::OperatorDefiniteness::positive_semidefinite, + preconditioning::OperatorRepresentation::assembled_sparse, + preconditioning::OperatorDistribution::distributed_true_dof, + preconditioning::OperatorFESpace::h1, + preconditioning::OperatorNullspace::supplied_basis>; + + using FixedAMG = backend::HypreBoomerAMG; + using AdaptiveAMG = backend::HypreBoomerAMG; + + using DiagonalComponent = preconditioning::ComponentDeclaration< + blocks::type_list, + blocks::type_list, + blocks::type_list<>, + DiagonalMass, + backend::Diagonal>; + using UnderdeclaredDiagonalComponent = preconditioning::ComponentDeclaration< + blocks::type_list, + blocks::type_list, + blocks::type_list<>, + DiagonalMass, + backend::Diagonal, + preconditioning::NoPreparationDependencies>; + + class TinyParallelH1Operator final { + public: + TinyParallelH1Operator() + : m_serialMesh(mfem::Mesh::MakeCartesian1D(4)), + m_parallelMesh( + MPI_COMM_WORLD, + m_serialMesh + ), + m_collection( + 1, + 1 + ), + m_space( + &m_parallelMesh, + &m_collection + ), + m_form(&m_space) { + m_form.AddDomainIntegrator(new mfem::DiffusionIntegrator()); + m_form.AddDomainIntegrator(new mfem::MassIntegrator()); + m_form.Assemble(); + m_form.Finalize(); + m_matrix.reset(m_form.ParallelAssemble()); + } + + [[nodiscard]] const mfem::HypreParMatrix &matrix() const { + return *m_matrix; + } + + private: + mfem::Mesh m_serialMesh; + mfem::ParMesh m_parallelMesh; + mfem::H1_FECollection m_collection; + mfem::ParFiniteElementSpace m_space; + mfem::ParBilinearForm m_form; + std::unique_ptr m_matrix; + }; + + class KnownMatrixFreeSPDOperator final : public mfem::Operator { + public: + KnownMatrixFreeSPDOperator() + : mfem::Operator(3), + m_matrix(3) { + m_matrix = 0.0; + m_matrix(0, 0) = 4.0; + m_matrix(0, 1) = 1.0; + m_matrix(1, 0) = 1.0; + m_matrix(1, 1) = 3.0; + m_matrix(1, 2) = 0.5; + m_matrix(2, 1) = 0.5; + m_matrix(2, 2) = 2.0; + } + + void Mult( + const mfem::Vector &input, + mfem::Vector &output + ) const override { + m_matrix.Mult(input, output); + } + + void AssembleDiagonal(mfem::Vector &diagonal) const override { + diagonal.SetSize(Height()); + for (int index = 0; index < Height(); ++index) { + diagonal(index) = m_matrix(index, index); + } + } + + private: + mfem::DenseMatrix m_matrix; + }; + + [[nodiscard]] double relativeResidual( + const mfem::HypreParMatrix &matrix, + const mfem::Vector &rightHandSide, + const mfem::Vector &action + ) { + mfem::Vector residual(rightHandSide.Size()); + matrix.Mult(action, residual); + residual -= rightHandSide; + return gravity_prepared_test_utils::global_norm(residual, matrix.GetComm()) / + gravity_prepared_test_utils::global_norm(rightHandSide, matrix.GetComm()); + } +} // namespace + +TEST_CASE( + "Preconditioning Backends Expose Complete Compile-Time Capabilities", + tags::preconditioning_backend_unit +) { + STATIC_CHECK(backend::Compatible); + STATIC_CHECK(backend::Compatible); + STATIC_CHECK(backend::Compatible); + STATIC_CHECK(backend::Compatible); + STATIC_CHECK(backend::Compatible); + STATIC_CHECK(backend::Compatible); + STATIC_CHECK_FALSE(backend::Compatible); + STATIC_CHECK_FALSE(backend::Compatible); + STATIC_CHECK_FALSE(backend::Compatible); + + STATIC_CHECK(backend::ArnoldiAdmissible); + STATIC_CHECK(backend::ArnoldiAdmissible); + STATIC_CHECK_FALSE(backend::ArnoldiAdmissible); + STATIC_CHECK(backend::requiresAssembledSparseSurrogate); + STATIC_CHECK_FALSE(backend::requiresAssembledSparseSurrogate); + STATIC_CHECK(backend::Traits::supportsSerialExecution); + STATIC_CHECK(backend::Traits::supportsDistributedExecution); + STATIC_CHECK(backend::Traits::supportsSerialExecution); + STATIC_CHECK_FALSE(backend::Traits::supportsDistributedExecution); + STATIC_CHECK_FALSE(backend::Traits::supportsSerialExecution); + STATIC_CHECK(backend::Traits::supportsDistributedExecution); + + STATIC_CHECK(preconditioning::PreconditionerComponent); + STATIC_CHECK( + DiagonalComponent::PreparationDependencies::contains(preconditioning::PreparationDependency::linearization) + ); + STATIC_CHECK_FALSE(preconditioning::PreconditionerComponent); +} + +TEST_CASE( + "Diagonal Backend Exactly Inverts A Known Diagonal Operator", + tags::preconditioning_backend_unit +) { + mfem::Vector diagonal(3); + diagonal(0) = 2.0; + diagonal(1) = -4.0; + diagonal(2) = 0.5; + + auto prepared = backend::prepare(backend::Diagonal{}, diagonal); + mfem::Vector rightHandSide(3); + rightHandSide(0) = 4.0; + rightHandSide(1) = 8.0; + rightHandSide(2) = -1.0; + mfem::Vector action(3); + prepared.Mult(rightHandSide, action); + + CHECK(action(0) == Catch::Approx(2.0)); + CHECK(action(1) == Catch::Approx(-2.0)); + CHECK(action(2) == Catch::Approx(-2.0)); + CHECK(prepared.GetStatistics().setups == 1); + CHECK(prepared.GetStatistics().applications == 1); + CHECK(prepared.GetStatistics().innerIterations == 0); + + diagonal(1) = 0.0; + CHECK_THROWS_AS(prepared.Refresh(diagonal), std::invalid_argument); +} + +TEST_CASE( + "Dense Direct Backend Exactly Solves And Refreshes A Known Border", + tags::preconditioning_backend_unit +) { + mfem::DenseMatrix matrix(2); + matrix(0, 0) = 4.0; + matrix(0, 1) = 1.0; + matrix(1, 0) = 2.0; + matrix(1, 1) = 3.0; + + auto prepared = backend::prepare(backend::DenseDirect{}, matrix); + mfem::Vector rightHandSide(2); + rightHandSide(0) = 7.0; + rightHandSide(1) = 1.0; + mfem::Vector action(2); + prepared.Mult(rightHandSide, action); + + CHECK(action(0) == Catch::Approx(2.0).margin(1.0e-14)); + CHECK(action(1) == Catch::Approx(-1.0).margin(1.0e-14)); + + matrix = 0.0; + matrix(0, 0) = 2.0; + matrix(1, 1) = 4.0; + prepared.Refresh(matrix); + prepared.Mult(rightHandSide, action); + + CHECK(action(0) == Catch::Approx(3.5).margin(1.0e-14)); + CHECK(action(1) == Catch::Approx(0.25).margin(1.0e-14)); + CHECK(prepared.GetStatistics().setups == 2); + CHECK(prepared.GetStatistics().applications == 2); +} + +TEST_CASE( + "Matrix-Free Chebyshev Backend Is A Fixed Linear Positive Approximate Inverse", + tags::preconditioning_backend_unit +) { + const KnownMatrixFreeSPDOperator operation; + auto prepared = + backend::prepare(backend::MatrixFreeChebyshev{.order = 3, .powerIterations = 20}, operation, MPI_COMM_WORLD); + + mfem::Vector first(3); + first(0) = 1.0; + first(1) = -2.0; + first(2) = 0.25; + mfem::Vector second(3); + second(0) = -0.5; + second(1) = 0.75; + second(2) = 3.0; + mfem::Vector combination(first); + combination *= 1.7; + combination.Add(-0.4, second); + + mfem::Vector firstAction(3); + mfem::Vector secondAction(3); + mfem::Vector combinationAction(3); + prepared.Mult(first, firstAction); + prepared.Mult(second, secondAction); + prepared.Mult(combination, combinationAction); + + mfem::Vector expected(firstAction); + expected *= 1.7; + expected.Add(-0.4, secondAction); + mfem::Vector linearityError(combinationAction); + linearityError -= expected; + CHECK(linearityError.Norml2() <= 2.0e-12 * std::max(1.0, expected.Norml2())); + CHECK((first * firstAction) > 0.0); + CHECK(prepared.GetStatistics().setups == 1); + CHECK(prepared.GetStatistics().applications == 3); + CHECK(prepared.GetStatistics().innerIterations == 9); + CHECK(prepared.GetStatistics().lastInnerIterations == 3); + CHECK_THROWS_AS( + backend::prepare(backend::MatrixFreeChebyshev{.order = 0}, operation, MPI_COMM_WORLD), std::invalid_argument + ); + CHECK_THROWS_AS( + backend::prepare(backend::MatrixFreeChebyshev{.order = 6}, operation, MPI_COMM_WORLD), std::invalid_argument + ); + CHECK_THROWS_AS( + backend::prepare(backend::MatrixFreeChebyshev{.powerTolerance = 1.0}, operation, MPI_COMM_WORLD), + std::invalid_argument + ); + CHECK_THROWS_AS( + backend::prepare(backend::MatrixFreeChebyshev{.powerSeed = 0}, operation, MPI_COMM_WORLD), std::invalid_argument + ); +} + +TEST_CASE( + "BoomerAMG Fixed Cycles Are Linear While Adaptive Application Meets Its Tolerance", + tags::preconditioning_backend_unit +) { + TinyParallelH1Operator problem; + const mfem::HypreParMatrix &matrix = problem.matrix(); + + mfem::Vector exact(matrix.Width()); + exact = 1.0; + mfem::Vector rightHandSide(matrix.Height()); + matrix.Mult(exact, rightHandSide); + + auto fixed = backend::prepare(FixedAMG{backend::FixedCycles{.cycles = 2}}, matrix); + mfem::Vector fixedAction(matrix.Width()); + fixedAction = 0.0; + fixed.Mult(rightHandSide, fixedAction); + + CHECK(relativeResidual(matrix, rightHandSide, fixedAction) < 1.0); + + mfem::Vector secondExact(matrix.Width()); + for (int index = 0; index < secondExact.Size(); ++index) { + secondExact(index) = 0.25 + static_cast(index); + } + mfem::Vector secondRightHandSide(matrix.Height()); + matrix.Mult(secondExact, secondRightHandSide); + mfem::Vector secondAction(matrix.Width()); + secondAction = 0.0; + fixed.Mult(secondRightHandSide, secondAction); + + mfem::Vector combinedRightHandSide(rightHandSide); + combinedRightHandSide *= 0.7; + combinedRightHandSide.Add(-0.2, secondRightHandSide); + mfem::Vector combinedAction(matrix.Width()); + combinedAction = 0.0; + fixed.Mult(combinedRightHandSide, combinedAction); + + mfem::Vector expectedCombinedAction(fixedAction); + expectedCombinedAction *= 0.7; + expectedCombinedAction.Add(-0.2, secondAction); + combinedAction -= expectedCombinedAction; + CHECK( + gravity_prepared_test_utils::global_norm(combinedAction, matrix.GetComm()) < + 1.0e-11 * gravity_prepared_test_utils::global_norm(expectedCombinedAction, matrix.GetComm()) + ); + + CHECK(fixed.GetStatistics().setups == 1); + CHECK(fixed.GetStatistics().applications == 3); + CHECK(fixed.GetStatistics().lastInnerIterations >= 1); + CHECK(fixed.GetStatistics().lastInnerIterations <= 2); + + auto adaptive = backend::prepare( + AdaptiveAMG{backend::SolveToTolerance{.relativeTolerance = 1.0e-10, .maximumCycles = 50}}, matrix + ); + mfem::Vector adaptiveAction(matrix.Width()); + adaptiveAction = 0.0; + adaptive.Mult(rightHandSide, adaptiveAction); + + CHECK(relativeResidual(matrix, rightHandSide, adaptiveAction) < 1.0e-8); + CHECK(adaptive.GetStatistics().setups == 1); + CHECK(adaptive.GetStatistics().applications == 1); + CHECK(adaptive.GetStatistics().lastInnerIterations >= 1); + CHECK(adaptive.GetStatistics().lastInnerIterations <= 50); +} diff --git a/tests/preconditioning/equilibrium_coordinates.cpp b/tests/preconditioning/equilibrium_coordinates.cpp new file mode 100644 index 0000000..1fe5e5e --- /dev/null +++ b/tests/preconditioning/equilibrium_coordinates.cpp @@ -0,0 +1,226 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +import mean_field; +import test_helpers; + +namespace { + namespace blocks = mean_field::utils::blocks; + namespace preconditioning = mean_field::preconditioning; + + using Form = blocks::central_density_bordered_stellar_equilibrium_form; + using GroupedComponent = preconditioning::ComponentDeclaration< + blocks::type_list< + blocks::density::mass::value, + blocks::surface_deformation::parameters::value, + blocks::enthalpy::specific::value, + blocks::gravity::gradient::value, + blocks::gravity::poisson::value, + blocks::fixed_total_mass::mass_normalization::value, + blocks::fixed_central_density::central_value::value>, + blocks::type_list< + blocks::density::mass::residual, + blocks::surface_deformation::shape_equilibrium::residual, + blocks::enthalpy::specific::residual, + blocks::gravity::gradient::residual, + blocks::gravity::poisson::residual, + blocks::fixed_total_mass::mass_normalization::residual, + blocks::fixed_central_density::central_value::residual>, + blocks::type_list<>, + preconditioning::IdentityOperatorCharacteristics, + preconditioning::backend::Identity>; + using IncompleteComponent = preconditioning::ComponentDeclaration< + blocks::type_list< + blocks::density::mass::value, + blocks::surface_deformation::parameters::value, + blocks::enthalpy::specific::value, + blocks::gravity::gradient::value, + blocks::gravity::poisson::value, + blocks::fixed_total_mass::mass_normalization::value>, + blocks::type_list< + blocks::density::mass::residual, + blocks::surface_deformation::shape_equilibrium::residual, + blocks::enthalpy::specific::residual, + blocks::gravity::gradient::residual, + blocks::gravity::poisson::residual, + blocks::fixed_total_mass::mass_normalization::residual>, + blocks::type_list<>, + preconditioning::IdentityOperatorCharacteristics, + preconditioning::backend::Identity>; + + [[nodiscard]] blocks::form_layout makeUnevenLayout() { + return { + std::array{2, 3, 4, 5, 6, 1, 1}, + std::array{4, 5, 2, 3, 6, 1, 1} + }; + } + + [[nodiscard]] double relativeError( + const mfem::Vector &left, + const mfem::Vector &right + ) { + mfem::Vector difference(left); + difference -= right; + return difference.Norml2() / std::max({1.0, left.Norml2(), right.Norml2()}); + } + + [[nodiscard]] mean_field::operators::StellarEquilibriumDependencies makeDependencies() { + return { + .discretization = {.identity = 9301, .revision = 1}, + .density = {.identity = 9303, .revision = 1}, + .surfaceDeformation = {.identity = 9307, .revision = 1}, + .gravityGradient = {.identity = 9311, .revision = 1}, + .gravityPotential = {.identity = 9317, .revision = 1}, + .enthalpy = {.identity = 9323, .revision = 1}, + .bernoulliConstant = {.identity = 9329, .revision = 1}, + .rotation = {.identity = 9331, .revision = 1}, + .targetMass = {.identity = 9337, .revision = 1} + }; + } + + [[nodiscard]] mean_field::physics::RigidRotation zeroRotation() { + mfem::Vector angularVelocity(3); + mfem::Vector center(3); + angularVelocity = 0.0; + center = 0.0; + return {angularVelocity, center}; + } +} // namespace + +TEST_CASE( + "Typed Equilibrium Coordinate Maps Preserve Every Uneven Block Without Scaling", + "[preconditioning][equilibrium_coordinates][unit]" +) { + STATIC_CHECK(preconditioning::EquilibriumCoordinateComponentFor); + STATIC_CHECK_FALSE(preconditioning::EquilibriumCoordinateComponentFor); + + const auto layout = makeUnevenLayout(); + preconditioning::EquilibriumPreconditionerCoordinateMap coordinates(layout); + + REQUIRE(coordinates.EquilibriumStateSize() == 22); + REQUIRE(coordinates.EquilibriumResidualSize() == 22); + REQUIRE(coordinates.PreconditionerCorrectionSize() == 22); + REQUIRE(coordinates.PreconditionerResidualSize() == 22); + + const auto &correctionRanges = coordinates.GetCorrectionRanges(); + CHECK(correctionRanges[0] == (preconditioning::EquilibriumCoordinateRange{0, 0, 2})); + CHECK(correctionRanges[1] == (preconditioning::EquilibriumCoordinateRange{2, 2, 3})); + CHECK(correctionRanges[2] == (preconditioning::EquilibriumCoordinateRange{14, 5, 6})); + CHECK(correctionRanges[3] == (preconditioning::EquilibriumCoordinateRange{5, 11, 4})); + CHECK(correctionRanges[4] == (preconditioning::EquilibriumCoordinateRange{9, 15, 5})); + CHECK(correctionRanges[5] == (preconditioning::EquilibriumCoordinateRange{20, 20, 1})); + CHECK(correctionRanges[6] == (preconditioning::EquilibriumCoordinateRange{21, 21, 1})); + + const auto &residualRanges = coordinates.GetResidualRanges(); + CHECK(residualRanges[0] == (preconditioning::EquilibriumCoordinateRange{9, 0, 2})); + CHECK(residualRanges[1] == (preconditioning::EquilibriumCoordinateRange{11, 2, 3})); + CHECK(residualRanges[2] == (preconditioning::EquilibriumCoordinateRange{14, 5, 6})); + CHECK(residualRanges[3] == (preconditioning::EquilibriumCoordinateRange{0, 11, 4})); + CHECK(residualRanges[4] == (preconditioning::EquilibriumCoordinateRange{4, 15, 5})); + CHECK(residualRanges[5] == (preconditioning::EquilibriumCoordinateRange{20, 20, 1})); + CHECK(residualRanges[6] == (preconditioning::EquilibriumCoordinateRange{21, 21, 1})); + + mfem::Vector equilibriumCorrection(22); + mfem::Vector equilibriumResidual(22); + for (int index = 0; index < 22; ++index) { + equilibriumCorrection(index) = 100.0 + static_cast(index); + equilibriumResidual(index) = -200.0 - static_cast(index); + } + + mfem::Vector groupedCorrection(22); + mfem::Vector groupedResidual(22); + const double *const groupedCorrectionStorage = groupedCorrection.GetData(); + const double *const groupedResidualStorage = groupedResidual.GetData(); + coordinates.PackCorrection(equilibriumCorrection, groupedCorrection); + coordinates.PackResidual(equilibriumResidual, groupedResidual); + CHECK(groupedCorrection.GetData() == groupedCorrectionStorage); + CHECK(groupedResidual.GetData() == groupedResidualStorage); + CHECK(groupedCorrection(5) == equilibriumCorrection(14)); + CHECK(groupedCorrection(11) == equilibriumCorrection(5)); + CHECK(groupedResidual(0) == equilibriumResidual(9)); + CHECK(groupedResidual(11) == equilibriumResidual(0)); + + mfem::Vector recoveredCorrection(22); + mfem::Vector recoveredResidual(22); + coordinates.UnpackCorrection(groupedCorrection, recoveredCorrection); + coordinates.UnpackResidual(groupedResidual, recoveredResidual); + CHECK(relativeError(recoveredCorrection, equilibriumCorrection) == 0.0); + CHECK(relativeError(recoveredResidual, equilibriumResidual) == 0.0); + + const auto &statistics = coordinates.GetStatistics(); + CHECK(statistics.correctionPacks == 1); + CHECK(statistics.correctionUnpacks == 1); + CHECK(statistics.residualPacks == 1); + CHECK(statistics.residualUnpacks == 1); + + mfem::Vector wrongSize(21); + CHECK_THROWS_AS(coordinates.PackResidual(wrongSize, groupedResidual), std::invalid_argument); + CHECK_THROWS_AS(coordinates.UnpackCorrection(wrongSize, recoveredCorrection), std::invalid_argument); +} + +TEST_CASE( + "Prepared Stellar Preconditioning Matches An Explicit Canonical Coordinate Transformation", + "[preconditioning][equilibrium_coordinates][integration]" +) { + using namespace mean_field; + + const utils::Args arguments = test_utils::setup_args(); + fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0); + REQUIRE(finiteElements.okay()); + + constexpr double radius = utils::RADIUS; + constexpr double mass = utils::MASS; + const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v; + const double centralDensity = std::numbers::pi_v * mass / (4.0 * radius * radius * radius); + auto model = model::StellarModel( + eos::Polytrope({.n = 1.0, .K = polytropicConstant}), + surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), + integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}), + constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}}) + ); + auto problem = equilibrium::discretize(model, finiteElements); + auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 512})); + problem.Prepare(projected.values, makeDependencies(), zeroRotation()); + + auto component = preconditioning::makePreconditioner(problem); + auto prepared = preconditioning::prepare(problem, component); + prepared.SetOperator(problem.GetLinearizationOperator()); + + mfem::Vector equilibriumResidual(problem.EquationSize()); + for (int index = 0; index < equilibriumResidual.Size(); ++index) { + equilibriumResidual(index) = 0.25 * std::cos(0.19 * static_cast(index + 1)); + } + + mfem::Vector groupedResidual(problem.EquationSize()); + mfem::Vector groupedCorrection(problem.StateSize()); + mfem::Vector expected(problem.StateSize()); + prepared.GetCoordinateMap().PackResidual(equilibriumResidual, groupedResidual); + prepared.GetGroupedPreconditioner().Mult(groupedResidual, groupedCorrection); + prepared.GetCoordinateMap().UnpackCorrection(groupedCorrection, expected); + + mfem::Vector actual(problem.StateSize()); + const double *const actionStorage = actual.GetData(); + prepared.Mult(equilibriumResidual, actual); + CHECK(actual.GetData() == actionStorage); + CHECK(relativeError(actual, expected) <= 2.0e-12); + + const auto &statistics = prepared.GetStatistics(); + CHECK(statistics.applications == 1); + CHECK(statistics.residualCoordinateMappings == 1); + CHECK(statistics.correctionCoordinateMappings == 1); + CHECK(prepared.GetCoordinateMap().GetStatistics().residualPacks == 2); + CHECK(prepared.GetCoordinateMap().GetStatistics().correctionUnpacks == 2); + + const auto unchanged = prepared.Refresh(); + CHECK_FALSE(unchanged.DidAnyWork()); + CHECK(prepared.IsCurrent()); +} diff --git a/tests/preconditioning/gravity_field.cpp b/tests/preconditioning/gravity_field.cpp new file mode 100644 index 0000000..53a0172 --- /dev/null +++ b/tests/preconditioning/gravity_field.cpp @@ -0,0 +1,393 @@ +#include +#include +#include +#include +#include + +#include +#include +#include + +import mean_field; +import test_helpers; + +namespace { + namespace backend = mean_field::preconditioning::backend; + namespace blocks = mean_field::utils::blocks; + namespace gravity_context = mean_field::operators::context::gravity_field; + namespace preconditioning = mean_field::preconditioning; + + using FixedAMG = backend::HypreBoomerAMG; + using AdaptiveAMG = backend::HypreBoomerAMG; + using FixedGravityLDU = + preconditioning::GravityFieldBlock; + using ChebyshevGravityLDU = preconditioning:: + GravityFieldBlock; + using AdaptiveGravityLDU = + preconditioning::GravityFieldBlock; + + using DensityIdentity = + preconditioning::IdentityBlock; + using SurfaceIdentity = preconditioning::IdentityBlock< + blocks::surface_deformation::parameters::value, + blocks::surface_deformation::shape_equilibrium::residual>; + using EnthalpyIdentity = + preconditioning::IdentityBlock; + using MassIdentity = preconditioning::IdentityBlock< + blocks::fixed_total_mass::mass_normalization::value, + blocks::fixed_total_mass::mass_normalization::residual>; + using FixedGravityPlan = preconditioning:: + PreconditionerPlan; + using AdaptiveGravityPlan = preconditioning:: + PreconditionerPlan; + + template + mfem::Vector applyKnownFactorization( + Policy policy, + const mfem::Vector &rightHandSide, + preconditioning::GravityFactorizationStatistics *statistics = nullptr + ) { + mfem::Vector massDiagonal(2); + massDiagonal = 1.0; + auto massInverse = backend::prepare(backend::Diagonal{}, massDiagonal); + + mfem::DenseMatrix schurMatrix(1); + schurMatrix(0, 0) = 5.0; + auto schurInverse = backend::prepare(backend::DenseDirect{}, schurMatrix); + + mfem::DenseMatrix divergence(1, 2); + divergence(0, 0) = 2.0; + divergence(0, 1) = -1.0; + + preconditioning::GravityFactorizationOperator factorization( + policy, massInverse, schurInverse, divergence + ); + mfem::Vector action(factorization.Height()); + action = std::numeric_limits::quiet_NaN(); + factorization.Mult(rightHandSide, action); + if (statistics != nullptr) { + *statistics = factorization.GetStatistics(); + } + return action; + } + + void checkVector( + const mfem::Vector &computed, + const std::array< + double, + 3> &expected + ) { + REQUIRE(computed.Size() == static_cast(expected.size())); + for (int index = 0; index < computed.Size(); ++index) { + CHECK(computed(index) == Catch::Approx(expected[static_cast(index)]).margin(2.0e-14)); + } + } + + template void checkExactDenseRecovery(Policy policy) { + mfem::DenseMatrix mass(2); + mass(0, 0) = 2.0; + mass(0, 1) = 0.5; + mass(1, 0) = 0.5; + mass(1, 1) = 1.5; + auto massInverse = backend::prepare(backend::DenseDirect{}, mass); + + mfem::DenseMatrix divergence(1, 2); + divergence(0, 0) = 1.0; + divergence(0, 1) = -2.0; + + mfem::Vector divergenceTranspose(2); + divergenceTranspose(0) = 1.0; + divergenceTranspose(1) = -2.0; + mfem::Vector massInverseDivergenceTranspose(2); + massInverse.Mult(divergenceTranspose, massInverseDivergenceTranspose); + + mfem::DenseMatrix schur(1); + schur(0, 0) = divergenceTranspose * massInverseDivergenceTranspose; + auto schurInverse = backend::prepare(backend::DenseDirect{}, schur); + + preconditioning::GravityFactorizationOperator factorization( + policy, massInverse, schurInverse, divergence + ); + + mfem::Vector exact(3); + exact(0) = 0.7; + exact(1) = -1.2; + exact(2) = 0.4; + + mfem::Vector rightHandSide(3); + mfem::Vector exactGradient(exact.GetData(), 2); + mfem::Vector gradientRightHandSide(rightHandSide.GetData(), 2); + mass.Mult(exactGradient, gradientRightHandSide); + gradientRightHandSide(0) += divergence(0, 0) * exact(2); + gradientRightHandSide(1) += divergence(0, 1) * exact(2); + rightHandSide(2) = divergence(0, 0) * exact(0) + divergence(0, 1) * exact(1); + + mfem::Vector action(3); + action = 0.0; + const double *const actionStorage = action.GetData(); + factorization.Mult(rightHandSide, action); + + CHECK(action.GetData() == actionStorage); + for (int index = 0; index < action.Size(); ++index) { + CHECK(action(index) == Catch::Approx(exact(index)).margin(2.0e-13)); + } + + mfem::Vector repeated(3); + repeated = 0.0; + factorization.Mult(rightHandSide, repeated); + for (int index = 0; index < repeated.Size(); ++index) { + CHECK(repeated(index) == action(index)); + } + } + + struct PreparedGeometry final { + mean_field::fem::FEM finiteElements; + gravity_context::GravityFieldGeometryContext context; + + explicit PreparedGeometry(const mean_field::utils::Args &arguments) + : finiteElements( + mean_field::fem::setup_fem( + arguments.mesh_file, + arguments, + 0 + ) + ), + context( + finiteElements, + *finiteElements.domainMapperStateless + ) { + mfem::Vector displacementTrue(finiteElements.displacementFes->GetTrueVSize()); + displacementTrue = 0.0; + const mfem::Vector displacement = context.GetDisplacementMap().gather(displacementTrue); + context.PreparePrimal(displacement, {.value = 1}, {.value = 1}); + } + }; +} // namespace + +TEST_CASE( + "Gravity Field Blocks Expose Complete Compile-Time Ownership And Backend Contracts", + tags::preconditioning_gravity_unit +) { + using Form = blocks::surface_deformed_stellar_equilibrium_form; + using JacobianForm = blocks::surface_deformed_stellar_equilibrium_jacobian_form; + + STATIC_CHECK(preconditioning::PreconditionerComponent); + STATIC_CHECK(preconditioning::PreconditionerComponent); + STATIC_CHECK(preconditioning::PreconditionerComponent); + STATIC_CHECK(preconditioning::backend::ArnoldiAdmissible); + STATIC_CHECK(preconditioning::backend::ArnoldiAdmissible); + STATIC_CHECK_FALSE(preconditioning::backend::ArnoldiAdmissible); + STATIC_CHECK(preconditioning::CompletePreconditionerFor); + STATIC_CHECK(preconditioning::CompatiblePreconditionerFor); + STATIC_CHECK(preconditioning::StationaryLinearPreconditionerPlan); + STATIC_CHECK(preconditioning::CompletePreconditionerFor); + STATIC_CHECK(preconditioning::CompatiblePreconditionerFor); + STATIC_CHECK_FALSE(preconditioning::StationaryLinearPreconditionerPlan); + STATIC_CHECK(FixedGravityLDU::RequiredCouplings::size == 3); +} + +TEST_CASE( + "Gravity Factorization Policies Preserve Their Signed Block Algebra", + tags::preconditioning_gravity_unit +) { + mfem::Vector rightHandSide(3); + rightHandSide(0) = 3.0; + rightHandSide(1) = 4.0; + rightHandSide(2) = 7.0; + + checkVector(applyKnownFactorization(preconditioning::GravityBlockDiagonal{}, rightHandSide), {3.0, 4.0, 1.4}); + checkVector(applyKnownFactorization(preconditioning::GravityLowerTriangular{}, rightHandSide), {3.0, 4.0, -1.0}); + checkVector(applyKnownFactorization(preconditioning::GravityUpperTriangular{}, rightHandSide), {5.8, 2.6, -1.4}); + + preconditioning::GravityFactorizationStatistics statistics; + checkVector( + applyKnownFactorization(preconditioning::GravityApproximateLDU{}, rightHandSide, &statistics), {5.0, 3.0, -1.0} + ); + CHECK(statistics.applications == 1); + CHECK(statistics.massInverseApplications == 2); + CHECK(statistics.potentialSchurApplications == 1); + CHECK(statistics.divergenceApplications == 1); + CHECK(statistics.transposeDivergenceApplications == 1); +} + +TEST_CASE( + "Exact Gravity LDU Recovers A Dense Coupled Saddle-Point System", + tags::preconditioning_gravity_unit +) { + checkExactDenseRecovery(preconditioning::GravityApproximateLDU{}); +} + +TEST_CASE( + "Assembled Gravity Divergence Matches The Prepared Matrix-Free Couplings", + tags::preconditioning_gravity_integration +) { + const auto arguments = test_utils::setup_args(); + PreparedGeometry geometry(arguments); + + const auto assembledDivergence = preconditioning::assembleGravityDivergenceSurrogate(geometry.finiteElements); + const mfem::Operator &preparedDivergence = geometry.context.GetDivergenceOperator(); + + const mfem::Vector flux = gravity_prepared_test_utils::make_deterministic_vector( + geometry.finiteElements.gravityFluxFes->GetTrueVSize(), 0.31 + ); + mfem::Vector assembledForward(assembledDivergence->Height()); + mfem::Vector preparedForward(preparedDivergence.Height()); + assembledDivergence->Mult(flux, assembledForward); + preparedDivergence.Mult(flux, preparedForward); + + const mfem::Vector potential = gravity_prepared_test_utils::make_deterministic_vector( + geometry.finiteElements.gravityPotentialFes->GetTrueVSize(), 0.73 + ); + mfem::Vector assembledTranspose(assembledDivergence->Width()); + mfem::Vector preparedTranspose(preparedDivergence.Width()); + assembledDivergence->MultTranspose(potential, assembledTranspose); + preparedDivergence.MultTranspose(potential, preparedTranspose); + + const MPI_Comm communicator = geometry.finiteElements.mesh->GetComm(); + CHECK(gravity_prepared_test_utils::relative_error(assembledForward, preparedForward, communicator) <= 2.0e-12); + CHECK(gravity_prepared_test_utils::relative_error(assembledTranspose, preparedTranspose, communicator) <= 2.0e-12); + + const auto &gradientMap = geometry.context.GetMassOperator().GetFluxMap(); + const auto &potentialMap = geometry.context.GetSourceOperator().GetPotentialMap(); + preconditioning::ReducedGravityDivergenceOperator reducedDivergence(preparedDivergence, gradientMap, potentialMap); + const mfem::Vector reducedFlux = + gravity_prepared_test_utils::make_deterministic_vector(gradientMap.reduced_size(), 0.47); + mfem::Vector reducedAction(reducedDivergence.Height()); + reducedDivergence.Mult(reducedFlux, reducedAction); + + const mfem::Vector trueFlux = gradientMap.scatter(reducedFlux); + mfem::Vector trueAction(potentialMap.full_size()); + preparedDivergence.Mult(trueFlux, trueAction); + const mfem::Vector expectedReducedAction = potentialMap.gather(trueAction); + CHECK(gravity_prepared_test_utils::relative_error(reducedAction, expectedReducedAction, communicator) <= 2.0e-14); +} + +TEST_CASE( + "Prepared Gravity Block Diagonal Is Legacy Equivalent And Allocation Stable", + tags::preconditioning_gravity_integration +) { + const auto arguments = test_utils::setup_args(); + PreparedGeometry geometry(arguments); + + mean_field::operators::ReducedGravityFieldPreconditioner legacy(geometry.finiteElements, geometry.context); + const auto block = preconditioning::GravityFieldBlock( + backend::Diagonal{}, FixedAMG{backend::FixedCycles{.cycles = 1}}, preconditioning::GravityBlockDiagonal{} + ); + auto prepared = preconditioning::prepare(geometry.finiteElements, geometry.context, block); + + const mfem::Vector rightHandSide = gravity_prepared_test_utils::make_deterministic_vector(prepared.Width(), 0.59); + mfem::Vector legacyAction(prepared.Height()); + mfem::Vector preparedAction(prepared.Height()); + legacyAction = 0.0; + preparedAction = 0.0; + double *const preparedStorage = preparedAction.GetData(); + + const std::uint64_t massPreparations = geometry.context.GetMassOperator().GetPreparationCount(); + const std::uint64_t sourcePreparations = geometry.context.GetSourceOperator().GetPreparationCount(); + legacy.Mult(rightHandSide, legacyAction); + prepared.Mult(rightHandSide, preparedAction); + + CHECK(preparedAction.GetData() == preparedStorage); + CHECK(geometry.context.GetMassOperator().GetPreparationCount() == massPreparations); + CHECK(geometry.context.GetSourceOperator().GetPreparationCount() == sourcePreparations); + CHECK( + gravity_prepared_test_utils::relative_error( + preparedAction, legacyAction, geometry.finiteElements.mesh->GetComm() + ) <= 2.0e-12 + ); +} + +TEST_CASE( + "Prepared Gravity Blocks Refresh Explicitly Without Repreparing Geometry", + tags::preconditioning_gravity_integration +) { + const auto arguments = test_utils::setup_args(); + PreparedGeometry geometry(arguments); + + const auto block = preconditioning::GravityFieldBlock( + backend::Diagonal{}, FixedAMG{backend::FixedCycles{.cycles = 1}}, preconditioning::GravityApproximateLDU{} + ); + auto prepared = preconditioning::prepare(geometry.finiteElements, geometry.context, block); + const auto chebyshevBlock = preconditioning::GravityFieldBlock( + backend::MatrixFreeChebyshev{.order = 2, .powerIterations = 10}, FixedAMG{backend::FixedCycles{.cycles = 1}}, + preconditioning::GravityApproximateLDU{} + ); + auto chebyshevPrepared = preconditioning::prepare(geometry.finiteElements, geometry.context, chebyshevBlock); + + const auto unchanged = prepared.Refresh(geometry.finiteElements, geometry.context); + const auto unchangedChebyshev = chebyshevPrepared.Refresh(geometry.finiteElements, geometry.context); + CHECK_FALSE(unchanged.DidAnyWork()); + CHECK_FALSE(unchangedChebyshev.DidAnyWork()); + CHECK(prepared.GetStatistics().refreshChecks == 1); + CHECK(prepared.GetStatistics().noOpRefreshes == 1); + + const mfem::Vector displacementTrue = gravity_prepared_test_utils::make_displacement(geometry.finiteElements, 0.4); + const mfem::Vector displacement = geometry.context.GetDisplacementMap().gather(displacementTrue); + geometry.context.PreparePrimal(displacement, {.value = 1}, {.value = 2}); + CHECK_FALSE(prepared.IsCurrent()); + CHECK_FALSE(chebyshevPrepared.IsCurrent()); + + mfem::Vector rightHandSide(prepared.Width()); + mfem::Vector action(prepared.Height()); + rightHandSide = 1.0; + action = 0.0; + CHECK_THROWS_AS(prepared.Mult(rightHandSide, action), std::logic_error); + CHECK_THROWS_AS(chebyshevPrepared.Mult(rightHandSide, action), std::logic_error); + + const std::uint64_t massPreparations = geometry.context.GetMassOperator().GetPreparationCount(); + const std::uint64_t sourcePreparations = geometry.context.GetSourceOperator().GetPreparationCount(); + const auto changed = prepared.Refresh(geometry.finiteElements, geometry.context); + const auto changedChebyshev = chebyshevPrepared.Refresh(geometry.finiteElements, geometry.context); + + CHECK(changed.geometryChanged); + CHECK_FALSE(changed.discretizationChanged); + CHECK(changed.rebuiltMassInverse); + CHECK_FALSE(changed.rebuiltDivergenceBinding); + CHECK(changed.rebuiltPotentialSchur); + CHECK(prepared.IsCurrent()); + CHECK(changedChebyshev.geometryChanged); + CHECK(changedChebyshev.rebuiltMassInverse); + CHECK(changedChebyshev.rebuiltPotentialSchur); + CHECK(chebyshevPrepared.IsCurrent()); + CHECK(chebyshevPrepared.GetMassInverse().GetStatistics().setups == 2); + CHECK(geometry.context.GetMassOperator().GetPreparationCount() == massPreparations); + CHECK(geometry.context.GetSourceOperator().GetPreparationCount() == sourcePreparations); + CHECK(prepared.GetStatistics().refreshes == 1); + + mfem::Vector refreshedAction(chebyshevPrepared.Height()); + refreshedAction = 0.0; + chebyshevPrepared.Mult(rightHandSide, refreshedAction); + for (int index = 0; index < refreshedAction.Size(); ++index) { + CHECK(std::isfinite(refreshedAction(index))); + } + + // A discretization revision reconstructs the matrix-free mass operator. The + // owning gravity block must reject every route to its now-stale inverse until + // refresh has rebound and rebuilt the Chebyshev smoother. + geometry.context.PreparePrimal(displacement, {.value = 2}, {.value = 2}); + CHECK_FALSE(chebyshevPrepared.IsCurrent()); + CHECK_THROWS_AS(chebyshevPrepared.Mult(rightHandSide, action), std::logic_error); + CHECK_THROWS_AS(chebyshevPrepared.GetMassInverse(), std::logic_error); + + const auto reconstructed = chebyshevPrepared.Refresh(geometry.finiteElements, geometry.context); + CHECK(reconstructed.discretizationChanged); + CHECK_FALSE(reconstructed.geometryChanged); + CHECK(reconstructed.rebuiltMassInverse); + CHECK(reconstructed.rebuiltDivergenceBinding); + CHECK(reconstructed.rebuiltPotentialSchur); + CHECK(chebyshevPrepared.IsCurrent()); + CHECK(chebyshevPrepared.GetMassInverse().GetStatistics().setups == 3); + + mfem::Vector firstReconstructedAction(chebyshevPrepared.Height()); + mfem::Vector secondReconstructedAction(chebyshevPrepared.Height()); + firstReconstructedAction = 0.0; + secondReconstructedAction = 0.0; + chebyshevPrepared.Mult(rightHandSide, firstReconstructedAction); + chebyshevPrepared.Mult(rightHandSide, secondReconstructedAction); + mfem::Vector repeatabilityError(firstReconstructedAction); + repeatabilityError -= secondReconstructedAction; + CHECK(repeatabilityError.Norml2() <= 2.0e-14 * std::max(1.0, firstReconstructedAction.Norml2())); + for (int index = 0; index < firstReconstructedAction.Size(); ++index) { + CHECK(std::isfinite(firstReconstructedAction(index))); + } +} diff --git a/tests/preconditioning/material_surface.cpp b/tests/preconditioning/material_surface.cpp new file mode 100644 index 0000000..cbedd7a --- /dev/null +++ b/tests/preconditioning/material_surface.cpp @@ -0,0 +1,614 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +import mean_field; +import test_helpers; + +namespace { + namespace backend = mean_field::preconditioning::backend; + namespace blocks = mean_field::utils::blocks; + namespace preconditioning = mean_field::preconditioning; + + using PolytropicModel = mean_field::model::StellarModel>; + using PolytropicProblem = mean_field::equilibrium::StellarEquilibriumProblem; + using PolytropicMaterialSurfaceDescriptor = preconditioning::MaterialSurfaceDescriptorFor; + using MaterialSurfaceDiagonal = preconditioning::MaterialSurfaceBlock< + PolytropicMaterialSurfaceDescriptor, + backend::Diagonal, + backend::Diagonal, + preconditioning::SurfaceThenMaterialTriangular>; + using FixedCycleAMG = backend::HypreBoomerAMG; + using MaterialSurfaceH1 = preconditioning::MaterialSurfaceBlock< + PolytropicMaterialSurfaceDescriptor, + backend::Diagonal, + FixedCycleAMG, + preconditioning::ApproximateMaterialSurfaceLDU, + preconditioning::SurfaceH1MassStiffness>; + using PreparedMaterialSurfaceDiagonal = preconditioning::PreparedMaterialSurfaceBlock< + PolytropicMaterialSurfaceDescriptor, + preconditioning::SurfaceThenMaterialTriangular>; + using PreparedMaterialSurfaceH1 = preconditioning::PreparedH1MaterialSurfaceBlock< + PolytropicMaterialSurfaceDescriptor, + preconditioning::ApproximateMaterialSurfaceLDU, + backend::FixedCycles>; + + class KnownCouplings final { + public: + KnownCouplings() : m_offsets(4) { + m_offsets[0] = 0; + m_offsets[1] = 1; + m_offsets[2] = 2; + m_offsets[3] = 3; + } + + [[nodiscard]] int Height() const noexcept { + return 3; + } + [[nodiscard]] const mfem::Array &GetOffsets() const noexcept { + return m_offsets; + } + + void ApplyEnthalpyToDensity( + const mfem::Vector &enthalpy, + mfem::Vector &density + ) const { + density(0) = 4.0 * enthalpy(0); + } + void ApplySurfaceToMaterial( + const mfem::Vector &surface, + mfem::Vector &density, + mfem::Vector &enthalpy + ) const { + density(0) = 3.0 * surface(0); + enthalpy(0) = 8.0 * surface(0); + } + void ApplyMaterialToSurface( + const mfem::Vector &density, + const mfem::Vector &enthalpy, + mfem::Vector &surface + ) const { + surface(0) = 5.0 * density(0) + 7.0 * enthalpy(0); + } + + private: + mfem::Array m_offsets; + }; + + template + [[nodiscard]] mfem::Vector applyKnownFactorization( + Policy policy, + const mfem::Vector &rightHandSide, + const double surfaceEntry = 6.0 + ) { + mfem::Vector densityDiagonal(1); + mfem::Vector surfaceDiagonal(1); + mfem::Vector enthalpyDiagonal(1); + densityDiagonal(0) = 2.0; + surfaceDiagonal(0) = surfaceEntry; + enthalpyDiagonal(0) = 9.0; + const auto densityInverse = backend::prepare(backend::Diagonal{}, densityDiagonal); + const auto surfaceInverse = backend::prepare(backend::Diagonal{}, surfaceDiagonal); + const auto enthalpyInverse = backend::prepare(backend::Diagonal{}, enthalpyDiagonal); + const KnownCouplings couplings; + preconditioning::MaterialSurfaceFactorizationOperator factorization( + policy, densityInverse, surfaceInverse, enthalpyInverse, couplings + ); + mfem::Vector action(3); + factorization.Mult(rightHandSide, action); + return action; + } + + [[nodiscard]] mean_field::operators::StellarEquilibriumDependencies makeDependencies(std::uint64_t revision = 1) { + return { + .discretization = {.identity = 8101, .revision = 1}, + .density = {.identity = 8103, .revision = revision}, + .surfaceDeformation = {.identity = 8107, .revision = revision}, + .gravityGradient = {.identity = 8111, .revision = revision}, + .gravityPotential = {.identity = 8117, .revision = revision}, + .enthalpy = {.identity = 8123, .revision = revision}, + .bernoulliConstant = {.identity = 8129, .revision = revision}, + .rotation = {.identity = 8131, .revision = revision}, + .targetMass = {.identity = 8137, .revision = 1} + }; + } + + [[nodiscard]] mean_field::physics::RigidRotation zeroRotation() { + mfem::Vector angularVelocity(3); + mfem::Vector center(3); + angularVelocity = 0.0; + center = 0.0; + return {angularVelocity, center}; + } + + [[nodiscard]] double relativeError( + const mfem::Vector &left, + const mfem::Vector &right + ) { + mfem::Vector difference(left); + difference -= right; + return difference.Norml2() / std::max({1.0, left.Norml2(), right.Norml2()}); + } +} // namespace + +TEST_CASE( + "Compiled Material Surface Blocks Derive Their Physical Ownership And Backend Requirements", + "[preconditioning][material_surface][unit][type_contract]" +) { + using Form = blocks::surface_deformed_stellar_equilibrium_form; + using JacobianForm = blocks::surface_deformed_stellar_equilibrium_jacobian_form; + using GravityIdentity = + preconditioning::IdentityBlock; + using PotentialIdentity = + preconditioning::IdentityBlock; + using MassIdentity = preconditioning::IdentityBlock< + blocks::fixed_total_mass::mass_normalization::value, blocks::fixed_total_mass::mass_normalization::residual>; + using Plan = + preconditioning::PreconditionerPlan; + + STATIC_CHECK(preconditioning::PreconditionerComponent); + STATIC_CHECK(preconditioning::PreconditionerComponent); + STATIC_CHECK(preconditioning::MaterialSurfaceDescriptor); + STATIC_CHECK( + mean_field::material::CompiledThermodynamicEquations + ); + STATIC_CHECK( + std::same_as< + typename PolytropicMaterialSurfaceDescriptor::SurfaceStateFields, + mean_field::field::TypeList> + ); + STATIC_CHECK(MaterialSurfaceDiagonal::CorrectionBlocks::size == 3); + STATIC_CHECK(MaterialSurfaceDiagonal::ResidualBlocks::size == 3); + STATIC_CHECK(MaterialSurfaceDiagonal::RequiredCouplings::size == 8); + STATIC_CHECK(preconditioning::CompletePreconditionerFor); + STATIC_CHECK(preconditioning::CompatiblePreconditionerFor); + STATIC_CHECK(preconditioning::backend::ArnoldiAdmissible); + STATIC_CHECK_FALSE(std::is_copy_constructible_v); + STATIC_CHECK_FALSE(std::is_copy_assignable_v); + STATIC_CHECK_FALSE(std::is_move_constructible_v); + STATIC_CHECK_FALSE(std::is_move_assignable_v); + STATIC_CHECK_FALSE(std::is_copy_constructible_v); + STATIC_CHECK_FALSE(std::is_copy_assignable_v); + STATIC_CHECK_FALSE(std::is_move_constructible_v); + STATIC_CHECK_FALSE(std::is_move_assignable_v); + STATIC_CHECK( + preconditioning::backend::Compatible + ); + STATIC_CHECK_FALSE( + preconditioning::backend::Compatible + ); + STATIC_CHECK( + preconditioning::backend::Compatible + ); + STATIC_CHECK_FALSE(std::same_as); + STATIC_CHECK(std::same_as); + STATIC_CHECK(preconditioning::backend::ArnoldiAdmissible); +} + +TEST_CASE( + "Surface H1 Calibration Recovers Signed Nonnegative Mass And Stiffness Fits", + "[preconditioning][material_surface][surface_h1][unit]" +) { + const preconditioning::SurfaceH1MassStiffness configuration{ + .calibration = + {.target = preconditioning::SurfaceRieszCalibrationTarget::approximate_material_schur, .probeCount = 4}, + .relativeMassCoefficientFloor = 1.0e-12, + .gramRelativeTolerance = 1.0e-12 + }; + const preconditioning::SurfaceH1NormalEquations exactPositive{ + .massMass = 2.0, + .massStiffness = 2.0, + .stiffnessStiffness = 5.0, + .massTarget = 10.0, + .stiffnessTarget = 19.0, + .targetTarget = 77.0 + }; + + const auto positive = preconditioning::detail::fitSurfaceH1Coefficients(exactPositive, configuration); + CHECK(positive.WasCalibrated()); + CHECK(positive.sign == 1.0); + CHECK(positive.massCoefficient == Catch::Approx(2.0).margin(2.0e-13)); + CHECK(positive.stiffnessCoefficient == Catch::Approx(3.0).margin(2.0e-13)); + CHECK(positive.relativeResidual == Catch::Approx(0.0).margin(2.0e-13)); + CHECK(positive.relativeGramDeterminant > configuration.gramRelativeTolerance); + CHECK(positive.normalEquations.targetTarget == Catch::Approx(77.0)); + + auto exactNegative = exactPositive; + exactNegative.massTarget = -exactNegative.massTarget; + exactNegative.stiffnessTarget = -exactNegative.stiffnessTarget; + const auto negative = preconditioning::detail::fitSurfaceH1Coefficients(exactNegative, configuration); + CHECK(negative.sign == -1.0); + CHECK(negative.massCoefficient == Catch::Approx(2.0).margin(2.0e-13)); + CHECK(negative.stiffnessCoefficient == Catch::Approx(3.0).margin(2.0e-13)); + CHECK(negative.relativeResidual == Catch::Approx(0.0).margin(2.0e-13)); + + const preconditioning::SurfaceH1NormalEquations massDominated{ + .massMass = 1.0, + .massStiffness = 0.0, + .stiffnessStiffness = 1.0, + .massTarget = 4.0, + .stiffnessTarget = -2.0, + .targetTarget = 20.0 + }; + const auto constrained = preconditioning::detail::fitSurfaceH1Coefficients(massDominated, configuration); + CHECK(constrained.sign == 1.0); + CHECK(constrained.massCoefficient == Catch::Approx(4.0).margin(2.0e-13)); + CHECK(constrained.stiffnessCoefficient == Catch::Approx(0.0).margin(2.0e-13)); + CHECK(constrained.relativeResidual == Catch::Approx(std::sqrt(0.2)).margin(2.0e-13)); + + auto rankDeficient = exactPositive; + rankDeficient.massMass = 1.0; + rankDeficient.massStiffness = 2.0; + rankDeficient.stiffnessStiffness = 4.0; + CHECK_THROWS_AS( + preconditioning::detail::fitSurfaceH1Coefficients(rankDeficient, configuration), std::runtime_error + ); +} + +TEST_CASE( + "Surface Riesz Scalar Calibration Distinguishes Operator And Right-Preconditioned Objectives", + "[preconditioning][material_surface][surface_riesz][unit]" +) { + using Objective = preconditioning::SurfaceRieszCalibrationObjective; + + const auto operatorFit = preconditioning::detail::fitSurfaceRieszScalar(6.0, 2.0, Objective::operator_action); + CHECK(operatorFit.surrogateScale == Catch::Approx(3.0)); + CHECK(operatorFit.inverseMultiplier == Catch::Approx(1.0 / 3.0)); + + const auto inverseFit = + preconditioning::detail::fitSurfaceRieszScalar(6.0, 2.0, Objective::right_preconditioned_action); + CHECK(inverseFit.surrogateScale == Catch::Approx(1.0 / 3.0)); + CHECK(inverseFit.inverseMultiplier == Catch::Approx(3.0)); + + CHECK_THROWS_AS( + preconditioning::detail::fitSurfaceRieszScalar(1.0, 0.0, Objective::operator_action), std::invalid_argument + ); + CHECK_THROWS_AS( + preconditioning::detail::fitSurfaceRieszScalar(0.0, 1.0, Objective::right_preconditioned_action), + std::runtime_error + ); +} + +TEST_CASE( + "Signed Surface Solver Adapts Boundary Coordinates Without Exposing An Indefinite Backend", + "[preconditioning][material_surface][surface_h1][unit]" +) { + mfem::DenseMatrix ambientMatrix(3); + ambientMatrix = 0.0; + ambientMatrix(0, 0) = 2.0; + ambientMatrix(1, 1) = 7.0; + ambientMatrix(2, 2) = 4.0; + const auto ambientInverse = backend::prepare(backend::DenseDirect{}, ambientMatrix); + + mfem::Array boundaryTrueDofs(2); + boundaryTrueDofs[0] = 0; + boundaryTrueDofs[1] = 2; + mean_field::field::ScalarBoundaryDofMap surfaceMap(3, boundaryTrueDofs, 0, 2); + preconditioning::SignedScalarBoundarySolverAdapter surfaceInverse(ambientInverse, surfaceMap, -1.0); + + mfem::Vector rightHandSide(2); + mfem::Vector action(2); + rightHandSide(0) = 2.0; + rightHandSide(1) = 4.0; + surfaceInverse.Mult(rightHandSide, action); + CHECK(action(0) == Catch::Approx(-1.0)); + CHECK(action(1) == Catch::Approx(-1.0)); + CHECK(surfaceInverse.GetSign() == -1.0); + CHECK_THROWS_AS(surfaceInverse.SetSign(0.0), std::invalid_argument); +} + +TEST_CASE( + "Material Surface Factorization Policies Preserve Their Signed Triangular Algebra", + "[preconditioning][material_surface][unit][factorization]" +) { + mfem::Vector rightHandSide(3); + rightHandSide(0) = 29.0; + rightHandSide(1) = 44.0; + rightHandSide(2) = 43.0; + + const auto check = [](const mfem::Vector &value, std::array expected) { + for (int index = 0; index < value.Size(); ++index) { + CHECK(value(index) == Catch::Approx(expected[static_cast(index)]).margin(2.0e-13)); + } + }; + + check( + applyKnownFactorization(preconditioning::MaterialSurfaceBlockDiagonal{}, rightHandSide), + {14.5, 44.0 / 6.0, 43.0 / 9.0} + ); + check( + applyKnownFactorization(preconditioning::CoupledMaterialIndependentSurface{}, rightHandSide), + {(29.0 - 4.0 * (43.0 / 9.0)) / 2.0, 44.0 / 6.0, 43.0 / 9.0} + ); + const double materialEnthalpy = 43.0 / 9.0; + const double materialDensity = (29.0 - 4.0 * materialEnthalpy) / 2.0; + check( + applyKnownFactorization(preconditioning::MaterialThenSurfaceTriangular{}, rightHandSide), + {materialDensity, (44.0 - 5.0 * materialDensity - 7.0 * materialEnthalpy) / 6.0, materialEnthalpy} + ); + const double surfaceFirst = 44.0 / 6.0; + const double surfaceCorrectedEnthalpy = (43.0 - 8.0 * surfaceFirst) / 9.0; + check( + applyKnownFactorization(preconditioning::SurfaceThenMaterialTriangular{}, rightHandSide), + {(29.0 - 3.0 * surfaceFirst - 4.0 * surfaceCorrectedEnthalpy) / 2.0, surfaceFirst, surfaceCorrectedEnthalpy} + ); + + const double firstMaterialEnthalpy = 43.0 / 9.0; + const double firstMaterialDensity = (29.0 - 4.0 * firstMaterialEnthalpy) / 2.0; + const double lduSurface = (44.0 - 5.0 * firstMaterialDensity - 7.0 * firstMaterialEnthalpy) / 6.0; + const double lduEnthalpy = (43.0 - 8.0 * lduSurface) / 9.0; + check( + applyKnownFactorization(preconditioning::ApproximateMaterialSurfaceLDU{}, rightHandSide), + {(29.0 - 3.0 * lduSurface - 4.0 * lduEnthalpy) / 2.0, lduSurface, lduEnthalpy} + ); + + // M = [[2,4],[0,9]], B = [3,8]^T, C = [5,7], so the exact + // scalar surface Schur complement is 6 - C M^{-1} B = 7/6. + const mfem::Vector exact = + applyKnownFactorization(preconditioning::ApproximateMaterialSurfaceLDU{}, rightHandSide, 7.0 / 6.0); + CHECK(2.0 * exact(0) + 3.0 * exact(1) + 4.0 * exact(2) == Catch::Approx(29.0).margin(2.0e-12)); + CHECK(5.0 * exact(0) + 6.0 * exact(1) + 7.0 * exact(2) == Catch::Approx(44.0).margin(2.0e-12)); + CHECK(8.0 * exact(1) + 9.0 * exact(2) == Catch::Approx(43.0).margin(2.0e-12)); +} + +TEST_CASE( + "Generated Material Surface Action Is The Exact Restricted Stellar Jacobian And Uses A Bounded Surrogate", + "[preconditioning][material_surface][surface_h1][integration]" +) { + using namespace mean_field; + const utils::Args arguments = test_utils::setup_args(); + fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0); + REQUIRE(finiteElements.okay()); + + constexpr double radius = utils::RADIUS; + constexpr double mass = utils::MASS; + const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v; + const double centralDensity = std::numbers::pi_v * mass / (4.0 * radius * radius * radius); + const auto stellarModel = model::StellarModel( + eos::Polytrope({.n = 1.0, .K = polytropicConstant}), + surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), + integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}), + constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}}) + ); + auto problem = equilibrium::discretize(stellarModel, finiteElements); + auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 512})); + const auto rotation = zeroRotation(); + problem.Prepare(projected.values, makeDependencies(), rotation); + const auto &physical = problem.GetPreparedOperator().GetPhysicalOperator(); + + const auto block = preconditioning::materialSurfaceBlock( + problem, backend::Diagonal{}, backend::Diagonal{}, preconditioning::SurfaceThenMaterialTriangular{} + ); + const auto defaultBlock = preconditioning::materialSurfaceBlock(problem); + STATIC_CHECK( + std::same_as< + typename std::remove_cvref_t::Descriptor, + preconditioning::MaterialSurfaceDescriptorFor> + ); + STATIC_CHECK(std::same_as, std::remove_cvref_t>); + auto prepared = preconditioning::prepare(problem, block); + const auto &restricted = prepared.GetCoupledOperator(); + mfem::Vector restrictedDirection(restricted.Width()); + for (int index = 0; index < restrictedDirection.Size(); ++index) { + restrictedDirection(index) = 0.01 * std::sin(0.37 * static_cast(index + 1)); + } + mfem::Vector restrictedAction(restricted.Height()); + restricted.Mult(restrictedDirection, restrictedAction); + + mfem::Vector fullDirection(physical.Width()); + fullDirection = 0.0; + const auto fullDirectionView = physical.GetRootManifest().directionView(fullDirection); + const auto &offsets = restricted.GetOffsets(); + const mfem::Vector densityDirection(restrictedDirection.GetData(), offsets[1]); + const mfem::Vector surfaceDirection(restrictedDirection.GetData() + offsets[1], offsets[2] - offsets[1]); + const mfem::Vector enthalpyDirection(restrictedDirection.GetData() + offsets[2], offsets[3] - offsets[2]); + mfem::Vector fullDensityDirection = fullDirectionView.block(blocks::density_field.mass_term); + mfem::Vector fullSurfaceDirection = fullDirectionView.block(blocks::surface_deformation_field.parameters_term); + mfem::Vector fullEnthalpyDirection = fullDirectionView.block(blocks::enthalpy_field.specific_term); + fullDensityDirection = densityDirection; + fullSurfaceDirection = surfaceDirection; + fullEnthalpyDirection = enthalpyDirection; + + mfem::Vector fullAction; + physical.Mult(fullDirection, fullAction); + const auto fullActionView = physical.GetRootManifest().residualView(fullAction); + mfem::Vector expected(restricted.Height()); + mfem::Vector expectedDensity(expected.GetData(), offsets[1]); + mfem::Vector expectedSurface(expected.GetData() + offsets[1], offsets[2] - offsets[1]); + mfem::Vector expectedEnthalpy(expected.GetData() + offsets[2], offsets[3] - offsets[2]); + const mfem::Vector fullDensityAction = fullActionView.block(blocks::density_field.mass_term); + const mfem::Vector fullSurfaceAction = + fullActionView.block(blocks::surface_deformation_field.shape_equilibrium_term); + const mfem::Vector fullEnthalpyAction = fullActionView.block(blocks::enthalpy_field.specific_term); + expectedDensity = fullDensityAction; + expectedSurface = fullSurfaceAction; + expectedEnthalpy = fullEnthalpyAction; + const mfem::Vector restrictedDensity(restrictedAction.GetData(), offsets[1]); + const mfem::Vector restrictedSurface(restrictedAction.GetData() + offsets[1], offsets[2] - offsets[1]); + const mfem::Vector restrictedEnthalpy(restrictedAction.GetData() + offsets[2], offsets[3] - offsets[2]); + INFO("Restricted density-row error = " << relativeError(restrictedDensity, expectedDensity)); + INFO("Restricted surface-row error = " << relativeError(restrictedSurface, expectedSurface)); + INFO("Restricted enthalpy-row error = " << relativeError(restrictedEnthalpy, expectedEnthalpy)); + CHECK(relativeError(restrictedDensity, expectedDensity) <= 2.0e-12); + CHECK(relativeError(restrictedSurface, expectedSurface) <= 2.0e-12); + CHECK(relativeError(restrictedEnthalpy, expectedEnthalpy) <= 2.0e-12); + CHECK(relativeError(restrictedAction, expected) <= 2.0e-12); + + CHECK(prepared.GetDensityDiagonalQuality().maximumAbsoluteEntryBeforeRegularization > 0.0); + CHECK(prepared.GetSurfaceDiagonalQuality().maximumAbsoluteEntryBeforeRegularization > 0.0); + CHECK(prepared.GetSurfaceDiagonalQuality().minimumAbsoluteEntryBeforeRegularization > 0.0); + CHECK(prepared.GetSurfaceDiagonalQuality().regularizedEntries == 0); + CHECK(prepared.GetEnthalpyDiagonalQuality().maximumAbsoluteEntryBeforeRegularization > 0.0); + CHECK(prepared.GetStatistics().surfaceJacobianProbes == 0); + CHECK(prepared.GetStatistics().surfaceRieszAssemblies == 1); + + const auto calibratedBlock = preconditioning::materialSurfaceBlock( + problem, backend::Diagonal{}, backend::Diagonal{}, preconditioning::ApproximateMaterialSurfaceLDU{}, + {.surfaceCalibration = { + .target = preconditioning::SurfaceRieszCalibrationTarget::approximate_material_schur, + .probeCount = 3, + .objective = preconditioning::SurfaceRieszCalibrationObjective::right_preconditioned_action + }} + ); + auto calibrated = preconditioning::prepare(problem, calibratedBlock); + CHECK(calibrated.GetSurfaceCalibration().WasCalibrated()); + CHECK( + calibrated.GetSurfaceCalibration().target == + preconditioning::SurfaceRieszCalibrationTarget::approximate_material_schur + ); + CHECK(calibrated.GetSurfaceCalibration().probeCount == 3); + CHECK( + calibrated.GetSurfaceCalibration().objective == + preconditioning::SurfaceRieszCalibrationObjective::right_preconditioned_action + ); + CHECK(std::isfinite(calibrated.GetSurfaceCalibration().scale)); + CHECK(calibrated.GetSurfaceCalibration().scale != 0.0); + CHECK(calibrated.GetStatistics().surfaceJacobianProbes == 3); + CHECK(calibrated.GetSurfaceDiagonalQuality().maximumAbsoluteEntryBeforeRegularization > 0.0); + + const auto frequencyAwareBlock = preconditioning::materialSurfaceBlock( + problem, backend::Diagonal{}, FixedCycleAMG{backend::FixedCycles{.cycles = 1}}, + preconditioning::ApproximateMaterialSurfaceLDU{}, + preconditioning::SurfaceH1MassStiffness{ + .calibration = + {.target = preconditioning::SurfaceRieszCalibrationTarget::approximate_material_schur, .probeCount = 4}, + .relativeMassCoefficientFloor = 1.0e-10, + .gramRelativeTolerance = 1.0e-12 + } + ); + STATIC_CHECK( + std::same_as< + typename std::remove_cvref_t::SurfaceSurrogate, + preconditioning::SurfaceH1MassStiffness> + ); + auto frequencyAware = preconditioning::prepare(problem, frequencyAwareBlock); + using PreparedFrequencyAware = std::remove_cvref_t; + STATIC_CHECK_FALSE(std::copy_constructible); + STATIC_CHECK_FALSE(std::move_constructible); + const auto &surfaceFit = frequencyAware.GetSurfaceFit(); + CHECK(surfaceFit.WasCalibrated()); + CHECK(surfaceFit.target == preconditioning::SurfaceRieszCalibrationTarget::approximate_material_schur); + CHECK(surfaceFit.probeCount == 4); + CHECK((surfaceFit.sign == -1.0 || surfaceFit.sign == 1.0)); + CHECK(std::isfinite(surfaceFit.massCoefficient)); + CHECK(surfaceFit.massCoefficient > 0.0); + CHECK(std::isfinite(surfaceFit.stiffnessCoefficient)); + CHECK(surfaceFit.stiffnessCoefficient >= 0.0); + CHECK(std::isfinite(surfaceFit.relativeResidual)); + CHECK(surfaceFit.relativeGramDeterminant > 1.0e-12); + CHECK(surfaceFit.normalEquations.targetTarget > 0.0); + CHECK(frequencyAware.GetSurfaceInverse().Height() == physical.GetDomainDeformation().parameterCount()); + CHECK(frequencyAware.GetSurfaceSurrogateMatrix().Height() == finiteElements.surfaceDeformationFes->GetTrueVSize()); + CHECK(frequencyAware.GetSurfaceBackend().GetStatistics().setups == 1); + CHECK(frequencyAware.GetStatistics().surfaceJacobianProbes == 4); + CHECK(frequencyAware.GetStatistics().surfaceH1Assemblies == 3); + const auto frequencyAwareNoChange = frequencyAware.Refresh(physical); + CHECK_FALSE(frequencyAwareNoChange.DidAnyWork()); + CHECK(frequencyAware.GetStatistics().noOpRefreshes == 1); + + mfem::Vector rightHandSide(prepared.Width()); + mfem::Vector correction(prepared.Height()); + mfem::Vector repeatedCorrection(prepared.Height()); + for (int index = 0; index < rightHandSide.Size(); ++index) { + rightHandSide(index) = std::cos(0.19 * static_cast(index + 1)); + } + correction = 0.0; + repeatedCorrection = 0.0; + double *const correctionStorage = correction.GetData(); + prepared.Mult(rightHandSide, correction); + prepared.Mult(rightHandSide, repeatedCorrection); + CHECK(correction.GetData() == correctionStorage); + CHECK(relativeError(correction, repeatedCorrection) <= 2.0e-15); + for (int index = 0; index < correction.Size(); ++index) { + REQUIRE(std::isfinite(correction(index))); + } + + mfem::Vector frequencyAwareCorrection(frequencyAware.Height()); + mfem::Vector repeatedFrequencyAwareCorrection(frequencyAware.Height()); + frequencyAwareCorrection = 0.0; + repeatedFrequencyAwareCorrection = 0.0; + frequencyAware.Mult(rightHandSide, frequencyAwareCorrection); + frequencyAware.Mult(rightHandSide, repeatedFrequencyAwareCorrection); + CHECK(relativeError(frequencyAwareCorrection, repeatedFrequencyAwareCorrection) <= 2.0e-13); + for (int index = 0; index < frequencyAwareCorrection.Size(); ++index) { + REQUIRE(std::isfinite(frequencyAwareCorrection(index))); + } + + const auto noChange = prepared.Refresh(physical); + CHECK_FALSE(noChange.DidAnyWork()); + + // Full stellar-Jacobian finite-difference accuracy is covered by the + // prepared-stellar-equilibrium tests. Here we change the state only to + // exercise the material-surface refresh contract without repeating two + // expensive nonlinear residual assemblies. + mfem::Vector changedState(projected.values); + mfem::Vector borderedDirection(problem.StateSize()); + borderedDirection = 0.0; + mfem::Vector physicalDirection(borderedDirection.GetData(), physical.Width()); + physicalDirection = fullDirection; + changedState.Add(1.0e-5, borderedDirection); + problem.Prepare(changedState, makeDependencies(2), rotation); + + CHECK_FALSE(prepared.IsCurrent()); + rightHandSide = 1.0; + correction = 0.0; + CHECK_THROWS_AS(prepared.Mult(rightHandSide, correction), std::logic_error); + const auto refreshed = prepared.Refresh(problem.GetPreparedOperator().GetPhysicalOperator()); + CHECK(refreshed.linearizationChanged); + CHECK(refreshed.rebuiltDensityInverse); + CHECK(refreshed.rebuiltSurfaceInverse); + CHECK(refreshed.rebuiltEnthalpyInverse); + CHECK(prepared.IsCurrent()); + CHECK(prepared.GetStatistics().surfaceJacobianProbes == 0); + CHECK(prepared.GetStatistics().surfaceRieszAssemblies == 2); + + auto densityOnlyDependencies = makeDependencies(2); + densityOnlyDependencies.density.revision = 3; + problem.Prepare(changedState, densityOnlyDependencies, rotation); + CHECK_FALSE(prepared.IsCurrent()); + const auto stateOnlyRefresh = prepared.Refresh(problem.GetPreparedOperator().GetPhysicalOperator()); + CHECK(stateOnlyRefresh.linearizationChanged); + CHECK_FALSE(stateOnlyRefresh.DidAnyWork()); + CHECK_FALSE(stateOnlyRefresh.rebuiltDensityInverse); + CHECK_FALSE(stateOnlyRefresh.rebuiltSurfaceInverse); + CHECK_FALSE(stateOnlyRefresh.rebuiltEnthalpyInverse); + CHECK(prepared.IsCurrent()); + CHECK(prepared.GetStatistics().surfaceRieszAssemblies == 2); + + CHECK_FALSE(calibrated.IsCurrent()); + CHECK_FALSE(frequencyAware.IsCurrent()); + const auto calibratedRefresh = calibrated.Refresh(problem.GetPreparedOperator().GetPhysicalOperator()); + CHECK(calibratedRefresh.DidAnyWork()); + CHECK(calibratedRefresh.rebuiltDensityInverse); + CHECK(calibratedRefresh.rebuiltSurfaceInverse); + CHECK(calibratedRefresh.rebuiltEnthalpyInverse); + CHECK(calibrated.IsCurrent()); + CHECK(calibrated.GetStatistics().surfaceJacobianProbes == 6); + CHECK(calibrated.GetStatistics().surfaceRieszAssemblies == 2); + + const auto frequencyAwareRefresh = frequencyAware.Refresh(problem.GetPreparedOperator().GetPhysicalOperator()); + CHECK(frequencyAwareRefresh.DidAnyWork()); + CHECK(frequencyAwareRefresh.rebuiltDensityInverse); + CHECK(frequencyAwareRefresh.rebuiltSurfaceInverse); + CHECK(frequencyAwareRefresh.rebuiltEnthalpyInverse); + CHECK(frequencyAware.IsCurrent()); + CHECK(frequencyAware.GetSurfaceBackend().GetStatistics().setups == 2); + CHECK(frequencyAware.GetStatistics().surfaceJacobianProbes == 8); + CHECK(frequencyAware.GetStatistics().surfaceH1Assemblies == 6); + frequencyAware.Mult(rightHandSide, frequencyAwareCorrection); + for (int index = 0; index < frequencyAwareCorrection.Size(); ++index) { + REQUIRE(std::isfinite(frequencyAwareCorrection(index))); + } +} diff --git a/tests/preconditioning/plan.cpp b/tests/preconditioning/plan.cpp new file mode 100644 index 0000000..22c8152 --- /dev/null +++ b/tests/preconditioning/plan.cpp @@ -0,0 +1,236 @@ +#include +#include + +#include + +import mean_field; +import test_helpers; + +namespace { + namespace blocks = mean_field::utils::blocks; + namespace preconditioning = mean_field::preconditioning; + + using Form = blocks::surface_deformed_stellar_equilibrium_form; + using JacobianForm = blocks::surface_deformed_stellar_equilibrium_jacobian_form; + using CentralForm = blocks::central_density_bordered_stellar_equilibrium_form; + using CentralJacobianForm = blocks::central_density_bordered_stellar_equilibrium_jacobian_form; + + using DensityIdentity = + preconditioning::IdentityBlock; + using SurfaceIdentity = preconditioning::IdentityBlock< + blocks::surface_deformation::parameters::value, + blocks::surface_deformation::shape_equilibrium::residual>; + using GravityGradientIdentity = + preconditioning::IdentityBlock; + using GravityPotentialIdentity = + preconditioning::IdentityBlock; + using EnthalpyIdentity = + preconditioning::IdentityBlock; + using MassIdentity = preconditioning::IdentityBlock< + blocks::fixed_total_mass::mass_normalization::value, + blocks::fixed_total_mass::mass_normalization::residual>; + using CentralDensityIdentity = preconditioning::IdentityBlock< + blocks::fixed_central_density::central_value::value, + blocks::fixed_central_density::central_value::residual>; + + using IdentityPlan = preconditioning::PreconditionerPlan< + DensityIdentity, + SurfaceIdentity, + GravityGradientIdentity, + GravityPotentialIdentity, + EnthalpyIdentity, + MassIdentity>; + + using CentralIdentityPlan = preconditioning::PreconditionerPlan< + DensityIdentity, + SurfaceIdentity, + GravityGradientIdentity, + GravityPotentialIdentity, + EnthalpyIdentity, + MassIdentity, + CentralDensityIdentity>; + + using IncompleteCentralPlan = IdentityPlan; + + struct AlternateDensityIdentity final : preconditioning::ComponentDeclaration< + blocks::type_list, + blocks::type_list, + blocks::type_list<>, + preconditioning::IdentityOperatorCharacteristics, + preconditioning::backend::Identity> { }; + + using DuplicateOwnershipPlan = preconditioning::PreconditionerPlan< + DensityIdentity, + SurfaceIdentity, + GravityGradientIdentity, + GravityPotentialIdentity, + EnthalpyIdentity, + MassIdentity, + CentralDensityIdentity, + AlternateDensityIdentity>; + + using ExplicitOverlapPlan = preconditioning::OverlappingPreconditionerPlan< + DensityIdentity, + SurfaceIdentity, + GravityGradientIdentity, + GravityPotentialIdentity, + EnthalpyIdentity, + MassIdentity, + CentralDensityIdentity, + AlternateDensityIdentity>; + + struct ExtraCorrection final : blocks::value_block_base { }; + struct ExtraResidual final : blocks::residual_block_base { }; + using ExtraIdentity = preconditioning::IdentityBlock; + using UnexpectedOwnershipPlan = preconditioning::PreconditionerPlan< + DensityIdentity, + SurfaceIdentity, + GravityGradientIdentity, + GravityPotentialIdentity, + EnthalpyIdentity, + MassIdentity, + ExtraIdentity>; + + using CoupledGravity = preconditioning::ComponentDeclaration< + blocks::type_list, + blocks::type_list, + blocks::type_list< + preconditioning::Coupling, + preconditioning::Coupling, + preconditioning::Coupling>, + preconditioning::IdentityOperatorCharacteristics, + preconditioning::backend::Identity>; + + using ValidCoupledPlan = preconditioning:: + PreconditionerPlan; + + using InvalidGravityCoupling = preconditioning::ComponentDeclaration< + blocks::type_list, + blocks::type_list, + blocks::type_list>, + preconditioning::IdentityOperatorCharacteristics, + preconditioning::backend::Identity>; + + using InvalidCoupledPlan = preconditioning:: + PreconditionerPlan; + + using IncompatibleBackendComponent = preconditioning::ComponentDeclaration< + blocks::type_list, + blocks::type_list, + blocks::type_list<>, + preconditioning::OperatorCharacteristics< + preconditioning::OperatorCategory::elliptic_like, + preconditioning::OperatorValueStructure::vector, + preconditioning::OperatorSymmetry::symmetric, + preconditioning::OperatorDefiniteness::positive_definite, + preconditioning::OperatorRepresentation::assembled_sparse, + preconditioning::OperatorDistribution::distributed_true_dof, + preconditioning::OperatorFESpace::h_div>, + preconditioning::backend::HypreBoomerAMG<>>; + + struct IncoherentPlanDeclaration final { + using ComponentTypes = blocks::type_list; + using CorrectionBlocks = blocks::type_list; + using ResidualBlocks = blocks::type_list; + using RequiredCouplings = blocks::type_list<>; + }; +} // namespace + +TEST_CASE( + "Preconditioning Backends Advertise Compile-Time Operator Compatibility", + tags::preconditioning_type_contract +) { + using ScalarElliptic = preconditioning::OperatorCharacteristics< + preconditioning::OperatorCategory::elliptic_like, preconditioning::OperatorValueStructure::scalar, + preconditioning::OperatorSymmetry::symmetric, preconditioning::OperatorDefiniteness::positive_definite, + preconditioning::OperatorRepresentation::assembled_sparse, + preconditioning::OperatorDistribution::distributed_true_dof, preconditioning::OperatorFESpace::h1>; + + using VectorElliptic = preconditioning::OperatorCharacteristics< + preconditioning::OperatorCategory::elliptic_like, preconditioning::OperatorValueStructure::vector, + preconditioning::OperatorSymmetry::symmetric, preconditioning::OperatorDefiniteness::positive_definite, + preconditioning::OperatorRepresentation::assembled_sparse, + preconditioning::OperatorDistribution::distributed_true_dof, preconditioning::OperatorFESpace::h_div>; + + using LocalDenseBorder = preconditioning::OperatorCharacteristics< + preconditioning::OperatorCategory::dense_border, preconditioning::OperatorValueStructure::block, + preconditioning::OperatorSymmetry::nonsymmetric, preconditioning::OperatorDefiniteness::indefinite, + preconditioning::OperatorRepresentation::assembled_dense, preconditioning::OperatorDistribution::local>; + + using FixedAMG = preconditioning::backend::HypreBoomerAMG; + + STATIC_CHECK(preconditioning::backend::Registered); + STATIC_CHECK(preconditioning::backend::Compatible); + STATIC_CHECK_FALSE(preconditioning::backend::Compatible); + STATIC_CHECK(preconditioning::backend::Compatible); + STATIC_CHECK_FALSE(preconditioning::PreconditionerComponent); + STATIC_CHECK( + preconditioning::backend::applicationContract == + preconditioning::ApplicationContract::stationary_linear + ); +} + +TEST_CASE( + "Preconditioner Plans Prove Complete Unique Ownership Of Every Equilibrium Block", + tags::preconditioning_type_contract +) { + STATIC_CHECK(preconditioning::PreconditionerComponent); + STATIC_CHECK(preconditioning::PreconditionerPlanType); + STATIC_CHECK_FALSE(preconditioning::PreconditionerPlanType); + STATIC_CHECK(preconditioning::CompletePreconditionerFor); + STATIC_CHECK(preconditioning::CompatiblePreconditionerFor); + STATIC_CHECK(preconditioning::CompletePreconditionerFor); + STATIC_CHECK(preconditioning::CompatiblePreconditionerFor); + STATIC_CHECK(preconditioning::StationaryLinearPreconditionerPlan); +} + +TEST_CASE( + "Preconditioner Coverage Reports Missing Generated Borders And Rejects Accidental Overlap", + tags::preconditioning_type_contract +) { + using IncompleteCoverage = preconditioning::PreconditionerCoverage; + using DuplicateCoverage = preconditioning::PreconditionerCoverage; + + STATIC_CHECK_FALSE(preconditioning::CompletePreconditionerFor); + STATIC_CHECK(IncompleteCoverage::MissingCorrectionBlocks::size == 1); + STATIC_CHECK(IncompleteCoverage::MissingResidualBlocks::size == 1); + STATIC_CHECK( + blocks::contains_type_v< + blocks::fixed_central_density::central_value::value, IncompleteCoverage::MissingCorrectionBlocks> + ); + STATIC_CHECK( + blocks::contains_type_v< + blocks::fixed_central_density::central_value::residual, IncompleteCoverage::MissingResidualBlocks> + ); + + STATIC_CHECK_FALSE(preconditioning::CompletePreconditionerFor); + STATIC_CHECK(DuplicateCoverage::RepeatedCorrectionBlocks::size == 1); + STATIC_CHECK(DuplicateCoverage::RepeatedResidualBlocks::size == 1); + STATIC_CHECK(preconditioning::CompletePreconditionerFor); +} + +TEST_CASE( + "Preconditioner Coverage Rejects Blocks Outside The Compiled Stellar Form", + tags::preconditioning_type_contract +) { + using Coverage = preconditioning::PreconditionerCoverage; + + STATIC_CHECK_FALSE(preconditioning::CompletePreconditionerFor); + STATIC_CHECK(Coverage::UnexpectedCorrectionBlocks::size == 1); + STATIC_CHECK(Coverage::UnexpectedResidualBlocks::size == 1); + STATIC_CHECK(blocks::contains_type_v); + STATIC_CHECK(blocks::contains_type_v); +} + +TEST_CASE( + "Preconditioner Component Dependencies Must Exist In The Compiled Jacobian Graph", + tags::preconditioning_type_contract +) { + STATIC_CHECK(preconditioning::PreconditionerComponent); + STATIC_CHECK(preconditioning::CompletePreconditionerFor); + STATIC_CHECK(preconditioning::CompatiblePreconditionerFor); + + STATIC_CHECK(preconditioning::CompletePreconditionerFor); + STATIC_CHECK_FALSE(preconditioning::requiredCouplingsExist); + STATIC_CHECK_FALSE(preconditioning::CompatiblePreconditionerFor); +} diff --git a/tests/preconditioning/specification_border.cpp b/tests/preconditioning/specification_border.cpp new file mode 100644 index 0000000..44b6451 --- /dev/null +++ b/tests/preconditioning/specification_border.cpp @@ -0,0 +1,519 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +import mean_field; +import test_helpers; + +namespace { + namespace backend = mean_field::preconditioning::backend; + namespace blocks = mean_field::utils::blocks; + namespace preconditioning = mean_field::preconditioning; + + using BaseModel = mean_field::operators::StellarEquilibriumSpecificationModel; + using CentralModel = mean_field::operators::CentralDensityStellarEquilibriumSpecificationModel; + using ReorderedCentralModel = mean_field::model::StellarModel>; + using BaseProblem = mean_field::equilibrium::StellarEquilibriumProblem; + using CentralProblem = mean_field::equilibrium::StellarEquilibriumProblem; + using BaseBorder = preconditioning::CompiledSpecificationBorderFor; + using CentralBorder = preconditioning::CompiledSpecificationBorderFor; + using BaseComponent = decltype(preconditioning::specificationBorderBlock(std::declval())); + using CentralComponent = + decltype(preconditioning::specificationBorderBlock(std::declval())); + using BasePlan = preconditioning::PreconditionerPlan; + using CentralPlan = preconditioning::PreconditionerPlan; + + class KnownBorderCouplings final { + public: + explicit KnownBorderCouplings(const int borderSize) + : m_borderSize(borderSize), + m_structureToBorder( + borderSize, + StructureSize() + ), + m_borderToStructure( + StructureSize(), + borderSize + ), + m_borderDiagonal(borderSize) { + if (borderSize <= 0) { + throw std::invalid_argument("The known border must have positive size."); + } + for (int row = 0; row < borderSize; ++row) { + for (int column = 0; column < StructureSize(); ++column) { + m_structureToBorder(row, column) = 0.04 * static_cast((row + 1) * (column + 2)); + m_borderToStructure(column, row) = -0.03 * static_cast((column + 1) * (row + 2)); + } + for (int column = 0; column < borderSize; ++column) { + m_borderDiagonal(row, column) = + row == column ? 2.0 + static_cast(row) : 0.01 * static_cast(row + column + 1); + } + } + } + + [[nodiscard]] static constexpr int StructureSize() noexcept { + return 3; + } + + [[nodiscard]] int BorderSize() const noexcept { + return m_borderSize; + } + + void ApplyStructureToBorder( + const mfem::Vector &direction, + mfem::Vector &action + ) const { + m_structureToBorder.Mult(direction, action); + } + + void ApplyBorderToStructure( + const mfem::Vector &direction, + mfem::Vector &action + ) const { + m_borderToStructure.Mult(direction, action); + } + + void ApplyBorderToBorder( + const mfem::Vector &direction, + mfem::Vector &action + ) const { + m_borderDiagonal.Mult(direction, action); + } + + void IncreaseBorderDiagonal(const double increment) { + for (int index = 0; index < m_borderSize; ++index) { + m_borderDiagonal(index, index) += increment; + } + } + + [[nodiscard]] const mfem::DenseMatrix &StructureToBorder() const noexcept { + return m_structureToBorder; + } + + [[nodiscard]] const mfem::DenseMatrix &BorderToStructure() const noexcept { + return m_borderToStructure; + } + + [[nodiscard]] const mfem::DenseMatrix &BorderDiagonal() const noexcept { + return m_borderDiagonal; + } + + private: + int m_borderSize; + mfem::DenseMatrix m_structureToBorder; + mfem::DenseMatrix m_borderToStructure; + mfem::DenseMatrix m_borderDiagonal; + }; + + [[nodiscard]] double relativeError( + const mfem::Vector &left, + const mfem::Vector &right + ) { + mfem::Vector difference(left); + difference -= right; + return difference.Norml2() / std::max({1.0, left.Norml2(), right.Norml2()}); + } + + template < + preconditioning::ApplicationContract StructureInverseContract = + preconditioning::ApplicationContract::stationary_linear> + void verifyKnownBorderFactorization(const int borderSize) { + mfem::Vector structureDiagonal(KnownBorderCouplings::StructureSize()); + structureDiagonal(0) = 2.0; + structureDiagonal(1) = 3.0; + structureDiagonal(2) = 5.0; + auto structureInverse = backend::prepare(backend::Diagonal{}, structureDiagonal); + KnownBorderCouplings couplings(borderSize); + using Factorization = + preconditioning::SpecificationBorderFactorizationOperator; + Factorization factorization(structureInverse, couplings); + constexpr bool cachesStructureResponse = Factorization::cachesStructureInverseBorderCoupling; + + const auto expectedSchurEntry = [&](const int row, const int column) { + double correction = 0.0; + for (int inner = 0; inner < KnownBorderCouplings::StructureSize(); ++inner) { + correction += couplings.StructureToBorder()(row, inner) * couplings.BorderToStructure()(inner, column) / + structureDiagonal(inner); + } + return couplings.BorderDiagonal()(row, column) - correction; + }; + for (int row = 0; row < borderSize; ++row) { + for (int column = 0; column < borderSize; ++column) { + CHECK( + factorization.GetSchurComplement()(row, column) == + Catch::Approx(expectedSchurEntry(row, column)).margin(2.0e-14) + ); + } + } + + const int completeSize = KnownBorderCouplings::StructureSize() + borderSize; + mfem::DenseMatrix completeMatrix(completeSize); + completeMatrix = 0.0; + for (int index = 0; index < KnownBorderCouplings::StructureSize(); ++index) { + completeMatrix(index, index) = structureDiagonal(index); + } + for (int row = 0; row < KnownBorderCouplings::StructureSize(); ++row) { + for (int column = 0; column < borderSize; ++column) { + completeMatrix(row, KnownBorderCouplings::StructureSize() + column) = + couplings.BorderToStructure()(row, column); + completeMatrix(KnownBorderCouplings::StructureSize() + column, row) = + couplings.StructureToBorder()(column, row); + } + } + for (int row = 0; row < borderSize; ++row) { + for (int column = 0; column < borderSize; ++column) { + completeMatrix( + KnownBorderCouplings::StructureSize() + row, KnownBorderCouplings::StructureSize() + column + ) = couplings.BorderDiagonal()(row, column); + } + } + + mfem::Vector rightHandSide(completeSize); + for (int index = 0; index < completeSize; ++index) { + rightHandSide(index) = 0.25 + 0.17 * static_cast(index + 1); + } + mfem::Vector actual(completeSize); + mfem::Vector expected(completeSize); + factorization.Mult(rightHandSide, actual); + mfem::DenseMatrixInverse exactInverse(completeMatrix); + exactInverse.Mult(rightHandSide, expected); + CHECK(relativeError(actual, expected) <= 2.0e-13); + + const auto statisticsBeforeRefresh = factorization.GetStatistics(); + CHECK(statisticsBeforeRefresh.setups == 1); + CHECK(statisticsBeforeRefresh.schurProbes == static_cast(borderSize)); + CHECK(statisticsBeforeRefresh.applications == 1); + CHECK( + statisticsBeforeRefresh.structureInverseApplications == + static_cast(borderSize + (cachesStructureResponse ? 1 : 2)) + ); + CHECK( + statisticsBeforeRefresh.cachedStructureInverseBorderApplications == + static_cast(cachesStructureResponse ? 1 : 0) + ); + CHECK(statisticsBeforeRefresh.structureToBorderApplications == static_cast(borderSize + 1)); + CHECK( + statisticsBeforeRefresh.borderToStructureApplications == + static_cast(borderSize + (cachesStructureResponse ? 0 : 1)) + ); + CHECK(statisticsBeforeRefresh.borderToBorderApplications == static_cast(borderSize)); + CHECK( + structureInverse.GetStatistics().applications == + static_cast(borderSize + (cachesStructureResponse ? 1 : 2)) + ); + + for (int index = 0; index < KnownBorderCouplings::StructureSize(); ++index) { + structureDiagonal(index) += 0.25 * static_cast(index + 1); + completeMatrix(index, index) = structureDiagonal(index); + } + structureInverse.Refresh(structureDiagonal); + couplings.IncreaseBorderDiagonal(0.5); + for (int index = 0; index < borderSize; ++index) { + completeMatrix( + KnownBorderCouplings::StructureSize() + index, KnownBorderCouplings::StructureSize() + index + ) += 0.5; + } + factorization.RefreshSchurComplement(); + CHECK(factorization.GetStatistics().setups == 2); + CHECK(factorization.GetStatistics().schurProbes == static_cast(2 * borderSize)); + CHECK(factorization.GetStatistics().borderToBorderApplications == static_cast(2 * borderSize)); + CHECK( + factorization.GetStatistics().structureInverseApplications == + static_cast(2 * borderSize + (cachesStructureResponse ? 1 : 2)) + ); + CHECK( + factorization.GetStatistics().cachedStructureInverseBorderApplications == + static_cast(cachesStructureResponse ? 1 : 0) + ); + CHECK( + factorization.GetStatistics().structureToBorderApplications == + static_cast(2 * borderSize + 1) + ); + CHECK( + factorization.GetStatistics().borderToStructureApplications == + static_cast(2 * borderSize + (cachesStructureResponse ? 0 : 1)) + ); + for (int row = 0; row < borderSize; ++row) { + for (int column = 0; column < borderSize; ++column) { + CHECK( + factorization.GetSchurComplement()(row, column) == + Catch::Approx(expectedSchurEntry(row, column)).margin(2.0e-14) + ); + } + } + + mfem::Vector refreshedActual(completeSize); + mfem::Vector refreshedExpected(completeSize); + factorization.Mult(rightHandSide, refreshedActual); + mfem::DenseMatrixInverse refreshedExactInverse(completeMatrix); + refreshedExactInverse.Mult(rightHandSide, refreshedExpected); + CHECK(relativeError(refreshedActual, refreshedExpected) <= 2.0e-13); + + const auto statisticsAfterRefreshApplication = factorization.GetStatistics(); + CHECK(statisticsAfterRefreshApplication.applications == 2); + CHECK( + statisticsAfterRefreshApplication.structureInverseApplications == + static_cast(2 * borderSize + (cachesStructureResponse ? 2 : 4)) + ); + CHECK( + statisticsAfterRefreshApplication.cachedStructureInverseBorderApplications == + static_cast(cachesStructureResponse ? 2 : 0) + ); + CHECK( + statisticsAfterRefreshApplication.structureToBorderApplications == + static_cast(2 * borderSize + 2) + ); + CHECK( + statisticsAfterRefreshApplication.borderToStructureApplications == + static_cast(2 * borderSize + (cachesStructureResponse ? 0 : 2)) + ); + } + + [[nodiscard]] mean_field::operators::StellarEquilibriumDependencies + makeDependencies(const std::uint64_t revision = 1) { + return { + .discretization = {.identity = 9201, .revision = 1}, + .density = {.identity = 9203, .revision = revision}, + .surfaceDeformation = {.identity = 9207, .revision = revision}, + .gravityGradient = {.identity = 9211, .revision = revision}, + .gravityPotential = {.identity = 9217, .revision = revision}, + .enthalpy = {.identity = 9223, .revision = revision}, + .bernoulliConstant = {.identity = 9229, .revision = revision}, + .rotation = {.identity = 9231, .revision = revision}, + .targetMass = {.identity = 9237, .revision = 1} + }; + } + + [[nodiscard]] mean_field::physics::RigidRotation zeroRotation() { + mfem::Vector angularVelocity(3); + mfem::Vector center(3); + angularVelocity = 0.0; + center = 0.0; + return {angularVelocity, center}; + } + + template < + typename View, + typename Term> + void assignStateBlock( + const View &view, + const Term &term, + const mfem::Vector &source, + mfem::Vector &state + ) { + mfem::Vector destination = view.block(term); + REQUIRE(destination.Size() == source.Size()); + destination = source; + destination.SyncAliasMemory(state); + } +} // namespace + +TEST_CASE( + "Model Specifications Compile Complete Canonical Preconditioning Borders", + "[preconditioning][specification_border][unit][type_contract]" +) { + using ExpectedBaseCorrections = blocks::type_list; + using ExpectedBaseResiduals = blocks::type_list; + using ExpectedCentralCorrections = blocks::type_list< + blocks::fixed_total_mass::mass_normalization::value, blocks::fixed_central_density::central_value::value>; + using ExpectedCentralResiduals = blocks::type_list< + blocks::fixed_total_mass::mass_normalization::residual, blocks::fixed_central_density::central_value::residual>; + + STATIC_CHECK(std::same_as); + STATIC_CHECK(BaseBorder::valueArity == 1); + STATIC_CHECK(BaseBorder::residualArity == 1); + STATIC_CHECK(BaseBorder::specificationCount == 1); + STATIC_CHECK(std::same_as); + STATIC_CHECK(std::same_as); + STATIC_CHECK(BaseBorder::RequiredCouplings::size == 3); + + STATIC_CHECK(CentralBorder::valueArity == 2); + STATIC_CHECK(CentralBorder::residualArity == 2); + STATIC_CHECK(CentralBorder::specificationCount == 2); + STATIC_CHECK(std::same_as); + STATIC_CHECK(std::same_as); + STATIC_CHECK(CentralBorder::RequiredCouplings::size == 5); + STATIC_CHECK( + preconditioning::specificationBorderValueOffset == 0 + ); + STATIC_CHECK( + preconditioning::specificationBorderValueOffset == 1 + ); + STATIC_CHECK( + preconditioning::specificationBorderResidualOffset == 0 + ); + STATIC_CHECK( + preconditioning::specificationBorderResidualOffset == 1 + ); + + STATIC_CHECK(preconditioning::PreconditionerComponent); + STATIC_CHECK(preconditioning::PreconditionerComponent); + STATIC_CHECK(BaseComponent::RequiredCouplings::size == 19); + STATIC_CHECK(CentralComponent::RequiredCouplings::size == 21); + STATIC_CHECK(preconditioning::CompletePreconditionerFor); + STATIC_CHECK( + preconditioning::CompatiblePreconditionerFor< + BasePlan, typename BaseProblem::FormType, typename BaseProblem::JacobianFormType> + ); + STATIC_CHECK(preconditioning::CompletePreconditionerFor); + STATIC_CHECK( + preconditioning::CompatiblePreconditionerFor< + CentralPlan, typename CentralProblem::FormType, typename CentralProblem::JacobianFormType> + ); + STATIC_CHECK(preconditioning::backend::ArnoldiAdmissible); +} + +TEST_CASE( + "Dense Specification Borders Cache Stationary Structure Responses And Reproduce Exact Block Factorizations", + "[preconditioning][specification_border][unit][factorization]" +) { + SECTION("one generated scalar") { + verifyKnownBorderFactorization(1); + } + SECTION("two generated scalars") { + verifyKnownBorderFactorization(2); + } + SECTION("four generated scalars") { + verifyKnownBorderFactorization(4); + } +} + +TEST_CASE( + "Flexible Specification Borders Preserve Per-Application Structure Solves", + "[preconditioning][specification_border][unit][factorization]" +) { + verifyKnownBorderFactorization(2); +} + +TEST_CASE( + "Generated Specification Border Actions Match The Authoritative Stellar Jacobian", + "[preconditioning][specification_border][integration]" +) { + using namespace mean_field; + const utils::Args arguments = test_utils::setup_args(); + fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0); + REQUIRE(finiteElements.okay()); + + constexpr double radius = utils::RADIUS; + constexpr double mass = utils::MASS; + const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v; + const double centralDensity = std::numbers::pi_v * mass / (4.0 * radius * radius * radius); + const auto stellarModel = model::StellarModel( + eos::Polytrope({.n = 1.0, .K = polytropicConstant}), + surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), + integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}), + constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}}) + ); + auto problem = equilibrium::discretize(stellarModel, finiteElements); + auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 512})); + problem.Prepare(projected.values, makeDependencies(), zeroRotation()); + + preconditioning::SpecificationBorderJacobianOperator coupling(problem); + REQUIRE(coupling.BorderSize() == 2); + REQUIRE(coupling.StructureSize() + coupling.BorderSize() == problem.StateSize()); + const auto &offsets = coupling.GetStructureOffsets(); + + mfem::Vector groupedDirection(coupling.Width()); + for (int index = 0; index < groupedDirection.Size(); ++index) { + groupedDirection(index) = 0.015 * std::sin(0.23 * static_cast(index + 1)); + } + const auto groupedBlock = [&](const int block) { + return mfem::Vector(groupedDirection.GetData() + offsets[block], offsets[block + 1] - offsets[block]); + }; + + mfem::Vector structureOnlyRoot(problem.StateSize()); + structureOnlyRoot = 0.0; + const auto structureView = problem.GetManifest().directionView(structureOnlyRoot); + assignStateBlock(structureView, blocks::density_field.mass_term, groupedBlock(0), structureOnlyRoot); + assignStateBlock( + structureView, blocks::surface_deformation_field.parameters_term, groupedBlock(1), structureOnlyRoot + ); + assignStateBlock(structureView, blocks::enthalpy_field.specific_term, groupedBlock(2), structureOnlyRoot); + assignStateBlock(structureView, blocks::gravity_field.gradient_term, groupedBlock(3), structureOnlyRoot); + assignStateBlock(structureView, blocks::gravity_field.poisson_term, groupedBlock(4), structureOnlyRoot); + + mfem::Vector borderOnlyRoot(problem.StateSize()); + borderOnlyRoot = 0.0; + const auto borderView = problem.GetManifest().directionView(borderOnlyRoot); + mfem::Vector massDirection(groupedDirection.GetData() + coupling.StructureSize(), 1); + mfem::Vector centralDirection(groupedDirection.GetData() + coupling.StructureSize() + 1, 1); + assignStateBlock( + borderView, blocks::fixed_total_mass_constraint.mass_normalization_term, massDirection, borderOnlyRoot + ); + assignStateBlock( + borderView, blocks::fixed_central_density_phase.central_value_term, centralDirection, borderOnlyRoot + ); + + mfem::Vector structureOnlyAction; + mfem::Vector borderOnlyAction; + problem.ApplyLinearization(structureOnlyRoot, structureOnlyAction); + problem.ApplyLinearization(borderOnlyRoot, borderOnlyAction); + auto structureOnlyResidual = problem.GetManifest().residualView(structureOnlyAction); + auto borderOnlyResidual = problem.GetManifest().residualView(borderOnlyAction); + + mfem::Vector expected(coupling.Height()); + expected = 0.0; + expected.SetVector(borderOnlyResidual.block(blocks::density_field.mass_term), offsets[0]); + expected.SetVector(borderOnlyResidual.block(blocks::surface_deformation_field.shape_equilibrium_term), offsets[1]); + expected.SetVector(borderOnlyResidual.block(blocks::enthalpy_field.specific_term), offsets[2]); + expected.SetVector(borderOnlyResidual.block(blocks::gravity_field.gradient_term), offsets[3]); + expected.SetVector(borderOnlyResidual.block(blocks::gravity_field.poisson_term), offsets[4]); + expected.SetVector( + structureOnlyResidual.block(blocks::fixed_total_mass_constraint.mass_normalization_term), + coupling.StructureSize() + ); + expected.SetVector( + structureOnlyResidual.block(blocks::fixed_central_density_phase.central_value_term), + coupling.StructureSize() + 1 + ); + mfem::Vector borderDiagonal(2); + borderDiagonal(0) = borderOnlyResidual.block(blocks::fixed_total_mass_constraint.mass_normalization_term)(0); + borderDiagonal(1) = borderOnlyResidual.block(blocks::fixed_central_density_phase.central_value_term)(0); + mfem::Vector expectedBorder(expected, coupling.StructureSize(), coupling.BorderSize()); + expectedBorder += borderDiagonal; + expectedBorder.SyncAliasMemory(expected); + + mfem::Vector actual(coupling.Height()); + coupling.Mult(groupedDirection, actual); + CHECK(relativeError(actual, expected) <= 2.0e-12); + + auto component = preconditioning::makePreconditioner(problem); + using Component = decltype(component); + STATIC_CHECK(std::same_as); + auto prepared = preconditioning::prepare(problem, component); + using GroupedPreconditioner = typename decltype(prepared)::GroupedPreconditioner; + using PreparedFactorization = typename GroupedPreconditioner::Factorization; + STATIC_CHECK(PreparedFactorization::cachesStructureInverseBorderCoupling); + mfem::Vector rightHandSide(prepared.Width()); + for (int index = 0; index < rightHandSide.Size(); ++index) { + rightHandSide(index) = std::cos(0.11 * static_cast(index + 1)); + } + mfem::Vector correction(prepared.Height()); + prepared.Mult(rightHandSide, correction); + for (int index = 0; index < correction.Size(); ++index) { + REQUIRE(std::isfinite(correction(index))); + } + const auto &factorizationStatistics = prepared.GetGroupedPreconditioner().GetFactorization().GetStatistics(); + CHECK(factorizationStatistics.setups == 1); + CHECK(factorizationStatistics.schurProbes == 2); + CHECK(factorizationStatistics.applications == 1); + CHECK(factorizationStatistics.structureInverseApplications == 3); + CHECK(factorizationStatistics.cachedStructureInverseBorderApplications == 1); + CHECK(factorizationStatistics.borderToStructureApplications == 2); + const auto unchanged = prepared.Refresh(); + CHECK_FALSE(unchanged.DidAnyWork()); + CHECK(prepared.IsCurrent()); +} diff --git a/tests/preconditioning/stellar_equilibrium.cpp b/tests/preconditioning/stellar_equilibrium.cpp new file mode 100644 index 0000000..0d36eb7 --- /dev/null +++ b/tests/preconditioning/stellar_equilibrium.cpp @@ -0,0 +1,336 @@ +#include +#include +#include +#include +#include +#include + +#include +#include + +import mean_field; +import test_helpers; + +namespace preconditioning_runtime_test { + namespace blocks = mean_field::utils::blocks; + + using Form = blocks::surface_deformed_stellar_equilibrium_form; + using JacobianForm = blocks::surface_deformed_stellar_equilibrium_jacobian_form; + using Layout = blocks::form_layout; + + class Manifest final { + public: + Manifest() + : m_layout( + std::array< + int, + Form::value_block_count>{ + 2, + 3, + 4, + 5, + 6, + 1 + }, + std::array< + int, + Form::residual_block_count>{ + 4, + 5, + 2, + 3, + 6, + 1 + } + ) { + } + + [[nodiscard]] const Layout &layout() const noexcept { + return m_layout; + } + + private: + Layout m_layout; + }; + + class Problem final { + public: + Problem() : m_linearization(m_manifest.layout().value_offsets().Last()) { + m_snapshot.discretization = {.identity = 11, .revision = 1}; + m_snapshot.geometry = {.identity = 12, .revision = 1}; + m_snapshot.equationOfStateIdentity = &m_equationOfStateToken; + m_snapshot.linearization.discretization = m_snapshot.discretization; + m_snapshot.linearization.density = {.identity = 21, .revision = 1}; + } + + void AdvanceDensity() noexcept { + ++m_snapshot.linearization.density.revision; + } + + void AdvanceGeometry() noexcept { + ++m_snapshot.geometry.revision; + } + + void SetPrepared(const bool prepared) noexcept { + m_prepared = prepared; + } + + [[nodiscard]] bool IsPrepared() const noexcept { + return m_prepared; + } + + [[nodiscard]] int StateSize() const noexcept { + return m_manifest.layout().value_offsets().Last(); + } + + [[nodiscard]] int EquationSize() const noexcept { + return m_manifest.layout().residual_offsets().Last(); + } + + [[nodiscard]] const Manifest &GetManifest() const noexcept { + return m_manifest; + } + + [[nodiscard]] const mfem::Operator &GetLinearizationOperator() const noexcept { + return m_linearization; + } + + [[nodiscard]] mean_field::preconditioning::StellarPreconditionerLifecycleSnapshot Snapshot() const { + return m_snapshot; + } + + private: + Manifest m_manifest; + mfem::IdentityOperator m_linearization; + std::uint8_t m_equationOfStateToken{0}; + mean_field::preconditioning::StellarPreconditionerLifecycleSnapshot m_snapshot; + bool m_prepared{true}; + }; +} // namespace preconditioning_runtime_test + +template <> struct mean_field::preconditioning::StellarEquilibriumProblemTraits { + using Problem = preconditioning_runtime_test::Problem; + using Form = preconditioning_runtime_test::Form; + using JacobianForm = preconditioning_runtime_test::JacobianForm; + using Manifest = preconditioning_runtime_test::Manifest; + + static constexpr bool registered = true; + + [[nodiscard]] static bool IsPrepared(const Problem &problem) noexcept { + return problem.IsPrepared(); + } + + [[nodiscard]] static int StateSize(const Problem &problem) noexcept { + return problem.StateSize(); + } + + [[nodiscard]] static int EquationSize(const Problem &problem) noexcept { + return problem.EquationSize(); + } + + [[nodiscard]] static const Manifest &ManifestOf(const Problem &problem) noexcept { + return problem.GetManifest(); + } + + [[nodiscard]] static const mfem::Operator &LinearizationOperator(const Problem &problem) noexcept { + return problem.GetLinearizationOperator(); + } + + [[nodiscard]] static mean_field::preconditioning::StellarPreconditionerLifecycleSnapshot + Snapshot(const Problem &problem) { + return problem.Snapshot(); + } +}; + +namespace { + namespace blocks = mean_field::utils::blocks; + namespace preconditioning = mean_field::preconditioning; + + using ModelWithoutPhase = mean_field::operators::StellarEquilibriumSpecificationModel; + using CentralDensityModel = mean_field::operators::CentralDensityStellarEquilibriumSpecificationModel; + + using ProblemWithoutPhase = mean_field::equilibrium::StellarEquilibriumProblem; + using CentralDensityProblem = mean_field::equilibrium::StellarEquilibriumProblem; + using PlanWithoutPhase = preconditioning::IdentityPreconditionerPlanFor; + using CentralDensityPlan = preconditioning::IdentityPreconditionerPlanFor; + + using RefreshingDensityIdentity = preconditioning::ComponentDeclaration< + blocks::type_list, + blocks::type_list, + blocks::type_list<>, + preconditioning::IdentityOperatorCharacteristics, + preconditioning::backend::Identity, + preconditioning::PreparationDependencies>; + using SurfaceIdentity = preconditioning::IdentityBlock< + blocks::surface_deformation::parameters::value, + blocks::surface_deformation::shape_equilibrium::residual>; + using GravityGradientIdentity = + preconditioning::IdentityBlock; + using GravityPotentialIdentity = + preconditioning::IdentityBlock; + using EnthalpyIdentity = + preconditioning::IdentityBlock; + using FixedMassIdentity = preconditioning::IdentityBlock< + blocks::fixed_total_mass::mass_normalization::value, + blocks::fixed_total_mass::mass_normalization::residual>; + using SelectiveRefreshPlan = preconditioning::PreconditionerPlan< + RefreshingDensityIdentity, + SurfaceIdentity, + GravityGradientIdentity, + GravityPotentialIdentity, + EnthalpyIdentity, + FixedMassIdentity>; + + [[nodiscard]] constexpr SelectiveRefreshPlan makeSelectiveRefreshPlan() { + return SelectiveRefreshPlan{RefreshingDensityIdentity{}, SurfaceIdentity{}, GravityGradientIdentity{}, + GravityPotentialIdentity{}, EnthalpyIdentity{}, FixedMassIdentity{}}; + } +} // namespace + +TEST_CASE( + "Stellar Identity Plans Follow The Compiled Equilibrium Problem Type", + tags::preconditioning_runtime_unit +) { + STATIC_CHECK(mean_field::equilibrium::DiscretizedStellarEquilibriumProblem); + STATIC_CHECK(mean_field::equilibrium::DiscretizedStellarEquilibriumProblem); + STATIC_CHECK(preconditioning::StellarPreconditionerProblem); + STATIC_CHECK(preconditioning::StellarPreconditionerProblem); + STATIC_CHECK(preconditioning::CompletePreconditionerFor); + STATIC_CHECK( + preconditioning::CompletePreconditionerFor + ); + STATIC_CHECK(PlanWithoutPhase::ComponentTypes::size == 6); + STATIC_CHECK(CentralDensityPlan::ComponentTypes::size == 7); +} + +TEST_CASE( + "Prepared Stellar Identity Preconditioning Is Bitwise Equivalent To The P0 Baseline", + tags::preconditioning_runtime_unit +) { + preconditioning_runtime_test::Problem problem; + auto plan = preconditioning::makeIdentityPlan(problem); + using Plan = decltype(plan); + using Problem = preconditioning_runtime_test::Problem; + auto preconditioner = preconditioning::prepare(problem, std::move(plan)); + + STATIC_CHECK(preconditioning::PreparedPreconditionerPlanFor); + CHECK(preconditioner.Height() == problem.StateSize()); + CHECK(preconditioner.Width() == problem.EquationSize()); + CHECK(preconditioner.IsCurrent()); + CHECK(&preconditioner.GetLinearizationOperator() == &problem.GetLinearizationOperator()); + + preconditioner.SetOperator(problem.GetLinearizationOperator()); + + mfem::Vector residual(problem.EquationSize()); + mfem::Vector correction(problem.StateSize()); + for (int index = 0; index < residual.Size(); ++index) { + residual(index) = static_cast(index) - 10.25; + } + correction = -1.0; + + const mfem::real_t *const correctionStorage = correction.GetData(); + const auto statisticsBefore = preconditioner.GetStatistics(); + preconditioner.Mult(residual, correction); + const auto statisticsAfter = preconditioner.GetStatistics(); + + CHECK(correction.GetData() == correctionStorage); + CHECK(std::memcmp(correction.GetData(), residual.GetData(), sizeof(mfem::real_t) * residual.Size()) == 0); + + const mfem::Vector densityCorrection = preconditioner.GetCorrectionBlock(correction); + const mfem::Vector densityResidual = preconditioner.GetResidualBlock(residual); + CHECK(densityCorrection.Size() == 2); + CHECK(densityCorrection.GetData() == correction.GetData()); + CHECK(densityResidual.Size() == 2); + CHECK(densityResidual.GetData() == residual.GetData() + 9); + CHECK(statisticsAfter.setups == statisticsBefore.setups); + CHECK(statisticsAfter.refreshes == statisticsBefore.refreshes); + CHECK(statisticsAfter.componentSetups == 6); + CHECK(statisticsAfter.applications == statisticsBefore.applications + 1); + CHECK(statisticsAfter.backendApplications == statisticsBefore.backendApplications + 1); + CHECK(statisticsAfter.innerIterations == 0); + CHECK(statisticsAfter.operatorBindings == 1); +} + +TEST_CASE( + "Stellar Preconditioner Refresh Is Explicit And Dependency Aware", + tags::preconditioning_runtime_unit +) { + preconditioning_runtime_test::Problem problem; + auto preconditioner = preconditioning::prepare(problem, preconditioning::makeIdentityPlan(problem)); + mfem::Vector residual(problem.EquationSize()); + mfem::Vector correction(problem.StateSize()); + residual = 1.0; + correction = 0.0; + + const auto noChange = preconditioner.Refresh(); + CHECK_FALSE(noChange.changes.Any()); + CHECK_FALSE(noChange.DidAnyWork()); + CHECK(preconditioner.GetStatistics().noOpRefreshes == 1); + + problem.AdvanceDensity(); + CHECK_FALSE(preconditioner.IsCurrent()); + CHECK_THROWS_AS(preconditioner.Mult(residual, correction), std::logic_error); + + const auto linearizationRefresh = preconditioner.Refresh(); + CHECK(linearizationRefresh.changes.linearization); + CHECK_FALSE(linearizationRefresh.changes.discretization); + CHECK_FALSE(linearizationRefresh.changes.geometry); + CHECK_FALSE(linearizationRefresh.DidAnyWork()); + CHECK(preconditioner.IsCurrent()); + CHECK(preconditioner.GetStatistics().refreshes == 1); + CHECK(preconditioner.GetStatistics().componentRefreshes == 0); + + problem.AdvanceGeometry(); + const auto geometryRefresh = preconditioner.Refresh(); + CHECK(geometryRefresh.changes.geometry); + CHECK_FALSE(geometryRefresh.changes.linearization); + CHECK(preconditioner.GetStatistics().refreshes == 2); + + problem.SetPrepared(false); + CHECK_FALSE(preconditioner.IsCurrent()); + CHECK_THROWS_AS(preconditioner.Refresh(), std::logic_error); +} + +TEST_CASE( + "Stellar Preconditioner Refresh Touches Only Components With Changed Dependencies", + tags::preconditioning_runtime_unit +) { + preconditioning_runtime_test::Problem problem; + auto preconditioner = preconditioning::prepare(problem, makeSelectiveRefreshPlan()); + + problem.AdvanceGeometry(); + const auto geometryRefresh = preconditioner.Refresh(); + CHECK(geometryRefresh.changes.geometry); + CHECK_FALSE(geometryRefresh.changes.linearization); + CHECK_FALSE(geometryRefresh.DidAnyWork()); + CHECK(geometryRefresh.refreshedComponents == 0); + + problem.AdvanceDensity(); + const auto linearizationRefresh = preconditioner.Refresh(); + CHECK_FALSE(linearizationRefresh.changes.geometry); + CHECK(linearizationRefresh.changes.linearization); + CHECK(linearizationRefresh.DidAnyWork()); + CHECK(linearizationRefresh.refreshedComponents == 1); + CHECK(preconditioner.GetStatistics().componentRefreshes == 1); +} + +TEST_CASE( + "Stellar Preconditioner Application Requires Preallocated Compatible Vectors", + tags::preconditioning_runtime_unit +) { + preconditioning_runtime_test::Problem problem; + auto preconditioner = preconditioning::prepare(problem, preconditioning::makeIdentityPlan(problem)); + mfem::Vector residual(problem.EquationSize()); + mfem::Vector missingCorrection; + mfem::IdentityOperator wrongOperator(problem.StateSize() - 1); + + CHECK_THROWS_AS(preconditioner.Mult(residual, missingCorrection), std::invalid_argument); + CHECK_THROWS_AS(preconditioner.SetOperator(wrongOperator), std::invalid_argument); + + preconditioning_runtime_test::Problem unpreparedProblem; + unpreparedProblem.SetPrepared(false); + CHECK_THROWS_AS( + preconditioning::prepare(unpreparedProblem, preconditioning::makeIdentityPlan(unpreparedProblem)), + std::logic_error + ); +} diff --git a/tests/preconditioning/stellar_structure.cpp b/tests/preconditioning/stellar_structure.cpp new file mode 100644 index 0000000..802f593 --- /dev/null +++ b/tests/preconditioning/stellar_structure.cpp @@ -0,0 +1,419 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +import mean_field; +import test_helpers; + +namespace { + namespace backend = mean_field::preconditioning::backend; + namespace blocks = mean_field::utils::blocks; + namespace preconditioning = mean_field::preconditioning; + + struct MaterialValue final : blocks::value_block_base { }; + struct GravityValue final : blocks::value_block_base { }; + struct MaterialResidual final : blocks::residual_block_base { }; + struct GravityResidual final : blocks::residual_block_base { }; + + using MockForm = blocks::block_form< + blocks::type_list, + blocks::type_list>; + using MockJacobian = blocks::type_list< + blocks::block_row, + blocks::block_row>; + using MockMaterialComponent = preconditioning::ComponentDeclaration< + blocks::type_list, + blocks::type_list, + blocks::type_list>, + preconditioning::IdentityOperatorCharacteristics, + backend::Identity, + preconditioning::NoPreparationDependencies>; + using MockGravityComponent = preconditioning::ComponentDeclaration< + blocks::type_list, + blocks::type_list, + blocks::type_list>, + preconditioning::IdentityOperatorCharacteristics, + backend::Identity, + preconditioning::NoPreparationDependencies>; + using MockStructure = preconditioning::StellarStructureBlock< + MockMaterialComponent, + MockGravityComponent, + MockForm, + MockJacobian, + preconditioning::ApproximateStellarBlockLDU>; + + template + concept MockComponentsCanCompose = requires { + typename preconditioning::StellarStructureBlock< + MaterialComponent, GravityComponent, MockForm, MockJacobian, preconditioning::IndependentStellarSubsystems>; + }; + + class KnownCrossCouplings final { + public: + [[nodiscard]] constexpr int MaterialSize() const noexcept { + return 1; + } + + [[nodiscard]] constexpr int GravitySize() const noexcept { + return 1; + } + + void ApplyMaterialToGravity( + const mfem::Vector &materialDirection, + mfem::Vector &gravityAction + ) const { + gravityAction(0) = 3.0 * materialDirection(0); + } + + void ApplyGravityToMaterial( + const mfem::Vector &gravityDirection, + mfem::Vector &materialAction + ) const { + materialAction(0) = 7.0 * gravityDirection(0); + } + }; + + template + [[nodiscard]] mfem::Vector applyKnownFactorization( + Policy policy, + preconditioning::StellarStructureFactorizationStatistics *statistics = nullptr + ) { + mfem::Vector materialDiagonal(1); + mfem::Vector gravityDiagonal(1); + materialDiagonal(0) = 2.0; + gravityDiagonal(0) = 5.0; + auto materialInverse = backend::prepare(backend::Diagonal{}, materialDiagonal); + auto gravityInverse = backend::prepare(backend::Diagonal{}, gravityDiagonal); + const KnownCrossCouplings couplings; + preconditioning::StellarStructureFactorizationOperator factorization( + policy, materialInverse, gravityInverse, couplings + ); + mfem::Vector rightHandSide(2); + mfem::Vector action(2); + rightHandSide(0) = 11.0; + rightHandSide(1) = 13.0; + factorization.Mult(rightHandSide, action); + if (statistics != nullptr) { + *statistics = factorization.GetStatistics(); + } + return action; + } + + using PolytropicModel = mean_field::model::StellarModel>; + using PolytropicProblem = mean_field::equilibrium::StellarEquilibriumProblem; + using MaterialComponent = + decltype(preconditioning::materialSurfaceBlock(std::declval())); + using FixedAMG = backend::HypreBoomerAMG; + using GravityComponent = + preconditioning::GravityFieldBlock; + using PolytropicStructure = decltype(preconditioning::stellarStructureBlock( + std::declval(), + std::declval(), + std::declval(), + preconditioning::IndependentStellarSubsystems{} + )); + + [[nodiscard]] mean_field::operators::StellarEquilibriumDependencies + makeDependencies(const std::uint64_t revision = 1) { + return { + .discretization = {.identity = 9101, .revision = 1}, + .density = {.identity = 9103, .revision = revision}, + .surfaceDeformation = {.identity = 9107, .revision = revision}, + .gravityGradient = {.identity = 9111, .revision = revision}, + .gravityPotential = {.identity = 9117, .revision = revision}, + .enthalpy = {.identity = 9123, .revision = revision}, + .bernoulliConstant = {.identity = 9129, .revision = revision}, + .rotation = {.identity = 9131, .revision = revision}, + .targetMass = {.identity = 9137, .revision = 1} + }; + } + + [[nodiscard]] mean_field::physics::RigidRotation zeroRotation() { + mfem::Vector angularVelocity(3); + mfem::Vector center(3); + angularVelocity = 0.0; + center = 0.0; + return {angularVelocity, center}; + } + + [[nodiscard]] double relativeError( + const mfem::Vector &left, + const mfem::Vector &right + ) { + mfem::Vector difference(left); + difference -= right; + return difference.Norml2() / std::max({1.0, left.Norml2(), right.Norml2()}); + } +} // namespace + +TEST_CASE( + "Stellar Structure Composition Derives Both Cross-Subsystem Graphs", + "[preconditioning][stellar_structure][unit][type_contract]" +) { + using ExpectedMaterialToGravity = blocks::type_list< + preconditioning::Coupling, + preconditioning::Coupling, + preconditioning::Coupling>; + using ExpectedGravityToMaterial = blocks::type_list< + preconditioning::Coupling< + blocks::surface_deformation::shape_equilibrium::residual, blocks::gravity::gradient::value>, + preconditioning::Coupling>; + + STATIC_CHECK(preconditioning::PreconditionerComponent); + STATIC_CHECK(std::same_as); + STATIC_CHECK(std::same_as); + STATIC_CHECK(PolytropicStructure::MaterialToGravityCouplings::size == 3); + STATIC_CHECK(PolytropicStructure::GravityToMaterialCouplings::size == 2); + STATIC_CHECK(PolytropicStructure::RequiredCouplings::size == 16); + STATIC_CHECK(preconditioning::backend::ArnoldiAdmissible); + + using FixedMassIdentity = preconditioning::IdentityBlock< + blocks::fixed_total_mass::mass_normalization::value, blocks::fixed_total_mass::mass_normalization::residual>; + using FixedCentralDensityIdentity = preconditioning::IdentityBlock< + blocks::fixed_central_density::central_value::value, blocks::fixed_central_density::central_value::residual>; + using CompletePlan = + preconditioning::PreconditionerPlan; + STATIC_CHECK(preconditioning::CompletePreconditionerFor); + STATIC_CHECK( + preconditioning::CompatiblePreconditionerFor< + CompletePlan, typename PolytropicProblem::FormType, typename PolytropicProblem::JacobianFormType> + ); + + STATIC_CHECK(MockComponentsCanCompose); + STATIC_CHECK_FALSE(MockComponentsCanCompose); + STATIC_CHECK(preconditioning::PreconditionerComponent); + STATIC_CHECK(MockStructure::MaterialToGravityCouplings::size == 1); + STATIC_CHECK(MockStructure::GravityToMaterialCouplings::size == 1); +} + +TEST_CASE( + "Stellar Structure Factorizations Preserve Independent Triangular And Approximate LDU Algebra", + "[preconditioning][stellar_structure][unit][factorization]" +) { + const auto check = [](const mfem::Vector &value, const std::array expected) { + REQUIRE(value.Size() == 2); + CHECK(value(0) == Catch::Approx(expected[0]).margin(2.0e-14)); + CHECK(value(1) == Catch::Approx(expected[1]).margin(2.0e-14)); + mfem::Vector expectedVector(2); + expectedVector(0) = expected[0]; + expectedVector(1) = expected[1]; + CHECK(relativeError(value, expectedVector) <= 2.0e-14); + }; + + check(applyKnownFactorization(preconditioning::IndependentStellarSubsystems{}), {5.5, 2.6}); + check(applyKnownFactorization(preconditioning::MaterialThenGravityTriangular{}), {5.5, -0.7}); + check(applyKnownFactorization(preconditioning::GravityThenMaterialTriangular{}), {-3.6, 2.6}); + + preconditioning::StellarStructureFactorizationStatistics statistics; + check(applyKnownFactorization(preconditioning::ApproximateStellarBlockLDU{}, &statistics), {7.95, -0.7}); + CHECK(statistics.applications == 1); + CHECK(statistics.materialSurfaceInverseApplications == 2); + CHECK(statistics.gravityInverseApplications == 1); + CHECK(statistics.materialToGravityApplications == 1); + CHECK(statistics.gravityToMaterialApplications == 1); +} + +TEST_CASE( + "Stellar Structure Cross Actions Are Exact Restricted Jacobian Actions And Compose Prepared Blocks", + "[preconditioning][stellar_structure][integration]" +) { + using namespace mean_field; + const utils::Args arguments = test_utils::setup_args(); + fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0); + REQUIRE(finiteElements.okay()); + + constexpr double radius = utils::RADIUS; + constexpr double mass = utils::MASS; + const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v; + const double centralDensity = std::numbers::pi_v * mass / (4.0 * radius * radius * radius); + const auto stellarModel = model::StellarModel( + eos::Polytrope({.n = 1.0, .K = polytropicConstant}), + surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), + integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}), + constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}}) + ); + auto problem = equilibrium::discretize(stellarModel, finiteElements); + auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 512})); + problem.Prepare(projected.values, makeDependencies(), zeroRotation()); + const auto &physical = problem.GetPreparedOperator().GetPhysicalOperator(); + + preconditioning::StellarStructureCrossJacobianOperator cross(physical); + mfem::Vector direction(cross.Width()); + for (int index = 0; index < direction.Size(); ++index) { + direction(index) = 0.01 * std::sin(0.29 * static_cast(index + 1)); + } + mfem::Vector crossAction(cross.Height()); + cross.Mult(direction, crossAction); + + const mfem::Vector materialDirection(direction.GetData(), cross.MaterialSize()); + const mfem::Vector gravityDirection(direction.GetData() + cross.MaterialSize(), cross.GravitySize()); + const auto &materialOffsets = cross.GetMaterialOffsets(); + const auto &gravityOffsets = cross.GetGravityOffsets(); + + mfem::Vector materialOnlyDirection(physical.Width()); + materialOnlyDirection = 0.0; + auto materialOnlyView = physical.GetRootManifest().directionView(materialOnlyDirection); + mfem::Vector materialDensity = materialOnlyView.block(blocks::density_field.mass_term); + mfem::Vector materialSurface = materialOnlyView.block(blocks::surface_deformation_field.parameters_term); + mfem::Vector materialEnthalpy = materialOnlyView.block(blocks::enthalpy_field.specific_term); + const mfem::Vector sourceDensity( + const_cast(materialDirection.GetData()) + materialOffsets[0], + materialOffsets[1] - materialOffsets[0] + ); + const mfem::Vector sourceSurface( + const_cast(materialDirection.GetData()) + materialOffsets[1], + materialOffsets[2] - materialOffsets[1] + ); + const mfem::Vector sourceEnthalpy( + const_cast(materialDirection.GetData()) + materialOffsets[2], + materialOffsets[3] - materialOffsets[2] + ); + materialDensity = sourceDensity; + materialSurface = sourceSurface; + materialEnthalpy = sourceEnthalpy; + mfem::Vector materialOnlyAction; + physical.Mult(materialOnlyDirection, materialOnlyAction); + const auto materialOnlyActionView = physical.GetRootManifest().residualView(materialOnlyAction); + + mfem::Vector gravityOnlyDirection(physical.Width()); + gravityOnlyDirection = 0.0; + auto gravityOnlyView = physical.GetRootManifest().directionView(gravityOnlyDirection); + mfem::Vector gravityGradient = gravityOnlyView.block(blocks::gravity_field.gradient_term); + mfem::Vector gravityPotential = gravityOnlyView.block(blocks::gravity_field.poisson_term); + const mfem::Vector sourceGravityGradient( + const_cast(gravityDirection.GetData()) + gravityOffsets[0], + gravityOffsets[1] - gravityOffsets[0] + ); + const mfem::Vector sourceGravityPotential( + const_cast(gravityDirection.GetData()) + gravityOffsets[1], + gravityOffsets[2] - gravityOffsets[1] + ); + gravityGradient = sourceGravityGradient; + gravityPotential = sourceGravityPotential; + mfem::Vector gravityOnlyAction; + physical.Mult(gravityOnlyDirection, gravityOnlyAction); + const auto gravityOnlyActionView = physical.GetRootManifest().residualView(gravityOnlyAction); + + mfem::Vector expected(cross.Height()); + expected = 0.0; + expected.SetVector(gravityOnlyActionView.block(blocks::density_field.mass_term), materialOffsets[0]); + expected.SetVector( + gravityOnlyActionView.block(blocks::surface_deformation_field.shape_equilibrium_term), materialOffsets[1] + ); + expected.SetVector(gravityOnlyActionView.block(blocks::enthalpy_field.specific_term), materialOffsets[2]); + expected.SetVector( + materialOnlyActionView.block(blocks::gravity_field.gradient_term), cross.MaterialSize() + gravityOffsets[0] + ); + expected.SetVector( + materialOnlyActionView.block(blocks::gravity_field.poisson_term), cross.MaterialSize() + gravityOffsets[1] + ); + + const mfem::Vector expectedMaterial(expected.GetData(), cross.MaterialSize()); + const mfem::Vector expectedGravity(expected.GetData() + cross.MaterialSize(), cross.GravitySize()); + const mfem::Vector expectedDensity( + expectedMaterial.GetData() + materialOffsets[0], materialOffsets[1] - materialOffsets[0] + ); + const mfem::Vector expectedSurface( + expectedMaterial.GetData() + materialOffsets[1], materialOffsets[2] - materialOffsets[1] + ); + const mfem::Vector expectedEnthalpy( + expectedMaterial.GetData() + materialOffsets[2], materialOffsets[3] - materialOffsets[2] + ); + const mfem::Vector expectedGravityGradient( + expectedGravity.GetData() + gravityOffsets[0], gravityOffsets[1] - gravityOffsets[0] + ); + const mfem::Vector expectedGravityPotential( + expectedGravity.GetData() + gravityOffsets[1], gravityOffsets[2] - gravityOffsets[1] + ); + + const mfem::Vector crossMaterial(crossAction.GetData(), cross.MaterialSize()); + const mfem::Vector crossGravity(crossAction.GetData() + cross.MaterialSize(), cross.GravitySize()); + const mfem::Vector crossDensity( + crossMaterial.GetData() + materialOffsets[0], materialOffsets[1] - materialOffsets[0] + ); + const mfem::Vector crossSurface( + crossMaterial.GetData() + materialOffsets[1], materialOffsets[2] - materialOffsets[1] + ); + const mfem::Vector crossEnthalpy( + crossMaterial.GetData() + materialOffsets[2], materialOffsets[3] - materialOffsets[2] + ); + const mfem::Vector crossGravityGradient( + crossGravity.GetData() + gravityOffsets[0], gravityOffsets[1] - gravityOffsets[0] + ); + const mfem::Vector crossGravityPotential( + crossGravity.GetData() + gravityOffsets[1], gravityOffsets[2] - gravityOffsets[1] + ); + + INFO( + "gravity-to-material density-row error = " << relativeError(crossDensity, expectedDensity) + << ", actual norm = " << crossDensity.Norml2() + << ", expected norm = " << expectedDensity.Norml2() + ); + INFO( + "gravity-to-material surface-row error = " << relativeError(crossSurface, expectedSurface) + << ", actual norm = " << crossSurface.Norml2() + << ", expected norm = " << expectedSurface.Norml2() + ); + INFO( + "gravity-to-material enthalpy-row error = " << relativeError(crossEnthalpy, expectedEnthalpy) + << ", actual norm = " << crossEnthalpy.Norml2() + << ", expected norm = " << expectedEnthalpy.Norml2() + ); + INFO( + "material-to-gravity gradient-row error = " << relativeError(crossGravityGradient, expectedGravityGradient) + << ", actual norm = " << crossGravityGradient.Norml2() + << ", expected norm = " << expectedGravityGradient.Norml2() + ); + INFO( + "material-to-gravity Poisson-row error = " << relativeError(crossGravityPotential, expectedGravityPotential) + << ", actual norm = " << crossGravityPotential.Norml2() + << ", expected norm = " << expectedGravityPotential.Norml2() + ); + CHECK(relativeError(crossDensity, expectedDensity) <= 2.0e-12); + CHECK(relativeError(crossSurface, expectedSurface) <= 2.0e-12); + CHECK(relativeError(crossEnthalpy, expectedEnthalpy) <= 2.0e-12); + CHECK(relativeError(crossGravityGradient, expectedGravityGradient) <= 2.0e-12); + CHECK(relativeError(crossGravityPotential, expectedGravityPotential) <= 2.0e-12); + CHECK(relativeError(crossAction, expected) <= 2.0e-12); + + auto materialBlock = preconditioning::materialSurfaceBlock(problem); + auto gravityBlock = preconditioning::GravityFieldBlock( + backend::Diagonal{}, FixedAMG{backend::FixedCycles{.cycles = 1}}, preconditioning::GravityApproximateLDU{} + ); + auto structure = preconditioning::stellarStructureBlock( + problem, materialBlock, gravityBlock, preconditioning::ApproximateStellarBlockLDU{} + ); + auto prepared = preconditioning::prepare(problem, structure); + mfem::Vector rightHandSide(prepared.Width()); + mfem::Vector correction(prepared.Height()); + for (int index = 0; index < rightHandSide.Size(); ++index) { + rightHandSide(index) = std::cos(0.17 * static_cast(index + 1)); + } + prepared.Mult(rightHandSide, correction); + for (int index = 0; index < correction.Size(); ++index) { + REQUIRE(std::isfinite(correction(index))); + } + const auto &statistics = prepared.GetFactorization().GetStatistics(); + CHECK(statistics.applications == 1); + CHECK(statistics.materialSurfaceInverseApplications == 2); + CHECK(statistics.gravityInverseApplications == 1); + CHECK(statistics.materialToGravityApplications == 1); + CHECK(statistics.gravityToMaterialApplications == 1); + + const auto unchanged = prepared.Refresh(); + CHECK_FALSE(unchanged.DidAnyWork()); + CHECK(prepared.IsCurrent()); +} diff --git a/tests/surface/constant_surface_compilation.cpp b/tests/surface/constant_surface_compilation.cpp index 18106cd..f1ff6b2 100644 --- a/tests/surface/constant_surface_compilation.cpp +++ b/tests/surface/constant_surface_compilation.cpp @@ -25,12 +25,21 @@ namespace { struct EntropyField final { static constexpr std::string_view name = "entropy"; + using PhysicalQuantity = Entropy; }; struct ElectronFractionField final { static constexpr std::string_view name = "electron_fraction"; + using PhysicalQuantity = ElectronFraction; }; + struct EntropyValueBlock final : mean_field::utils::blocks::value_block_base { }; + struct EntropyResidualBlock final : mean_field::utils::blocks::residual_block_base { }; + struct ElectronFractionValueBlock final : mean_field::utils::blocks::value_block_base { }; + struct ElectronFractionResidualBlock final : mean_field::utils::blocks::residual_block_base { }; + struct GeneralEnthalpyValueBlock final : mean_field::utils::blocks::value_block_base { }; + struct GeneralEnthalpyResidualBlock final : mean_field::utils::blocks::residual_block_base { }; + using SpecificEnthalpyFromPressureEntropyAndElectronFraction = eos::Relation; @@ -151,13 +160,46 @@ namespace { } }; - using GeneralSurfaceFormulation = surface::SurfaceConstraintFormulation< - eos::quantity::SpecificEnthalpy, - field::Enthalpy, - surface::SurfaceStateBindings< - surface::SurfaceStateBinding, - surface::SurfaceStateBinding, - surface::SurfaceStateBinding>>; + using GeneralAvailableThermodynamicEquations = mean_field::material::ThermodynamicEquationCatalog< + mean_field::material::ThermodynamicEquation, + mean_field::material:: + ThermodynamicEquation, + mean_field::material:: + ThermodynamicEquation>; + + using GeneralMaterialSurfaceForm = mean_field::utils::blocks::block_form< + mean_field::utils::blocks::type_list< + EntropyValueBlock, + ElectronFractionValueBlock, + mean_field::utils::blocks::surface_deformation::parameters::value, + GeneralEnthalpyValueBlock>, + mean_field::utils::blocks::type_list< + EntropyResidualBlock, + ElectronFractionResidualBlock, + mean_field::utils::blocks::surface_deformation::shape_equilibrium::residual, + GeneralEnthalpyResidualBlock>>; + using GeneralMaterialSurfaceJacobian = mean_field::utils::blocks::type_list< + mean_field::utils::blocks::block_row, + mean_field::utils::blocks:: + block_row, + mean_field::utils::blocks::block_row< + mean_field::utils::blocks::surface_deformation::shape_equilibrium::residual, + EntropyValueBlock, + ElectronFractionValueBlock, + mean_field::utils::blocks::surface_deformation::parameters::value, + GeneralEnthalpyValueBlock>, + mean_field::utils::blocks::block_row< + GeneralEnthalpyResidualBlock, + EntropyValueBlock, + ElectronFractionValueBlock, + mean_field::utils::blocks::surface_deformation::parameters::value, + GeneralEnthalpyValueBlock>>; + + using GeneralThermodynamicEquations = mean_field::material::CompiledThermodynamicEquationsT< + GeneralStellarMatterEquationOfState, + GeneralMaterialSurfaceForm, + GeneralAvailableThermodynamicEquations>; + using GeneralSurfaceFormulation = GeneralThermodynamicEquations::PressureSurfaceFormulation; struct PolytropicSurfaceState final { double specificEnthalpy; @@ -187,6 +229,9 @@ namespace { template concept HasTargetEnthalpy = requires(const Candidate &candidate) { candidate.targetEnthalpy; }; + + template + concept SelectsThermodynamicCarrier = requires { typename Candidate::CarrierField; }; } // namespace TEST_CASE( @@ -198,6 +243,7 @@ TEST_CASE( STATIC_CHECK_FALSE(std::constructible_from); STATIC_CHECK_FALSE(std::constructible_from); STATIC_CHECK(std::same_as); + STATIC_CHECK_FALSE(SelectsThermodynamicCarrier); STATIC_CHECK(std::is_trivially_copyable_v); STATIC_CHECK(std::is_trivially_copyable_v); STATIC_CHECK(std::is_trivially_copyable_v); @@ -218,8 +264,12 @@ TEST_CASE( "Polytropic EOS Resolves Constant Surface Pressure Through Its Enthalpy Relation", tags::surface_constraint_compilation ) { - using Formulation = surface::BarotropicSurfaceFormulation; + using ThermodynamicEquations = mean_field::material::CompiledThermodynamicEquationsT< + eos::Polytrope, mean_field::utils::blocks::surface_deformed_stellar_equilibrium_form, + mean_field::material::StellarEquilibriumThermodynamicEquations>; + using Formulation = ThermodynamicEquations::PressureSurfaceFormulation; + STATIC_CHECK(mean_field::material::CompiledThermodynamicEquations); STATIC_CHECK(surface::PressureSurfaceCompilable); STATIC_CHECK_FALSE(surface::PressureSurfaceCompilable); @@ -255,7 +305,10 @@ TEST_CASE( "General EOS Resolves Constant Surface Pressure With Local Composition", tags::surface_constraint_compilation ) { - STATIC_CHECK(surface::PressureSurfaceCompilable); + using Formulation = GeneralThermodynamicEquations::PressureSurfaceFormulation; + + STATIC_CHECK(mean_field::material::CompiledThermodynamicEquations); + STATIC_CHECK(surface::PressureSurfaceCompilable); STATIC_CHECK_FALSE( surface::PressureSurfaceCompilable ); @@ -267,11 +320,10 @@ TEST_CASE( const GeneralStellarMatterEquationOfState equationOfState; const surface::ConstantPressureSurface pressureSurface{eos::PressureValue{0.4}}; - const auto constraint = - surface::compilePressureSurfaceConstraint(pressureSurface, equationOfState); + const auto constraint = surface::compilePressureSurfaceConstraint(pressureSurface, equationOfState); - using Constraint = std::remove_cvref_t; - using Dependencies = Constraint::SurfaceDependencies; + using Constraint = std::remove_cvref_t; + using Dependencies = Constraint::SurfaceDependencies; STATIC_CHECK(std::same_as); STATIC_CHECK( @@ -297,6 +349,50 @@ TEST_CASE( CHECK(runtimeDependencies.stateFields[2] == surface::surfaceFieldId); } +TEST_CASE( + "Compiled Material Surface Descriptors Change With The Material State Basis", + tags::surface_constraint_compilation +) { + using Constraint = surface::CompiledPressureSurfaceConstraintT< + GeneralThermodynamicEquations::PressureSurfaceFormulation, GeneralStellarMatterEquationOfState>; + using Descriptor = mean_field::preconditioning::CompiledMaterialSurfaceDescriptor< + GeneralThermodynamicEquations, Constraint, GeneralMaterialSurfaceForm, GeneralMaterialSurfaceJacobian>; + using IncompleteForm = mean_field::utils::blocks::block_form< + mean_field::utils::blocks::type_list< + EntropyValueBlock, mean_field::utils::blocks::surface_deformation::parameters::value, + GeneralEnthalpyValueBlock>, + mean_field::utils::blocks::type_list< + EntropyResidualBlock, mean_field::utils::blocks::surface_deformation::shape_equilibrium::residual, + GeneralEnthalpyResidualBlock>>; + using IncompleteJacobian = mean_field::utils::blocks::type_list< + mean_field::utils::blocks::block_row, + mean_field::utils::blocks::block_row< + mean_field::utils::blocks::surface_deformation::shape_equilibrium::residual, EntropyValueBlock, + mean_field::utils::blocks::surface_deformation::parameters::value, GeneralEnthalpyValueBlock>, + mean_field::utils::blocks::block_row< + GeneralEnthalpyResidualBlock, EntropyValueBlock, + mean_field::utils::blocks::surface_deformation::parameters::value, GeneralEnthalpyValueBlock>>; + using IncompleteDescriptor = mean_field::preconditioning::CompiledMaterialSurfaceDescriptor< + GeneralThermodynamicEquations, Constraint, IncompleteForm, IncompleteJacobian>; + + STATIC_CHECK(mean_field::preconditioning::MaterialSurfaceDescriptor); + STATIC_CHECK_FALSE(mean_field::preconditioning::MaterialSurfaceDescriptor); + STATIC_CHECK(Descriptor::CorrectionBlocks::size == 4); + STATIC_CHECK(Descriptor::ResidualBlocks::size == 4); + STATIC_CHECK(Descriptor::RequiredCouplings::size == 12); + STATIC_CHECK( + std::same_as< + Descriptor::SurfaceStateFields, field::TypeList> + ); + STATIC_CHECK( + std::same_as< + Descriptor::CorrectionBlocks, + mean_field::utils::blocks::type_list< + EntropyValueBlock, ElectronFractionValueBlock, + mean_field::utils::blocks::surface_deformation::parameters::value, GeneralEnthalpyValueBlock>> + ); +} + TEST_CASE( "General EOS Pressure Surface Jacobian Includes Every Local State Dependency", tags::surface_constraint_jacobian diff --git a/tests/test_helpers.cppm b/tests/test_helpers.cppm index 547132d..4246d05 100644 --- a/tests/test_helpers.cppm +++ b/tests/test_helpers.cppm @@ -477,7 +477,14 @@ export namespace tags { inline constexpr auto stellar_seed_projection = model & initialization & solver & make_tag("seed_projection"); inline constexpr auto stellar_seed_projection_type_contract = stellar_seed_projection & unit & make_tag("type_contract"); - inline constexpr auto preconditioning_diagnostics = solver & make_tag("preconditioning") & make_tag("diagnostics"); + inline constexpr auto preconditioning = solver & make_tag("preconditioning"); + inline constexpr auto preconditioning_type_contract = preconditioning & unit & make_tag("type_contract"); + inline constexpr auto preconditioning_runtime_unit = preconditioning & unit & make_tag("runtime"); + inline constexpr auto preconditioning_backend_unit = preconditioning & unit & make_tag("backend"); + inline constexpr auto preconditioning_gravity_unit = preconditioning & gravity & unit & make_tag("gravity_block"); + inline constexpr auto preconditioning_gravity_integration = + preconditioning & gravity & integration & make_tag("gravity_block"); + inline constexpr auto preconditioning_diagnostics = preconditioning & make_tag("diagnostics"); inline constexpr auto preconditioning_diagnostics_unit = preconditioning_diagnostics & unit; inline constexpr auto preconditioning_spectral_unit = preconditioning_diagnostics_unit & make_tag("spectrum"); inline constexpr auto root_manifest_type_contract = diff --git a/tests/test_main.cpp b/tests/test_main.cpp index 4e44ba1..6ee8bb2 100644 --- a/tests/test_main.cpp +++ b/tests/test_main.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -235,6 +236,23 @@ class CheckReporter : public Catch::StreamingReporterBase { std::vector m_testRunData; std::chrono::time_point m_testStartTime; + static bool isRootProcess() { + int initialized = 0; + int finalized = 0; + MPI_Initialized(&initialized); + if (initialized == 0) { + return true; + } + MPI_Finalized(&finalized); + if (finalized != 0) { + return true; + } + + int rank = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + return rank == 0; + } + void captureInfoMessages(Catch::AssertionStats const &assertionStats) { for (auto const &message : assertionStats.infoMessages) { if (m_currentInfoSequences.insert(message.sequence).second) { @@ -263,6 +281,10 @@ public: void testRunStarting(Catch::TestRunInfo const &_testRunInfo) override { StreamingReporterBase::testRunStarting(_testRunInfo); + if (!isRootProcess()) { + return; + } + std::cout << '\n'; std::cout << std::left << std::setw(85) << "Test Case Name" << "Status " << std::right << std::setw(8) << "Passed" << std::setw(8) << "Failed" << std::setw(12) @@ -273,7 +295,11 @@ public: void testCaseStarting(Catch::TestCaseInfo const &testInfo) override { StreamingReporterBase::testCaseStarting(testInfo); - m_testStartTime = std::chrono::steady_clock::now(); + m_testStartTime = std::chrono::steady_clock::now(); + if (!isRootProcess()) { + return; + } + std::string name = testInfo.name; auto wrappedName = wrapText(name, 83); @@ -324,31 +350,36 @@ public: std::string name = stats.testInfo->name; auto wrappedName = wrapText(name, 83); - // Overwrite the loading line with the actual result - std::cout << "\r\033[K" << std::left << std::setw(85) << wrappedName[0] << mark << " " << std::right - << std::setw(8) << stats.totals.assertions.passed << std::setw(8) << stats.totals.assertions.failed - << std::setw(11) << std::fixed << std::setprecision(3) << duration_s << "s\n"; + if (isRootProcess()) { + // Overwrite the loading line with the actual result + std::cout << "\r\033[K" << std::left << std::setw(85) << wrappedName[0] << mark << " " << std::right + << std::setw(8) << stats.totals.assertions.passed << std::setw(8) + << stats.totals.assertions.failed << std::setw(11) << std::fixed << std::setprecision(3) + << duration_s << "s\n"; - for (size_t i = 1; i < wrappedName.size(); ++i) { - std::cout << " \033[90m↳ \033[0m" // Dim indent arrow - << std::left << std::setw(81) << wrappedName[i] << '\n'; + for (size_t i = 1; i < wrappedName.size(); ++i) { + std::cout << " \033[90m↳ \033[0m" // Dim indent arrow + << std::left << std::setw(81) << wrappedName[i] << '\n'; + } + + std::string tagsStr = stats.testInfo->tagsAsString(); + if (!tagsStr.empty()) { + auto wrappedTags = wrapText("Tags: " + tagsStr, 83); + for (const auto &line : wrappedTags) { + std::cout << " \033[36m" << line << "\033[0m\n"; // Cyan + } + } + + if (!m_currentFailures.empty()) { + std::cout << '\n'; + for (auto const &failure : m_currentFailures) { + std::cout << failure << '\n'; + } + std::cout << std::string(133, '-') << '\n'; + } } std::string tagsStr = stats.testInfo->tagsAsString(); - if (!tagsStr.empty()) { - auto wrappedTags = wrapText("Tags: " + tagsStr, 83); - for (const auto &line : wrappedTags) { - std::cout << " \033[36m" << line << "\033[0m\n"; // Cyan - } - } - - if (!m_currentFailures.empty()) { - std::cout << '\n'; - for (auto const &failure : m_currentFailures) { - std::cout << failure << '\n'; - } - std::cout << std::string(133, '-') << '\n'; - } m_testRunData.push_back( {name, tagsStr, passed, stats.totals.assertions.passed, stats.totals.assertions.failed, duration_s, @@ -363,6 +394,10 @@ public: void testRunEnded(Catch::TestRunStats const &_testRunStats) override { StreamingReporterBase::testRunEnded(_testRunStats); + if (!isRootProcess()) { + return; + } + std::cout << std::string(133, '=') << '\n'; auto const &tc = _testRunStats.totals.testCases; @@ -587,12 +622,18 @@ int main( mfem::Mpi::Init(argc, argv); + std::uint32_t synchronized_seed = session.configData().rngSeed; + MPI_Bcast(&synchronized_seed, 1, MPI_UINT32_T, 0, MPI_COMM_WORLD); + session.configData().rngSeed = synchronized_seed; + constexpr std::string device_config = "cpu"; mfem::Device device(device_config); const int hdiv_max_q1d = mfem::DeviceDofQuadLimits::Get().HDIV_MAX_Q1D; - std::cout << "H(div) maximum Q1D = " << hdiv_max_q1d << '\n'; - std::cout << "Approximate maximum safe integration order = " << 2 * hdiv_max_q1d - 1 << '\n'; + if (mfem::Mpi::Root()) { + std::cout << "H(div) maximum Q1D = " << hdiv_max_q1d << '\n'; + std::cout << "Approximate maximum safe integration order = " << 2 * hdiv_max_q1d - 1 << '\n'; + } mean_field::utils::Args test_args = cfg.main(); @@ -611,4 +652,4 @@ int main( test_utils::set_args(std::move(test_args)); return session.run(); -} \ No newline at end of file +} diff --git a/tests/user-api/stellar_equilibrium.cpp b/tests/user-api/stellar_equilibrium.cpp new file mode 100644 index 0000000..f4a288b --- /dev/null +++ b/tests/user-api/stellar_equilibrium.cpp @@ -0,0 +1,116 @@ +#include +#include + +#include + +import mean_field; +import test_helpers; + +namespace { + [[nodiscard]] mean_field::fem::FEM makeFiniteElements() { + const mean_field::utils::Args arguments = test_utils::setup_args(); + return mean_field::fem::setup_fem(arguments.mesh_file, arguments, 0); + } +} // namespace + +TEST_CASE( + "Simple User API Builds A Stellar Model With Its Default Preconditioner", + "[user-api][simple]" +) { + using namespace mean_field; + + auto finiteElements = makeFiniteElements(); + REQUIRE(finiteElements.okay()); + + auto model = model::StellarModel( + eos::Polytrope({.n = 1.0, .K = 0.25}), surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), + integral::FixedTotalMass({.Mtotal = dimensions::MassValue{1.0}}) + ); + + auto problem = equilibrium::discretize(model, finiteElements); + auto preconditioner = preconditioning::makePreconditioner(problem); + const auto &gravity = preconditioner.structureComponent().gravityComponent(); + + STATIC_CHECK(preconditioning::PreconditionerComponent); + STATIC_CHECK( + std::same_as< + typename std::remove_cvref_t::MassBackend, preconditioning::backend::MatrixFreeChebyshev> + ); + CHECK(problem.StateSize() == problem.EquationSize()); + CHECK(decltype(preconditioner)::borderValueArity == 1); + CHECK(gravity.massInverseBackend().order == 5); + CHECK(gravity.potentialSchurBackend().application.cycles == 3); +} + +TEST_CASE( + "Intermediate User API Selects A Coupled Stellar Factorization", + "[user-api][intermediate]" +) { + using namespace mean_field; + + auto finiteElements = makeFiniteElements(); + REQUIRE(finiteElements.okay()); + + auto model = model::StellarModel( + eos::Polytrope({.n = 3.0, .K = 0.25}), surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), + integral::FixedTotalMass({.Mtotal = dimensions::MassValue{1.0}}), + constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{1.0}}) + ); + auto problem = equilibrium::discretize(model, finiteElements); + + auto material = preconditioning::materialSurfaceBlock(problem); + auto gravity = preconditioning::GravityFieldBlock( + preconditioning::backend::Diagonal{}, + preconditioning::backend::HypreBoomerAMG(preconditioning::backend::FixedCycles{.cycles = 2}), + preconditioning::GravityApproximateLDU{} + ); + auto structure = preconditioning::stellarStructureBlock( + problem, material, gravity, preconditioning::ApproximateStellarBlockLDU{} + ); + auto preconditioner = preconditioning::specificationBorderBlock(problem, structure); + + STATIC_CHECK(preconditioning::PreconditionerComponent); + STATIC_CHECK(preconditioning::backend::ArnoldiAdmissible); + CHECK(decltype(preconditioner)::borderValueArity == 2); +} + +TEST_CASE( + "Advanced User API Composes Explicit Blocks Backends And Application Modes", + "[user-api][advanced]" +) { + using namespace mean_field; + + auto finiteElements = makeFiniteElements(); + REQUIRE(finiteElements.okay()); + + auto model = model::StellarModel( + eos::Polytrope({.n = 3.0, .K = 0.25}), surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), + integral::FixedTotalMass({.Mtotal = dimensions::MassValue{1.0}}), + constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{1.0}}) + ); + auto problem = equilibrium::discretize(model, finiteElements); + + auto material = preconditioning::materialSurfaceBlock( + problem, preconditioning::backend::Diagonal{}, preconditioning::backend::Diagonal{}, + preconditioning::MaterialThenSurfaceTriangular{}, {.relativeFloor = 1.0e-10, .absoluteFloor = 1.0e-13} + ); + auto gravity = preconditioning::GravityFieldBlock( + preconditioning::backend::Diagonal{}, + preconditioning::backend::HypreBoomerAMG( + preconditioning::backend::SolveToTolerance{.relativeTolerance = 1.0e-8, .maximumCycles = 20} + ), + preconditioning::GravityUpperTriangular{} + ); + auto structure = preconditioning::stellarStructureBlock( + problem, material, gravity, preconditioning::MaterialThenGravityTriangular{} + ); + auto preconditioner = + preconditioning::specificationBorderBlock(problem, structure, preconditioning::backend::DenseDirect{}); + + STATIC_CHECK(preconditioning::PreconditionerComponent); + STATIC_CHECK_FALSE(preconditioning::backend::ArnoldiAdmissible); + CHECK(preconditioner.structureComponent().materialSurfaceComponent().diagonalOptions().relativeFloor == 1.0e-10); + CHECK( + preconditioner.structureComponent().gravityComponent().potentialSchurBackend().application.maximumCycles == 20 + ); +} diff --git a/tests/utils/profiling.cpp b/tests/utils/profiling.cpp new file mode 100644 index 0000000..1572168 --- /dev/null +++ b/tests/utils/profiling.cpp @@ -0,0 +1,91 @@ +#include "profile.h" + +#include +#include +#include +#include +#include + +import test_helpers; + +TEST_CASE( + "Profiling Registry Tracks Warmups Timings And Work", + tags::unit &tags::utils +) { + mean_field::profiling::Registry ®istry = mean_field::profiling::Registry::Get(); + registry.Reset(); + + registry.Record("deterministic-region", 1.0, 2); + registry.Record("deterministic-region", 2.0, 2); + registry.Record("deterministic-region", 3.0, 2); + registry.AddCount("deterministic-region", 7); + registry.AddCount("deterministic-region", 5); + + const auto snapshot = registry.Snapshot(); + REQUIRE(snapshot.contains("deterministic-region")); + + const mean_field::profiling::Statistics &statistics = snapshot.at("deterministic-region"); + CHECK(statistics.observations == 3); + CHECK(statistics.warmups == 2); + CHECK(statistics.samples == 1); + CHECK(statistics.warmup_target == 2); + CHECK(statistics.work_units == 12); + CHECK(statistics.total_seconds == 3.0); + CHECK(statistics.minimum_seconds == 3.0); + CHECK(statistics.maximum_seconds == 3.0); +} + +TEST_CASE( + "Profiling Registry Rejects Invalid Inputs Explicitly", + tags::unit &tags::utils +) { + mean_field::profiling::Registry ®istry = mean_field::profiling::Registry::Get(); + + CHECK_THROWS_AS(registry.Record("", 1.0), std::invalid_argument); + CHECK_THROWS_AS(registry.Record("negative-duration", -1.0), std::invalid_argument); + CHECK_THROWS_AS( + registry.Record("infinite-duration", std::numeric_limits::infinity()), std::invalid_argument + ); +} + +TEST_CASE( + "Profiling Registry Produces Deterministic Human And CSV Reports", + tags::unit &tags::utils +) { + mean_field::profiling::Registry ®istry = mean_field::profiling::Registry::Get(); + registry.Reset(); + registry.Record("report-region", 0.25); + registry.AddCount("report-region", 9); + + std::ostringstream human_report; + registry.Print(MPI_COMM_WORLD, human_report); + CHECK(human_report.str().find("report-region") != std::string::npos); + CHECK(human_report.str().find("MPI ranks: 1") != std::string::npos); + + std::ostringstream csv_report; + registry.PrintCsv(MPI_COMM_WORLD, csv_report); + CHECK(csv_report.str().find("maximum_rank_total_seconds") != std::string::npos); + CHECK(csv_report.str().find("\"report-region\"") != std::string::npos); +} + +#if MEAN_FIELD_ENABLE_PROFILING +TEST_CASE( + "Profiling Scope Macros Preserve Warmup Semantics", + tags::unit &tags::utils +) { + mean_field::profiling::Registry ®istry = mean_field::profiling::Registry::Get(); + registry.Reset(); + + for (int observation = 0; observation < 3; ++observation) { + MEAN_FIELD_PROFILE_SCOPE_WARMUP("macro-region", 1); + } + MEAN_FIELD_PROFILE_COUNT("macro-region", 4); + + const auto snapshot = registry.Snapshot(); + REQUIRE(snapshot.contains("macro-region")); + CHECK(snapshot.at("macro-region").observations == 3); + CHECK(snapshot.at("macro-region").warmups == 1); + CHECK(snapshot.at("macro-region").samples == 2); + CHECK(snapshot.at("macro-region").work_units == 4); +} +#endif