#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include import experiment; import mean_field; import test_helpers; namespace { using Clock = std::chrono::steady_clock; namespace backend = mean_field::preconditioning::backend; namespace preconditioning = mean_field::preconditioning; namespace solver = mean_field::solver; struct MaterialBlockMeasurements final { double density{0.0}; double surface{0.0}; double enthalpy{0.0}; }; [[nodiscard]] const char *buildConfiguration() noexcept { #ifdef NDEBUG return "release"; #else return "debug"; #endif } [[nodiscard]] double maximumRankSeconds( const Clock::time_point start, const MPI_Comm communicator ) { const double localSeconds = std::chrono::duration(Clock::now() - start).count(); double maximumSeconds = 0.0; MPI_Allreduce(&localSeconds, &maximumSeconds, 1, MPI_DOUBLE, MPI_MAX, communicator); return maximumSeconds; } [[nodiscard]] double globalNorm( const mfem::Vector &vector, const MPI_Comm communicator ) { const double localSquaredNorm = vector * vector; double globalSquaredNorm = 0.0; MPI_Allreduce(&localSquaredNorm, &globalSquaredNorm, 1, MPI_DOUBLE, MPI_SUM, communicator); return std::sqrt(std::max(globalSquaredNorm, 0.0)); } void announce( const MPI_Comm communicator, const std::string &message ) { int rank = 0; MPI_Comm_rank(communicator, &rank); if (rank == 0) { std::cout << "[P9 material-surface] " << message << std::endl; } } [[nodiscard]] mean_field::operators::StellarEquilibriumDependencies makeDependencies() { return { .discretization = {.identity = 10103, .revision = 1}, .density = {.identity = 10111, .revision = 1}, .surfaceDeformation = {.identity = 10133, .revision = 1}, .gravityGradient = {.identity = 10139, .revision = 1}, .gravityPotential = {.identity = 10141, .revision = 1}, .enthalpy = {.identity = 10151, .revision = 1}, .bernoulliConstant = {.identity = 10159, .revision = 1}, .rotation = {.identity = 10163, .revision = 1}, .targetMass = {.identity = 10169, .revision = 1} }; } [[nodiscard]] mean_field::physics::RigidRotation zeroRotation() { mfem::Vector angularVelocity(3); mfem::Vector center(3); angularVelocity = 0.0; center = 0.0; return {angularVelocity, center}; } [[nodiscard]] mfem::Vector blockBalancedDirection( const mfem::Array &offsets, const double phase, const MPI_Comm communicator ) { mfem::Vector direction(offsets.Last()); direction = 0.0; for (int block = 0; block < offsets.Size() - 1; ++block) { mfem::Vector values(direction, offsets[block], offsets[block + 1] - offsets[block]); for (int index = 0; index < values.Size(); ++index) { const double ordinal = static_cast(index + 1); values(index) = std::sin(0.371 * ordinal + phase + static_cast(block)) + 0.29 * std::cos(0.173 * ordinal - 0.5 * phase); } const double norm = globalNorm(values, communicator); REQUIRE(norm > 0.0); values /= norm; values.SyncAliasMemory(direction); } return direction; } [[nodiscard]] MaterialBlockMeasurements blockNorms( const mfem::Vector &vector, const mfem::Array &offsets, const MPI_Comm communicator ) { REQUIRE(offsets.Size() == 4); const mfem::Vector density(const_cast(vector.GetData()) + offsets[0], offsets[1] - offsets[0]); const mfem::Vector surface(const_cast(vector.GetData()) + offsets[1], offsets[2] - offsets[1]); const mfem::Vector enthalpy(const_cast(vector.GetData()) + offsets[2], offsets[3] - offsets[2]); return { .density = globalNorm(density, communicator), .surface = globalNorm(surface, communicator), .enthalpy = globalNorm(enthalpy, communicator) }; } [[nodiscard]] MaterialBlockMeasurements relativeBlockNorms( const mfem::Vector &numerator, const mfem::Vector &denominator, const mfem::Array &offsets, const MPI_Comm communicator ) { const MaterialBlockMeasurements numeratorNorms = blockNorms(numerator, offsets, communicator); const MaterialBlockMeasurements denominatorNorms = blockNorms(denominator, offsets, communicator); constexpr double floor = 1.0e-300; return { .density = numeratorNorms.density / std::max(denominatorNorms.density, floor), .surface = numeratorNorms.surface / std::max(denominatorNorms.surface, floor), .enthalpy = numeratorNorms.enthalpy / std::max(denominatorNorms.enthalpy, floor) }; } [[nodiscard]] std::map< std::string, std::string> commonParameters( const std::string &candidate, const std::string &measurement, const int dimension ) { return { {"build_configuration", buildConfiguration()}, {"candidate", candidate}, {"equation_of_state", "Polytrope(n=1)"}, {"experiment_schema", "p9_material_surface_v1"}, {"factorization", candidate}, {"linearization_state", "projected_lane_emden"}, {"measurement", measurement}, {"mesh_file", test_utils::setup_args().mesh_file}, {"operator", "restricted_material_surface_jacobian"}, {"preconditioned_product", "A_material_surface M^-1"}, {"root_dimension", std::to_string(dimension)}, {"rotation", "zero"} }; } void recordSpectrum( const std::string &candidate, const solver::ArnoldiSpectralMeasurement &spectrum, const int dimension, const double setupSeconds ) { experiment::record_experiment_result( "material_surface_preconditioning_p9", candidate + "_arnoldi_summary", commonParameters(candidate, "arnoldi_summary", dimension), {{"setup_seconds_maximum_rank", setupSeconds}, {"requested_dimension", static_cast(spectrum.requestedDimension)}, {"achieved_dimension", static_cast(spectrum.achievedDimension)}, {"invariant_subspace_found", spectrum.invariantSubspaceFound ? 1.0 : 0.0}, {"operator_applications", static_cast(spectrum.operatorApplications)}, {"measurement_seconds_maximum_rank", spectrum.measurementSecondsMaximumRank}, {"operator_application_seconds_maximum_rank", spectrum.operatorApplicationSecondsMaximumRank}, {"projected_condition_proxy", spectrum.projectedConditionProxy}, {"projected_largest_singular_value", spectrum.projectedLargestSingularValue}, {"projected_smallest_singular_value", spectrum.projectedSmallestSingularValue}, {"centroid_real_part", spectrum.centroidRealPart}, {"centroid_imaginary_part", spectrum.centroidImaginaryPart}, {"rms_distance_from_one", spectrum.rmsDistanceFromOne}, {"rms_cluster_radius", spectrum.rmsClusterRadius}, {"minimum_magnitude", spectrum.minimumMagnitude}, {"maximum_magnitude", spectrum.maximumMagnitude}, {"minimum_real_part", spectrum.minimumRealPart}, {"maximum_real_part", spectrum.maximumRealPart}, {"maximum_absolute_imaginary_part", spectrum.maximumAbsoluteImaginaryPart}, {"negative_real_part_count", static_cast(spectrum.negativeRealPartCount)}, {"converged_ritz_value_count", static_cast(spectrum.convergedRitzValueCount)}, {"conjugate_pair_defect", spectrum.conjugatePairDefect}, {"projected_departure_from_normality", spectrum.projectedDepartureFromNormality}, {"field_of_values_minimum_real_part", spectrum.projectedFieldOfValuesMinimumRealPart}, {"field_of_values_maximum_real_part", spectrum.projectedFieldOfValuesMaximumRealPart}} ); std::vector ordered = spectrum.ritzValues; std::ranges::sort(ordered, [](const auto &left, const auto &right) { if (left.realPart != right.realPart) { return left.realPart < right.realPart; } return left.imaginaryPart < right.imaginaryPart; }); for (std::size_t index = 0; index < ordered.size(); ++index) { const auto &value = ordered[index]; experiment::record_experiment_result( "material_surface_preconditioning_p9", candidate + "_ritz_" + std::to_string(index), commonParameters(candidate, "ritz_value", dimension), {{"ritz_index", static_cast(index)}, {"real_part", value.realPart}, {"imaginary_part", value.imaginaryPart}, {"magnitude", value.magnitude}, {"distance_from_one", value.distanceFromOne}, {"residual_estimate", value.residualEstimate}, {"relative_residual_estimate", value.relativeResidualEstimate}, {"converged", value.converged ? 1.0 : 0.0}} ); } } template void measureCandidate( const std::string &candidate, Preconditioner &inversePreconditioner, const double setupSeconds, const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation, const mfem::Vector &exactCorrection, const mfem::Vector &rightHandSide, const mfem::Vector &arnoldiDirection, const MPI_Comm communicator, std::map< std::string, double> preparationMetrics = {} ) { constexpr int maximumIterations = 40; constexpr int restartDimension = 20; constexpr int arnoldiDimension = 16; solver::InstrumentedOperator instrumentedOperation(operation); solver::InstrumentedPreconditioner instrumentedPreconditioner(inversePreconditioner); solver::ResidualHistoryMonitor monitor; mfem::FGMRESSolver krylov(communicator); krylov.SetPreconditioner(instrumentedPreconditioner); krylov.SetOperator(instrumentedOperation); krylov.SetMonitor(monitor); krylov.SetRelTol(1.0e-8); krylov.SetAbsTol(1.0e-12); krylov.SetMaxIter(maximumIterations); krylov.SetKDim(restartDimension); krylov.SetPrintLevel(0); mfem::Vector solution(operation.Width()); solution = 0.0; announce(communicator, "solving manufactured system with " + candidate); const Clock::time_point solveStart = Clock::now(); krylov.Mult(rightHandSide, solution); const double solveSeconds = maximumRankSeconds(solveStart, communicator); mfem::Vector trueResidual(operation.Height()); operation.Mult(solution, trueResidual); trueResidual -= rightHandSide; mfem::Vector solutionError(solution); solutionError -= exactCorrection; const double trueRelativeResidual = globalNorm(trueResidual, communicator) / std::max(globalNorm(rightHandSide, communicator), std::numeric_limits::min()); const double relativeSolutionError = globalNorm(solutionError, communicator) / std::max(globalNorm(exactCorrection, communicator), std::numeric_limits::min()); const MaterialBlockMeasurements relativeResidualBlocks = relativeBlockNorms(trueResidual, rightHandSide, operation.GetOffsets(), communicator); mfem::Vector preconditionedDirection(operation.Height()); inversePreconditioner.Mult(arnoldiDirection, preconditionedDirection); mfem::Vector defect(operation.Height()); operation.Mult(preconditionedDirection, defect); defect -= arnoldiDirection; const MaterialBlockMeasurements defectBlocks = blockNorms(defect, operation.GetOffsets(), communicator); const double defectNorm = globalNorm(defect, communicator) / std::max(globalNorm(arnoldiDirection, communicator), std::numeric_limits::min()); std::map solveMetrics{ {"setup_seconds_maximum_rank", setupSeconds}, {"maximum_iterations", static_cast(maximumIterations)}, {"restart_dimension", static_cast(restartDimension)}, {"solver_converged", krylov.GetConverged() ? 1.0 : 0.0}, {"outer_iterations", static_cast(krylov.GetNumIterations())}, {"reported_initial_residual_norm", std::abs(krylov.GetInitialNorm())}, {"reported_final_residual_norm", std::abs(krylov.GetFinalNorm())}, {"true_relative_residual", trueRelativeResidual}, {"relative_solution_error", relativeSolutionError}, {"density_relative_residual", relativeResidualBlocks.density}, {"surface_relative_residual", relativeResidualBlocks.surface}, {"enthalpy_relative_residual", relativeResidualBlocks.enthalpy}, {"right_preconditioned_defect", defectNorm}, {"density_defect_norm", defectBlocks.density}, {"surface_defect_norm", defectBlocks.surface}, {"enthalpy_defect_norm", defectBlocks.enthalpy}, {"solve_seconds_maximum_rank", solveSeconds}, {"jacobian_applications", static_cast(instrumentedOperation.GetStatistics().applications)}, {"jacobian_application_seconds", instrumentedOperation.GetStatistics().totalSeconds}, {"preconditioner_applications", static_cast(instrumentedPreconditioner.GetStatistics().applications)}, {"preconditioner_application_seconds", instrumentedPreconditioner.GetStatistics().totalSeconds}, {"preconditioner_maximum_application_seconds", instrumentedPreconditioner.GetStatistics().maximumSeconds} }; solveMetrics.insert(preparationMetrics.begin(), preparationMetrics.end()); experiment::record_experiment_result( "material_surface_preconditioning_p9", candidate + "_linear_solve", commonParameters(candidate, "manufactured_linear_solve", operation.Width()), std::move(solveMetrics) ); const double initialNorm = std::max(std::abs(krylov.GetInitialNorm()), 1.0e-300); const auto &history = monitor.GetHistory(); for (std::size_t index = 0; index < history.size(); ++index) { const auto &sample = history[index]; experiment::record_experiment_result( "material_surface_preconditioning_p9", candidate + "_history_" + std::to_string(index), commonParameters(candidate, "fgmres_residual_history", operation.Width()), {{"history_sample", static_cast(index)}, {"iteration", static_cast(sample.iteration)}, {"reported_residual_norm", sample.reportedNorm}, {"reported_relative_residual", std::abs(sample.reportedNorm) / initialNorm}, {"final_measurement", sample.final ? 1.0 : 0.0}} ); } instrumentedOperation.ResetStatistics(); instrumentedPreconditioner.ResetStatistics(); solver::FixedRightPreconditionedOperator product(instrumentedOperation, instrumentedPreconditioner); announce(communicator, "measuring " + candidate + " with 16-vector Arnoldi"); const solver::ArnoldiSpectralMeasurement spectrum = solver::measureArnoldiSpectrum( product, arnoldiDirection, communicator, {.krylovDimension = arnoldiDimension, .breakdownRelativeTolerance = 1.0e-13, .ritzConvergenceRelativeTolerance = 1.0e-7, .reorthogonalize = true} ); REQUIRE(std::isfinite(trueRelativeResidual)); REQUIRE(std::isfinite(relativeSolutionError)); REQUIRE(std::isfinite(defectNorm)); REQUIRE(std::isfinite(spectrum.projectedConditionProxy)); recordSpectrum(candidate, spectrum, operation.Width(), setupSeconds); int rank = 0; MPI_Comm_rank(communicator, &rank); if (rank == 0) { std::cout << "[P9 material-surface] " << candidate << ": iterations=" << krylov.GetNumIterations() << ", converged=" << (krylov.GetConverged() ? "yes" : "no") << ", true residual=" << trueRelativeResidual << ", defect=" << defectNorm << ", projected condition=" << spectrum.projectedConditionProxy << '\n'; } } template void prepareAndMeasure( const std::string &candidate, const Policy policy, const auto &problem, const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation, const mfem::Vector &exactCorrection, const mfem::Vector &rightHandSide, const mfem::Vector &arnoldiDirection, const MPI_Comm communicator, const preconditioning::MaterialSurfaceDiagonalOptions diagonalOptions = {} ) { const Clock::time_point setupStart = Clock::now(); auto block = preconditioning::materialSurfaceBlock( problem, backend::Diagonal{}, backend::Diagonal{}, policy, diagonalOptions ); auto prepared = preconditioning::prepare(problem, block); const double setupTime = maximumRankSeconds(setupStart, communicator); const auto &density = prepared.GetDensityDiagonalQuality(); const auto &surface = prepared.GetSurfaceDiagonalQuality(); const auto &enthalpy = prepared.GetEnthalpyDiagonalQuality(); const auto &calibration = prepared.GetSurfaceCalibration(); measureCandidate( candidate, prepared, setupTime, operation, exactCorrection, rightHandSide, arnoldiDirection, communicator, {{"density_diagonal_minimum", density.minimumAbsoluteEntryBeforeRegularization}, {"density_diagonal_maximum", density.maximumAbsoluteEntryBeforeRegularization}, {"density_diagonal_floor", density.appliedFloor}, {"density_regularized_entries", static_cast(density.regularizedEntries)}, {"surface_diagonal_minimum", surface.minimumAbsoluteEntryBeforeRegularization}, {"surface_diagonal_maximum", surface.maximumAbsoluteEntryBeforeRegularization}, {"surface_diagonal_floor", surface.appliedFloor}, {"surface_regularized_entries", static_cast(surface.regularizedEntries)}, {"surface_calibration_target", static_cast(calibration.target)}, {"surface_calibration_probes", static_cast(calibration.probeCount)}, {"surface_calibration_objective", static_cast(calibration.objective)}, {"surface_calibration_scale", calibration.scale}, {"surface_calibration_inverse_multiplier", calibration.inverseMultiplier}, {"surface_calibration_numerator", calibration.leastSquaresNumerator}, {"surface_calibration_denominator", calibration.leastSquaresDenominator}, {"enthalpy_diagonal_minimum", enthalpy.minimumAbsoluteEntryBeforeRegularization}, {"enthalpy_diagonal_maximum", enthalpy.maximumAbsoluteEntryBeforeRegularization}, {"enthalpy_diagonal_floor", enthalpy.appliedFloor}, {"enthalpy_regularized_entries", static_cast(enthalpy.regularizedEntries)}} ); } template void prepareAndMeasureH1( const std::string &candidate, const Policy policy, const int fixedAMGCycles, const int calibrationProbeCount, const auto &problem, const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation, const mfem::Vector &exactCorrection, const mfem::Vector &rightHandSide, const mfem::Vector &arnoldiDirection, const MPI_Comm communicator ) { REQUIRE(fixedAMGCycles > 0); REQUIRE(calibrationProbeCount >= 3); const Clock::time_point setupStart = Clock::now(); auto block = preconditioning::materialSurfaceBlock( problem, backend::Diagonal{}, backend::HypreBoomerAMG{backend::FixedCycles{.cycles = fixedAMGCycles}}, policy, preconditioning::SurfaceH1MassStiffness{ .calibration = { .target = preconditioning::SurfaceRieszCalibrationTarget::surface_jacobian, .probeCount = calibrationProbeCount } } ); auto prepared = preconditioning::prepare(problem, std::move(block)); const double setupTime = maximumRankSeconds(setupStart, communicator); const auto &density = prepared.GetDensityDiagonalQuality(); const auto &enthalpy = prepared.GetEnthalpyDiagonalQuality(); const auto &fit = prepared.GetSurfaceFit(); measureCandidate( candidate, prepared, setupTime, operation, exactCorrection, rightHandSide, arnoldiDirection, communicator, {{"density_diagonal_minimum", density.minimumAbsoluteEntryBeforeRegularization}, {"density_diagonal_maximum", density.maximumAbsoluteEntryBeforeRegularization}, {"density_diagonal_floor", density.appliedFloor}, {"density_regularized_entries", static_cast(density.regularizedEntries)}, {"surface_h1_calibration_target", static_cast(fit.target)}, {"surface_h1_calibration_probes", static_cast(fit.probeCount)}, {"surface_h1_fit_sign", fit.sign}, {"surface_h1_mass_coefficient", fit.massCoefficient}, {"surface_h1_stiffness_coefficient", fit.stiffnessCoefficient}, {"surface_h1_fit_relative_residual", fit.relativeResidual}, {"surface_h1_fit_relative_gram_determinant", fit.relativeGramDeterminant}, {"surface_amg_fixed_cycles", static_cast(fixedAMGCycles)}, {"enthalpy_diagonal_minimum", enthalpy.minimumAbsoluteEntryBeforeRegularization}, {"enthalpy_diagonal_maximum", enthalpy.maximumAbsoluteEntryBeforeRegularization}, {"enthalpy_diagonal_floor", enthalpy.appliedFloor}, {"enthalpy_regularized_entries", static_cast(enthalpy.regularizedEntries)}} ); const auto &surfaceBackendStatistics = prepared.GetSurfaceBackend().GetStatistics(); const auto &factorizationStatistics = prepared.GetFactorization().GetStatistics(); const auto &preparationStatistics = prepared.GetStatistics(); auto parameters = commonParameters(candidate, "surface_h1_backend_statistics", operation.Width()); parameters["experiment_schema"] = "p9_material_surface_h1_v1"; parameters["surface_surrogate"] = "h1_mass_plus_tangential_stiffness"; parameters["surface_calibration_target"] = "surface_jacobian"; parameters["surface_calibration_probes"] = std::to_string(calibrationProbeCount); parameters["surface_amg_fixed_cycles"] = std::to_string(fixedAMGCycles); experiment::record_experiment_result( "material_surface_preconditioning_p9", candidate + "_surface_h1_backend_statistics", std::move(parameters), {{"setup_seconds_maximum_rank", setupTime}, {"surface_h1_fit_sign", fit.sign}, {"surface_h1_mass_coefficient", fit.massCoefficient}, {"surface_h1_stiffness_coefficient", fit.stiffnessCoefficient}, {"surface_h1_fit_relative_residual", fit.relativeResidual}, {"surface_h1_fit_relative_gram_determinant", fit.relativeGramDeterminant}, {"surface_backend_setups", static_cast(surfaceBackendStatistics.setups)}, {"surface_backend_applications", static_cast(surfaceBackendStatistics.applications)}, {"surface_backend_inner_iterations", static_cast(surfaceBackendStatistics.innerIterations)}, {"surface_backend_last_inner_iterations", static_cast(surfaceBackendStatistics.lastInnerIterations)}, {"factorization_applications", static_cast(factorizationStatistics.applications)}, {"surface_inverse_applications", static_cast(factorizationStatistics.surfaceInverseApplications)}, {"block_setups", static_cast(preparationStatistics.setups)}, {"surface_jacobian_probes", static_cast(preparationStatistics.surfaceJacobianProbes)}, {"surface_h1_assemblies", static_cast(preparationStatistics.surfaceH1Assemblies)}} ); } enum class SurfaceProbeMode { constant, ordered_low, alternating_high, deterministic_mixed }; [[nodiscard]] const char *surfaceProbeModeName(const SurfaceProbeMode mode) noexcept { switch (mode) { case SurfaceProbeMode::constant: return "constant"; case SurfaceProbeMode::ordered_low: return "ordered_low"; case SurfaceProbeMode::alternating_high: return "alternating_high"; case SurfaceProbeMode::deterministic_mixed: return "deterministic_mixed"; } return "unknown"; } [[nodiscard]] mfem::Vector normalizedSurfaceProbe( const int localSize, const SurfaceProbeMode mode, const MPI_Comm communicator ) { int globalSize = 0; int offset = 0; MPI_Allreduce(&localSize, &globalSize, 1, MPI_INT, MPI_SUM, communicator); MPI_Exscan(&localSize, &offset, 1, MPI_INT, MPI_SUM, communicator); int rank = 0; MPI_Comm_rank(communicator, &rank); if (rank == 0) { offset = 0; } REQUIRE(globalSize > 0); mfem::Vector probe(localSize); for (int index = 0; index < localSize; ++index) { const int globalIndex = offset + index; const double position = (static_cast(globalIndex) + 0.5) / static_cast(globalSize); switch (mode) { case SurfaceProbeMode::constant: probe(index) = 1.0; break; case SurfaceProbeMode::ordered_low: probe(index) = std::cos(std::numbers::pi_v * position); break; case SurfaceProbeMode::alternating_high: probe(index) = globalIndex % 2 == 0 ? 1.0 : -1.0; break; case SurfaceProbeMode::deterministic_mixed: probe(index) = 0.41 * std::cos(std::numbers::pi_v * position) + std::sin(5.0 * std::numbers::pi_v * position) + 0.23 * (globalIndex % 2 == 0 ? 1.0 : -1.0); break; } } const double norm = globalNorm(probe, communicator); REQUIRE(norm > 0.0); probe /= norm; return probe; } void applyParameterOverrides( std::map< std::string, std::string> ¶meters, const std::map< std::string, std::string> &overrides ) { for (const auto &[key, value] : overrides) { parameters.insert_or_assign(key, value); } } template void recordSurfaceInverseRecovery( const std::string &candidate, SurfaceInverse &surfaceInverse, const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation, const MPI_Comm communicator, const std::map< std::string, std::string> ¶meterOverrides ) { constexpr std::array modes{ SurfaceProbeMode::constant, SurfaceProbeMode::ordered_low, SurfaceProbeMode::alternating_high, SurfaceProbeMode::deterministic_mixed }; const int surfaceSize = operation.GetOffsets()[2] - operation.GetOffsets()[1]; REQUIRE(surfaceInverse.Width() == surfaceSize); REQUIRE(surfaceInverse.Height() == surfaceSize); for (const SurfaceProbeMode mode : modes) { const mfem::Vector probe = normalizedSurfaceProbe(surfaceSize, mode, communicator); mfem::Vector surfaceAction(surfaceSize); mfem::Vector recovered(surfaceSize); operation.ApplySurfaceToSurface(probe, surfaceAction); const Clock::time_point inverseStart = Clock::now(); surfaceInverse.Mult(surfaceAction, recovered); const double inverseSeconds = maximumRankSeconds(inverseStart, communicator); mfem::Vector recoveryError(recovered); recoveryError -= probe; const double probeNorm = globalNorm(probe, communicator); const double actionNorm = globalNorm(surfaceAction, communicator); const double recoveredNorm = globalNorm(recovered, communicator); const double relativeError = globalNorm(recoveryError, communicator) / probeNorm; constexpr double nonzeroFloor = 1.0e-300; auto parameters = commonParameters(candidate, "surface_inverse_recovery", operation.Width()); applyParameterOverrides(parameters, parameterOverrides); parameters["surface_probe_mode"] = surfaceProbeModeName(mode); experiment::record_experiment_result( "material_surface_preconditioning_p9", candidate + "_" + surfaceProbeModeName(mode), std::move(parameters), {{"surface_probe_norm", probeNorm}, {"surface_action_norm", actionNorm}, {"surface_recovered_norm", recoveredNorm}, {"surface_recovery_relative_error", relativeError}, {"surface_operator_gain", actionNorm / probeNorm}, {"surface_inverse_gain", recoveredNorm / std::max(actionNorm, nonzeroFloor)}, {"surface_recovered_gain", recoveredNorm / probeNorm}, {"surface_inverse_seconds_maximum_rank", inverseSeconds}} ); } } template void recordBalancedPreconditionedDefect( const std::string &candidate, PreparedPreconditioner &prepared, const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation, const MPI_Comm communicator, const std::map< std::string, std::string> ¶meterOverrides ) { const mfem::Vector direction = blockBalancedDirection(operation.GetOffsets(), 1.37, communicator); mfem::Vector correction(operation.Width()); mfem::Vector defect(operation.Height()); const Clock::time_point applicationStart = Clock::now(); prepared.Mult(direction, correction); const double applicationSeconds = maximumRankSeconds(applicationStart, communicator); operation.Mult(correction, defect); defect -= direction; const MaterialBlockMeasurements blockDefects = blockNorms(defect, operation.GetOffsets(), communicator); const double relativeDefect = globalNorm(defect, communicator) / std::max(globalNorm(direction, communicator), std::numeric_limits::min()); auto parameters = commonParameters(candidate, "balanced_right_preconditioned_defect", operation.Width()); applyParameterOverrides(parameters, parameterOverrides); experiment::record_experiment_result( "material_surface_preconditioning_p9", candidate + "_balanced_defect", std::move(parameters), {{"right_preconditioned_defect", relativeDefect}, {"density_defect_norm", blockDefects.density}, {"surface_defect_norm", blockDefects.surface}, {"enthalpy_defect_norm", blockDefects.enthalpy}, {"preconditioner_application_seconds_maximum_rank", applicationSeconds}} ); } void measureH1CalibrationFloor( const std::string &candidate, const std::string &relativeMassCoefficientFloorLabel, const double relativeMassCoefficientFloor, const auto &problem, const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation, const MPI_Comm communicator ) { const Clock::time_point setupStart = Clock::now(); auto block = preconditioning::materialSurfaceBlock( problem, backend::Diagonal{}, backend::HypreBoomerAMG{backend::FixedCycles{.cycles = 1}}, preconditioning::ApproximateMaterialSurfaceLDU{}, preconditioning::SurfaceH1MassStiffness{ .calibration = {.target = preconditioning::SurfaceRieszCalibrationTarget::surface_jacobian, .probeCount = 4}, .relativeMassCoefficientFloor = relativeMassCoefficientFloor } ); auto prepared = preconditioning::prepare(problem, std::move(block)); const double setupTime = maximumRankSeconds(setupStart, communicator); const auto &fit = prepared.GetSurfaceFit(); const std::map parameters{ {"experiment_schema", "p9_material_surface_h1_tuning_v1"}, {"surface_surrogate", "h1_mass_plus_tangential_stiffness"}, {"surface_calibration_target", "surface_jacobian"}, {"surface_calibration_probes", "4"}, {"surface_amg_fixed_cycles", "1"}, {"relative_mass_coefficient_floor", relativeMassCoefficientFloorLabel} }; recordSurfaceInverseRecovery(candidate, prepared.GetSurfaceInverse(), operation, communicator, parameters); recordBalancedPreconditionedDefect(candidate, prepared, operation, communicator, parameters); const auto &backendStatistics = prepared.GetSurfaceBackend().GetStatistics(); const auto &factorizationStatistics = prepared.GetFactorization().GetStatistics(); const auto &preparationStatistics = prepared.GetStatistics(); auto summaryParameters = commonParameters(candidate, "surface_h1_floor_summary", operation.Width()); applyParameterOverrides(summaryParameters, parameters); experiment::record_experiment_result( "material_surface_preconditioning_p9", candidate + "_summary", std::move(summaryParameters), {{"setup_seconds_maximum_rank", setupTime}, {"relative_mass_coefficient_floor", relativeMassCoefficientFloor}, {"surface_h1_fit_sign", fit.sign}, {"surface_h1_mass_coefficient", fit.massCoefficient}, {"surface_h1_stiffness_coefficient", fit.stiffnessCoefficient}, {"surface_h1_fit_relative_residual", fit.relativeResidual}, {"surface_h1_fit_relative_gram_determinant", fit.relativeGramDeterminant}, {"surface_backend_setups", static_cast(backendStatistics.setups)}, {"surface_backend_applications", static_cast(backendStatistics.applications)}, {"surface_backend_inner_iterations", static_cast(backendStatistics.innerIterations)}, {"surface_backend_last_inner_iterations", static_cast(backendStatistics.lastInnerIterations)}, {"factorization_applications", static_cast(factorizationStatistics.applications)}, {"surface_inverse_applications", static_cast(factorizationStatistics.surfaceInverseApplications)}, {"surface_jacobian_probes", static_cast(preparationStatistics.surfaceJacobianProbes)}, {"surface_h1_assemblies", static_cast(preparationStatistics.surfaceH1Assemblies)}} ); } void measureScalarSurfaceControl( const std::string &candidate, const preconditioning::SurfaceRieszCalibrationObjective objective, const int calibrationProbeCount, const auto &problem, const mean_field::preconditioning::MaterialSurfaceJacobianOperator &operation, const MPI_Comm communicator ) { REQUIRE(calibrationProbeCount > 0); const preconditioning::MaterialSurfaceDiagonalOptions calibration{ .surfaceCalibration = { .target = preconditioning::SurfaceRieszCalibrationTarget::surface_jacobian, .probeCount = calibrationProbeCount, .objective = objective } }; const Clock::time_point setupStart = Clock::now(); auto block = preconditioning::materialSurfaceBlock( problem, backend::Diagonal{}, backend::Diagonal{}, preconditioning::ApproximateMaterialSurfaceLDU{}, calibration ); auto prepared = preconditioning::prepare(problem, std::move(block)); auto directSurfaceInverse = backend::prepare(backend::Diagonal{}, prepared.GetSurfaceDiagonal()); const double setupTime = maximumRankSeconds(setupStart, communicator); const auto &calibrationData = prepared.GetSurfaceCalibration(); const std::map parameters{ {"experiment_schema", "p9_material_surface_h1_tuning_v1"}, {"surface_surrogate", "scalar_mass_diagonal"}, {"surface_calibration_target", "surface_jacobian"}, {"surface_calibration_probes", std::to_string(calibrationProbeCount)}, {"surface_calibration_objective", objective == preconditioning::SurfaceRieszCalibrationObjective::operator_action ? "operator_action" : "right_preconditioned_action"} }; recordSurfaceInverseRecovery(candidate, directSurfaceInverse, operation, communicator, parameters); recordBalancedPreconditionedDefect(candidate, prepared, operation, communicator, parameters); const auto &directStatistics = directSurfaceInverse.GetStatistics(); const auto &factorizationStatistics = prepared.GetFactorization().GetStatistics(); auto summaryParameters = commonParameters(candidate, "scalar_surface_control_summary", operation.Width()); applyParameterOverrides(summaryParameters, parameters); experiment::record_experiment_result( "material_surface_preconditioning_p9", candidate + "_summary", std::move(summaryParameters), {{"setup_seconds_maximum_rank", setupTime}, {"surface_calibration_scale", calibrationData.scale}, {"surface_calibration_inverse_multiplier", calibrationData.inverseMultiplier}, {"surface_calibration_numerator", calibrationData.leastSquaresNumerator}, {"surface_calibration_denominator", calibrationData.leastSquaresDenominator}, {"surface_backend_setups", static_cast(directStatistics.setups)}, {"surface_backend_applications", static_cast(directStatistics.applications)}, {"factorization_applications", static_cast(factorizationStatistics.applications)}, {"surface_inverse_applications", static_cast(factorizationStatistics.surfaceInverseApplications)}} ); } } // namespace TEST_CASE( "Material Surface P9 Numerical Factorization Comparison", "[preconditioning][material_surface][diagnostics][experiment][spectrum][p9][p9_baseline]" ) { using namespace mean_field; const utils::Args arguments = test_utils::setup_args(); fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0); REQUIRE(finiteElements.okay()); const MPI_Comm communicator = finiteElements.mesh->GetComm(); constexpr double radius = utils::RADIUS; constexpr double mass = utils::MASS; const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v; const double centralDensity = std::numbers::pi_v * mass / (4.0 * radius * radius * radius); auto model = model::StellarModel( eos::Polytrope({.n = 1.0, .K = polytropicConstant}), surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}), constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}}) ); auto problem = equilibrium::discretize(model, finiteElements); auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 1024})); problem.Prepare(projected.values, makeDependencies(), zeroRotation()); const auto &physical = problem.GetPreparedOperator().GetPhysicalOperator(); preconditioning::MaterialSurfaceJacobianOperator materialSurfaceOperator(physical); const auto exactCorrection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.23, communicator); mfem::Vector rightHandSide(materialSurfaceOperator.Height()); materialSurfaceOperator.Mult(exactCorrection, rightHandSide); const auto arnoldiDirection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.79, communicator); solver::IdentityPreconditioner identity(materialSurfaceOperator.Width()); measureCandidate( "identity", identity, 0.0, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator ); prepareAndMeasure( "block_diagonal", preconditioning::MaterialSurfaceBlockDiagonal{}, problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator ); prepareAndMeasure( "material_independent_surface", preconditioning::CoupledMaterialIndependentSurface{}, problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator ); prepareAndMeasure( "material_then_surface", preconditioning::MaterialThenSurfaceTriangular{}, problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator ); prepareAndMeasure( "surface_then_material", preconditioning::SurfaceThenMaterialTriangular{}, problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator ); prepareAndMeasure( "approximate_ldu", preconditioning::ApproximateMaterialSurfaceLDU{}, problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator ); } TEST_CASE( "Material Surface P9 Calibrated LDU Comparison", "[preconditioning][material_surface][diagnostics][experiment][spectrum][p9][p9_refinement]" ) { using namespace mean_field; const utils::Args arguments = test_utils::setup_args(); fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0); REQUIRE(finiteElements.okay()); const MPI_Comm communicator = finiteElements.mesh->GetComm(); constexpr double radius = utils::RADIUS; constexpr double mass = utils::MASS; const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v; const double centralDensity = std::numbers::pi_v * mass / (4.0 * radius * radius * radius); auto model = model::StellarModel( eos::Polytrope({.n = 1.0, .K = polytropicConstant}), surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}), constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}}) ); auto problem = equilibrium::discretize(model, finiteElements); auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 1024})); problem.Prepare(projected.values, makeDependencies(), zeroRotation()); const auto &physical = problem.GetPreparedOperator().GetPhysicalOperator(); preconditioning::MaterialSurfaceJacobianOperator materialSurfaceOperator(physical); const auto exactCorrection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.23, communicator); mfem::Vector rightHandSide(materialSurfaceOperator.Height()); materialSurfaceOperator.Mult(exactCorrection, rightHandSide); const auto arnoldiDirection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.79, communicator); constexpr preconditioning::MaterialSurfaceDiagonalOptions surfaceJacobianCalibration{ .surfaceCalibration = { .target = preconditioning::SurfaceRieszCalibrationTarget::surface_jacobian, .probeCount = 4 } }; constexpr preconditioning::MaterialSurfaceDiagonalOptions surfaceSchurCalibration{ .surfaceCalibration = { .target = preconditioning::SurfaceRieszCalibrationTarget::approximate_material_schur, .probeCount = 4 } }; prepareAndMeasure( "surface_then_material_calibrated_aqq", preconditioning::SurfaceThenMaterialTriangular{}, problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator, surfaceJacobianCalibration ); prepareAndMeasure( "approximate_ldu", preconditioning::ApproximateMaterialSurfaceLDU{}, problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator ); prepareAndMeasure( "approximate_ldu_calibrated_aqq", preconditioning::ApproximateMaterialSurfaceLDU{}, problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator, surfaceJacobianCalibration ); prepareAndMeasure( "approximate_ldu_calibrated_schur", preconditioning::ApproximateMaterialSurfaceLDU{}, problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator, surfaceSchurCalibration ); } TEST_CASE( "Material Surface P9 Frequency-Aware Surface Refinement", "[preconditioning][material_surface][diagnostics][experiment][spectrum][p9][p9_h1_refinement]" ) { using namespace mean_field; const utils::Args arguments = test_utils::setup_args(); fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0); REQUIRE(finiteElements.okay()); const MPI_Comm communicator = finiteElements.mesh->GetComm(); constexpr double radius = utils::RADIUS; constexpr double mass = utils::MASS; const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v; const double centralDensity = std::numbers::pi_v * mass / (4.0 * radius * radius * radius); auto model = model::StellarModel( eos::Polytrope({.n = 1.0, .K = polytropicConstant}), surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}), constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}}) ); auto problem = equilibrium::discretize(model, finiteElements); auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 1024})); problem.Prepare(projected.values, makeDependencies(), zeroRotation()); const auto &physical = problem.GetPreparedOperator().GetPhysicalOperator(); preconditioning::MaterialSurfaceJacobianOperator materialSurfaceOperator(physical); const auto exactCorrection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.23, communicator); mfem::Vector rightHandSide(materialSurfaceOperator.Height()); materialSurfaceOperator.Mult(exactCorrection, rightHandSide); const auto arnoldiDirection = blockBalancedDirection(materialSurfaceOperator.GetOffsets(), 0.79, communicator); constexpr int calibrationProbeCount = 4; prepareAndMeasureH1( "h1_aqq_surface_then_material_amg1", preconditioning::SurfaceThenMaterialTriangular{}, 1, calibrationProbeCount, problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator ); prepareAndMeasureH1( "h1_aqq_approximate_ldu_amg1", preconditioning::ApproximateMaterialSurfaceLDU{}, 1, calibrationProbeCount, problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator ); prepareAndMeasureH1( "h1_aqq_approximate_ldu_amg2", preconditioning::ApproximateMaterialSurfaceLDU{}, 2, calibrationProbeCount, problem, materialSurfaceOperator, exactCorrection, rightHandSide, arnoldiDirection, communicator ); } TEST_CASE( "Material Surface P9 H1 Calibration Floor Tuning", "[preconditioning][material_surface][diagnostics][experiment][p9][p9_h1_tuning]" ) { using namespace mean_field; const utils::Args arguments = test_utils::setup_args(); fem::FEM finiteElements = fem::setup_fem(arguments.mesh_file, arguments, 0); REQUIRE(finiteElements.okay()); const MPI_Comm communicator = finiteElements.mesh->GetComm(); constexpr double radius = utils::RADIUS; constexpr double mass = utils::MASS; const double polytropicConstant = 2.0 * utils::G * radius * radius / std::numbers::pi_v; const double centralDensity = std::numbers::pi_v * mass / (4.0 * radius * radius * radius); auto model = model::StellarModel( eos::Polytrope({.n = 1.0, .K = polytropicConstant}), surface::Isobaric({.Psurf = dimensions::PressureValue{0.0}}), integral::FixedTotalMass({.Mtotal = dimensions::MassValue{mass}}), constraint::FixedCentralDensity({.RhoC = dimensions::DensityValue{centralDensity}}) ); auto problem = equilibrium::discretize(model, finiteElements); auto projected = seed::makeProjectedEquilibriumState(problem, seed::LaneEmden({.radialSampleCount = 1024})); problem.Prepare(projected.values, makeDependencies(), zeroRotation()); const auto &physical = problem.GetPreparedOperator().GetPhysicalOperator(); preconditioning::MaterialSurfaceJacobianOperator materialSurfaceOperator(physical); constexpr std::array floorCases{ std::pair{"1e-10", 1.0e-10}, std::pair{"1e-4", 1.0e-4}, std::pair{"1e-2", 1.0e-2}, std::pair{"1e-1", 1.0e-1}, std::pair{"1", 1.0} }; for (const auto &[label, floor] : floorCases) { measureH1CalibrationFloor( std::string("h1_aqq_floor_") + label, label, floor, problem, materialSurfaceOperator, communicator ); } measureScalarSurfaceControl( "scalar_aqq_operator_calibrated_diagonal_control", preconditioning::SurfaceRieszCalibrationObjective::operator_action, 4, problem, materialSurfaceOperator, communicator ); constexpr std::array inverseProbeCounts{1, 2, 4, 8, 16}; for (const int probeCount : inverseProbeCounts) { measureScalarSurfaceControl( "scalar_aqq_right_calibrated_diagonal_" + std::to_string(probeCount) + "_probes", preconditioning::SurfaceRieszCalibrationObjective::right_preconditioned_action, probeCount, problem, materialSurfaceOperator, communicator ); } }