From 46249d0758e1c30f7585d35821f78cd558197cec Mon Sep 17 00:00:00 2001 From: Emily Boudreaux Date: Fri, 18 Sep 2026 10:21:19 -0400 Subject: [PATCH] Record coupled correction benefit cost and nonlinear curvature limits --- .../COUPLED_TRIAL_CORRECTION_2026-09-18.md | 207 +++++++++++++++++- experiments/check_coupled_trial.py | 44 ++++ experiments/test_coupled_trial_checker.py | 33 +++ 3 files changed, 282 insertions(+), 2 deletions(-) diff --git a/experiments/COUPLED_TRIAL_CORRECTION_2026-09-18.md b/experiments/COUPLED_TRIAL_CORRECTION_2026-09-18.md index 891e56f..353d9da 100644 --- a/experiments/COUPLED_TRIAL_CORRECTION_2026-09-18.md +++ b/experiments/COUPLED_TRIAL_CORRECTION_2026-09-18.md @@ -23,7 +23,8 @@ then prepare and evaluate y+c. All equations, normalization factors and ordinary full-residual acceptance thresholds remain unchanged. A separate auxiliary true-defect check is required; the base Newton report is never replaced by a correction report. The locally second-order argument requires -a bounded inverse, which is not assumed to hold uniformly in this model. +sufficient residual smoothness and a bounded inverse, neither of which is +assumed to hold uniformly in this model. ## Implementation contracts @@ -77,4 +78,206 @@ Useful frozen progress is a prerequisite for a bounded nonlinear campaign. Any accepted model must also be checked for virial balance and internal admissibility; a lower residual alone is insufficient. -Implementation and test results are pending. +## Result and present recommendation + +The opt-in implementation is complete and tested. It creates a newly +Armijo-acceptable frozen candidate at alpha = 1/256 for both trial paths, +but does not establish a production convergence or speed improvement. Do not +enable it by default on the strength of these measurements. + +The nominal fourfold increase over the previously accepted alpha = 1/1024 +overstates the realized displacement. The coupled correction largely reverses +the original Newton direction. Its own nonlinear curvature is much larger +than the checked auxiliary linear error, and the useful candidate remains +internally nonphysical despite passing enthalpy-based virial checks. + +Each frozen alpha has a fresh one-attempt budget. In a live search that spends +the default budget at the unsuccessful 1/64 trial, the later 1/256 trial +cannot use this correction. Thus these independent samples are not a measured +live line-search speedup. The existing seed protocol projects radial profiles +and regenerates fields and shape; it cannot faithfully resume the arbitrary +saved late state. No diagnostic mutation of accepted state was used to bypass +that restriction, and no new initialization API was added. + +## Numerical results + +The common late state has ||F0|| = 1.3966991043973936e-5 and the saved outer +direction again passes the full-J gate with relative defect 0.02955918315. +All four distinct auxiliary solves pass the independent 0.03 relative gate +and the norm cap. No auxiliary failure or relaxed acceptance threshold +explains the outcomes below. + +| Prepared path | alpha | Before correction: ||F||/||F0|| | Corrected: ||F||/||F0|| | Selection | Auxiliary iterations | Added transaction time | +|---|---:|---:|---:|---|---:|---:| +| Raw | 1/64 | 1.80726810 | 1.03360191 | Improved, still fails Armijo | 34 | 27.33 s | +| Raw | 1/256 | 1.00250966 | 0.99746464 | Passes Armijo | 52 | 37.59 s | +| Gravity-corrected | 1/64 | 1.08032296 | 1.09566246 | Worse; original restored | 55 | 47.60 s | +| Gravity-corrected | 1/256 | 1.00202482 | 0.99810801 | Passes Armijo | 62 | 43.53 s | + +At 1/16, both raw preparations fail at the pressure-force thermodynamic-domain +check, before any coupled solve. At 1/1024, the original candidates already +pass Armijo and receive no coupled solve: their residual ratios are +0.9991695118 (raw) and 0.9993941386 (gravity-corrected). + +The Lane–Emden seed's gravity-corrected full step likewise remains acceptable +without a coupled solve. All frozen probes preserve accepted buffers, +normalization factors and direction buffers; the final residual and Jp at +the restored base agree exactly with the initially captured values. + +Raw timings above come from the uncontended repeat. An initially launched +artifact audit accidentally rebuilt a gravity-index set for every vector +entry and overlapped the first raw case. It was stopped, the checker was +made linear in state size, and the raw case was repeated. Its trial vectors +and residuals reproduce the first run. The original evidence is retained; +its raw timing is excluded. All 50 saved vector files are byte-identical in +the repeat. See `AUDIT_RUN_NOTES.md` in the evidence folder. + +The added times include refresh, solve, independent verification, candidate +preparation, and any fallback restoration. They do not include the original +trial preparation or captured base action. At 1/256 the full candidate-path +costs are 43.86 s (raw) and 63.45 s (gravity-first), versus 6.30 s and 18.59 s +for their already acceptable 1/1024 controls. Outer Newton work, previous +rejected trials, base capture and physical postprocessing are separate. +Timings are single observations, not statistical performance estimates. + +## Why a verified solve does not ensure useful correction + +For the actual candidate y, let b = L - F(y), delta = J(y)c - b and +E = F(y+c) - F(y) - J(y)c. The exact saved-vector identity is + + F(y+c) - L = delta + E. + +The auxiliary gate bounds delta. It does not bound E. At alpha = 1/256, +||E|| is about 11.28 times the verified defect for the raw path and 254 times +for the gravity-first path. For the two gravity-first attempts, +||E||/||c||^2 is 13.5756 and 13.5358; the raw 1/256 value is 13.5271. +This is evidence that the correction's own quadratic curvature, rather +than inaccurate solution of its linear equation, dominates its remaining +nonlinear error. + +On the gravity-first path at 1/256, the affine-target remainder actually +grows from 8.87747e-8 to 6.23769e-7 (7.03 times), despite the improved merit. +The raw path reduces that remainder by about 3.165 times. Gravity completion +before the coupled solve therefore does not help this particular comparison. + +The correction/direction cosines are -0.9999330 (raw) and -0.9999942 +(gravity-first). The realized displacement norms at 1/256 are only 1.7596 +and 1.2342 times their respective raw-direction 1/1024 displacement norms. +On the gravity-first path, ||c||/||b|| increases from 480.8 to 2418.1 when +alpha decreases fourfold. These samples do not demonstrate a stable small +second-order correction regime. They do not identify a null mode either. +Tightening the auxiliary tolerance is consequently a weak next target. + +Near collinearity is insufficient to conclude that scalar damping is +equivalent: small transverse displacements can have large Jacobian action. +The final scalar controls below address that attribution directly. + +## Matched scalar controls + +For each selected 1/256 candidate z, compute +t = dot(z-x,p)/dot(p,p), then freshly prepare x+t*p through the existing +`--replay-candidate` path. The input generator validates and hashes the +source artifacts before writing normalized candidates. No new linear solve, +repair, or acceptance threshold is introduced. + +| Coupled candidate being matched | Scalar t | Scalar ||F||/||F0|| | Coupled ||F||/||F0|| | Scalar residual decrease | Coupled residual decrease | +|---|---:|---:|---:|---:|---:| +| Raw + coupled | 0.00171818301767 | 0.9988437736 | 0.9974646400 | 0.115623% | 0.253536% | +| Gravity-first + coupled | 0.00120521815790 | 0.9990306561 | 0.9981080052 | 0.096934% | 0.189199% | + +Both scalar controls prepare and pass the ordinary Armijo criterion at t. +Their normalized base and full base residual reproduce the coupled screen +byte for byte. The replay uses the independent preconditioner configuration +because that is the existing replay interface; no inverse or linear solve is +used in evaluating these candidates. The equation operator and normalization +are the same. The campaign executable is byte-identical to both screens. + +The coupled candidate produces 2.19 times the residual decrease of the +matched scalar control on the raw path, and 1.95 times on the gravity-first +path. Thus the small transverse changes have measured nonlinear benefit. +The result cannot be dismissed as scalar damping alone, but it is a small +absolute gain bought with an extra full coupled solve. This does not measure +an entire Newton iteration or prove amortized speedup. The scalar replay +records preparation and residuals; it is not an additional physical-model +certification or quadrature study. + +The next bounded question is whether damping or otherwise limiting the +already computed correction can retain this transverse benefit while +reducing its own nonlinear curvature, using additional preparations instead +of additional full solves. Test that together with a work-aware attempt +trigger; simply increasing the attempt budget can spend multiple outer-solve +equivalents on rejected candidates. Keep normalization, equations, physical +validation and true-defect gates fixed during that comparison. This follow-up +is not implemented here. + +## Physical validation + +Every available selected state is freshly prepared and evaluated at +quadrature orders 14 and 18, using the existing field reconstruction, +rotation, Piola transforms and physical diagnostics. This postprocessing is +separate from timed trial selection and does not change acceptance. + +At alpha = 1/256 and quadrature order 18: + +| Quantity | Raw + coupled | Gravity-first + coupled | +|---|---:|---:| +| Enthalpy-pressure energy virial error | 4.05889e-6 | 4.06134e-6 | +| Enthalpy-pressure force virial error | 2.56495e-6 | 2.56840e-6 | +| Absolute mass error | 3.13178e-6 | 3.13420e-6 | +| Angular momentum Jz | 0 | 0 | +| Minimum sampled density | -8.28793e-4 | -8.24696e-4 | +| Negative density samples | 9600 | 9600 | +| Negative-density mass fraction | 2.89684e-5 | 2.89654e-5 | +| Scaled EOS density mismatch RMS | 1.28264e-4 | 1.28231e-4 | + +Enthalpy-pressure virial errors are below 1e-5, with order-14 to order-18 +differences near 1.83e-9. However, pressure inferred from the negative density +is undefined for this EOS, so the corresponding density-pressure virials +are undefined too. A finite enthalpy-pressure virial is not a substitute for +internal admissibility or pointwise EOS agreement. Negative-density volume +fractions also change with quadrature (about 2.83% to 1.87% on the +gravity-first candidate). No internally correct, converged stellar model, +continuum accuracy, or mesh-convergence claim follows from this trial. + +## Tests and reproduction + +- The Release `tests` and `plateau_diagnostic` targets build successfully. +- Fifteen focused correction/transaction cases pass in serial (584 + assertions) and on two MPI ranks (586 assertions per rank). These include + analytic mixed curvature, a lying backend, bounded real FGMRES failure, + norm caps, stale preparation, zero remainders, nonfinite/overflow handling, + empty-rank collectives, budgets, rollback and exceptions. +- Thirty-five relevant stellar/Newton/gravity regression cases pass with + 356,933 assertions, including typed polynomial composition, optional + borders, original-report preservation and observer/rollback behavior. +- The independent checker passes 27 synthetic tests. The completed screen + and uncontended repeat pass artifact, provenance and physical-replay + consistency checks. Its observational mechanism statistics distinguish + the checked linear defect from the correction's own nonlinear remainder. + Their final audits pass 2,105 and 885 checks; the separate scalar-control + audit passes 68 hash, projection, shared-base and replay checks. +- Full stellar MPI operation remains outside this validation; its existing + interpolation restriction is unchanged. + +Evidence root: +`/Users/tboudreaux/.codex/.chatgpt-projects/g-p-6a6b3f2194ec8191bebcd59075897f1d/diagnostics/coupled_trial_2026-09-18`. +`tests_provenance.json` records the test executable and log hashes. The +campaign directories contain complete commands, source revisions, immutable +executables and hashes of mesh, inputs and outputs. All three campaign +executables are byte-identical (SHA256 begins `620d625636bc77b1`). + +Source checkpoints: implementation `0d87b8b`; initial campaign/checker +`2d0d32b`; checker fix and raw timing repeat `5aa48dc`; scalar-control +generator `0c926f4`. The original checkout and its unrelated quadrature +formatting change are preserved. + +From this worktree, reproduce the initial screen with +`experiments/run_plateau_campaign.py`, the committed +`coupled_trial_screen_2026-09-18.json`, the configured Release build, +`sandbox.smesh`, and a new output directory. Use +`coupled_trial_raw_repeat_2026-09-18.json` for the raw repeat. Run +`check_coupled_trial.py CAMPAIGN --output OUTSIDE_CAMPAIGN.json` afterward. +`make_coupled_scalar_controls.py SCREEN FRESH_OUTPUT` generates the two +normalized scalar inputs and `cases.json` for the existing campaign runner. +All runs use one FEM rank; the seed and outer direction inputs are saved +from the preceding investigation and are hashed into the new evidence. diff --git a/experiments/check_coupled_trial.py b/experiments/check_coupled_trial.py index 66ac2cc..81cf827 100644 --- a/experiments/check_coupled_trial.py +++ b/experiments/check_coupled_trial.py @@ -26,6 +26,47 @@ def near_vector(a,b,scale): return len(a)==len(b) and norm(minus(a,b)) <= 1024*EPS*max(scale,1.e-28) +def finite_ratio(numerator,denominator): + if not math.isfinite(numerator) or not math.isfinite(denominator) or denominator==0: + return None + result=numerator/denominator + return result if math.isfinite(result) else None + + +def correction_mechanism(base,direction,alpha,prediction,before_residual,corrected_state, + corrected_residual,correction,rhs,action): + """Describe a saved correction; large remainders are observations, not failed audits. + + The candidate Jacobian action gives R_new = (J_candidate c - rhs) + E, + where E is the correction's own Taylor remainder. Improving merit does not + imply that either term or the affine-target remainder becomes smaller. + """ + cn,pn,rn=norm(correction),norm(direction),norm(rhs) + displacement=minus(corrected_state,base) + before=minus(before_residual,prediction) + after=minus(corrected_residual,prediction) + defect=minus(action,rhs) + own_taylor=minus(minus(corrected_residual,before_residual),action) + en,dn=norm(own_taylor),norm(defect) + reconstruction=[v+w for v,w in zip(defect,own_taylor)] + return sanitize({ + 'correction_direction_cosine':finite_ratio(dot(correction,direction),cn*pn), + 'correction_over_rhs_norm':finite_ratio(cn,rn), + 'correction_over_alpha_squared':finite_ratio(cn,alpha*alpha), + 'corrected_displacement_norm':norm(displacement), + 'corrected_displacement_over_outer_step_norm':finite_ratio(norm(displacement),alpha*pn), + 'effective_direction_projection_t':finite_ratio(dot(displacement,direction),pn*pn), + 'remainder_before_norm':norm(before), + 'remainder_after_norm':norm(after), + 'verified_linear_defect_norm':dn, + 'own_taylor_remainder_norm':en, + 'own_taylor_over_correction_squared':finite_ratio(en,cn*cn), + 'own_taylor_over_verified_defect':finite_ratio(en,dn), + 'linear_defect_dot_own_taylor':dot(defect,own_taylor), + 'remainder_decomposition_error_norm':norm(minus(after,reconstruction)), + 'scope':'Saved candidate-J vector algebra; normalized coefficient norms, with no new FEM action or asymptotic-order claim.'}) + + def inspect_case(folder): try: return inspect_case_artifacts(Path(folder)) @@ -270,6 +311,9 @@ def inspect_case_artifacts(folder): 'correction_norm':float(row['correction_norm']),'correction_cap':float(row['correction_cap']), 'auxiliary_iterations':int(row['aux_iterations']),'auxiliary_verified_defect_norm':float(row['aux_verified_true_norm']), 'corrected_affine_target_defect_norm':norm(minus(vectors['corrected'][1],prediction)) if flags['corrected_metric_observed'] else None, + 'correction_mechanism':correction_mechanism(x,p,a,prediction,vectors['before'][1],vectors['corrected'][0], + vectors['corrected'][1],correction,rhs,action) if attempted and flags['corrected_metric_observed'] and + correction is not None and rhs is not None and action is not None else None, 'auxiliary_seconds':{k:float(row['soc_'+k]) for k in ('refresh_seconds','solve_seconds','verification_seconds','preparation_seconds')}, 'residual_norms_by_block':{stage:{block:norm(value[1][start:start+n]) for block,(start,n) in rl.items()} for stage,value in vectors.items()}}) diff --git a/experiments/test_coupled_trial_checker.py b/experiments/test_coupled_trial_checker.py index 2f15522..616cc15 100644 --- a/experiments/test_coupled_trial_checker.py +++ b/experiments/test_coupled_trial_checker.py @@ -182,6 +182,39 @@ class CoupledTrialCheckerTests(unittest.TestCase): def test_exact_mixed_correction_and_provenance(self): self.valid(c.inspect_campaign(self.campaign)) + def test_saved_exact_correction_reports_geometry_without_dividing_by_zero_defect(self): + result=c.inspect_case(self.folder);self.valid(result) + mechanism=result['trial_outcomes'][0]['correction_mechanism'] + self.assertAlmostEqual(mechanism['correction_direction_cosine'],1/math.sqrt(3.)) + self.assertAlmostEqual(mechanism['correction_over_rhs_norm'],1.) + self.assertAlmostEqual(mechanism['correction_over_alpha_squared'],10.) + self.assertAlmostEqual(mechanism['corrected_displacement_over_outer_step_norm'],math.sqrt(41.)) + self.assertAlmostEqual(mechanism['effective_direction_projection_t'],13/3) + self.assertEqual(mechanism['remainder_before_norm'],10.) + self.assertEqual(mechanism['remainder_after_norm'],0.) + self.assertEqual(mechanism['remainder_decomposition_error_norm'],0.) + self.assertIsNone(mechanism['own_taylor_over_verified_defect']) + + def test_quadratic_correction_separates_linear_defect_from_own_curvature(self): + # F(x)=1+x+x^2 at x0=0 has p=-1. At x=-1 the Jacobian + # is -1; the inexact c=.98 passes eta=.03 but nearly undoes p. + # Its own E=c^2=.9604 dominates the .02 verified linear defect. + corrected_x=-.02;corrected_f=1+corrected_x+corrected_x**2 + mechanism=c.correction_mechanism([0.],[-1.],1.,[0.],[1.],[corrected_x], + [corrected_f],[.98],[-1.],[-.98]) + self.assertAlmostEqual(mechanism['correction_direction_cosine'],-1.) + self.assertAlmostEqual(mechanism['correction_over_rhs_norm'],.98) + self.assertAlmostEqual(mechanism['corrected_displacement_over_outer_step_norm'],.02) + self.assertAlmostEqual(mechanism['effective_direction_projection_t'],.02) + self.assertAlmostEqual(mechanism['remainder_after_norm'],.9804) + self.assertAlmostEqual(mechanism['verified_linear_defect_norm'],.02) + self.assertAlmostEqual(mechanism['own_taylor_remainder_norm'],.9604) + self.assertAlmostEqual(mechanism['own_taylor_over_correction_squared'],1.) + self.assertAlmostEqual(mechanism['own_taylor_over_verified_defect'],48.02) + self.assertAlmostEqual(mechanism['linear_defect_dot_own_taylor'],.019208) + self.assertLess(mechanism['remainder_decomposition_error_norm'],1.e-15) + json.dumps(mechanism,allow_nan=False) + def test_failed_verified_gate_is_valid_evidence_but_not_a_correction_success(self): self.make_case('failed_gate');r=c.inspect_case(self.folder);self.valid(r) self.assertFalse(r['trial_outcomes'][0]['aux_verified_gate']);self.assertFalse(r['trial_outcomes'][0]['full_merit_improved'])