tboudreaux/MeanField
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Validate coarse preconditioner evidence and fallback comparisons

Browse Source
This commit is contained in:
Emily Boudreaux
2026-09-17 16:42:22 -04:00
parent 370dbbf73b
commit 48cb8c527f
2 changed files with 271 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
experiments/check_preconditioner_benchmark.py
View File

@@ -24,6 +24,7 @@ REQUIRED_ARTIFACTS = {
} | {role + suffix for role in ('baseline', 'candidate') for suffix in (
'_normalized_correction.txt', '_physical_correction.txt',
'_normalized_linear_defect.txt', '_physical_linear_defect.txt')}
COARSE_VARIANTS = {'coupled-shape-coarse', 'gravity-schur-coarse'}
def digest(path):
@@ -68,7 +69,8 @@ def invalidate_comparison(result):
comparison = result['comparison']
comparison['comparison_valid'] = False
comparison['outcome'] = 'invalid_artifacts'
for key in ('candidate_over_baseline_defect', 'candidate_over_baseline_measured_cost', 'equal_gate_speedup'):
for key in ('candidate_over_baseline_defect', 'candidate_over_baseline_measured_cost',
'equal_gate_speedup', 'effective_base_equal_gate_speedup'):
comparison[key] = None
@@ -86,6 +88,106 @@ def inspect_case(folder):
return invalid_case(error)
def inspect_coarse(folder, by_role, check):
coarse_roles = {role for role, row in by_role.items() if row['variant'] in COARSE_VARIANTS}
if not coarse_roles:
return {}
records = rows(folder / 'preconditioner_coarse.csv')
check('coarse_only_requested_roles', {r['role'] for r in records} == coarse_roles)
report = {}
stage_names = ('constructed', 'before_krylov', 'after_krylov')
stable_names = {
'input_modes', 'independent_modes', 'active_modes', 'fallback',
'minimum_overlap_lower_bound', 'reciprocal_overlap_condition',
'setup_jacobian_applications', 'setup_inverse_applications', 'setup_seconds',
'persistent_vector_bytes',
}
counter_names = {'applications', 'inverse_applications', 'failed_applications'}
construction_names = {'physical_inverse_construction_seconds', 'basis_construction_seconds',
'complete_coarse_construction_seconds'}
fallback_names = {0: 'none', 1: 'no_independent_modes', 2: 'insufficient_subspace_overlap'}
for role in coarse_roles:
row = by_role[role]
prefix = role + '_coarse_'
selected = [r for r in records if r['role'] == role]
check(prefix + 'variant_matches', all(r['variant'] == row['variant'] for r in selected))
check(prefix + 'unique_metrics', len(selected) == len({(r['stage'], r['metric']) for r in selected}))
stages = {name: {r['metric']: r['value'] for r in selected if r['stage'] == name}
for name in (*stage_names, 'validation')}
check(prefix + 'known_stages', {r['stage'] for r in selected} == set(stages))
for stage in stage_names:
check(prefix + stage + '_complete', stable_names | counter_names <= stages[stage].keys())
constructed = stages['constructed']
check(prefix + 'construction_components_recorded', construction_names <= constructed.keys())
counts = {name: int(constructed[name]) for name in (
'input_modes', 'independent_modes', 'active_modes', 'fallback',
'setup_jacobian_applications', 'setup_inverse_applications', 'persistent_vector_bytes')}
input_modes, independent, active, fallback = (counts[name] for name in
('input_modes', 'independent_modes', 'active_modes', 'fallback'))
check(prefix + 'consistent_mode_counts', 0 <= active <= independent <= input_modes)
check(prefix + 'consistent_fallback',
(fallback == 0 and active == independent and active > 0) or
(fallback == 1 and independent == active == 0) or
(fallback == 2 and independent > 0 and active == 0))
setup_inverse, setup_jacobian = (counts[name] for name in
('setup_inverse_applications', 'setup_jacobian_applications'))
check(prefix + 'consistent_setup_work', independent <= setup_inverse <= input_modes and
2 * setup_inverse <= setup_jacobian <= input_modes + setup_inverse and
counts['persistent_vector_bytes'] > 0)
for stage in stage_names:
# NaN overlap estimates are expected when rank rejection prevents
# forming a coarse system. Preserve their strings in the report.
check(prefix + stage + '_unchanged_preparation', all(
float(stages[stage][name]) == float(constructed[name]) or
(math.isnan(float(stages[stage][name])) and math.isnan(float(constructed[name])))
for name in stable_names))
calls = int(stages[stage]['applications'])
inverse_calls = int(stages[stage]['inverse_applications'])
check(prefix + stage + '_successful_applications', calls >= 0 and inverse_calls == calls and
int(stages[stage]['failed_applications']) == 0)
check(prefix + 'constructed_without_applications', int(constructed['applications']) == 0)
check(prefix + 'all_active_modes_checked', int(stages['before_krylov']['applications']) == active + 1)
check(prefix + 'solve_work_matches', int(stages['after_krylov']['applications']) -
int(stages['before_krylov']['applications']) == int(row['preconditioner_applications']))
if fallback == 0:
check(prefix + 'usable_overlap', all(math.isfinite(float(constructed[name])) and
float(constructed[name]) > 0 for name in
('minimum_overlap_lower_bound', 'reciprocal_overlap_condition')) and
float(constructed['minimum_overlap_lower_bound']) >= 1e-8)
elif fallback == 2:
overlap = float(constructed['minimum_overlap_lower_bound'])
check(prefix + 'overlap_fallback_reason', not math.isfinite(overlap) or overlap < 1e-8)
times = {name: float(constructed[name]) for name in construction_names | {'setup_seconds'}}
check(prefix + 'finite_nonnegative_construction_times', all(math.isfinite(v) and v >= 0 for v in times.values()))
check(prefix + 'complete_coarse_construction_cost', times['complete_coarse_construction_seconds'] + 1e-8 >=
times['basis_construction_seconds'] + times['setup_seconds'])
check(prefix + 'physical_and_coarse_cost_included', close(float(row['construction_seconds']),
times['physical_inverse_construction_seconds'] + times['complete_coarse_construction_seconds']))
error = float(stages['validation']['maximum_exact_coarse_relative_error'])
check(prefix + 'exact_subspace_error_within_tolerance', math.isfinite(error) and 0 <= error <= 1e-8)
check(prefix + 'empty_validation_is_vacuous', active > 0 or error == 0)
# An empty validation loop reports zero by construction. It proves no
# coarse identity and cannot substantiate a coarse-method speedup.
refined_base = row['variant'] == 'gravity-schur-coarse' and int(row['schur_correction_sweeps']) > 0
effective_base = 'gravity-schur' if refined_base else 'independent'
report[role] = {
'active': active > 0, 'input_modes': input_modes, 'independent_modes': independent,
'active_modes': active, 'fallback_code': fallback,
'fallback': fallback_names.get(fallback, 'invalid'),
'exact_subspace_validation': ('verified' if math.isfinite(error) and 0 <= error <= 1e-8
else 'failed') if active > 0 else 'not_applicable_empty_subspace',
'maximum_exact_coarse_relative_error': stages['validation']['maximum_exact_coarse_relative_error'],
'effective_base_variant': effective_base,
'effective_method': row['variant'] if active > 0 else effective_base,
'method_status': 'active_coarse' if active > 0 else
'baseline_fallback' if effective_base == 'independent' else 'gravity_schur_fallback',
'recorded_stages': stages,
'timing_scope': 'Construction includes physical inverse, basis, and coarse setup. '
'Extra exact-subspace checks are diagnostics outside solve totals.',
}
return report
def inspect_case_artifacts(folder):
checks = []
def check(name, result):
@@ -173,6 +275,7 @@ def inspect_case_artifacts(folder):
check(role + '_' + block['block'] + '_rhs_norm',
close(float(measured['normalized_rhs_norm']), norm(rhs[start:start + count])))
baseline, candidate = by_role['baseline'], by_role['candidate']
coarse = inspect_coarse(folder, by_role, check)
for key in ('restart', 'maximum_iterations', 'relative_tolerance', 'rhs_norm', 'absolute_target'):
check('matched_' + key, baseline[key] == candidate[key])
both_pass = baseline['true_gate_pass'] == candidate['true_gate_pass'] == '1'
@@ -182,7 +285,12 @@ def inspect_case_artifacts(folder):
'candidate_over_baseline_defect': ratio(float(candidate['true_relative']), float(baseline['true_relative'])),
'candidate_over_baseline_measured_cost': ratio(float(candidate['setup_solve_and_true_check_seconds']), float(baseline['setup_solve_and_true_check_seconds'])),
'equal_gate_speedup': float(baseline['setup_solve_and_true_check_seconds']) / float(candidate['setup_solve_and_true_check_seconds']) if both_pass else None,
'baseline': baseline, 'candidate': candidate}
'effective_base_equal_gate_speedup': None,
'baseline': baseline, 'candidate': candidate, 'coarse': coarse}
if 'candidate' in coarse and not coarse['candidate']['active']:
if coarse['candidate']['effective_base_variant'] == 'gravity-schur':
comparison['effective_base_equal_gate_speedup'] = comparison['equal_gate_speedup']
comparison['equal_gate_speedup'] = None
result = {'passed': all(c['passed'] for c in checks), 'checks': checks, 'comparison': comparison}
if not result['passed']:
invalidate_comparison(result)
@@ -217,9 +325,14 @@ def inspect_campaign(campaign):
for path, expected in record.get('output_hashes', {}).items():
checks.append({'check': name + '_output_hash_' + path, 'passed': hash_matches(campaign / name / path, expected)})
results[name] = inspect_case(campaign / name)
candidate = results[name]['comparison'].get('candidate', {})
requested_variant = (arguments[arguments.index('--preconditioner-variant') + 1]
if '--preconditioner-variant' in arguments else 'independent')
if requested_variant in COARSE_VARIANTS or candidate.get('variant') in COARSE_VARIANTS:
checks.append({'check': name + '_coarse_report_hashed',
'passed': 'preconditioner_coarse.csv' in record.get('output_hashes', {})})
if '--preconditioner-variant' in arguments:
expected_variant = arguments[arguments.index('--preconditioner-variant') + 1]
candidate = results[name]['comparison'].get('candidate', {})
checks.append({'check': name + '_requested_variant', 'passed': candidate.get('variant') == expected_variant})
passed = all(c['passed'] for c in checks) and all(r['passed'] for r in results.values())
if not all(c['passed'] for c in checks):
@@ -246,6 +359,11 @@ def main():
c = result['comparison']
print(name, 'PASS' if result['passed'] else 'FAIL', c['outcome'],
'defect_ratio=', c['candidate_over_baseline_defect'], 'equal_gate_speedup=', c['equal_gate_speedup'])
coarse = c.get('coarse', {}).get('candidate')
if coarse:
print(' coarse_status=', coarse['method_status'], 'effective_method=', coarse['effective_method'],
'identity=', coarse['exact_subspace_validation'],
'effective_base_equal_gate_speedup=', c['effective_base_equal_gate_speedup'])
return 0 if assessment['passed'] else 1

150
experiments/test_preconditioner_benchmark_checker.py
View File

@@ -127,6 +127,45 @@ class BenchmarkCheckerTests(unittest.TestCase):
change(data)
write_csv(path, data)
def refresh_output_hashes(self):
path = self.campaign / 'screen_run.json'
record = json.loads(path.read_text())
record['output_hashes'] = {p.name: checker.digest(p) for p in self.folder.iterdir()}
write_json(path, record)
def make_coarse(self, variant='coupled-shape-coarse', fallback=0, sweeps=1):
def summary(data):
data[0]['schur_correction_sweeps'] = 0
data[1].update(variant=variant, schur_correction_sweeps=sweeps)
self.alter_rows('preconditioner_summary.csv', summary)
self.alter_rows('preconditioner_actions.csv', lambda data: data[1].update(variant=variant))
active = 2 if fallback == 0 else 0
independent = 0 if fallback == 1 else 2
stable = {
'input_modes': 3, 'independent_modes': independent, 'active_modes': active,
'fallback': fallback, 'minimum_overlap_lower_bound': 'nan' if fallback == 1 else
1e-10 if fallback == 2 else 0.1,
'reciprocal_overlap_condition': 'nan' if fallback == 1 else 0.1,
'setup_jacobian_applications': 5, 'setup_inverse_applications': 2,
'setup_seconds': 0.15, 'persistent_vector_bytes': 128,
}
records = []
for stage, calls in [('constructed', 0), ('before_krylov', active + 1), ('after_krylov', active + 5)]:
metrics = stable | {'applications': calls, 'inverse_applications': calls, 'failed_applications': 0}
if stage == 'constructed':
metrics.update(physical_inverse_construction_seconds=0.7, basis_construction_seconds=0.1,
complete_coarse_construction_seconds=0.3)
records.extend({'role': 'candidate', 'variant': variant, 'stage': stage, 'metric': key, 'value': value}
for key, value in metrics.items())
records.append({'role': 'candidate', 'variant': variant, 'stage': 'validation',
'metric': 'maximum_exact_coarse_relative_error', 'value': 1e-12 if active else 0.0})
write_csv(self.folder / 'preconditioner_coarse.csv', records)
path = self.campaign / 'provenance.json'
provenance = json.loads(path.read_text())
provenance['cases'][0]['arguments'][-1] = variant
write_json(path, provenance)
self.refresh_output_hashes()
def assert_no_claim(self, result):
self.assertFalse(result['passed'])
self.assertFalse(result['comparison']['comparison_valid'])
@@ -259,6 +298,117 @@ class BenchmarkCheckerTests(unittest.TestCase):
self.assertFalse(result['passed'])
self.assert_no_claim(result['cases']['screen'])
def test_active_coarse_requires_real_subspace_and_complete_construction_cost(self):
self.make_coarse()
result = checker.inspect_campaign(self.campaign)
self.assertTrue(result['passed'], result)
comparison = result['cases']['screen']['comparison']
self.assertAlmostEqual(comparison['equal_gate_speedup'], 10.1 / 5.1)
coarse = comparison['coarse']['candidate']
self.assertEqual(coarse['active_modes'], 2)
self.assertEqual(coarse['exact_subspace_validation'], 'verified')
self.assertEqual(coarse['method_status'], 'active_coarse')
def test_invalid_coarse_identity_cost_or_work_cannot_support_speedup(self):
for metric, value, stage in [
('maximum_exact_coarse_relative_error', 2e-8, 'validation'),
('maximum_exact_coarse_relative_error', 'nan', 'validation'),
('physical_inverse_construction_seconds', 0.1, 'constructed'),
('basis_construction_seconds', 0.9, 'constructed'),
('applications', 2, 'before_krylov'),
('failed_applications', 1, 'after_krylov')]:
with self.subTest(metric=metric, value=value):
self.make_case()
self.make_coarse()
def corrupt(data):
for row in data:
if row['metric'] == metric and row['stage'] == stage:
row['value'] = value
self.alter_rows('preconditioner_coarse.csv', corrupt)
self.assert_no_claim(checker.inspect_case(self.folder))
def test_coarse_missing_duplicate_or_changing_preparation_is_rejected(self):
for corruption in ('missing', 'duplicate', 'changed'):
with self.subTest(corruption=corruption):
self.make_case()
self.make_coarse()
def corrupt(data):
if corruption == 'missing':
data.pop(0)
elif corruption == 'duplicate':
data.append(dict(data[0]))
else:
next(row for row in data if row['stage'] == 'after_krylov' and
row['metric'] == 'active_modes')['value'] = 0
self.alter_rows('preconditioner_coarse.csv', corrupt)
self.assert_no_claim(checker.inspect_case(self.folder))
def test_coarse_fallback_is_valid_but_empty_identity_proves_nothing(self):
for fallback in (1, 2):
with self.subTest(fallback=fallback):
self.make_case()
self.make_coarse(fallback=fallback)
result = checker.inspect_campaign(self.campaign)
self.assertTrue(result['passed'], result)
comparison = result['cases']['screen']['comparison']
self.assertIsNone(comparison['equal_gate_speedup'])
self.assertIsNone(comparison['effective_base_equal_gate_speedup'])
coarse = comparison['coarse']['candidate']
self.assertEqual(coarse['method_status'], 'baseline_fallback')
self.assertEqual(coarse['effective_method'], 'independent')
self.assertEqual(coarse['exact_subspace_validation'], 'not_applicable_empty_subspace')
json.dumps(result, allow_nan=False)
def test_schur_coarse_fallback_credits_only_the_actual_base(self):
for sweeps in (0, 1):
with self.subTest(sweeps=sweeps):
self.make_case()
self.make_coarse('gravity-schur-coarse', fallback=2, sweeps=sweeps)
result = checker.inspect_campaign(self.campaign)
self.assertTrue(result['passed'], result)
comparison = result['cases']['screen']['comparison']
self.assertIsNone(comparison['equal_gate_speedup'])
if sweeps:
self.assertAlmostEqual(comparison['effective_base_equal_gate_speedup'], 10.1 / 5.1)
self.assertEqual(comparison['coarse']['candidate']['effective_method'], 'gravity-schur')
else:
self.assertIsNone(comparison['effective_base_equal_gate_speedup'])
self.assertEqual(comparison['coarse']['candidate']['effective_method'], 'independent')
def test_contradictory_coarse_modes_or_fallback_are_rejected(self):
for metric, value in [('fallback', 1), ('fallback', 8), ('active_modes', 0), ('independent_modes', 4),
('minimum_overlap_lower_bound', 1e-10)]:
with self.subTest(metric=metric, value=value):
self.make_case()
self.make_coarse()
def corrupt(data):
for row in data:
if row['metric'] == metric:
row['value'] = value
self.alter_rows('preconditioner_coarse.csv', corrupt)
self.assert_no_claim(checker.inspect_case(self.folder))
def test_schur_fallback_does_not_claim_speedup_when_baseline_misses_gate(self):
self.make_case(baseline_defect=0.2)
self.make_coarse('gravity-schur-coarse', fallback=2)
result = checker.inspect_campaign(self.campaign)
self.assertTrue(result['passed'], result)
comparison = result['cases']['screen']['comparison']
self.assertEqual(comparison['outcome'], 'candidate_only_pass')
self.assertIsNone(comparison['equal_gate_speedup'])
self.assertIsNone(comparison['effective_base_equal_gate_speedup'])
def test_missing_coarse_hash_invalidates_even_a_schur_fallback_speedup(self):
self.make_coarse('gravity-schur-coarse', fallback=2)
path = self.campaign / 'screen_run.json'
record = json.loads(path.read_text())
record['output_hashes'].pop('preconditioner_coarse.csv')
write_json(path, record)
result = checker.inspect_campaign(self.campaign)
self.assertFalse(result['passed'])
self.assert_no_claim(result['cases']['screen'])
self.assertIsNone(result['cases']['screen']['comparison']['effective_base_equal_gate_speedup'])
if __name__ == '__main__':
unittest.main()