415 lines
23 KiB
Python
415 lines
23 KiB
Python
#!/usr/bin/env python3
|
|
"""Small synthetic linear experiments testing whether a speedup is supportable.
|
|
|
|
No executable, FEM setup, or solver is launched. The fixture uses J=I and keeps
|
|
the exact solution/defect relationship while varying convergence and evidence.
|
|
"""
|
|
import csv
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
|
|
import check_preconditioner_benchmark as checker
|
|
|
|
|
|
def write_csv(path, records):
|
|
with path.open('w', newline='') as stream:
|
|
writer = csv.DictWriter(stream, fieldnames=list(records[0]))
|
|
writer.writeheader()
|
|
writer.writerows(records)
|
|
|
|
|
|
def write_vector(path, values):
|
|
path.write_text(str(len(values)) + '\n' + '\n'.join(repr(v) for v in values) + '\n')
|
|
|
|
|
|
def write_json(path, value):
|
|
path.write_text(json.dumps(value, indent=2) + '\n')
|
|
|
|
|
|
class BenchmarkCheckerTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temporary.cleanup)
|
|
self.campaign = Path(self.temporary.name).resolve()
|
|
self.folder = self.campaign / 'screen'
|
|
self.folder.mkdir()
|
|
self.rhs = [3.0, 4.0]
|
|
self.make_case()
|
|
|
|
def make_case(self, baseline_defect=0.1, candidate_defect=0.05):
|
|
layout = [{'kind': 'residual', 'block': 'equilibrium', 'offset': 0, 'size': 2, 'scale': 1},
|
|
{'kind': 'value', 'block': 'state', 'offset': 0, 'size': 2, 'scale': 1}]
|
|
write_csv(self.folder / 'layout.csv', layout)
|
|
summary, blocks, actions, correction_blocks, history = [], [], [], [], []
|
|
for role, variant, defect, cost in (
|
|
('baseline', 'independent', baseline_defect, 10.0),
|
|
('candidate', 'stellar-ldu', candidate_defect, 5.0)):
|
|
# The saved defect convention is Jp-rhs, hence p=rhs+defect for J=I.
|
|
solution = [self.rhs[0] + defect, self.rhs[1]]
|
|
exact_defect = [defect, 0.0]
|
|
for coordinates in ('normalized', 'physical'):
|
|
write_vector(self.folder / f'{role}_{coordinates}_correction.txt', solution)
|
|
write_vector(self.folder / f'{role}_{coordinates}_linear_defect.txt', exact_defect)
|
|
summary.append({
|
|
'role': role, 'variant': variant, 'restart': 10, 'maximum_iterations': 20,
|
|
'relative_tolerance': 0.03, 'rhs_norm': 5.0, 'absolute_target': 0.15,
|
|
'true_residual_norm': defect, 'true_relative': defect / 5.0,
|
|
'true_gate_pass': int(defect <= 0.15), 'solver_converged': int(defect <= 0.15),
|
|
'reported_initial_norm': 5.0, 'normalized_correction_norm': checker.norm(solution),
|
|
'iterations': 4, 'preconditioner_applications': 4,
|
|
'construction_seconds': 1.0, 'normalization_binding_seconds': 0.1,
|
|
'krylov_setup_seconds': 0.2, 'krylov_seconds': cost - 1.3,
|
|
'setup_plus_solve_seconds': cost, 'true_check_seconds': 0.1,
|
|
'setup_solve_and_true_check_seconds': cost + 0.1,
|
|
'jacobian_seconds': 1.0, 'preconditioner_seconds': 1.0,
|
|
'true_check_jacobian_applications': 1,
|
|
})
|
|
blocks.append({'role': role, 'variant': variant, 'block': 'equilibrium',
|
|
'normalized_rhs_norm': 5.0, 'normalized_defect_norm': defect})
|
|
actions.append({'role': role, 'variant': variant,
|
|
'context_equivalence_required': int(role == 'baseline'),
|
|
'context_action_relative_difference': 0.0})
|
|
correction_blocks.append({'role': role, 'variant': variant, 'block': 'state',
|
|
'normalized_norm': checker.norm(solution),
|
|
'physical_norm': checker.norm(solution)})
|
|
history.append({'role': role, 'variant': variant, 'iteration': 4,
|
|
'reported_norm': defect, 'final': 1})
|
|
for file, data in [('preconditioner_summary.csv', summary), ('preconditioner_blocks.csv', blocks),
|
|
('preconditioner_actions.csv', actions), ('preconditioner_history.csv', history),
|
|
('preconditioner_correction_blocks.csv', correction_blocks)]:
|
|
write_csv(self.folder / file, data)
|
|
invariants = []
|
|
for role in ('context_reference', 'baseline', 'candidate'):
|
|
for metric in ('accepted_physical_bitwise_identical', 'accepted_normalized_bitwise_identical',
|
|
'accepted_residual_bitwise_identical', 'correction_bitwise_identical',
|
|
'trial_state_bitwise_identical', 'trial_residual_bitwise_identical',
|
|
'candidate_physical_bitwise_identical', 'state_factors_bitwise_identical',
|
|
'residual_factors_bitwise_identical', 'preparation_generation_unchanged',
|
|
'normalized_operator_current', 'jacobian_action_bitwise_identical'):
|
|
invariants.append({'role': role, 'metric': metric, 'value': 1})
|
|
invariants.append({'role': role, 'metric': 'jacobian_action_difference_norm', 'value': 0})
|
|
write_csv(self.folder / 'preconditioner_invariants.csv', invariants)
|
|
write_csv(self.folder / 'preconditioner_reference.csv', [{'metric': 'context_inverse_correction_norm', 'value': 5.0}])
|
|
for name, values in {
|
|
'benchmark_normalized_rhs.txt': self.rhs,
|
|
'benchmark_normalized_base.txt': [2.0, 3.0], 'benchmark_physical_base.txt': [2.0, 3.0],
|
|
'benchmark_state_factors.txt': [1.0, 1.0], 'benchmark_residual_factors.txt': [1.0, 1.0],
|
|
'benchmark_invariant_direction.txt': [1.0, 0.0], 'benchmark_invariant_action.txt': [1.0, 0.0],
|
|
'context_reference_normalized_correction.txt': self.rhs,
|
|
}.items():
|
|
write_vector(self.folder / name, values)
|
|
(self.folder / 'metadata.txt').write_text('n=1\nJ=0\ncentral_phase=0\npreconditioner_probe=1\n')
|
|
(self.folder / 'preconditioner_scope.txt').write_text('Synthetic J=I accuracy-gate comparison.\n')
|
|
self.snapshot = self.campaign / 'snapshot.txt'
|
|
write_vector(self.snapshot, [2.0, 3.0])
|
|
self.mesh = self.campaign / 'mesh.smesh'
|
|
self.mesh.write_text('Synthetic fixture; never opened by a solver.\n')
|
|
executable = self.campaign / 'plateau_diagnostic'
|
|
executable.write_bytes(b'Synthetic executable provenance; not executable.\n')
|
|
write_json(self.campaign / 'provenance.json', {
|
|
'commit': 'synthetic-test', 'source': str(self.campaign),
|
|
'binary_sha256': checker.digest(executable), 'mesh': str(self.mesh),
|
|
'mesh_sha256': checker.digest(self.mesh), 'cases': [{
|
|
'id': 'screen', 'arguments': ['--probe-state', str(self.snapshot),
|
|
'--preconditioner-variant', 'stellar-ldu']}],
|
|
})
|
|
write_json(self.campaign / 'screen_run.json', {
|
|
'exit_code': 0, 'finished_utc': '2026-09-18T00:00:00Z',
|
|
'input_state_hashes': {str(self.snapshot): checker.digest(self.snapshot)},
|
|
'output_hashes': {p.name: checker.digest(p) for p in self.folder.iterdir()},
|
|
})
|
|
|
|
def alter_rows(self, filename, change):
|
|
path = self.folder / filename
|
|
data = checker.rows(path)
|
|
change(data)
|
|
write_csv(path, data)
|
|
|
|
def refresh_output_hashes(self):
|
|
path = self.campaign / 'screen_run.json'
|
|
record = json.loads(path.read_text())
|
|
record['output_hashes'] = {p.name: checker.digest(p) for p in self.folder.iterdir()}
|
|
write_json(path, record)
|
|
|
|
def make_coarse(self, variant='coupled-shape-coarse', fallback=0, sweeps=1):
|
|
def summary(data):
|
|
data[0]['schur_correction_sweeps'] = 0
|
|
data[1].update(variant=variant, schur_correction_sweeps=sweeps)
|
|
self.alter_rows('preconditioner_summary.csv', summary)
|
|
self.alter_rows('preconditioner_actions.csv', lambda data: data[1].update(variant=variant))
|
|
active = 2 if fallback == 0 else 0
|
|
independent = 0 if fallback == 1 else 2
|
|
stable = {
|
|
'input_modes': 3, 'independent_modes': independent, 'active_modes': active,
|
|
'fallback': fallback, 'minimum_overlap_lower_bound': 'nan' if fallback == 1 else
|
|
1e-10 if fallback == 2 else 0.1,
|
|
'reciprocal_overlap_condition': 'nan' if fallback == 1 else 0.1,
|
|
'setup_jacobian_applications': 5, 'setup_inverse_applications': 2,
|
|
'setup_seconds': 0.15, 'persistent_vector_bytes': 128,
|
|
}
|
|
records = []
|
|
for stage, calls in [('constructed', 0), ('before_krylov', active + 1), ('after_krylov', active + 5)]:
|
|
metrics = stable | {'applications': calls, 'inverse_applications': calls, 'failed_applications': 0}
|
|
if stage == 'constructed':
|
|
metrics.update(physical_inverse_construction_seconds=0.7, basis_construction_seconds=0.1,
|
|
complete_coarse_construction_seconds=0.3)
|
|
records.extend({'role': 'candidate', 'variant': variant, 'stage': stage, 'metric': key, 'value': value}
|
|
for key, value in metrics.items())
|
|
records.append({'role': 'candidate', 'variant': variant, 'stage': 'validation',
|
|
'metric': 'maximum_exact_coarse_relative_error', 'value': 1e-12 if active else 0.0})
|
|
write_csv(self.folder / 'preconditioner_coarse.csv', records)
|
|
path = self.campaign / 'provenance.json'
|
|
provenance = json.loads(path.read_text())
|
|
provenance['cases'][0]['arguments'][-1] = variant
|
|
write_json(path, provenance)
|
|
self.refresh_output_hashes()
|
|
|
|
def assert_no_claim(self, result):
|
|
self.assertFalse(result['passed'])
|
|
self.assertFalse(result['comparison']['comparison_valid'])
|
|
self.assertIsNone(result['comparison']['equal_gate_speedup'])
|
|
|
|
def test_matched_verified_gates_allow_measured_speedup(self):
|
|
result = checker.inspect_campaign(self.campaign)
|
|
self.assertTrue(result['passed'], result)
|
|
comparison = result['cases']['screen']['comparison']
|
|
self.assertEqual(comparison['outcome'], 'both_pass')
|
|
self.assertAlmostEqual(comparison['equal_gate_speedup'], 10.1 / 5.1)
|
|
|
|
def test_failed_gates_are_valid_observations_without_speedup(self):
|
|
for baseline, candidate, outcome in ((0.2, 0.3, 'neither_passes'),
|
|
(0.2, 0.1, 'candidate_only_pass'),
|
|
(0.1, 0.2, 'baseline_only_pass')):
|
|
with self.subTest(outcome=outcome):
|
|
self.make_case(baseline, candidate)
|
|
result = checker.inspect_campaign(self.campaign)
|
|
self.assertTrue(result['passed'], result)
|
|
comparison = result['cases']['screen']['comparison']
|
|
self.assertEqual(comparison['outcome'], outcome)
|
|
self.assertIsNone(comparison['equal_gate_speedup'])
|
|
|
|
def test_exact_zero_defect_does_not_invent_a_defect_ratio(self):
|
|
self.make_case(0.0, 0.0)
|
|
result = checker.inspect_case(self.folder)
|
|
self.assertTrue(result['passed'], result)
|
|
self.assertIsNone(result['comparison']['candidate_over_baseline_defect'])
|
|
self.assertIsNotNone(result['comparison']['equal_gate_speedup'])
|
|
json.dumps(result, allow_nan=False)
|
|
|
|
def test_missing_invariant_cannot_support_speedup(self):
|
|
self.alter_rows('preconditioner_invariants.csv', lambda data: data.pop(0))
|
|
self.assert_no_claim(checker.inspect_case(self.folder))
|
|
|
|
def test_duplicate_invariant_cannot_hide_a_failed_check(self):
|
|
def corrupt(data):
|
|
duplicate = dict(data[0])
|
|
data[0]['value'] = 0
|
|
data.append(duplicate)
|
|
self.alter_rows('preconditioner_invariants.csv', corrupt)
|
|
self.assert_no_claim(checker.inspect_case(self.folder))
|
|
|
|
def test_false_reported_gate_is_rejected(self):
|
|
self.make_case(0.2, 0.1)
|
|
self.alter_rows('preconditioner_summary.csv', lambda data: data[0].update(true_gate_pass=1))
|
|
self.assert_no_claim(checker.inspect_case(self.folder))
|
|
|
|
def test_faster_total_cannot_omit_verification_or_setup(self):
|
|
self.alter_rows('preconditioner_summary.csv',
|
|
lambda data: data[1].update(setup_solve_and_true_check_seconds=0.01))
|
|
self.assert_no_claim(checker.inspect_case(self.folder))
|
|
|
|
def test_saved_defect_and_csv_must_agree(self):
|
|
write_vector(self.folder / 'candidate_normalized_linear_defect.txt', [1.0, 0.0])
|
|
self.assert_no_claim(checker.inspect_case(self.folder))
|
|
|
|
def test_same_vector_norm_does_not_hide_wrong_block_distribution(self):
|
|
layout = [
|
|
{'kind': 'residual', 'block': 'gravity', 'offset': 0, 'size': 1, 'scale': 1},
|
|
{'kind': 'residual', 'block': 'material', 'offset': 1, 'size': 1, 'scale': 1},
|
|
]
|
|
write_csv(self.folder / 'layout.csv', layout)
|
|
block_rows = []
|
|
for role, defect in [('baseline', 0.1), ('candidate', 0.05)]:
|
|
block_rows.extend([{'role': role, 'variant': 'unused', 'block': 'gravity',
|
|
'normalized_rhs_norm': 3.0, 'normalized_defect_norm': defect},
|
|
{'role': role, 'variant': 'unused', 'block': 'material',
|
|
'normalized_rhs_norm': 4.0, 'normalized_defect_norm': 0.0}])
|
|
write_csv(self.folder / 'preconditioner_blocks.csv', block_rows)
|
|
write_vector(self.folder / 'candidate_normalized_linear_defect.txt', [0.0, 0.05])
|
|
self.assert_no_claim(checker.inspect_case(self.folder))
|
|
|
|
def test_default_equivalence_cannot_be_disabled_in_csv(self):
|
|
self.alter_rows('preconditioner_actions.csv',
|
|
lambda data: data[0].update(context_equivalence_required=0))
|
|
self.assert_no_claim(checker.inspect_case(self.folder))
|
|
|
|
def test_zero_sweep_gravity_control_requires_default_equivalence(self):
|
|
for variant, sweeps, equivalent in (('gravity-schur', 0, True),
|
|
('gravity-schur', 1, False),
|
|
('stellar-ldu-gravity-schur', 0, False)):
|
|
with self.subTest(variant=variant, sweeps=sweeps):
|
|
self.make_case()
|
|
def summary(data):
|
|
data[0]['schur_correction_sweeps'] = 0
|
|
data[1].update(variant=variant, schur_correction_sweeps=sweeps)
|
|
self.alter_rows('preconditioner_summary.csv', summary)
|
|
self.alter_rows('preconditioner_actions.csv', lambda data:
|
|
data[1].update(variant=variant, context_equivalence_required=int(equivalent)))
|
|
result = checker.inspect_case(self.folder)
|
|
self.assertTrue(result['passed'], result)
|
|
if equivalent:
|
|
self.alter_rows('preconditioner_actions.csv', lambda data:
|
|
data[1].update(context_equivalence_required=0))
|
|
self.assert_no_claim(checker.inspect_case(self.folder))
|
|
|
|
def test_omitted_input_or_output_hash_cannot_support_speedup(self):
|
|
for key, remove in [('input_state_hashes', str(self.snapshot)),
|
|
('output_hashes', 'candidate_normalized_linear_defect.txt')]:
|
|
with self.subTest(key=key):
|
|
self.make_case()
|
|
path = self.campaign / 'screen_run.json'
|
|
record = json.loads(path.read_text())
|
|
record[key].pop(remove)
|
|
write_json(path, record)
|
|
result = checker.inspect_campaign(self.campaign)
|
|
self.assertFalse(result['passed'])
|
|
self.assert_no_claim(result['cases']['screen'])
|
|
|
|
def test_changed_provenance_input_executable_mesh_or_output_blocks_claim(self):
|
|
for name in ('input', 'executable', 'mesh', 'output'):
|
|
with self.subTest(name=name):
|
|
self.make_case()
|
|
path = {'input': self.snapshot, 'executable': self.campaign / 'plateau_diagnostic',
|
|
'mesh': self.mesh, 'output': self.folder / 'preconditioner_summary.csv'}[name]
|
|
with path.open('a') as stream:
|
|
stream.write('\n')
|
|
result = checker.inspect_campaign(self.campaign)
|
|
self.assertFalse(result['passed'])
|
|
self.assert_no_claim(result['cases']['screen'])
|
|
|
|
def test_in_progress_record_is_not_a_completed_comparison(self):
|
|
path = self.campaign / 'screen_run.json'
|
|
record = json.loads(path.read_text())
|
|
record.pop('finished_utc')
|
|
write_json(path, record)
|
|
result = checker.inspect_campaign(self.campaign)
|
|
self.assertFalse(result['passed'])
|
|
self.assert_no_claim(result['cases']['screen'])
|
|
|
|
def test_active_coarse_requires_real_subspace_and_complete_construction_cost(self):
|
|
self.make_coarse()
|
|
result = checker.inspect_campaign(self.campaign)
|
|
self.assertTrue(result['passed'], result)
|
|
comparison = result['cases']['screen']['comparison']
|
|
self.assertAlmostEqual(comparison['equal_gate_speedup'], 10.1 / 5.1)
|
|
coarse = comparison['coarse']['candidate']
|
|
self.assertEqual(coarse['active_modes'], 2)
|
|
self.assertEqual(coarse['exact_subspace_validation'], 'verified')
|
|
self.assertEqual(coarse['method_status'], 'active_coarse')
|
|
|
|
def test_invalid_coarse_identity_cost_or_work_cannot_support_speedup(self):
|
|
for metric, value, stage in [
|
|
('maximum_exact_coarse_relative_error', 2e-8, 'validation'),
|
|
('maximum_exact_coarse_relative_error', 'nan', 'validation'),
|
|
('physical_inverse_construction_seconds', 0.1, 'constructed'),
|
|
('basis_construction_seconds', 0.9, 'constructed'),
|
|
('applications', 2, 'before_krylov'),
|
|
('failed_applications', 1, 'after_krylov')]:
|
|
with self.subTest(metric=metric, value=value):
|
|
self.make_case()
|
|
self.make_coarse()
|
|
def corrupt(data):
|
|
for row in data:
|
|
if row['metric'] == metric and row['stage'] == stage:
|
|
row['value'] = value
|
|
self.alter_rows('preconditioner_coarse.csv', corrupt)
|
|
self.assert_no_claim(checker.inspect_case(self.folder))
|
|
|
|
def test_coarse_missing_duplicate_or_changing_preparation_is_rejected(self):
|
|
for corruption in ('missing', 'duplicate', 'changed'):
|
|
with self.subTest(corruption=corruption):
|
|
self.make_case()
|
|
self.make_coarse()
|
|
def corrupt(data):
|
|
if corruption == 'missing':
|
|
data.pop(0)
|
|
elif corruption == 'duplicate':
|
|
data.append(dict(data[0]))
|
|
else:
|
|
next(row for row in data if row['stage'] == 'after_krylov' and
|
|
row['metric'] == 'active_modes')['value'] = 0
|
|
self.alter_rows('preconditioner_coarse.csv', corrupt)
|
|
self.assert_no_claim(checker.inspect_case(self.folder))
|
|
|
|
def test_coarse_fallback_is_valid_but_empty_identity_proves_nothing(self):
|
|
for fallback in (1, 2):
|
|
with self.subTest(fallback=fallback):
|
|
self.make_case()
|
|
self.make_coarse(fallback=fallback)
|
|
result = checker.inspect_campaign(self.campaign)
|
|
self.assertTrue(result['passed'], result)
|
|
comparison = result['cases']['screen']['comparison']
|
|
self.assertIsNone(comparison['equal_gate_speedup'])
|
|
self.assertIsNone(comparison['effective_base_equal_gate_speedup'])
|
|
coarse = comparison['coarse']['candidate']
|
|
self.assertEqual(coarse['method_status'], 'baseline_fallback')
|
|
self.assertEqual(coarse['effective_method'], 'independent')
|
|
self.assertEqual(coarse['exact_subspace_validation'], 'not_applicable_empty_subspace')
|
|
json.dumps(result, allow_nan=False)
|
|
|
|
def test_schur_coarse_fallback_credits_only_the_actual_base(self):
|
|
for sweeps in (0, 1):
|
|
with self.subTest(sweeps=sweeps):
|
|
self.make_case()
|
|
self.make_coarse('gravity-schur-coarse', fallback=2, sweeps=sweeps)
|
|
result = checker.inspect_campaign(self.campaign)
|
|
self.assertTrue(result['passed'], result)
|
|
comparison = result['cases']['screen']['comparison']
|
|
self.assertIsNone(comparison['equal_gate_speedup'])
|
|
if sweeps:
|
|
self.assertAlmostEqual(comparison['effective_base_equal_gate_speedup'], 10.1 / 5.1)
|
|
self.assertEqual(comparison['coarse']['candidate']['effective_method'], 'gravity-schur')
|
|
else:
|
|
self.assertIsNone(comparison['effective_base_equal_gate_speedup'])
|
|
self.assertEqual(comparison['coarse']['candidate']['effective_method'], 'independent')
|
|
|
|
def test_contradictory_coarse_modes_or_fallback_are_rejected(self):
|
|
for metric, value in [('fallback', 1), ('fallback', 8), ('active_modes', 0), ('independent_modes', 4),
|
|
('minimum_overlap_lower_bound', 1e-10)]:
|
|
with self.subTest(metric=metric, value=value):
|
|
self.make_case()
|
|
self.make_coarse()
|
|
def corrupt(data):
|
|
for row in data:
|
|
if row['metric'] == metric:
|
|
row['value'] = value
|
|
self.alter_rows('preconditioner_coarse.csv', corrupt)
|
|
self.assert_no_claim(checker.inspect_case(self.folder))
|
|
|
|
def test_schur_fallback_does_not_claim_speedup_when_baseline_misses_gate(self):
|
|
self.make_case(baseline_defect=0.2)
|
|
self.make_coarse('gravity-schur-coarse', fallback=2)
|
|
result = checker.inspect_campaign(self.campaign)
|
|
self.assertTrue(result['passed'], result)
|
|
comparison = result['cases']['screen']['comparison']
|
|
self.assertEqual(comparison['outcome'], 'candidate_only_pass')
|
|
self.assertIsNone(comparison['equal_gate_speedup'])
|
|
self.assertIsNone(comparison['effective_base_equal_gate_speedup'])
|
|
|
|
def test_missing_coarse_hash_invalidates_even_a_schur_fallback_speedup(self):
|
|
self.make_coarse('gravity-schur-coarse', fallback=2)
|
|
path = self.campaign / 'screen_run.json'
|
|
record = json.loads(path.read_text())
|
|
record['output_hashes'].pop('preconditioner_coarse.csv')
|
|
write_json(path, record)
|
|
result = checker.inspect_campaign(self.campaign)
|
|
self.assertFalse(result['passed'])
|
|
self.assert_no_claim(result['cases']['screen'])
|
|
self.assertIsNone(result['cases']['screen']['comparison']['effective_base_equal_gate_speedup'])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|