diff --git a/experiments/check_preconditioner_benchmark.py b/experiments/check_preconditioner_benchmark.py index 08b5cc0..29be32b 100644 --- a/experiments/check_preconditioner_benchmark.py +++ b/experiments/check_preconditioner_benchmark.py @@ -24,6 +24,7 @@ REQUIRED_ARTIFACTS = { } | {role + suffix for role in ('baseline', 'candidate') for suffix in ( '_normalized_correction.txt', '_physical_correction.txt', '_normalized_linear_defect.txt', '_physical_linear_defect.txt')} +COARSE_VARIANTS = {'coupled-shape-coarse', 'gravity-schur-coarse'} def digest(path): @@ -68,7 +69,8 @@ def invalidate_comparison(result): comparison = result['comparison'] comparison['comparison_valid'] = False comparison['outcome'] = 'invalid_artifacts' - for key in ('candidate_over_baseline_defect', 'candidate_over_baseline_measured_cost', 'equal_gate_speedup'): + for key in ('candidate_over_baseline_defect', 'candidate_over_baseline_measured_cost', + 'equal_gate_speedup', 'effective_base_equal_gate_speedup'): comparison[key] = None @@ -86,6 +88,106 @@ def inspect_case(folder): return invalid_case(error) +def inspect_coarse(folder, by_role, check): + coarse_roles = {role for role, row in by_role.items() if row['variant'] in COARSE_VARIANTS} + if not coarse_roles: + return {} + records = rows(folder / 'preconditioner_coarse.csv') + check('coarse_only_requested_roles', {r['role'] for r in records} == coarse_roles) + report = {} + stage_names = ('constructed', 'before_krylov', 'after_krylov') + stable_names = { + 'input_modes', 'independent_modes', 'active_modes', 'fallback', + 'minimum_overlap_lower_bound', 'reciprocal_overlap_condition', + 'setup_jacobian_applications', 'setup_inverse_applications', 'setup_seconds', + 'persistent_vector_bytes', + } + counter_names = {'applications', 'inverse_applications', 'failed_applications'} + construction_names = {'physical_inverse_construction_seconds', 'basis_construction_seconds', + 'complete_coarse_construction_seconds'} + fallback_names = {0: 'none', 1: 'no_independent_modes', 2: 'insufficient_subspace_overlap'} + for role in coarse_roles: + row = by_role[role] + prefix = role + '_coarse_' + selected = [r for r in records if r['role'] == role] + check(prefix + 'variant_matches', all(r['variant'] == row['variant'] for r in selected)) + check(prefix + 'unique_metrics', len(selected) == len({(r['stage'], r['metric']) for r in selected})) + stages = {name: {r['metric']: r['value'] for r in selected if r['stage'] == name} + for name in (*stage_names, 'validation')} + check(prefix + 'known_stages', {r['stage'] for r in selected} == set(stages)) + for stage in stage_names: + check(prefix + stage + '_complete', stable_names | counter_names <= stages[stage].keys()) + constructed = stages['constructed'] + check(prefix + 'construction_components_recorded', construction_names <= constructed.keys()) + counts = {name: int(constructed[name]) for name in ( + 'input_modes', 'independent_modes', 'active_modes', 'fallback', + 'setup_jacobian_applications', 'setup_inverse_applications', 'persistent_vector_bytes')} + input_modes, independent, active, fallback = (counts[name] for name in + ('input_modes', 'independent_modes', 'active_modes', 'fallback')) + check(prefix + 'consistent_mode_counts', 0 <= active <= independent <= input_modes) + check(prefix + 'consistent_fallback', + (fallback == 0 and active == independent and active > 0) or + (fallback == 1 and independent == active == 0) or + (fallback == 2 and independent > 0 and active == 0)) + setup_inverse, setup_jacobian = (counts[name] for name in + ('setup_inverse_applications', 'setup_jacobian_applications')) + check(prefix + 'consistent_setup_work', independent <= setup_inverse <= input_modes and + 2 * setup_inverse <= setup_jacobian <= input_modes + setup_inverse and + counts['persistent_vector_bytes'] > 0) + for stage in stage_names: + # NaN overlap estimates are expected when rank rejection prevents + # forming a coarse system. Preserve their strings in the report. + check(prefix + stage + '_unchanged_preparation', all( + float(stages[stage][name]) == float(constructed[name]) or + (math.isnan(float(stages[stage][name])) and math.isnan(float(constructed[name]))) + for name in stable_names)) + calls = int(stages[stage]['applications']) + inverse_calls = int(stages[stage]['inverse_applications']) + check(prefix + stage + '_successful_applications', calls >= 0 and inverse_calls == calls and + int(stages[stage]['failed_applications']) == 0) + check(prefix + 'constructed_without_applications', int(constructed['applications']) == 0) + check(prefix + 'all_active_modes_checked', int(stages['before_krylov']['applications']) == active + 1) + check(prefix + 'solve_work_matches', int(stages['after_krylov']['applications']) - + int(stages['before_krylov']['applications']) == int(row['preconditioner_applications'])) + if fallback == 0: + check(prefix + 'usable_overlap', all(math.isfinite(float(constructed[name])) and + float(constructed[name]) > 0 for name in + ('minimum_overlap_lower_bound', 'reciprocal_overlap_condition')) and + float(constructed['minimum_overlap_lower_bound']) >= 1e-8) + elif fallback == 2: + overlap = float(constructed['minimum_overlap_lower_bound']) + check(prefix + 'overlap_fallback_reason', not math.isfinite(overlap) or overlap < 1e-8) + times = {name: float(constructed[name]) for name in construction_names | {'setup_seconds'}} + check(prefix + 'finite_nonnegative_construction_times', all(math.isfinite(v) and v >= 0 for v in times.values())) + check(prefix + 'complete_coarse_construction_cost', times['complete_coarse_construction_seconds'] + 1e-8 >= + times['basis_construction_seconds'] + times['setup_seconds']) + check(prefix + 'physical_and_coarse_cost_included', close(float(row['construction_seconds']), + times['physical_inverse_construction_seconds'] + times['complete_coarse_construction_seconds'])) + error = float(stages['validation']['maximum_exact_coarse_relative_error']) + check(prefix + 'exact_subspace_error_within_tolerance', math.isfinite(error) and 0 <= error <= 1e-8) + check(prefix + 'empty_validation_is_vacuous', active > 0 or error == 0) + # An empty validation loop reports zero by construction. It proves no + # coarse identity and cannot substantiate a coarse-method speedup. + refined_base = row['variant'] == 'gravity-schur-coarse' and int(row['schur_correction_sweeps']) > 0 + effective_base = 'gravity-schur' if refined_base else 'independent' + report[role] = { + 'active': active > 0, 'input_modes': input_modes, 'independent_modes': independent, + 'active_modes': active, 'fallback_code': fallback, + 'fallback': fallback_names.get(fallback, 'invalid'), + 'exact_subspace_validation': ('verified' if math.isfinite(error) and 0 <= error <= 1e-8 + else 'failed') if active > 0 else 'not_applicable_empty_subspace', + 'maximum_exact_coarse_relative_error': stages['validation']['maximum_exact_coarse_relative_error'], + 'effective_base_variant': effective_base, + 'effective_method': row['variant'] if active > 0 else effective_base, + 'method_status': 'active_coarse' if active > 0 else + 'baseline_fallback' if effective_base == 'independent' else 'gravity_schur_fallback', + 'recorded_stages': stages, + 'timing_scope': 'Construction includes physical inverse, basis, and coarse setup. ' + 'Extra exact-subspace checks are diagnostics outside solve totals.', + } + return report + + def inspect_case_artifacts(folder): checks = [] def check(name, result): @@ -173,6 +275,7 @@ def inspect_case_artifacts(folder): check(role + '_' + block['block'] + '_rhs_norm', close(float(measured['normalized_rhs_norm']), norm(rhs[start:start + count]))) baseline, candidate = by_role['baseline'], by_role['candidate'] + coarse = inspect_coarse(folder, by_role, check) for key in ('restart', 'maximum_iterations', 'relative_tolerance', 'rhs_norm', 'absolute_target'): check('matched_' + key, baseline[key] == candidate[key]) both_pass = baseline['true_gate_pass'] == candidate['true_gate_pass'] == '1' @@ -182,7 +285,12 @@ def inspect_case_artifacts(folder): 'candidate_over_baseline_defect': ratio(float(candidate['true_relative']), float(baseline['true_relative'])), 'candidate_over_baseline_measured_cost': ratio(float(candidate['setup_solve_and_true_check_seconds']), float(baseline['setup_solve_and_true_check_seconds'])), 'equal_gate_speedup': float(baseline['setup_solve_and_true_check_seconds']) / float(candidate['setup_solve_and_true_check_seconds']) if both_pass else None, - 'baseline': baseline, 'candidate': candidate} + 'effective_base_equal_gate_speedup': None, + 'baseline': baseline, 'candidate': candidate, 'coarse': coarse} + if 'candidate' in coarse and not coarse['candidate']['active']: + if coarse['candidate']['effective_base_variant'] == 'gravity-schur': + comparison['effective_base_equal_gate_speedup'] = comparison['equal_gate_speedup'] + comparison['equal_gate_speedup'] = None result = {'passed': all(c['passed'] for c in checks), 'checks': checks, 'comparison': comparison} if not result['passed']: invalidate_comparison(result) @@ -217,9 +325,14 @@ def inspect_campaign(campaign): for path, expected in record.get('output_hashes', {}).items(): checks.append({'check': name + '_output_hash_' + path, 'passed': hash_matches(campaign / name / path, expected)}) results[name] = inspect_case(campaign / name) + candidate = results[name]['comparison'].get('candidate', {}) + requested_variant = (arguments[arguments.index('--preconditioner-variant') + 1] + if '--preconditioner-variant' in arguments else 'independent') + if requested_variant in COARSE_VARIANTS or candidate.get('variant') in COARSE_VARIANTS: + checks.append({'check': name + '_coarse_report_hashed', + 'passed': 'preconditioner_coarse.csv' in record.get('output_hashes', {})}) if '--preconditioner-variant' in arguments: expected_variant = arguments[arguments.index('--preconditioner-variant') + 1] - candidate = results[name]['comparison'].get('candidate', {}) checks.append({'check': name + '_requested_variant', 'passed': candidate.get('variant') == expected_variant}) passed = all(c['passed'] for c in checks) and all(r['passed'] for r in results.values()) if not all(c['passed'] for c in checks): @@ -246,6 +359,11 @@ def main(): c = result['comparison'] print(name, 'PASS' if result['passed'] else 'FAIL', c['outcome'], 'defect_ratio=', c['candidate_over_baseline_defect'], 'equal_gate_speedup=', c['equal_gate_speedup']) + coarse = c.get('coarse', {}).get('candidate') + if coarse: + print(' coarse_status=', coarse['method_status'], 'effective_method=', coarse['effective_method'], + 'identity=', coarse['exact_subspace_validation'], + 'effective_base_equal_gate_speedup=', c['effective_base_equal_gate_speedup']) return 0 if assessment['passed'] else 1 diff --git a/experiments/test_preconditioner_benchmark_checker.py b/experiments/test_preconditioner_benchmark_checker.py index f3820a4..bb687ed 100644 --- a/experiments/test_preconditioner_benchmark_checker.py +++ b/experiments/test_preconditioner_benchmark_checker.py @@ -127,6 +127,45 @@ class BenchmarkCheckerTests(unittest.TestCase): change(data) write_csv(path, data) + def refresh_output_hashes(self): + path = self.campaign / 'screen_run.json' + record = json.loads(path.read_text()) + record['output_hashes'] = {p.name: checker.digest(p) for p in self.folder.iterdir()} + write_json(path, record) + + def make_coarse(self, variant='coupled-shape-coarse', fallback=0, sweeps=1): + def summary(data): + data[0]['schur_correction_sweeps'] = 0 + data[1].update(variant=variant, schur_correction_sweeps=sweeps) + self.alter_rows('preconditioner_summary.csv', summary) + self.alter_rows('preconditioner_actions.csv', lambda data: data[1].update(variant=variant)) + active = 2 if fallback == 0 else 0 + independent = 0 if fallback == 1 else 2 + stable = { + 'input_modes': 3, 'independent_modes': independent, 'active_modes': active, + 'fallback': fallback, 'minimum_overlap_lower_bound': 'nan' if fallback == 1 else + 1e-10 if fallback == 2 else 0.1, + 'reciprocal_overlap_condition': 'nan' if fallback == 1 else 0.1, + 'setup_jacobian_applications': 5, 'setup_inverse_applications': 2, + 'setup_seconds': 0.15, 'persistent_vector_bytes': 128, + } + records = [] + for stage, calls in [('constructed', 0), ('before_krylov', active + 1), ('after_krylov', active + 5)]: + metrics = stable | {'applications': calls, 'inverse_applications': calls, 'failed_applications': 0} + if stage == 'constructed': + metrics.update(physical_inverse_construction_seconds=0.7, basis_construction_seconds=0.1, + complete_coarse_construction_seconds=0.3) + records.extend({'role': 'candidate', 'variant': variant, 'stage': stage, 'metric': key, 'value': value} + for key, value in metrics.items()) + records.append({'role': 'candidate', 'variant': variant, 'stage': 'validation', + 'metric': 'maximum_exact_coarse_relative_error', 'value': 1e-12 if active else 0.0}) + write_csv(self.folder / 'preconditioner_coarse.csv', records) + path = self.campaign / 'provenance.json' + provenance = json.loads(path.read_text()) + provenance['cases'][0]['arguments'][-1] = variant + write_json(path, provenance) + self.refresh_output_hashes() + def assert_no_claim(self, result): self.assertFalse(result['passed']) self.assertFalse(result['comparison']['comparison_valid']) @@ -259,6 +298,117 @@ class BenchmarkCheckerTests(unittest.TestCase): self.assertFalse(result['passed']) self.assert_no_claim(result['cases']['screen']) + def test_active_coarse_requires_real_subspace_and_complete_construction_cost(self): + self.make_coarse() + result = checker.inspect_campaign(self.campaign) + self.assertTrue(result['passed'], result) + comparison = result['cases']['screen']['comparison'] + self.assertAlmostEqual(comparison['equal_gate_speedup'], 10.1 / 5.1) + coarse = comparison['coarse']['candidate'] + self.assertEqual(coarse['active_modes'], 2) + self.assertEqual(coarse['exact_subspace_validation'], 'verified') + self.assertEqual(coarse['method_status'], 'active_coarse') + + def test_invalid_coarse_identity_cost_or_work_cannot_support_speedup(self): + for metric, value, stage in [ + ('maximum_exact_coarse_relative_error', 2e-8, 'validation'), + ('maximum_exact_coarse_relative_error', 'nan', 'validation'), + ('physical_inverse_construction_seconds', 0.1, 'constructed'), + ('basis_construction_seconds', 0.9, 'constructed'), + ('applications', 2, 'before_krylov'), + ('failed_applications', 1, 'after_krylov')]: + with self.subTest(metric=metric, value=value): + self.make_case() + self.make_coarse() + def corrupt(data): + for row in data: + if row['metric'] == metric and row['stage'] == stage: + row['value'] = value + self.alter_rows('preconditioner_coarse.csv', corrupt) + self.assert_no_claim(checker.inspect_case(self.folder)) + + def test_coarse_missing_duplicate_or_changing_preparation_is_rejected(self): + for corruption in ('missing', 'duplicate', 'changed'): + with self.subTest(corruption=corruption): + self.make_case() + self.make_coarse() + def corrupt(data): + if corruption == 'missing': + data.pop(0) + elif corruption == 'duplicate': + data.append(dict(data[0])) + else: + next(row for row in data if row['stage'] == 'after_krylov' and + row['metric'] == 'active_modes')['value'] = 0 + self.alter_rows('preconditioner_coarse.csv', corrupt) + self.assert_no_claim(checker.inspect_case(self.folder)) + + def test_coarse_fallback_is_valid_but_empty_identity_proves_nothing(self): + for fallback in (1, 2): + with self.subTest(fallback=fallback): + self.make_case() + self.make_coarse(fallback=fallback) + result = checker.inspect_campaign(self.campaign) + self.assertTrue(result['passed'], result) + comparison = result['cases']['screen']['comparison'] + self.assertIsNone(comparison['equal_gate_speedup']) + self.assertIsNone(comparison['effective_base_equal_gate_speedup']) + coarse = comparison['coarse']['candidate'] + self.assertEqual(coarse['method_status'], 'baseline_fallback') + self.assertEqual(coarse['effective_method'], 'independent') + self.assertEqual(coarse['exact_subspace_validation'], 'not_applicable_empty_subspace') + json.dumps(result, allow_nan=False) + + def test_schur_coarse_fallback_credits_only_the_actual_base(self): + for sweeps in (0, 1): + with self.subTest(sweeps=sweeps): + self.make_case() + self.make_coarse('gravity-schur-coarse', fallback=2, sweeps=sweeps) + result = checker.inspect_campaign(self.campaign) + self.assertTrue(result['passed'], result) + comparison = result['cases']['screen']['comparison'] + self.assertIsNone(comparison['equal_gate_speedup']) + if sweeps: + self.assertAlmostEqual(comparison['effective_base_equal_gate_speedup'], 10.1 / 5.1) + self.assertEqual(comparison['coarse']['candidate']['effective_method'], 'gravity-schur') + else: + self.assertIsNone(comparison['effective_base_equal_gate_speedup']) + self.assertEqual(comparison['coarse']['candidate']['effective_method'], 'independent') + + def test_contradictory_coarse_modes_or_fallback_are_rejected(self): + for metric, value in [('fallback', 1), ('fallback', 8), ('active_modes', 0), ('independent_modes', 4), + ('minimum_overlap_lower_bound', 1e-10)]: + with self.subTest(metric=metric, value=value): + self.make_case() + self.make_coarse() + def corrupt(data): + for row in data: + if row['metric'] == metric: + row['value'] = value + self.alter_rows('preconditioner_coarse.csv', corrupt) + self.assert_no_claim(checker.inspect_case(self.folder)) + + def test_schur_fallback_does_not_claim_speedup_when_baseline_misses_gate(self): + self.make_case(baseline_defect=0.2) + self.make_coarse('gravity-schur-coarse', fallback=2) + result = checker.inspect_campaign(self.campaign) + self.assertTrue(result['passed'], result) + comparison = result['cases']['screen']['comparison'] + self.assertEqual(comparison['outcome'], 'candidate_only_pass') + self.assertIsNone(comparison['equal_gate_speedup']) + self.assertIsNone(comparison['effective_base_equal_gate_speedup']) + + def test_missing_coarse_hash_invalidates_even_a_schur_fallback_speedup(self): + self.make_coarse('gravity-schur-coarse', fallback=2) + path = self.campaign / 'screen_run.json' + record = json.loads(path.read_text()) + record['output_hashes'].pop('preconditioner_coarse.csv') + write_json(path, record) + result = checker.inspect_campaign(self.campaign) + self.assertFalse(result['passed']) + self.assert_no_claim(result['cases']['screen']) + self.assertIsNone(result['cases']['screen']['comparison']['effective_base_equal_gate_speedup']) + if __name__ == '__main__': unittest.main()