#!/usr/bin/env python3 """Audit frozen preconditioner artifacts and compare equal-accuracy solve costs. A completed experiment is valid even when neither inverse reaches the requested linear tolerance. Those observations must survive the audit, without acquiring an invented time-to-convergence speedup. """ import argparse import csv import hashlib import json import math from pathlib import Path REQUIRED_ARTIFACTS = { 'metadata.txt', 'layout.csv', 'preconditioner_summary.csv', 'preconditioner_actions.csv', 'preconditioner_blocks.csv', 'preconditioner_correction_blocks.csv', 'preconditioner_history.csv', 'preconditioner_invariants.csv', 'preconditioner_reference.csv', 'preconditioner_scope.txt', 'benchmark_physical_base.txt', 'benchmark_normalized_base.txt', 'benchmark_normalized_rhs.txt', 'benchmark_state_factors.txt', 'benchmark_residual_factors.txt', 'benchmark_invariant_direction.txt', 'benchmark_invariant_action.txt', 'context_reference_normalized_correction.txt', } | {role + suffix for role in ('baseline', 'candidate') for suffix in ( '_normalized_correction.txt', '_physical_correction.txt', '_normalized_linear_defect.txt', '_physical_linear_defect.txt')} def digest(path): return hashlib.sha256(path.read_bytes()).hexdigest() def hash_matches(path, expected): return path.is_file() and digest(path) == expected def rows(path): with path.open() as stream: return list(csv.DictReader(stream)) def vector(path): values = path.read_text().split() if not values: raise ValueError(f'Empty vector: {path}') count = int(values[0]) result = [float(v) for v in values[1:]] if count <= 0 or count != len(result) or not all(math.isfinite(v) for v in result): raise ValueError(f'Invalid finite vector: {path}') return result def norm(values): return math.sqrt(math.fsum(v * v for v in values)) def close(left, right): return math.isfinite(left) and math.isfinite(right) and math.isclose(left, right, rel_tol=2e-12, abs_tol=1e-28) def ratio(numerator, denominator): # Exact residual zero is a valid outcome. Do not invent a finite ratio or # serialize infinity when the comparison denominator is exactly zero. return numerator / denominator if denominator > 0 else None def invalidate_comparison(result): comparison = result['comparison'] comparison['comparison_valid'] = False comparison['outcome'] = 'invalid_artifacts' for key in ('candidate_over_baseline_defect', 'candidate_over_baseline_measured_cost', 'equal_gate_speedup'): comparison[key] = None def invalid_case(error): result = {'passed': False, 'checks': [{'check': 'readable_complete_artifacts', 'passed': False, 'error': str(error)}], 'comparison': {}} invalidate_comparison(result) return result def inspect_case(folder): try: return inspect_case_artifacts(folder) except (OSError, ValueError, KeyError, TypeError, OverflowError, ZeroDivisionError) as error: return invalid_case(error) def inspect_case_artifacts(folder): checks = [] def check(name, result): checks.append({'check': name, 'passed': bool(result)}) summary = rows(folder / 'preconditioner_summary.csv') check('exactly_one_baseline_and_candidate', len(summary) == 2 and {r['role'] for r in summary} == {'baseline', 'candidate'}) by_role = {r['role']: r for r in summary} rhs = vector(folder / 'benchmark_normalized_rhs.txt') rhs_norm = norm(rhs) check('finite_nonzero_rhs', math.isfinite(rhs_norm) and rhs_norm > 0) layout = rows(folder / 'layout.csv') residual_blocks = [r for r in layout if r['kind'] == 'residual'] covered = [] for row in residual_blocks: covered.extend(range(int(row['offset']), int(row['offset']) + int(row['size']))) check('residual_layout_covers_vector_once', covered == list(range(len(rhs)))) invariants = rows(folder / 'preconditioner_invariants.csv') required = {'accepted_physical_bitwise_identical', 'accepted_normalized_bitwise_identical', 'accepted_residual_bitwise_identical', 'correction_bitwise_identical', 'trial_state_bitwise_identical', 'trial_residual_bitwise_identical', 'candidate_physical_bitwise_identical', 'state_factors_bitwise_identical', 'residual_factors_bitwise_identical', 'preparation_generation_unchanged', 'normalized_operator_current', 'jacobian_action_bitwise_identical'} for role in ('context_reference', 'baseline', 'candidate'): role_rows = [r for r in invariants if r['role'] == role] metrics = {r['metric']: float(r['value']) for r in role_rows} check(role + '_unique_invariants', len(role_rows) == len(metrics)) check(role + '_all_invariants_recorded', required <= metrics.keys()) check(role + '_all_invariants_pass', all(metrics.get(name) == 1.0 for name in required)) check(role + '_exact_jacobian_unchanged', metrics.get('jacobian_action_difference_norm') == 0.0) action_rows = rows(folder / 'preconditioner_actions.csv') check('action_checks_present', len(action_rows) == 2 and {row['role'] for row in action_rows} == {'baseline', 'candidate'}) for row in action_rows: measured = by_role[row['role']] expected = measured['variant'] == 'independent' or ( measured['variant'] == 'gravity-schur' and int(measured['schur_correction_sweeps']) == 0) check(row['role'] + '_action_variant_matches', row['variant'] == by_role[row['role']]['variant']) check(row['role'] + '_equivalence_requirement_recorded', row['context_equivalence_required'] == ('1' if expected else '0')) if expected: difference = float(row['context_action_relative_difference']) check(row['role'] + '_fresh_inverse_matches_default', math.isfinite(difference) and 0 <= difference <= 256 * 2.220446049250313e-16) check('baseline_is_independent', by_role['baseline']['variant'] == 'independent') block_rows = rows(folder / 'preconditioner_blocks.csv') for role, row in by_role.items(): defect = vector(folder / (role + '_normalized_linear_defect.txt')) solution = vector(folder / (role + '_normalized_correction.txt')) true_norm = norm(defect) check(role + '_compatible_vector_sizes', len(defect) == len(rhs) == len(solution)) check(role + '_rhs_norm', close(float(row['rhs_norm']), rhs_norm)) check(role + '_defect_norm', close(float(row['true_residual_norm']), true_norm)) check(role + '_relative_defect', close(float(row['true_relative']), true_norm / rhs_norm)) check(role + '_solution_norm', close(float(row['normalized_correction_norm']), norm(solution))) target = float(row['relative_tolerance']) * rhs_norm check(role + '_valid_positive_tolerance', math.isfinite(target) and target > 0) check(role + '_unchanged_target', close(float(row['absolute_target']), target)) check(role + '_valid_gate_flag', row['true_gate_pass'] in ('0', '1')) check(role + '_honest_gate', (row['true_gate_pass'] == '1') == (true_norm <= target)) check(role + '_cold_start', close(float(row['reported_initial_norm']), rhs_norm)) check(role + '_iteration_count', int(row['iterations']) == int(row['preconditioner_applications']) and 0 < int(row['iterations']) <= int(row['maximum_iterations'])) timing_names = ['construction_seconds', 'normalization_binding_seconds', 'krylov_setup_seconds', 'krylov_seconds'] check(role + '_valid_timing', all(math.isfinite(float(row[k])) and float(row[k]) >= 0 for k in timing_names)) check(role + '_complete_setup_solve_cost', close(float(row['setup_plus_solve_seconds']), sum(float(row[k]) for k in timing_names))) extra_timings = ['true_check_seconds', 'jacobian_seconds', 'preconditioner_seconds'] check(role + '_valid_verification_application_timing', all(math.isfinite(float(row[k])) and float(row[k]) >= 0 for k in extra_timings)) full_cost = float(row['setup_solve_and_true_check_seconds']) check(role + '_complete_verified_solve_cost', full_cost > 0 and close(full_cost, float(row['setup_plus_solve_seconds']) + float(row['true_check_seconds']))) check(role + '_one_independent_true_check', int(row['true_check_jacobian_applications']) == 1) check(role + '_nonoverlapping_application_timers', float(row['jacobian_seconds']) + float(row['preconditioner_seconds']) <= float(row['krylov_seconds']) + 1e-8) role_blocks = [r for r in block_rows if r['role'] == role] recorded = {r['block']: r for r in role_blocks} check(role + '_unique_residual_blocks', len(role_blocks) == len(recorded)) check(role + '_all_residual_blocks_present', set(recorded) == {b['block'] for b in residual_blocks}) for block in residual_blocks: start, count = int(block['offset']), int(block['size']) measured = recorded[block['block']] check(role + '_' + block['block'] + '_defect_norm', close(float(measured['normalized_defect_norm']), norm(defect[start:start + count]))) check(role + '_' + block['block'] + '_rhs_norm', close(float(measured['normalized_rhs_norm']), norm(rhs[start:start + count]))) baseline, candidate = by_role['baseline'], by_role['candidate'] for key in ('restart', 'maximum_iterations', 'relative_tolerance', 'rhs_norm', 'absolute_target'): check('matched_' + key, baseline[key] == candidate[key]) both_pass = baseline['true_gate_pass'] == candidate['true_gate_pass'] == '1' outcome = 'both_pass' if both_pass else ('candidate_only_pass' if candidate['true_gate_pass'] == '1' else 'baseline_only_pass' if baseline['true_gate_pass'] == '1' else 'neither_passes') comparison = {'outcome': outcome, 'recorded_gate_outcome': outcome, 'comparison_valid': True, 'candidate_over_baseline_defect': ratio(float(candidate['true_relative']), float(baseline['true_relative'])), 'candidate_over_baseline_measured_cost': ratio(float(candidate['setup_solve_and_true_check_seconds']), float(baseline['setup_solve_and_true_check_seconds'])), 'equal_gate_speedup': float(baseline['setup_solve_and_true_check_seconds']) / float(candidate['setup_solve_and_true_check_seconds']) if both_pass else None, 'baseline': baseline, 'candidate': candidate} result = {'passed': all(c['passed'] for c in checks), 'checks': checks, 'comparison': comparison} if not result['passed']: invalidate_comparison(result) return result def inspect_campaign(campaign): provenance = json.loads((campaign / 'provenance.json').read_text()) checks = [{'check': 'immutable_executable_hash', 'passed': hash_matches(campaign / 'plateau_diagnostic', provenance['binary_sha256'])}, {'check': 'mesh_hash', 'passed': hash_matches(Path(provenance['mesh']), provenance['mesh_sha256'])}, {'check': 'nonempty_unique_cases', 'passed': bool(provenance['cases']) and len(provenance['cases']) == len({case['id'] for case in provenance['cases']})}] results = {} for case in provenance['cases']: name = case['id'] record = json.loads((campaign / (name + '_run.json')).read_text()) checks.append({'check': name + '_completed', 'passed': bool(record.get('finished_utc')) and record.get('exit_code') == 0 and not record.get('timed_out', False)}) checks.append({'check': name + '_required_outputs_hashed', 'passed': REQUIRED_ARTIFACTS <= record.get('output_hashes', {}).keys()}) arguments = case.get('arguments', []) expected_inputs = set() for index, argument in enumerate(arguments): if argument in ('--probe-state', '--probe-direction', '--replay-candidate'): path = Path(arguments[index + 1]) if not path.is_absolute(): path = Path(provenance['source']) / path expected_inputs.add(str(path.resolve())) checks.append({'check': name + '_required_inputs_hashed', 'passed': expected_inputs <= record.get('input_state_hashes', {}).keys()}) for path, expected in record.get('input_state_hashes', {}).items(): checks.append({'check': name + '_input_hash_' + path, 'passed': hash_matches(Path(path), expected)}) for path, expected in record.get('output_hashes', {}).items(): checks.append({'check': name + '_output_hash_' + path, 'passed': hash_matches(campaign / name / path, expected)}) results[name] = inspect_case(campaign / name) if '--preconditioner-variant' in arguments: expected_variant = arguments[arguments.index('--preconditioner-variant') + 1] candidate = results[name]['comparison'].get('candidate', {}) checks.append({'check': name + '_requested_variant', 'passed': candidate.get('variant') == expected_variant}) passed = all(c['passed'] for c in checks) and all(r['passed'] for r in results.values()) if not all(c['passed'] for c in checks): # A valid-looking CSV cannot substantiate a speedup when its producing # executable or input/output provenance failed verification. for result in results.values(): result['passed'] = False invalidate_comparison(result) assessment = {'passed': passed, 'source_commit': provenance['commit'], 'provenance_checks': checks, 'cases': results, 'scope': 'Frozen linear-solve comparison only. A failed baseline has no finite equal-gate speedup.'} return assessment def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('campaign', type=Path) parser.add_argument('--output', type=Path) args = parser.parse_args() campaign = args.campaign.resolve() assessment = inspect_campaign(campaign) path = args.output or campaign / 'preconditioner_assessment.json' path.write_text(json.dumps(assessment, indent=2, allow_nan=False) + '\n') for name, result in assessment['cases'].items(): c = result['comparison'] print(name, 'PASS' if result['passed'] else 'FAIL', c['outcome'], 'defect_ratio=', c['candidate_over_baseline_defect'], 'equal_gate_speedup=', c['equal_gate_speedup']) return 0 if assessment['passed'] else 1 if __name__ == '__main__': raise SystemExit(main())