372 lines
23 KiB
Python
372 lines
23 KiB
Python
#!/usr/bin/env python3
|
|
"""Audit frozen preconditioner artifacts and compare equal-accuracy solve costs.
|
|
|
|
A completed experiment is valid even when neither inverse reaches the requested
|
|
linear tolerance. Those observations must survive the audit, without acquiring
|
|
an invented time-to-convergence speedup.
|
|
"""
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
|
|
|
|
REQUIRED_ARTIFACTS = {
|
|
'metadata.txt', 'layout.csv', 'preconditioner_summary.csv', 'preconditioner_actions.csv',
|
|
'preconditioner_blocks.csv', 'preconditioner_correction_blocks.csv', 'preconditioner_history.csv',
|
|
'preconditioner_invariants.csv', 'preconditioner_reference.csv', 'preconditioner_scope.txt',
|
|
'benchmark_physical_base.txt', 'benchmark_normalized_base.txt', 'benchmark_normalized_rhs.txt',
|
|
'benchmark_state_factors.txt', 'benchmark_residual_factors.txt',
|
|
'benchmark_invariant_direction.txt', 'benchmark_invariant_action.txt',
|
|
'context_reference_normalized_correction.txt',
|
|
} | {role + suffix for role in ('baseline', 'candidate') for suffix in (
|
|
'_normalized_correction.txt', '_physical_correction.txt',
|
|
'_normalized_linear_defect.txt', '_physical_linear_defect.txt')}
|
|
COARSE_VARIANTS = {'coupled-shape-coarse', 'gravity-schur-coarse'}
|
|
|
|
|
|
def digest(path):
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def hash_matches(path, expected):
|
|
return path.is_file() and digest(path) == expected
|
|
|
|
|
|
def rows(path):
|
|
with path.open() as stream:
|
|
return list(csv.DictReader(stream))
|
|
|
|
|
|
def vector(path):
|
|
values = path.read_text().split()
|
|
if not values:
|
|
raise ValueError(f'Empty vector: {path}')
|
|
count = int(values[0])
|
|
result = [float(v) for v in values[1:]]
|
|
if count <= 0 or count != len(result) or not all(math.isfinite(v) for v in result):
|
|
raise ValueError(f'Invalid finite vector: {path}')
|
|
return result
|
|
|
|
|
|
def norm(values):
|
|
return math.sqrt(math.fsum(v * v for v in values))
|
|
|
|
|
|
def close(left, right):
|
|
return math.isfinite(left) and math.isfinite(right) and math.isclose(left, right, rel_tol=2e-12, abs_tol=1e-28)
|
|
|
|
|
|
def ratio(numerator, denominator):
|
|
# Exact residual zero is a valid outcome. Do not invent a finite ratio or
|
|
# serialize infinity when the comparison denominator is exactly zero.
|
|
return numerator / denominator if denominator > 0 else None
|
|
|
|
|
|
def invalidate_comparison(result):
|
|
comparison = result['comparison']
|
|
comparison['comparison_valid'] = False
|
|
comparison['outcome'] = 'invalid_artifacts'
|
|
for key in ('candidate_over_baseline_defect', 'candidate_over_baseline_measured_cost',
|
|
'equal_gate_speedup', 'effective_base_equal_gate_speedup'):
|
|
comparison[key] = None
|
|
|
|
|
|
def invalid_case(error):
|
|
result = {'passed': False, 'checks': [{'check': 'readable_complete_artifacts', 'passed': False,
|
|
'error': str(error)}], 'comparison': {}}
|
|
invalidate_comparison(result)
|
|
return result
|
|
|
|
|
|
def inspect_case(folder):
|
|
try:
|
|
return inspect_case_artifacts(folder)
|
|
except (OSError, ValueError, KeyError, TypeError, OverflowError, ZeroDivisionError) as error:
|
|
return invalid_case(error)
|
|
|
|
|
|
def inspect_coarse(folder, by_role, check):
|
|
coarse_roles = {role for role, row in by_role.items() if row['variant'] in COARSE_VARIANTS}
|
|
if not coarse_roles:
|
|
return {}
|
|
records = rows(folder / 'preconditioner_coarse.csv')
|
|
check('coarse_only_requested_roles', {r['role'] for r in records} == coarse_roles)
|
|
report = {}
|
|
stage_names = ('constructed', 'before_krylov', 'after_krylov')
|
|
stable_names = {
|
|
'input_modes', 'independent_modes', 'active_modes', 'fallback',
|
|
'minimum_overlap_lower_bound', 'reciprocal_overlap_condition',
|
|
'setup_jacobian_applications', 'setup_inverse_applications', 'setup_seconds',
|
|
'persistent_vector_bytes',
|
|
}
|
|
counter_names = {'applications', 'inverse_applications', 'failed_applications'}
|
|
construction_names = {'physical_inverse_construction_seconds', 'basis_construction_seconds',
|
|
'complete_coarse_construction_seconds'}
|
|
fallback_names = {0: 'none', 1: 'no_independent_modes', 2: 'insufficient_subspace_overlap'}
|
|
for role in coarse_roles:
|
|
row = by_role[role]
|
|
prefix = role + '_coarse_'
|
|
selected = [r for r in records if r['role'] == role]
|
|
check(prefix + 'variant_matches', all(r['variant'] == row['variant'] for r in selected))
|
|
check(prefix + 'unique_metrics', len(selected) == len({(r['stage'], r['metric']) for r in selected}))
|
|
stages = {name: {r['metric']: r['value'] for r in selected if r['stage'] == name}
|
|
for name in (*stage_names, 'validation')}
|
|
check(prefix + 'known_stages', {r['stage'] for r in selected} == set(stages))
|
|
for stage in stage_names:
|
|
check(prefix + stage + '_complete', stable_names | counter_names <= stages[stage].keys())
|
|
constructed = stages['constructed']
|
|
check(prefix + 'construction_components_recorded', construction_names <= constructed.keys())
|
|
counts = {name: int(constructed[name]) for name in (
|
|
'input_modes', 'independent_modes', 'active_modes', 'fallback',
|
|
'setup_jacobian_applications', 'setup_inverse_applications', 'persistent_vector_bytes')}
|
|
input_modes, independent, active, fallback = (counts[name] for name in
|
|
('input_modes', 'independent_modes', 'active_modes', 'fallback'))
|
|
check(prefix + 'consistent_mode_counts', 0 <= active <= independent <= input_modes)
|
|
check(prefix + 'consistent_fallback',
|
|
(fallback == 0 and active == independent and active > 0) or
|
|
(fallback == 1 and independent == active == 0) or
|
|
(fallback == 2 and independent > 0 and active == 0))
|
|
setup_inverse, setup_jacobian = (counts[name] for name in
|
|
('setup_inverse_applications', 'setup_jacobian_applications'))
|
|
check(prefix + 'consistent_setup_work', independent <= setup_inverse <= input_modes and
|
|
2 * setup_inverse <= setup_jacobian <= input_modes + setup_inverse and
|
|
counts['persistent_vector_bytes'] > 0)
|
|
for stage in stage_names:
|
|
# NaN overlap estimates are expected when rank rejection prevents
|
|
# forming a coarse system. Preserve their strings in the report.
|
|
check(prefix + stage + '_unchanged_preparation', all(
|
|
float(stages[stage][name]) == float(constructed[name]) or
|
|
(math.isnan(float(stages[stage][name])) and math.isnan(float(constructed[name])))
|
|
for name in stable_names))
|
|
calls = int(stages[stage]['applications'])
|
|
inverse_calls = int(stages[stage]['inverse_applications'])
|
|
check(prefix + stage + '_successful_applications', calls >= 0 and inverse_calls == calls and
|
|
int(stages[stage]['failed_applications']) == 0)
|
|
check(prefix + 'constructed_without_applications', int(constructed['applications']) == 0)
|
|
check(prefix + 'all_active_modes_checked', int(stages['before_krylov']['applications']) == active + 1)
|
|
check(prefix + 'solve_work_matches', int(stages['after_krylov']['applications']) -
|
|
int(stages['before_krylov']['applications']) == int(row['preconditioner_applications']))
|
|
if fallback == 0:
|
|
check(prefix + 'usable_overlap', all(math.isfinite(float(constructed[name])) and
|
|
float(constructed[name]) > 0 for name in
|
|
('minimum_overlap_lower_bound', 'reciprocal_overlap_condition')) and
|
|
float(constructed['minimum_overlap_lower_bound']) >= 1e-8)
|
|
elif fallback == 2:
|
|
overlap = float(constructed['minimum_overlap_lower_bound'])
|
|
check(prefix + 'overlap_fallback_reason', not math.isfinite(overlap) or overlap < 1e-8)
|
|
times = {name: float(constructed[name]) for name in construction_names | {'setup_seconds'}}
|
|
check(prefix + 'finite_nonnegative_construction_times', all(math.isfinite(v) and v >= 0 for v in times.values()))
|
|
check(prefix + 'complete_coarse_construction_cost', times['complete_coarse_construction_seconds'] + 1e-8 >=
|
|
times['basis_construction_seconds'] + times['setup_seconds'])
|
|
check(prefix + 'physical_and_coarse_cost_included', close(float(row['construction_seconds']),
|
|
times['physical_inverse_construction_seconds'] + times['complete_coarse_construction_seconds']))
|
|
error = float(stages['validation']['maximum_exact_coarse_relative_error'])
|
|
check(prefix + 'exact_subspace_error_within_tolerance', math.isfinite(error) and 0 <= error <= 1e-8)
|
|
check(prefix + 'empty_validation_is_vacuous', active > 0 or error == 0)
|
|
# An empty validation loop reports zero by construction. It proves no
|
|
# coarse identity and cannot substantiate a coarse-method speedup.
|
|
refined_base = row['variant'] == 'gravity-schur-coarse' and int(row['schur_correction_sweeps']) > 0
|
|
effective_base = 'gravity-schur' if refined_base else 'independent'
|
|
report[role] = {
|
|
'active': active > 0, 'input_modes': input_modes, 'independent_modes': independent,
|
|
'active_modes': active, 'fallback_code': fallback,
|
|
'fallback': fallback_names.get(fallback, 'invalid'),
|
|
'exact_subspace_validation': ('verified' if math.isfinite(error) and 0 <= error <= 1e-8
|
|
else 'failed') if active > 0 else 'not_applicable_empty_subspace',
|
|
'maximum_exact_coarse_relative_error': stages['validation']['maximum_exact_coarse_relative_error'],
|
|
'effective_base_variant': effective_base,
|
|
'effective_method': row['variant'] if active > 0 else effective_base,
|
|
'method_status': 'active_coarse' if active > 0 else
|
|
'baseline_fallback' if effective_base == 'independent' else 'gravity_schur_fallback',
|
|
'recorded_stages': stages,
|
|
'timing_scope': 'Construction includes physical inverse, basis, and coarse setup. '
|
|
'Extra exact-subspace checks are diagnostics outside solve totals.',
|
|
}
|
|
return report
|
|
|
|
|
|
def inspect_case_artifacts(folder):
|
|
checks = []
|
|
def check(name, result):
|
|
checks.append({'check': name, 'passed': bool(result)})
|
|
summary = rows(folder / 'preconditioner_summary.csv')
|
|
check('exactly_one_baseline_and_candidate', len(summary) == 2 and {r['role'] for r in summary} == {'baseline', 'candidate'})
|
|
by_role = {r['role']: r for r in summary}
|
|
rhs = vector(folder / 'benchmark_normalized_rhs.txt')
|
|
rhs_norm = norm(rhs)
|
|
check('finite_nonzero_rhs', math.isfinite(rhs_norm) and rhs_norm > 0)
|
|
layout = rows(folder / 'layout.csv')
|
|
residual_blocks = [r for r in layout if r['kind'] == 'residual']
|
|
covered = []
|
|
for row in residual_blocks:
|
|
covered.extend(range(int(row['offset']), int(row['offset']) + int(row['size'])))
|
|
check('residual_layout_covers_vector_once', covered == list(range(len(rhs))))
|
|
invariants = rows(folder / 'preconditioner_invariants.csv')
|
|
required = {'accepted_physical_bitwise_identical', 'accepted_normalized_bitwise_identical',
|
|
'accepted_residual_bitwise_identical', 'correction_bitwise_identical',
|
|
'trial_state_bitwise_identical', 'trial_residual_bitwise_identical',
|
|
'candidate_physical_bitwise_identical', 'state_factors_bitwise_identical',
|
|
'residual_factors_bitwise_identical', 'preparation_generation_unchanged',
|
|
'normalized_operator_current', 'jacobian_action_bitwise_identical'}
|
|
for role in ('context_reference', 'baseline', 'candidate'):
|
|
role_rows = [r for r in invariants if r['role'] == role]
|
|
metrics = {r['metric']: float(r['value']) for r in role_rows}
|
|
check(role + '_unique_invariants', len(role_rows) == len(metrics))
|
|
check(role + '_all_invariants_recorded', required <= metrics.keys())
|
|
check(role + '_all_invariants_pass', all(metrics.get(name) == 1.0 for name in required))
|
|
check(role + '_exact_jacobian_unchanged', metrics.get('jacobian_action_difference_norm') == 0.0)
|
|
action_rows = rows(folder / 'preconditioner_actions.csv')
|
|
check('action_checks_present', len(action_rows) == 2 and
|
|
{row['role'] for row in action_rows} == {'baseline', 'candidate'})
|
|
for row in action_rows:
|
|
measured = by_role[row['role']]
|
|
expected = measured['variant'] == 'independent' or (
|
|
measured['variant'] == 'gravity-schur' and int(measured['schur_correction_sweeps']) == 0)
|
|
check(row['role'] + '_action_variant_matches', row['variant'] == by_role[row['role']]['variant'])
|
|
check(row['role'] + '_equivalence_requirement_recorded',
|
|
row['context_equivalence_required'] == ('1' if expected else '0'))
|
|
if expected:
|
|
difference = float(row['context_action_relative_difference'])
|
|
check(row['role'] + '_fresh_inverse_matches_default',
|
|
math.isfinite(difference) and 0 <= difference <= 256 * 2.220446049250313e-16)
|
|
check('baseline_is_independent', by_role['baseline']['variant'] == 'independent')
|
|
block_rows = rows(folder / 'preconditioner_blocks.csv')
|
|
for role, row in by_role.items():
|
|
defect = vector(folder / (role + '_normalized_linear_defect.txt'))
|
|
solution = vector(folder / (role + '_normalized_correction.txt'))
|
|
true_norm = norm(defect)
|
|
check(role + '_compatible_vector_sizes', len(defect) == len(rhs) == len(solution))
|
|
check(role + '_rhs_norm', close(float(row['rhs_norm']), rhs_norm))
|
|
check(role + '_defect_norm', close(float(row['true_residual_norm']), true_norm))
|
|
check(role + '_relative_defect', close(float(row['true_relative']), true_norm / rhs_norm))
|
|
check(role + '_solution_norm', close(float(row['normalized_correction_norm']), norm(solution)))
|
|
target = float(row['relative_tolerance']) * rhs_norm
|
|
check(role + '_valid_positive_tolerance', math.isfinite(target) and target > 0)
|
|
check(role + '_unchanged_target', close(float(row['absolute_target']), target))
|
|
check(role + '_valid_gate_flag', row['true_gate_pass'] in ('0', '1'))
|
|
check(role + '_honest_gate', (row['true_gate_pass'] == '1') == (true_norm <= target))
|
|
check(role + '_cold_start', close(float(row['reported_initial_norm']), rhs_norm))
|
|
check(role + '_iteration_count', int(row['iterations']) == int(row['preconditioner_applications'])
|
|
and 0 < int(row['iterations']) <= int(row['maximum_iterations']))
|
|
timing_names = ['construction_seconds', 'normalization_binding_seconds', 'krylov_setup_seconds', 'krylov_seconds']
|
|
check(role + '_valid_timing', all(math.isfinite(float(row[k])) and float(row[k]) >= 0 for k in timing_names))
|
|
check(role + '_complete_setup_solve_cost', close(float(row['setup_plus_solve_seconds']), sum(float(row[k]) for k in timing_names)))
|
|
extra_timings = ['true_check_seconds', 'jacobian_seconds', 'preconditioner_seconds']
|
|
check(role + '_valid_verification_application_timing',
|
|
all(math.isfinite(float(row[k])) and float(row[k]) >= 0 for k in extra_timings))
|
|
full_cost = float(row['setup_solve_and_true_check_seconds'])
|
|
check(role + '_complete_verified_solve_cost', full_cost > 0 and
|
|
close(full_cost, float(row['setup_plus_solve_seconds']) + float(row['true_check_seconds'])))
|
|
check(role + '_one_independent_true_check', int(row['true_check_jacobian_applications']) == 1)
|
|
check(role + '_nonoverlapping_application_timers',
|
|
float(row['jacobian_seconds']) + float(row['preconditioner_seconds']) <= float(row['krylov_seconds']) + 1e-8)
|
|
role_blocks = [r for r in block_rows if r['role'] == role]
|
|
recorded = {r['block']: r for r in role_blocks}
|
|
check(role + '_unique_residual_blocks', len(role_blocks) == len(recorded))
|
|
check(role + '_all_residual_blocks_present', set(recorded) == {b['block'] for b in residual_blocks})
|
|
for block in residual_blocks:
|
|
start, count = int(block['offset']), int(block['size'])
|
|
measured = recorded[block['block']]
|
|
check(role + '_' + block['block'] + '_defect_norm',
|
|
close(float(measured['normalized_defect_norm']), norm(defect[start:start + count])))
|
|
check(role + '_' + block['block'] + '_rhs_norm',
|
|
close(float(measured['normalized_rhs_norm']), norm(rhs[start:start + count])))
|
|
baseline, candidate = by_role['baseline'], by_role['candidate']
|
|
coarse = inspect_coarse(folder, by_role, check)
|
|
for key in ('restart', 'maximum_iterations', 'relative_tolerance', 'rhs_norm', 'absolute_target'):
|
|
check('matched_' + key, baseline[key] == candidate[key])
|
|
both_pass = baseline['true_gate_pass'] == candidate['true_gate_pass'] == '1'
|
|
outcome = 'both_pass' if both_pass else ('candidate_only_pass' if candidate['true_gate_pass'] == '1'
|
|
else 'baseline_only_pass' if baseline['true_gate_pass'] == '1' else 'neither_passes')
|
|
comparison = {'outcome': outcome, 'recorded_gate_outcome': outcome, 'comparison_valid': True,
|
|
'candidate_over_baseline_defect': ratio(float(candidate['true_relative']), float(baseline['true_relative'])),
|
|
'candidate_over_baseline_measured_cost': ratio(float(candidate['setup_solve_and_true_check_seconds']), float(baseline['setup_solve_and_true_check_seconds'])),
|
|
'equal_gate_speedup': float(baseline['setup_solve_and_true_check_seconds']) / float(candidate['setup_solve_and_true_check_seconds']) if both_pass else None,
|
|
'effective_base_equal_gate_speedup': None,
|
|
'baseline': baseline, 'candidate': candidate, 'coarse': coarse}
|
|
if 'candidate' in coarse and not coarse['candidate']['active']:
|
|
if coarse['candidate']['effective_base_variant'] == 'gravity-schur':
|
|
comparison['effective_base_equal_gate_speedup'] = comparison['equal_gate_speedup']
|
|
comparison['equal_gate_speedup'] = None
|
|
result = {'passed': all(c['passed'] for c in checks), 'checks': checks, 'comparison': comparison}
|
|
if not result['passed']:
|
|
invalidate_comparison(result)
|
|
return result
|
|
|
|
|
|
def inspect_campaign(campaign):
|
|
provenance = json.loads((campaign / 'provenance.json').read_text())
|
|
checks = [{'check': 'immutable_executable_hash', 'passed': hash_matches(campaign / 'plateau_diagnostic', provenance['binary_sha256'])},
|
|
{'check': 'mesh_hash', 'passed': hash_matches(Path(provenance['mesh']), provenance['mesh_sha256'])},
|
|
{'check': 'nonempty_unique_cases', 'passed': bool(provenance['cases']) and
|
|
len(provenance['cases']) == len({case['id'] for case in provenance['cases']})}]
|
|
results = {}
|
|
for case in provenance['cases']:
|
|
name = case['id']
|
|
record = json.loads((campaign / (name + '_run.json')).read_text())
|
|
checks.append({'check': name + '_completed', 'passed': bool(record.get('finished_utc')) and
|
|
record.get('exit_code') == 0 and not record.get('timed_out', False)})
|
|
checks.append({'check': name + '_required_outputs_hashed',
|
|
'passed': REQUIRED_ARTIFACTS <= record.get('output_hashes', {}).keys()})
|
|
arguments = case.get('arguments', [])
|
|
expected_inputs = set()
|
|
for index, argument in enumerate(arguments):
|
|
if argument in ('--probe-state', '--probe-direction', '--replay-candidate'):
|
|
path = Path(arguments[index + 1])
|
|
if not path.is_absolute(): path = Path(provenance['source']) / path
|
|
expected_inputs.add(str(path.resolve()))
|
|
checks.append({'check': name + '_required_inputs_hashed',
|
|
'passed': expected_inputs <= record.get('input_state_hashes', {}).keys()})
|
|
for path, expected in record.get('input_state_hashes', {}).items():
|
|
checks.append({'check': name + '_input_hash_' + path, 'passed': hash_matches(Path(path), expected)})
|
|
for path, expected in record.get('output_hashes', {}).items():
|
|
checks.append({'check': name + '_output_hash_' + path, 'passed': hash_matches(campaign / name / path, expected)})
|
|
results[name] = inspect_case(campaign / name)
|
|
candidate = results[name]['comparison'].get('candidate', {})
|
|
requested_variant = (arguments[arguments.index('--preconditioner-variant') + 1]
|
|
if '--preconditioner-variant' in arguments else 'independent')
|
|
if requested_variant in COARSE_VARIANTS or candidate.get('variant') in COARSE_VARIANTS:
|
|
checks.append({'check': name + '_coarse_report_hashed',
|
|
'passed': 'preconditioner_coarse.csv' in record.get('output_hashes', {})})
|
|
if '--preconditioner-variant' in arguments:
|
|
expected_variant = arguments[arguments.index('--preconditioner-variant') + 1]
|
|
checks.append({'check': name + '_requested_variant', 'passed': candidate.get('variant') == expected_variant})
|
|
passed = all(c['passed'] for c in checks) and all(r['passed'] for r in results.values())
|
|
if not all(c['passed'] for c in checks):
|
|
# A valid-looking CSV cannot substantiate a speedup when its producing
|
|
# executable or input/output provenance failed verification.
|
|
for result in results.values():
|
|
result['passed'] = False
|
|
invalidate_comparison(result)
|
|
assessment = {'passed': passed, 'source_commit': provenance['commit'], 'provenance_checks': checks, 'cases': results,
|
|
'scope': 'Frozen linear-solve comparison only. A failed baseline has no finite equal-gate speedup.'}
|
|
return assessment
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('campaign', type=Path)
|
|
parser.add_argument('--output', type=Path)
|
|
args = parser.parse_args()
|
|
campaign = args.campaign.resolve()
|
|
assessment = inspect_campaign(campaign)
|
|
path = args.output or campaign / 'preconditioner_assessment.json'
|
|
path.write_text(json.dumps(assessment, indent=2, allow_nan=False) + '\n')
|
|
for name, result in assessment['cases'].items():
|
|
c = result['comparison']
|
|
print(name, 'PASS' if result['passed'] else 'FAIL', c['outcome'],
|
|
'defect_ratio=', c['candidate_over_baseline_defect'], 'equal_gate_speedup=', c['equal_gate_speedup'])
|
|
coarse = c.get('coarse', {}).get('candidate')
|
|
if coarse:
|
|
print(' coarse_status=', coarse['method_status'], 'effective_method=', coarse['effective_method'],
|
|
'identity=', coarse['exact_subspace_validation'],
|
|
'effective_base_equal_gate_speedup=', c['effective_base_equal_gate_speedup'])
|
|
return 0 if assessment['passed'] else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|