feat(linux): portable build works on linux + benchmarks
This commit is contained in:
@@ -589,15 +589,7 @@ def build_zlib(
|
||||
build.mkdir(parents=True, exist_ok=True)
|
||||
cmake_configure(args, args.zlib_source, build, options, env)
|
||||
run(
|
||||
[
|
||||
args.cmake,
|
||||
"--build",
|
||||
build,
|
||||
"--target",
|
||||
"zlibstatic",
|
||||
"--parallel",
|
||||
str(args.jobs),
|
||||
],
|
||||
[args.cmake, "--build", build, "--parallel", str(args.jobs)],
|
||||
env=env,
|
||||
)
|
||||
run([args.cmake, "--install", build], env=env)
|
||||
@@ -658,7 +650,7 @@ def build_hypre(
|
||||
options = common_cmake_args(
|
||||
args,
|
||||
build_type,
|
||||
c_compiler=mpicc if features["mpi"] else args.cc,
|
||||
c_compiler=args.cc,
|
||||
) + [
|
||||
f"-DHYPRE_ENABLE_MPI={'ON' if features['mpi'] else 'OFF'}",
|
||||
f"-DHYPRE_ENABLE_OPENMP={'ON' if features['openmp'] else 'OFF'}",
|
||||
@@ -668,6 +660,12 @@ def build_hypre(
|
||||
"-DHYPRE_BUILD_EXAMPLES=OFF",
|
||||
"-DHYPRE_BUILD_TESTS=OFF",
|
||||
]
|
||||
if features["mpi"]:
|
||||
# Keep CMake's compiler identity on the host compiler and let FindMPI
|
||||
# interrogate the wrapper. Treating mpicc itself as CMAKE_C_COMPILER
|
||||
# makes MPI's include path look implicit, so CUDA translation units
|
||||
# compiled by nvcc do not receive mpi.h.
|
||||
options.append(f"-DMPI_C_COMPILER={mpicc}")
|
||||
if features["cuda"]:
|
||||
options.extend(
|
||||
[
|
||||
@@ -1336,7 +1334,10 @@ def build_mfem(
|
||||
f"-DGSLIB_DIR={bundle_root(args, 'include/gslib/gslib.h')}"
|
||||
)
|
||||
if features["zlib"]:
|
||||
options.append(f"-DZLIB_ROOT={bundle_root(args, 'include/zlib.h')}")
|
||||
zlib_root = bundle_root(args, "include/zlib.h")
|
||||
options.append(f"-DZLIB_ROOT={zlib_root}")
|
||||
if args.host_system == "emscripten":
|
||||
options.append(f"-DZLIB_LIBRARY={zlib_root / 'lib' / 'libz.a'}")
|
||||
if features["sundials"]:
|
||||
options.append(
|
||||
f"-DSUNDIALS_DIR={bundle_root(args, 'include/sundials/sundials_config.h')}"
|
||||
|
||||
667
tools/run_backend_benchmarks.py
Executable file
667
tools/run_backend_benchmarks.py
Executable file
@@ -0,0 +1,667 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import platform
|
||||
import random
|
||||
import shutil
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
|
||||
JSON_PREFIX = "MFEM_BENCHMARK_JSON "
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Case:
|
||||
name: str
|
||||
device: str
|
||||
ranks: int
|
||||
threads: int
|
||||
|
||||
|
||||
def positive_int(value: str) -> int:
|
||||
parsed = int(value)
|
||||
if parsed < 1:
|
||||
raise argparse.ArgumentTypeError("must be at least one")
|
||||
return parsed
|
||||
|
||||
|
||||
def integer_list(value: str) -> list[int]:
|
||||
try:
|
||||
parsed = [int(item.strip()) for item in value.split(",") if item.strip()]
|
||||
except ValueError as error:
|
||||
raise argparse.ArgumentTypeError("expected comma-separated integers") from error
|
||||
if not parsed or any(item < 1 for item in parsed):
|
||||
raise argparse.ArgumentTypeError("all entries must be positive")
|
||||
return parsed
|
||||
|
||||
|
||||
def command_output(command: list[str]) -> str | None:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
if result.returncode != 0:
|
||||
return None
|
||||
output = result.stdout.strip()
|
||||
return output if output else None
|
||||
|
||||
|
||||
def resolve_program(value: str) -> str | None:
|
||||
candidate = pathlib.Path(value)
|
||||
if candidate.is_file():
|
||||
return str(candidate.resolve())
|
||||
discovered = shutil.which(value)
|
||||
return str(pathlib.Path(discovered).resolve()) if discovered else None
|
||||
|
||||
|
||||
def compact_cpu_list(cpus: list[int]) -> str:
|
||||
if not cpus:
|
||||
return ""
|
||||
groups: list[str] = []
|
||||
start = previous = cpus[0]
|
||||
for current in cpus[1:]:
|
||||
if current == previous + 1:
|
||||
previous = current
|
||||
continue
|
||||
groups.append(str(start) if start == previous else f"{start}-{previous}")
|
||||
start = previous = current
|
||||
groups.append(str(start) if start == previous else f"{start}-{previous}")
|
||||
return ",".join(groups)
|
||||
|
||||
|
||||
def affinity_prefix(case: Case, setting: str) -> tuple[list[str], str | None]:
|
||||
if setting == "none" or shutil.which("taskset") is None:
|
||||
return [], None
|
||||
if setting != "auto":
|
||||
return ["taskset", "-c", setting], setting
|
||||
if not hasattr(os, "sched_getaffinity"):
|
||||
return [], None
|
||||
available = sorted(os.sched_getaffinity(0))
|
||||
count = min(len(available), max(case.ranks, case.threads))
|
||||
chosen = compact_cpu_list(available[:count])
|
||||
return (["taskset", "-c", chosen] if chosen else []), chosen or None
|
||||
|
||||
|
||||
def expand_cases(args: argparse.Namespace) -> list[Case]:
|
||||
requested = [item.strip() for item in args.cases.split(",") if item.strip()]
|
||||
valid = {"cpu", "ceed-cpu", "omp", "mpi", "cuda", "ceed-cuda"}
|
||||
unknown = sorted(set(requested) - valid)
|
||||
if unknown:
|
||||
raise ValueError(f"unknown benchmark case(s): {', '.join(unknown)}")
|
||||
cases: list[Case] = []
|
||||
for requested_case in requested:
|
||||
if requested_case == "cpu":
|
||||
cases.append(Case("cpu-1r-1t", "cpu", 1, 1))
|
||||
elif requested_case == "ceed-cpu":
|
||||
cases.append(Case("ceed-cpu-1r-1t", args.ceed_cpu_device, 1, 1))
|
||||
elif requested_case == "omp":
|
||||
cases.extend(
|
||||
Case(f"omp-1r-{threads}t", "omp", 1, threads)
|
||||
for threads in args.omp_threads
|
||||
)
|
||||
elif requested_case == "mpi":
|
||||
cases.extend(
|
||||
Case(f"mpi-{ranks}r-1t", "cpu", ranks, 1)
|
||||
for ranks in args.mpi_ranks
|
||||
)
|
||||
elif requested_case == "cuda":
|
||||
cases.append(Case("cuda-1r-1t", "cuda", 1, 1))
|
||||
elif requested_case == "ceed-cuda":
|
||||
cases.append(Case("ceed-cuda-1r-1t", args.ceed_cuda_device, 1, 1))
|
||||
deduplicated: list[Case] = []
|
||||
for case in cases:
|
||||
if case not in deduplicated:
|
||||
deduplicated.append(case)
|
||||
return deduplicated
|
||||
|
||||
|
||||
def run_case(
|
||||
args: argparse.Namespace, case: Case, trial: int, warmup_trial: bool
|
||||
) -> dict[str, Any]:
|
||||
prefix, affinity = affinity_prefix(case, args.affinity)
|
||||
command = prefix + [args.launcher, "-n", str(case.ranks)]
|
||||
command.extend(args.launcher_arg)
|
||||
command.extend(
|
||||
[
|
||||
args.executable,
|
||||
"--device",
|
||||
case.device,
|
||||
"--mesh-n",
|
||||
str(args.mesh_n),
|
||||
"--order",
|
||||
str(args.order),
|
||||
"--applications",
|
||||
str(args.applications),
|
||||
"--max-applications",
|
||||
str(args.max_applications),
|
||||
"--minimum-apply-seconds",
|
||||
str(args.minimum_apply_seconds),
|
||||
"--warmup-applications",
|
||||
str(args.warmup_applications),
|
||||
"--relative-tolerance",
|
||||
str(args.relative_tolerance),
|
||||
"--max-iterations",
|
||||
str(args.max_iterations),
|
||||
"--trial",
|
||||
str(trial),
|
||||
]
|
||||
)
|
||||
if args.solve:
|
||||
command.append("--solve")
|
||||
environment = os.environ.copy()
|
||||
environment.update(
|
||||
{
|
||||
"OMP_NUM_THREADS": str(case.threads),
|
||||
"OMP_DYNAMIC": "FALSE",
|
||||
"OMP_PROC_BIND": "close",
|
||||
"OMP_PLACES": "cores",
|
||||
"FI_PROVIDER": environment.get("FI_PROVIDER", "sockets"),
|
||||
}
|
||||
)
|
||||
environment.setdefault("OMPI_ALLOW_RUN_AS_ROOT", "1")
|
||||
environment.setdefault("OMPI_ALLOW_RUN_AS_ROOT_CONFIRM", "1")
|
||||
selected_interface = args.network_interface
|
||||
if selected_interface == "local":
|
||||
selected_interface = "lo0" if sys.platform == "darwin" else "lo"
|
||||
if selected_interface == "auto":
|
||||
environment.pop("FI_SOCKETS_IFACE", None)
|
||||
environment.pop("HYDRA_IFACE", None)
|
||||
environment.pop("MPICH_INTERFACE_HOSTNAME", None)
|
||||
else:
|
||||
environment["FI_SOCKETS_IFACE"] = selected_interface
|
||||
environment["HYDRA_IFACE"] = selected_interface
|
||||
environment.pop("MPICH_INTERFACE_HOSTNAME", None)
|
||||
if sys.platform == "darwin" and args.network_interface == "local":
|
||||
environment["MPICH_INTERFACE_HOSTNAME"] = "127.0.0.1"
|
||||
if case.device.startswith("cuda") or case.device.startswith("ceed-cuda"):
|
||||
if args.gpu_aware_mpi:
|
||||
environment["MFEM_GPU_AWARE_MPI"] = "1"
|
||||
else:
|
||||
environment.pop("MFEM_GPU_AWARE_MPI", None)
|
||||
if args.cuda_visible_devices is not None:
|
||||
environment["CUDA_VISIBLE_DEVICES"] = args.cuda_visible_devices
|
||||
|
||||
started = time.perf_counter()
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
timeout=args.timeout,
|
||||
)
|
||||
wall_seconds = time.perf_counter() - started
|
||||
combined_output = "\n".join((completed.stdout, completed.stderr))
|
||||
records = [
|
||||
json.loads(line[len(JSON_PREFIX) :])
|
||||
for line in combined_output.splitlines()
|
||||
if line.startswith(JSON_PREFIX)
|
||||
]
|
||||
if completed.returncode != 0 or len(records) != 1:
|
||||
tail = "\n".join(combined_output.splitlines()[-30:])
|
||||
raise RuntimeError(
|
||||
f"{case.name} trial {trial} failed with exit code "
|
||||
f"{completed.returncode}; JSON records={len(records)}\n{tail}"
|
||||
)
|
||||
record = records[0]
|
||||
record.update(
|
||||
{
|
||||
"case": case.name,
|
||||
"requested_device": case.device,
|
||||
"runner_trial": trial,
|
||||
"discarded_warmup_trial": warmup_trial,
|
||||
"wall_seconds": wall_seconds,
|
||||
"cpu_affinity": affinity,
|
||||
"network_interface": environment.get("HYDRA_IFACE"),
|
||||
"cuda_visible_devices": environment.get("CUDA_VISIBLE_DEVICES"),
|
||||
"gpu_aware_mpi": environment.get("MFEM_GPU_AWARE_MPI") == "1",
|
||||
"command": command,
|
||||
}
|
||||
)
|
||||
if not record.get("valid", False):
|
||||
raise RuntimeError(f"{case.name} trial {trial} failed numerical validation")
|
||||
return record
|
||||
|
||||
|
||||
def median_absolute_deviation(values: list[float]) -> float:
|
||||
center = statistics.median(values)
|
||||
return statistics.median(abs(value - center) for value in values)
|
||||
|
||||
|
||||
def validate_cross_backend(
|
||||
records: list[dict[str, Any]], relative_tolerance: float
|
||||
) -> dict[str, float]:
|
||||
measured = [row for row in records if not row["discarded_warmup_trial"]]
|
||||
baseline = [row for row in measured if row["case"] == "cpu-1r-1t"]
|
||||
if not baseline:
|
||||
return {}
|
||||
validation_fields = ["probe_norm"]
|
||||
if measured and all(row.get("solve_ran", False) for row in measured):
|
||||
validation_fields.append("solution_norm")
|
||||
references = {
|
||||
field: statistics.median(float(row[field]) for row in baseline)
|
||||
for field in validation_fields
|
||||
}
|
||||
maximum_errors = {field: 0.0 for field in references}
|
||||
for row in measured:
|
||||
for field, reference in references.items():
|
||||
relative_error = abs(float(row[field]) - reference) / max(
|
||||
abs(reference), 1.0e-300
|
||||
)
|
||||
maximum_errors[field] = max(maximum_errors[field], relative_error)
|
||||
if relative_error > relative_tolerance:
|
||||
raise RuntimeError(
|
||||
f"{row['case']} disagrees with CPU for {field}: relative "
|
||||
f"error {relative_error:.3e} > {relative_tolerance:.3e}"
|
||||
)
|
||||
return maximum_errors
|
||||
|
||||
|
||||
def checkpoint_results(
|
||||
output_dir: pathlib.Path,
|
||||
metadata: dict[str, Any],
|
||||
records: list[dict[str, Any]],
|
||||
) -> None:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
destination = output_dir / "partial-results.json"
|
||||
temporary = output_dir / "partial-results.json.tmp"
|
||||
temporary.write_text(
|
||||
json.dumps({"schema_version": 1, "metadata": metadata, "raw": records}, indent=2)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
temporary.replace(destination)
|
||||
|
||||
|
||||
def summarize(cases: list[Case], records: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
summaries: list[dict[str, Any]] = []
|
||||
for case in cases:
|
||||
rows = [
|
||||
row
|
||||
for row in records
|
||||
if row["case"] == case.name and not row["discarded_warmup_trial"]
|
||||
]
|
||||
if not rows:
|
||||
continue
|
||||
throughput = [float(row["apply_mdof_per_second"]) for row in rows]
|
||||
apply_seconds = [float(row["apply_seconds"]) for row in rows]
|
||||
solve_seconds = [float(row["solve_seconds"]) for row in rows]
|
||||
total_seconds = [float(row["total_seconds"]) for row in rows]
|
||||
timed_applications = [int(row["applications"]) for row in rows]
|
||||
summaries.append(
|
||||
{
|
||||
"case": case.name,
|
||||
"device": case.device,
|
||||
"ranks": case.ranks,
|
||||
"threads": case.threads,
|
||||
"trials": len(rows),
|
||||
"global_true_dofs": rows[0]["global_true_dofs"],
|
||||
"requested_applications": rows[0]["requested_applications"],
|
||||
"applications_min": min(timed_applications),
|
||||
"applications_max": max(timed_applications),
|
||||
"apply_mdof_per_second_median": statistics.median(throughput),
|
||||
"apply_mdof_per_second_mad": median_absolute_deviation(throughput),
|
||||
"apply_mdof_per_second_min": min(throughput),
|
||||
"apply_mdof_per_second_max": max(throughput),
|
||||
"apply_seconds_median": statistics.median(apply_seconds),
|
||||
"solve_seconds_median": statistics.median(solve_seconds),
|
||||
"total_seconds_median": statistics.median(total_seconds),
|
||||
"cg_iterations": sorted({row["cg_iterations"] for row in rows}),
|
||||
"solve_ran": all(bool(row["solve_ran"]) for row in rows),
|
||||
"verified_relative_residual_max": max(
|
||||
float(row["verified_relative_residual"]) for row in rows
|
||||
),
|
||||
}
|
||||
)
|
||||
baseline = next((row for row in summaries if row["case"] == "cpu-1r-1t"), None)
|
||||
baseline_rate = (
|
||||
float(baseline["apply_mdof_per_second_median"]) if baseline else None
|
||||
)
|
||||
for row in summaries:
|
||||
row["speedup_vs_cpu"] = (
|
||||
float(row["apply_mdof_per_second_median"]) / baseline_rate
|
||||
if baseline_rate
|
||||
else None
|
||||
)
|
||||
return summaries
|
||||
|
||||
|
||||
def markdown_table(summaries: list[dict[str, Any]]) -> str:
|
||||
lines = [
|
||||
"| Case | Device | Ranks | Threads | Timed apps | Apply window (s) | Apply MDoF/s, median [range] | MAD | Speedup | Solve (s) | CG iters |",
|
||||
"|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|",
|
||||
]
|
||||
for row in summaries:
|
||||
speedup = row["speedup_vs_cpu"]
|
||||
speedup_text = f"{speedup:.2f}x" if speedup is not None else "n/a"
|
||||
solve_text = f"{row['solve_seconds_median']:.4f}" if row["solve_ran"] else "—"
|
||||
iteration_text = (
|
||||
",".join(str(value) for value in row["cg_iterations"])
|
||||
if row["solve_ran"]
|
||||
else "—"
|
||||
)
|
||||
applications_text = (
|
||||
str(row["applications_min"])
|
||||
if row["applications_min"] == row["applications_max"]
|
||||
else f"{row['applications_min']}-{row['applications_max']}"
|
||||
)
|
||||
lines.append(
|
||||
f"| {row['case']} | `{row['device']}` | {row['ranks']} | "
|
||||
f"{row['threads']} | {applications_text} | "
|
||||
f"{row['apply_seconds_median']:.3f} | "
|
||||
f"{row['apply_mdof_per_second_median']:.2f} "
|
||||
f"[{row['apply_mdof_per_second_min']:.2f}, "
|
||||
f"{row['apply_mdof_per_second_max']:.2f}] | "
|
||||
f"{row['apply_mdof_per_second_mad']:.2f} | "
|
||||
f"{speedup_text} | {solve_text} | {iteration_text} |"
|
||||
)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def collect_metadata(args: argparse.Namespace) -> dict[str, Any]:
|
||||
executable = pathlib.Path(args.executable).resolve()
|
||||
source_root = pathlib.Path(__file__).resolve().parents[1]
|
||||
return {
|
||||
"timestamp_utc": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"hostname": platform.node(),
|
||||
"platform": platform.platform(),
|
||||
"python": sys.version,
|
||||
"cpu_count": os.cpu_count(),
|
||||
"lscpu": command_output(["lscpu"]) if shutil.which("lscpu") else None,
|
||||
"executable": str(executable),
|
||||
"launcher": str(pathlib.Path(args.launcher).resolve()),
|
||||
"launcher_version": command_output([args.launcher, "--version"]),
|
||||
"compiler": command_output(["c++", "--version"]),
|
||||
"nvidia_smi": command_output(
|
||||
[
|
||||
"nvidia-smi",
|
||||
"--query-gpu=name,driver_version,temperature.gpu,pstate,clocks.sm,power.draw",
|
||||
"--format=csv,noheader",
|
||||
]
|
||||
)
|
||||
if shutil.which("nvidia-smi")
|
||||
else None,
|
||||
"git_commit": command_output(
|
||||
["git", "-C", str(source_root), "rev-parse", "HEAD"]
|
||||
),
|
||||
"git_dirty": bool(
|
||||
command_output(["git", "-C", str(source_root), "status", "--porcelain"])
|
||||
),
|
||||
"parameters": {
|
||||
"mesh_n": args.mesh_n,
|
||||
"order": args.order,
|
||||
"applications": args.applications,
|
||||
"max_applications": args.max_applications,
|
||||
"minimum_apply_seconds": args.minimum_apply_seconds,
|
||||
"warmup_applications": args.warmup_applications,
|
||||
"trials": args.trials,
|
||||
"discarded_warmup_trials": args.warmup_trials,
|
||||
"relative_tolerance": args.relative_tolerance,
|
||||
"max_iterations": args.max_iterations,
|
||||
"solve": args.solve,
|
||||
"affinity": args.affinity,
|
||||
"network_interface": args.network_interface,
|
||||
"cuda_visible_devices": args.cuda_visible_devices,
|
||||
"gpu_aware_mpi": args.gpu_aware_mpi,
|
||||
"cross_backend_relative_tolerance": args.cross_backend_relative_tolerance,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def write_results(
|
||||
output_dir: pathlib.Path,
|
||||
metadata: dict[str, Any],
|
||||
records: list[dict[str, Any]],
|
||||
summaries: list[dict[str, Any]],
|
||||
) -> None:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
(output_dir / "results.json").write_text(
|
||||
json.dumps(
|
||||
{"schema_version": 1, "metadata": metadata, "raw": records, "summary": summaries},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
fieldnames = [
|
||||
"case",
|
||||
"device",
|
||||
"ranks",
|
||||
"threads",
|
||||
"trials",
|
||||
"global_true_dofs",
|
||||
"requested_applications",
|
||||
"applications_min",
|
||||
"applications_max",
|
||||
"apply_mdof_per_second_median",
|
||||
"apply_mdof_per_second_mad",
|
||||
"apply_mdof_per_second_min",
|
||||
"apply_mdof_per_second_max",
|
||||
"speedup_vs_cpu",
|
||||
"apply_seconds_median",
|
||||
"solve_seconds_median",
|
||||
"solve_ran",
|
||||
"total_seconds_median",
|
||||
"cg_iterations",
|
||||
"verified_relative_residual_max",
|
||||
]
|
||||
with (output_dir / "results.csv").open("w", newline="", encoding="utf-8") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
for summary in summaries:
|
||||
writer.writerow(summary)
|
||||
first = records[0]
|
||||
if first["minimum_apply_seconds"] > 0.0:
|
||||
timing_description = (
|
||||
"Application counts were selected with a "
|
||||
f"{first['minimum_apply_seconds']:.3g}-second calibration window, then "
|
||||
"measured in a fresh timing window."
|
||||
)
|
||||
else:
|
||||
timing_description = (
|
||||
"The requested application count was measured directly in one timing "
|
||||
"window."
|
||||
)
|
||||
report = "# MFEM backend benchmark\n\n"
|
||||
report += (
|
||||
f"Host: `{metadata['hostname']}` \n"
|
||||
f"UTC timestamp: `{metadata['timestamp_utc']}` \n"
|
||||
f"Workload: {first['global_elements']:,} 3D hex elements, H1 order "
|
||||
f"{first['order']}, {first['global_true_dofs']:,} global true DoFs, "
|
||||
f"at least {first['requested_applications']} timed operator applications "
|
||||
f"per trial. {timing_description}\n\n"
|
||||
)
|
||||
report += markdown_table(summaries)
|
||||
report += (
|
||||
"\nThe table reports medians from fresh processes. Operator throughput uses "
|
||||
"the same global partial-assembly diffusion operator and true-DOF field in "
|
||||
"every case. MPI timings use the slowest rank.\n"
|
||||
)
|
||||
(output_dir / "results.md").write_text(report, encoding="utf-8")
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--executable", required=True)
|
||||
parser.add_argument("--launcher", required=True)
|
||||
parser.add_argument(
|
||||
"--cases",
|
||||
default="cpu,ceed-cpu,omp,mpi,cuda,ceed-cuda",
|
||||
help="Comma-separated subset of cpu, ceed-cpu, omp, mpi, cuda, ceed-cuda.",
|
||||
)
|
||||
parser.add_argument("--omp-threads", type=integer_list, default=[8])
|
||||
parser.add_argument("--mpi-ranks", type=integer_list, default=[4])
|
||||
parser.add_argument("--trials", type=positive_int, default=5)
|
||||
parser.add_argument("--warmup-trials", type=int, default=0)
|
||||
parser.add_argument("--mesh-n", type=positive_int, default=48)
|
||||
parser.add_argument("--order", type=positive_int, default=3)
|
||||
parser.add_argument("--applications", type=positive_int, default=50)
|
||||
parser.add_argument("--max-applications", type=positive_int, default=1_000_000)
|
||||
parser.add_argument(
|
||||
"--minimum-apply-seconds",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help=(
|
||||
"Double the requested applications until a calibration window is this "
|
||||
"long, then measure a fresh window."
|
||||
),
|
||||
)
|
||||
parser.add_argument("--warmup-applications", type=int, default=5)
|
||||
parser.add_argument("--relative-tolerance", type=float, default=1.0e-6)
|
||||
parser.add_argument("--max-iterations", type=positive_int, default=1000)
|
||||
parser.add_argument(
|
||||
"--solve",
|
||||
action="store_true",
|
||||
help="Also run and time the Jacobi-preconditioned CG validation solve.",
|
||||
)
|
||||
parser.add_argument("--timeout", type=positive_int, default=900)
|
||||
parser.add_argument(
|
||||
"--affinity",
|
||||
default="auto",
|
||||
help="CPU list for taskset, 'auto' for a compact set, or 'none'.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--network-interface",
|
||||
default="local",
|
||||
help="'local' for platform loopback, 'auto' for routing, or an interface name.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cuda-visible-devices",
|
||||
default=None,
|
||||
help="Optional CUDA_VISIBLE_DEVICES value for GPU cases.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--gpu-aware-mpi",
|
||||
action="store_true",
|
||||
help="Opt in only when the selected MPI implementation is CUDA-aware.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cross-backend-relative-tolerance",
|
||||
type=float,
|
||||
default=1.0e-5,
|
||||
help="Maximum relative difference from CPU for probe and solution norms.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ceed-cpu-device",
|
||||
default="ceed-cpu",
|
||||
help="Full libCEED CPU resource or portable ceed-cpu alias.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ceed-cuda-device",
|
||||
default="ceed-cuda",
|
||||
help="Full libCEED CUDA resource or portable ceed-cuda alias.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--launcher-arg",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Additional MPI launcher argument; repeat as needed.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=pathlib.Path,
|
||||
default=pathlib.Path("benchmark-results"),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.warmup_trials < 0 or args.warmup_applications < 0:
|
||||
parser.error("warm-up counts may not be negative")
|
||||
if args.relative_tolerance <= 0.0 or args.cross_backend_relative_tolerance <= 0.0:
|
||||
parser.error("tolerances must be positive")
|
||||
if args.minimum_apply_seconds < 0.0:
|
||||
parser.error("minimum apply seconds may not be negative")
|
||||
if args.max_applications < args.applications:
|
||||
parser.error("max applications must be at least the requested applications")
|
||||
for attribute in ("executable", "launcher"):
|
||||
value = getattr(args, attribute)
|
||||
resolved = resolve_program(value)
|
||||
if resolved is None:
|
||||
parser.error(f"program not found: {value}")
|
||||
setattr(args, attribute, resolved)
|
||||
return args
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_arguments()
|
||||
try:
|
||||
cases = expand_cases(args)
|
||||
except ValueError as error:
|
||||
print(f"error: {error}", file=sys.stderr)
|
||||
return 2
|
||||
metadata = collect_metadata(args)
|
||||
records: list[dict[str, Any]] = []
|
||||
|
||||
for warmup in range(args.warmup_trials):
|
||||
for case in cases:
|
||||
print(f"warm-up {warmup + 1}/{args.warmup_trials}: {case.name}", flush=True)
|
||||
records.append(run_case(args, case, -(warmup + 1), True))
|
||||
checkpoint_results(args.output_dir, metadata, records)
|
||||
|
||||
schedule = [(trial, case) for trial in range(1, args.trials + 1) for case in cases]
|
||||
random.Random(20260905).shuffle(schedule)
|
||||
for index, (trial, case) in enumerate(schedule, start=1):
|
||||
print(
|
||||
f"run {index}/{len(schedule)}: {case.name}, trial {trial}/{args.trials}",
|
||||
flush=True,
|
||||
)
|
||||
records.append(run_case(args, case, trial, False))
|
||||
checkpoint_results(args.output_dir, metadata, records)
|
||||
|
||||
summaries = summarize(cases, records)
|
||||
workload_signatures = {
|
||||
(
|
||||
row["global_elements"],
|
||||
row["global_true_dofs"],
|
||||
row["mesh_n"],
|
||||
row["order"],
|
||||
row["requested_applications"],
|
||||
row["minimum_apply_seconds"],
|
||||
)
|
||||
for row in records
|
||||
if not row["discarded_warmup_trial"]
|
||||
}
|
||||
if len(workload_signatures) != 1:
|
||||
raise RuntimeError(
|
||||
f"cases did not use one fixed global workload: {workload_signatures}"
|
||||
)
|
||||
metadata["cross_backend_maximum_relative_errors"] = validate_cross_backend(
|
||||
records, args.cross_backend_relative_tolerance
|
||||
)
|
||||
if shutil.which("nvidia-smi"):
|
||||
metadata["nvidia_smi_after"] = command_output(
|
||||
[
|
||||
"nvidia-smi",
|
||||
"--query-gpu=name,driver_version,temperature.gpu,pstate,clocks.sm,power.draw",
|
||||
"--format=csv,noheader",
|
||||
]
|
||||
)
|
||||
write_results(args.output_dir, metadata, records, summaries)
|
||||
table = markdown_table(summaries)
|
||||
print("\n" + table)
|
||||
print(f"JSON, CSV, and Markdown written to {args.output_dir.resolve()}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except (OSError, subprocess.TimeoutExpired, RuntimeError) as error:
|
||||
print(f"benchmark failed: {error}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
Reference in New Issue
Block a user