feat(linux): portable build works on linux + benchmarks

benchmarks/backend_operator.cpp

@@ -0,0 +1,373 @@
+#include <meson_mfem_template/config.hpp>
+#include <mfem.hpp>
+
+#include <cmath>
+#include <cstdlib>
+#include <functional>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <sstream>
+#include <string>
+
+#if !MESON_MFEM_HAS_MPI || !MESON_MFEM_HAS_HYPRE
+#error "The backend benchmark requires MFEM with MPI and Hypre"
+#endif
+
+namespace
+{
+
+constexpr const char *json_prefix = "MFEM_BENCHMARK_JSON ";
+
+double AnalyticField(const mfem::Vector &point)
+{
+   constexpr double pi = 3.141592653589793238462643383279502884;
+   return std::sin(pi * point[0]) * std::sin(pi * point[1]) *
+          std::sin(pi * point[2]);
+}
+
+void SynchronizeDevice(bool uses_gpu)
+{
+   if (uses_gpu) { MFEM_DEVICE_SYNC; }
+}
+
+double MaxTimedRegion(MPI_Comm communicator, bool uses_gpu,
+                      const std::function<void()> &operation)
+{
+   SynchronizeDevice(uses_gpu);
+   MPI_Barrier(communicator);
+   const double begin = MPI_Wtime();
+   operation();
+   SynchronizeDevice(uses_gpu);
+   const double local_seconds = MPI_Wtime() - begin;
+   double maximum_seconds = 0.0;
+   MPI_Allreduce(&local_seconds, &maximum_seconds, 1, MPI_DOUBLE, MPI_MAX,
+                 communicator);
+   return maximum_seconds;
+}
+
+std::string JsonEscape(const std::string &value)
+{
+   std::ostringstream escaped;
+   for (const char character : value)
+   {
+      switch (character)
+      {
+         case '\\': escaped << "\\\\"; break;
+         case '"': escaped << "\\\""; break;
+         case '\n': escaped << "\\n"; break;
+         case '\r': escaped << "\\r"; break;
+         case '\t': escaped << "\\t"; break;
+         default: escaped << character; break;
+      }
+   }
+   return escaped.str();
+}
+
+int PositiveEnvironmentInteger(const char *name, int fallback)
+{
+   const char *value = std::getenv(name);
+   if (value == nullptr) { return fallback; }
+   char *end = nullptr;
+   const long parsed = std::strtol(value, &end, 10);
+   return end != value && *end == '\0' && parsed > 0
+             ? static_cast<int>(parsed)
+             : fallback;
+}
+
+}  // namespace
+
+int main(int argc, char **argv)
+{
+   mfem::Mpi::Init(argc, argv);
+   mfem::Hypre::Init();
+   MPI_Comm communicator = MPI_COMM_WORLD;
+   const int rank = mfem::Mpi::WorldRank();
+   const int ranks = mfem::Mpi::WorldSize();
+   const double process_begin = MPI_Wtime();
+
+   const char *device_name = "cpu";
+   int mesh_n = 32;
+   int order = 3;
+   int applications = 50;
+   int warmup_applications = 5;
+   int maximum_applications = 1000000;
+   int maximum_iterations = 1000;
+   int trial = 0;
+   double relative_tolerance = 1.0e-6;
+   double minimum_apply_seconds = 0.0;
+   bool run_solve = false;
+
+   mfem::OptionsParser options(argc, argv);
+   options.AddOption(&device_name, "-d", "--device",
+                     "MFEM device string (cpu, omp, ceed-cpu, cuda, ...). ");
+   options.AddOption(&mesh_n, "-n", "--mesh-n",
+                     "Elements per dimension in the fixed global mesh.");
+   options.AddOption(&order, "-o", "--order", "H1 polynomial order.");
+   options.AddOption(&applications, "-a", "--applications",
+                     "Number of timed distributed operator applications.");
+   options.AddOption(&warmup_applications, "-w", "--warmup-applications",
+                     "Untimed operator applications before measurement.");
+   options.AddOption(&maximum_applications, "-ma", "--max-applications",
+                     "Safety cap for automatically calibrated applications.");
+   options.AddOption(&minimum_apply_seconds, "-mt", "--minimum-apply-seconds",
+                     "Minimum final timing window; zero keeps the requested count.");
+   options.AddOption(&maximum_iterations, "-m", "--max-iterations",
+                     "Maximum iterations for the identically configured CG solve.");
+   options.AddOption(&relative_tolerance, "-r", "--relative-tolerance",
+                     "Relative tolerance for the identically configured CG solve.");
+   options.AddOption(&run_solve, "-s", "--solve", "-no-s", "--no-solve",
+                     "Run the optional Jacobi-preconditioned CG validation solve.");
+   options.AddOption(&trial, "-t", "--trial",
+                     "Trial identifier copied into the JSON record.");
+   options.Parse();
+   if (!options.Good())
+   {
+      if (rank == 0) { options.PrintUsage(std::cerr); }
+      return 2;
+   }
+   if (mesh_n < 2 || order < 1 || applications < 1 ||
+       warmup_applications < 0 || maximum_applications < applications ||
+       maximum_iterations < 1 || relative_tolerance <= 0.0 ||
+       minimum_apply_seconds < 0.0)
+   {
+      if (rank == 0)
+      {
+         std::cerr << "All sizes/counts must be positive (warmups may be zero).\n";
+      }
+      return 2;
+   }
+
+   std::unique_ptr<mfem::Device> device;
+   MPI_Barrier(communicator);
+   const double device_begin = MPI_Wtime();
+   device = std::make_unique<mfem::Device>(device_name);
+   const bool uses_gpu = mfem::Device::Allows(
+      mfem::Backend::CUDA_MASK | mfem::Backend::HIP_MASK);
+   SynchronizeDevice(uses_gpu);
+   const double local_device_seconds = MPI_Wtime() - device_begin;
+   double device_seconds = 0.0;
+   MPI_Allreduce(&local_device_seconds, &device_seconds, 1, MPI_DOUBLE, MPI_MAX,
+                 communicator);
+
+   std::unique_ptr<mfem::ParMesh> mesh;
+   const double mesh_seconds = MaxTimedRegion(communicator, uses_gpu, [&]() {
+      mfem::Mesh serial_mesh = mfem::Mesh::MakeCartesian3D(
+         mesh_n, mesh_n, mesh_n, mfem::Element::HEXAHEDRON);
+      mfem::Array<int> partitioning(serial_mesh.GetNE());
+      for (int element = 0; element < serial_mesh.GetNE(); ++element)
+      {
+         partitioning[element] = static_cast<long long>(element) * ranks /
+                                 serial_mesh.GetNE();
+      }
+      mesh = std::make_unique<mfem::ParMesh>(communicator, serial_mesh,
+                                             partitioning.GetData());
+   });
+
+   std::unique_ptr<mfem::H1_FECollection> elements;
+   std::unique_ptr<mfem::ParFiniteElementSpace> space;
+   const double space_seconds = MaxTimedRegion(communicator, uses_gpu, [&]() {
+      elements = std::make_unique<mfem::H1_FECollection>(
+         order, mesh->Dimension(), mfem::BasisType::GaussLobatto);
+      space = std::make_unique<mfem::ParFiniteElementSpace>(mesh.get(),
+                                                            elements.get());
+   });
+
+   mfem::Array<int> essential_boundary(mesh->bdr_attributes.Max());
+   essential_boundary = 1;
+   mfem::Array<int> essential_dofs;
+   space->GetEssentialTrueDofs(essential_boundary, essential_dofs);
+
+   mfem::ConstantCoefficient one(1.0);
+   auto right_hand_side = std::make_unique<mfem::ParLinearForm>(space.get());
+   auto solution = std::make_unique<mfem::ParGridFunction>(space.get());
+   auto diffusion = std::make_unique<mfem::ParBilinearForm>(space.get());
+   const double assembly_seconds = MaxTimedRegion(communicator, uses_gpu, [&]() {
+      right_hand_side->AddDomainIntegrator(new mfem::DomainLFIntegrator(one));
+      right_hand_side->Assemble();
+      *solution = 0.0;
+      diffusion->SetAssemblyLevel(mfem::AssemblyLevel::PARTIAL);
+      diffusion->AddDomainIntegrator(new mfem::DiffusionIntegrator(one));
+      diffusion->Assemble();
+   });
+
+   mfem::OperatorPtr system_operator;
+   mfem::Vector linear_right_hand_side;
+   mfem::Vector linear_solution;
+   const double form_seconds = MaxTimedRegion(communicator, uses_gpu, [&]() {
+      diffusion->FormLinearSystem(essential_dofs, *solution, *right_hand_side,
+                                  system_operator, linear_solution,
+                                  linear_right_hand_side);
+   });
+
+   std::unique_ptr<mfem::OperatorJacobiSmoother> preconditioner;
+   double preconditioner_seconds = 0.0;
+   if (run_solve)
+   {
+      preconditioner_seconds = MaxTimedRegion(
+         communicator, uses_gpu, [&]() {
+         preconditioner = std::make_unique<mfem::OperatorJacobiSmoother>(
+            *diffusion, essential_dofs);
+      });
+   }
+
+   mfem::FunctionCoefficient analytic(AnalyticField);
+   mfem::ParGridFunction probe_field(space.get());
+   probe_field.ProjectCoefficient(analytic);
+   mfem::Vector probe_input;
+   probe_field.GetTrueDofs(probe_input);
+   mfem::Vector probe_output(system_operator->Height());
+   probe_input.UseDevice(true);
+   probe_output.UseDevice(true);
+
+   for (int application = 0; application < warmup_applications; ++application)
+   {
+      system_operator->Mult(probe_input, probe_output);
+   }
+   SynchronizeDevice(uses_gpu);
+
+   const int requested_applications = applications;
+   int timed_applications = requested_applications;
+   if (minimum_apply_seconds > 0.0)
+   {
+      while (true)
+      {
+         const double calibration_seconds = MaxTimedRegion(
+            communicator, uses_gpu, [&]() {
+               for (int application = 0; application < timed_applications;
+                    ++application)
+               {
+                  system_operator->Mult(probe_input, probe_output);
+               }
+            });
+         if (calibration_seconds >= minimum_apply_seconds ||
+             timed_applications == maximum_applications)
+         {
+            break;
+         }
+         timed_applications = timed_applications > maximum_applications / 2
+            ? maximum_applications
+            : timed_applications * 2;
+      }
+   }
+   const double apply_seconds = MaxTimedRegion(
+      communicator, uses_gpu, [&]() {
+         for (int application = 0; application < timed_applications;
+              ++application)
+         {
+            system_operator->Mult(probe_input, probe_output);
+         }
+      });
+
+   const double probe_norm = std::sqrt(
+      mfem::InnerProduct(communicator, probe_output, probe_output));
+   double solve_seconds = 0.0;
+   int cg_iterations = 0;
+   bool cg_converged = false;
+   double cg_final_norm = 0.0;
+   double relative_residual = 0.0;
+   double solution_norm = 0.0;
+   if (run_solve)
+   {
+      mfem::CGSolver solver(communicator);
+      solver.SetPreconditioner(*preconditioner);
+      solver.SetOperator(*system_operator);
+      solver.SetRelTol(relative_tolerance);
+      solver.SetAbsTol(0.0);
+      solver.SetPrintLevel(-1);
+      solver.iterative_mode = false;
+
+      linear_solution = 0.0;
+      solver.SetMaxIter(2);
+      solver.Mult(linear_right_hand_side, linear_solution);
+      SynchronizeDevice(uses_gpu);
+
+      linear_solution = 0.0;
+      solver.SetMaxIter(maximum_iterations);
+      solve_seconds = MaxTimedRegion(communicator, uses_gpu, [&]() {
+         solver.Mult(linear_right_hand_side, linear_solution);
+      });
+      cg_iterations = solver.GetNumIterations();
+      cg_converged = solver.GetConverged();
+      cg_final_norm = solver.GetFinalNorm();
+
+      mfem::Vector applied_solution(system_operator->Height());
+      mfem::Vector residual(linear_right_hand_side);
+      system_operator->Mult(linear_solution, applied_solution);
+      residual -= applied_solution;
+      SynchronizeDevice(uses_gpu);
+
+      const double right_hand_side_norm_squared = mfem::InnerProduct(
+         communicator, linear_right_hand_side, linear_right_hand_side);
+      const double residual_norm_squared = mfem::InnerProduct(
+         communicator, residual, residual);
+      relative_residual = right_hand_side_norm_squared > 0.0
+         ? std::sqrt(residual_norm_squared / right_hand_side_norm_squared)
+         : std::sqrt(residual_norm_squared);
+      solution_norm = std::sqrt(
+         mfem::InnerProduct(communicator, linear_solution, linear_solution));
+   }
+
+   const auto global_dofs = space->GlobalTrueVSize();
+   const auto global_elements = mesh->GetGlobalNE();
+   const double mdof_per_second =
+      1.0e-6 * static_cast<double>(global_dofs) * timed_applications /
+      apply_seconds;
+   const int omp_threads = PositiveEnvironmentInteger("OMP_NUM_THREADS", 1);
+   const bool solve_valid = !run_solve ||
+      (cg_converged && std::isfinite(relative_residual) &&
+       relative_residual <= 10.0 * relative_tolerance &&
+       std::isfinite(solution_norm) && solution_norm > 0.0);
+   const bool valid = solve_valid && std::isfinite(probe_norm) &&
+                      probe_norm > 0.0;
+
+   SynchronizeDevice(uses_gpu);
+   MPI_Barrier(communicator);
+   const double local_total_seconds = MPI_Wtime() - process_begin;
+   double total_seconds = 0.0;
+   MPI_Allreduce(&local_total_seconds, &total_seconds, 1, MPI_DOUBLE, MPI_MAX,
+                 communicator);
+
+   if (rank == 0)
+   {
+      std::cout << std::setprecision(17) << json_prefix
+                << '{'
+                << "\"schema_version\":1,"
+                << "\"trial\":" << trial << ','
+                << "\"device\":\"" << JsonEscape(device_name) << "\","
+                << "\"ranks\":" << ranks << ','
+                << "\"omp_threads\":" << omp_threads << ','
+                << "\"dimension\":3,"
+                << "\"mesh_n\":" << mesh_n << ','
+                << "\"order\":" << order << ','
+                << "\"global_elements\":" << global_elements << ','
+                << "\"global_true_dofs\":" << global_dofs << ','
+                << "\"assembly\":\"partial\","
+                << "\"warmup_applications\":" << warmup_applications << ','
+                << "\"requested_applications\":" << requested_applications << ','
+                << "\"applications\":" << timed_applications << ','
+                << "\"minimum_apply_seconds\":" << minimum_apply_seconds << ','
+                << "\"device_seconds\":" << device_seconds << ','
+                << "\"mesh_seconds\":" << mesh_seconds << ','
+                << "\"space_seconds\":" << space_seconds << ','
+                << "\"assembly_seconds\":" << assembly_seconds << ','
+                << "\"form_seconds\":" << form_seconds << ','
+                << "\"preconditioner_seconds\":" << preconditioner_seconds << ','
+                << "\"apply_seconds\":" << apply_seconds << ','
+                << "\"apply_mdof_per_second\":" << mdof_per_second << ','
+                << "\"solve_ran\":" << (run_solve ? "true" : "false") << ','
+                << "\"solve_seconds\":" << solve_seconds << ','
+                << "\"cg_iterations\":" << cg_iterations << ','
+                << "\"cg_converged\":"
+                << (cg_converged ? "true" : "false") << ','
+                << "\"cg_final_norm\":" << cg_final_norm << ','
+                << "\"verified_relative_residual\":" << relative_residual << ','
+                << "\"probe_norm\":" << probe_norm << ','
+                << "\"solution_norm\":" << solution_norm << ','
+                << "\"total_seconds\":" << total_seconds << ','
+                << "\"valid\":" << (valid ? "true" : "false")
+                << "}\n";
+   }
+
+   return valid ? 0 : 3;
+}

benchmarks/meson.build

@@ -0,0 +1,142 @@
+benchmark_executable = []
+benchmark_mpi_launcher = ''
+
+if get_option('build_benchmarks')
+  if is_wasm
+    error('build_benchmarks requires a native target; use the web demo for Emscripten')
+  endif
+  if not mfem_has_mpi
+    error('build_benchmarks requires -Dmfem_mpi=enabled (and therefore Hypre)')
+  endif
+
+  benchmark_accelerator_deps = []
+  if mfem_has_cuda
+    benchmark_nvcc = dependency_has_nvcc ? (
+      dependency_prefix / 'bin' / 'nvcc'
+    ) : nvcc_program.full_path()
+    benchmark_cuda_root_result = run_command(
+      python_build,
+      '-c',
+      'import os, sys; print(os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[1]))))',
+      benchmark_nvcc,
+      check: true,
+    )
+    benchmark_cuda_root = benchmark_cuda_root_result.stdout().strip()
+    benchmark_accelerator_deps += [cpp.find_library(
+      'cudart',
+      dirs: [
+        benchmark_cuda_root / 'lib64',
+        benchmark_cuda_root / 'lib',
+        benchmark_cuda_root / 'lib' / 'x64',
+        benchmark_cuda_root / 'targets' / 'x86_64-linux' / 'lib',
+        benchmark_cuda_root / 'targets' / 'aarch64-linux' / 'lib',
+        benchmark_cuda_root / 'targets' / 'ppc64le-linux' / 'lib',
+      ],
+      required: true,
+    )]
+  endif
+  if mfem_has_hip
+    benchmark_hipcc = dependency_has_hipcc ? (
+      dependency_prefix / 'bin' / 'hipcc'
+    ) : hipcc_program.full_path()
+    benchmark_hip_root_result = run_command(
+      python_build,
+      '-c',
+      'import os, sys; print(os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[1]))))',
+      benchmark_hipcc,
+      check: true,
+    )
+    benchmark_hip_root = benchmark_hip_root_result.stdout().strip()
+    benchmark_accelerator_deps += [cpp.find_library(
+      'amdhip64',
+      dirs: [benchmark_hip_root / 'lib', benchmark_hip_root / 'lib64'],
+      required: true,
+    )]
+  endif
+
+  benchmark_build_rpath = mfem_runtime_prefix == '' ? '' : mfem_runtime_prefix / 'lib'
+  benchmark_install_rpath = ''
+  if mfem_runtime_prefix != '' and not is_windows
+    benchmark_install_rpath = host_machine.system() == 'darwin' ? '@loader_path/../lib' : '$ORIGIN/../lib'
+  endif
+
+  benchmark_executable = executable(
+    'mfem-backend-benchmark',
+    'backend_operator.cpp',
+    dependencies: [mfem_dep, template_include_dep] + benchmark_accelerator_deps,
+    override_options: ['cpp_std=' + mfem_consumer_cpp_std],
+    build_rpath: benchmark_build_rpath,
+    install_rpath: benchmark_install_rpath,
+    install: true,
+    install_tag: 'benchmarks',
+  )
+
+  if mpi_launcher_from_dependency and not wheel_carries_native_bundle
+    benchmark_launcher_program = find_program(
+      dependency_prefix / 'bin' / 'mpiexec',
+      dependency_prefix / 'bin' / 'mpirun',
+      required: true,
+    )
+    benchmark_mpi_launcher = benchmark_launcher_program.full_path()
+  elif mfem_runtime_prefix != ''
+    benchmark_mpi_launcher = mfem_runtime_prefix / 'bin' / 'mpiexec'
+  else
+    benchmark_launcher_program = find_program('mpiexec', 'mpirun', required: true)
+    benchmark_mpi_launcher = benchmark_launcher_program.full_path()
+  endif
+
+  benchmark_cases = ['cpu']
+  if mfem_features.get('ceed')
+    benchmark_cases += ['ceed-cpu']
+  endif
+  if mfem_has_openmp
+    benchmark_cases += ['omp']
+  endif
+  benchmark_cases += ['mpi']
+  if mfem_has_cuda
+    benchmark_cases += ['cuda']
+    if mfem_features.get('ceed')
+      benchmark_cases += ['ceed-cuda']
+    endif
+  endif
+
+  benchmark_environment = environment()
+  if (
+    mfem_runtime_prefix != '' and dependency_prefix != '' and
+    not wheel_carries_native_bundle
+  )
+    benchmark_environment.prepend('PATH', dependency_prefix / 'bin')
+    if host_machine.system() == 'darwin'
+      benchmark_environment.prepend('DYLD_LIBRARY_PATH', dependency_prefix / 'lib64')
+      benchmark_environment.prepend('DYLD_LIBRARY_PATH', dependency_prefix / 'lib')
+    elif not is_windows
+      benchmark_environment.prepend('LD_LIBRARY_PATH', dependency_prefix / 'lib64')
+      benchmark_environment.prepend('LD_LIBRARY_PATH', dependency_prefix / 'lib')
+    endif
+  endif
+  if mfem_runtime_prefix != ''
+    benchmark_environment.prepend('PATH', mfem_runtime_prefix / 'bin')
+    if host_machine.system() == 'darwin'
+      benchmark_environment.prepend('DYLD_LIBRARY_PATH', mfem_runtime_prefix / 'lib64')
+      benchmark_environment.prepend('DYLD_LIBRARY_PATH', mfem_runtime_prefix / 'lib')
+    elif not is_windows
+      benchmark_environment.prepend('LD_LIBRARY_PATH', mfem_runtime_prefix / 'lib64')
+      benchmark_environment.prepend('LD_LIBRARY_PATH', mfem_runtime_prefix / 'lib')
+    endif
+  endif
+
+  benchmark(
+    'mfem-backend-comparison',
+    python_build,
+    args: [
+      files('../tools/run_backend_benchmarks.py'),
+      '--executable', benchmark_executable,
+      '--launcher', benchmark_mpi_launcher,
+      '--cases', ','.join(benchmark_cases),
+      '--output-dir', meson.current_build_dir(),
+    ],
+    depends: benchmark_executable,
+    env: benchmark_environment,
+    timeout: 3600,
+  )
+endif

build-config/mfem/meson.build

@@ -82,8 +82,8 @@ source_capable = {
   'fms': not is_windows,
   'ceed': not is_windows,
   'algoim': not meson.is_cross_build() and not is_windows and not is_wasm,
-  'cuda': cuda_toolkit_available and not is_wasm and not host_machine.system() == 'darwin',
-  'hip': hip_toolkit_available and not is_wasm and not host_machine.system() == 'darwin',
+  'cuda': cuda_toolkit_available and not is_wasm and host_machine.system() != 'darwin',
+  'hip': hip_toolkit_available and not is_wasm and host_machine.system() != 'darwin',
   'simd': simd_arch_supported and not is_wasm,
 }
 
@@ -567,6 +567,23 @@ else
     '-L' + mfem_runtime_prefix / 'lib',
     '-L' + mfem_runtime_prefix / 'lib64',
   ]
+  if mfem_has_cuda
+    cuda_compiler_path = dependency_has_nvcc ? (
+      dependency_prefix / 'bin' / 'nvcc'
+    ) : nvcc_program.full_path()
+    cuda_root_result = run_command(
+      python_build,
+      '-c',
+      'import os, sys; print(os.path.dirname(os.path.dirname(os.path.realpath(sys.argv[1]))))',
+      cuda_compiler_path,
+      check: true,
+    )
+    cuda_root = cuda_root_result.stdout().strip()
+    if not fs.exists(cuda_root / 'include' / 'cusparse.h')
+      error('CUDA public headers were not found under ' + cuda_root + '/include.')
+    endif
+    bundle_compile_args += ['-I' + cuda_root / 'include']
+  endif
   if dependency_prefix != '' and not wheel_carries_native_bundle
     bundle_compile_args += ['-I' + dependency_prefix / 'include']
     bundle_link_args += [
@@ -585,7 +602,9 @@ else
     bundle_link_args += ['-lgs']
   endif
   if mfem_has_zlib
-    bundle_link_args += ['-lz']
+    bundle_link_args += is_wasm ? [
+      mfem_runtime_prefix / 'lib' / 'libz.a',
+    ] : ['-lz']
   endif
   if mfem_has_sundials
     bundle_link_args += [

examples/parallel_hypre.cpp

@@ -13,12 +13,19 @@ int main(int argc, char **argv)
    mfem::Mpi::Init(argc, argv);
    mfem::Hypre::Init();
    const int rank = mfem::Mpi::WorldRank();
+   const int ranks = mfem::Mpi::WorldSize();
    const char *device_name = argc > 1 ? argv[1] : "cpu";
    mfem::Device device(device_name);
 
    mfem::Mesh serial_mesh = mfem::Mesh::MakeCartesian2D(
       8, 8, mfem::Element::QUADRILATERAL, true, 1.0, 1.0);
-   mfem::ParMesh mesh(MPI_COMM_WORLD, serial_mesh);
+   mfem::Array<int> partitioning(serial_mesh.GetNE());
+   for (int element = 0; element < serial_mesh.GetNE(); ++element)
+   {
+      partitioning[element] = static_cast<long long>(element) * ranks /
+                              serial_mesh.GetNE();
+   }
+   mfem::ParMesh mesh(MPI_COMM_WORLD, serial_mesh, partitioning.GetData());
    serial_mesh.Clear();
 
    mfem::H1_FECollection elements(2, mesh.Dimension());
@@ -66,10 +73,11 @@ int main(int argc, char **argv)
 
    const double local_norm = solution.Norml2();
    double global_norm = 0.0;
-   MPI_Allreduce(&local_norm, &global_norm, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+   MPI_Allreduce(&local_norm, &global_norm, 1, MPI_DOUBLE, MPI_SUM,
+                 MPI_COMM_WORLD);
    if (rank == 0)
    {
-      std::cout << "MFEM parallel Poisson: ranks=" << mfem::Mpi::WorldSize()
+      std::cout << "MFEM parallel Poisson: ranks=" << ranks
                 << " global_dofs=" << space.GlobalTrueVSize()
                 << " norm_sum=" << global_norm
                 << " device=" << device_name << '\n';

justfile

@@ -5,6 +5,8 @@ python_bin := env_var_or_default("PYTHON", "python3")
 jobs := env_var_or_default("MFEM_JOBS", "4")
 allow_preinstalled := env_var_or_default("MFEM_ALLOW_PREINSTALLED", "false")
 dependency_prefix := env_var_or_default("MFEM_DEPENDENCY_PREFIX", "")
+cuda_home := env_var_or_default("CUDA_HOME", "")
+cuda_arch := env_var_or_default("MFEM_CUDA_ARCH", "native")
 web_port := env_var_or_default("MFEM_WEB_PORT", "8000")
 macos_target := env_var_or_default("MACOSX_DEPLOYMENT_TARGET", "11.0")
 
@@ -25,6 +27,14 @@ parallel_install := install_root + "/parallel"
 debug_install := install_root + "/debug"
 cuda_install := install_root + "/cuda"
 hip_install := install_root + "/hip"
+benchmark_trials := env_var_or_default("MFEM_BENCH_TRIALS", "5")
+benchmark_mesh_n := env_var_or_default("MFEM_BENCH_MESH_N", "48")
+benchmark_order := env_var_or_default("MFEM_BENCH_ORDER", "3")
+benchmark_applications := env_var_or_default("MFEM_BENCH_APPLICATIONS", "50")
+benchmark_minimum_apply_seconds := env_var_or_default("MFEM_BENCH_MIN_SECONDS", "1.0")
+benchmark_max_applications := env_var_or_default("MFEM_BENCH_MAX_APPLICATIONS", "1000000")
+benchmark_omp_threads := env_var_or_default("MFEM_BENCH_OMP_THREADS", "8")
+benchmark_mpi_ranks := env_var_or_default("MFEM_BENCH_MPI_RANKS", "4")
 
 default:
     @just --list
@@ -58,11 +68,65 @@ basic: (configure basic_dir basic_install "release" "minimal" allow_preinstalled
 
 serial: (configure serial_dir serial_install "release" "portable" allow_preinstalled "-Dmfem_mpi=disabled" "-Dmfem_metis=disabled" "-Dmfem_cuda=disabled" "-Dmfem_hip=disabled") (build-and-test serial_dir)
 
-parallel: (configure parallel_dir parallel_install "release" "portable" allow_preinstalled "-Dmfem_mpi=enabled" "-Dmfem_sundials=enabled") (build-and-test parallel_dir)
+parallel: (configure parallel_dir parallel_install "release" "portable" allow_preinstalled "-Dmfem_mpi=enabled" "-Dmfem_sundials=enabled" "-Dmfem_cuda=disabled" "-Dmfem_hip=disabled" "-Dbuild_benchmarks=true") (build-and-test parallel_dir)
 
 debug: (configure debug_dir debug_install "debug" "portable" allow_preinstalled) (build-and-test debug_dir)
 
-cuda: (configure cuda_dir cuda_install "release" "portable" allow_preinstalled "-Dmfem_cuda=enabled" "-Dmfem_hip=disabled") (build-and-test cuda_dir)
+cuda:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    cuda_root="{{ cuda_home }}"
+    if [[ -z "$cuda_root" ]]; then
+      if command -v nvcc >/dev/null 2>&1; then
+        nvcc_path="$(command -v nvcc)"
+        while [[ -L "$nvcc_path" ]]; do
+          nvcc_target="$(readlink "$nvcc_path")"
+          if [[ "$nvcc_target" == /* ]]; then
+            nvcc_path="$nvcc_target"
+          else
+            nvcc_path="$(dirname "$nvcc_path")/$nvcc_target"
+          fi
+        done
+        cuda_root="$(cd "$(dirname "$nvcc_path")/.." && pwd -P)"
+      elif [[ -x /usr/local/cuda/bin/nvcc ]]; then
+        cuda_root=/usr/local/cuda
+      fi
+    fi
+    if [[ -n "$cuda_root" ]]; then
+      export CUDA_HOME="$cuda_root"
+      export PATH="$cuda_root/bin:$PATH"
+    fi
+    if ! command -v nvcc >/dev/null 2>&1; then
+      echo "CUDA was requested, but nvcc was not found; set CUDA_HOME or PATH." >&2
+      exit 2
+    fi
+    selected_arch="{{ cuda_arch }}"
+    if [[ "$selected_arch" == native ]] && command -v nvidia-smi >/dev/null 2>&1; then
+      if detected_arch="$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | sed -n '1{s/[[:space:].]//g;p;}')" && [[ "$detected_arch" =~ ^[0-9]+$ ]]; then
+        selected_arch="$detected_arch"
+      fi
+    fi
+    setup_args=(
+      "{{ cuda_dir }}"
+      "--prefix={{ cuda_install }}"
+      "--buildtype=release"
+      "-Dfeature_profile=portable"
+      "-Dallow_preinstalled={{ allow_preinstalled }}"
+      "-Ddependency_prefix={{ dependency_prefix }}"
+      "-Djobs={{ jobs }}"
+      "-Dbuild_examples=true"
+      "-Dbuild_benchmarks=true"
+      "-Dbuild_tests=true"
+      "-Dmfem_cuda=enabled"
+      "-Dmfem_hip=disabled"
+      "-Dmfem_cuda_arch=${selected_arch}"
+    )
+    if [[ -f "{{ cuda_dir }}/meson-private/coredata.dat" ]]; then
+      setup_args=(--reconfigure "${setup_args[@]}")
+    fi
+    "{{ meson_bin }}" setup "${setup_args[@]}"
+    "{{ meson_bin }}" compile -C "{{ cuda_dir }}"
+    "{{ meson_bin }}" test -C "{{ cuda_dir }}" --print-errorlogs
 
 hip: (configure hip_dir hip_install "release" "portable" allow_preinstalled "-Dmfem_hip=enabled" "-Dmfem_cuda=disabled") (build-and-test hip_dir)
 
@@ -103,6 +167,115 @@ run-serial: serial
 run-parallel ranks="4": parallel
     "{{ python_bin }}" "{{ project_root }}/tools/run_mpi_test.py" --launcher "{{ parallel_dir }}/build-config/mfem/mfem-prefix/bin/mpiexec" --processes "{{ ranks }}" "{{ parallel_dir }}/examples/mfem-parallel-hypre"
 
+run-cuda ranks="2": cuda
+    "{{ python_bin }}" "{{ project_root }}/tools/run_mpi_test.py" --launcher "{{ cuda_dir }}/build-config/mfem/mfem-prefix/bin/mpiexec" --processes "{{ ranks }}" "{{ cuda_dir }}/examples/mfem-parallel-hypre" cuda
+
+benchmark: benchmark-cpu
+
+benchmark-cpu: parallel
+    #!/usr/bin/env bash
+    set -euo pipefail
+    launcher="{{ parallel_dir }}/build-config/mfem/mfem-prefix/bin/mpiexec"
+    dep_prefix="{{ dependency_prefix }}"
+    if [[ ! -x "$launcher" && -n "$dep_prefix" ]]; then
+      for candidate in "$dep_prefix/bin/mpiexec" "$dep_prefix/bin/mpirun"; do
+        [[ -x "$candidate" ]] && launcher="$candidate" && break
+      done
+    fi
+    if [[ ! -x "$launcher" ]]; then
+      launcher="$(command -v mpiexec || command -v mpirun || true)"
+    fi
+    [[ -n "$launcher" ]] || { echo "No MPI launcher was found." >&2; exit 2; }
+    if [[ -n "$dep_prefix" ]]; then
+      export PATH="$dep_prefix/bin:$PATH"
+      if [[ "$(uname -s)" == Darwin ]]; then
+        export DYLD_LIBRARY_PATH="$dep_prefix/lib:$dep_prefix/lib64:${DYLD_LIBRARY_PATH:-}"
+      else
+        export LD_LIBRARY_PATH="$dep_prefix/lib:$dep_prefix/lib64:${LD_LIBRARY_PATH:-}"
+      fi
+    fi
+    "{{ python_bin }}" "{{ project_root }}/tools/run_backend_benchmarks.py" \
+      --executable "{{ parallel_dir }}/benchmarks/mfem-backend-benchmark" \
+      --launcher "$launcher" \
+      --cases cpu,ceed-cpu,omp,mpi \
+      --omp-threads "{{ benchmark_omp_threads }}" \
+      --mpi-ranks "{{ benchmark_mpi_ranks }}" \
+      --trials "{{ benchmark_trials }}" \
+      --mesh-n "{{ benchmark_mesh_n }}" \
+      --order "{{ benchmark_order }}" \
+      --applications "{{ benchmark_applications }}" \
+      --minimum-apply-seconds "{{ benchmark_minimum_apply_seconds }}" \
+      --max-applications "{{ benchmark_max_applications }}" \
+      --output-dir "{{ parallel_dir }}/benchmarks/results"
+
+benchmark-cuda: cuda
+    #!/usr/bin/env bash
+    set -euo pipefail
+    launcher="{{ cuda_dir }}/build-config/mfem/mfem-prefix/bin/mpiexec"
+    dep_prefix="{{ dependency_prefix }}"
+    if [[ ! -x "$launcher" && -n "$dep_prefix" ]]; then
+      for candidate in "$dep_prefix/bin/mpiexec" "$dep_prefix/bin/mpirun"; do
+        [[ -x "$candidate" ]] && launcher="$candidate" && break
+      done
+    fi
+    if [[ ! -x "$launcher" ]]; then
+      launcher="$(command -v mpiexec || command -v mpirun || true)"
+    fi
+    [[ -n "$launcher" ]] || { echo "No MPI launcher was found." >&2; exit 2; }
+    if [[ -n "$dep_prefix" ]]; then
+      export PATH="$dep_prefix/bin:$PATH"
+      if [[ "$(uname -s)" == Darwin ]]; then
+        export DYLD_LIBRARY_PATH="$dep_prefix/lib:$dep_prefix/lib64:${DYLD_LIBRARY_PATH:-}"
+      else
+        export LD_LIBRARY_PATH="$dep_prefix/lib:$dep_prefix/lib64:${LD_LIBRARY_PATH:-}"
+      fi
+    fi
+    "{{ python_bin }}" "{{ project_root }}/tools/run_backend_benchmarks.py" \
+      --executable "{{ cuda_dir }}/benchmarks/mfem-backend-benchmark" \
+      --launcher "$launcher" \
+      --cases cpu,ceed-cpu,omp,mpi,cuda,ceed-cuda \
+      --omp-threads "{{ benchmark_omp_threads }}" \
+      --mpi-ranks "{{ benchmark_mpi_ranks }}" \
+      --trials "{{ benchmark_trials }}" \
+      --mesh-n "{{ benchmark_mesh_n }}" \
+      --order "{{ benchmark_order }}" \
+      --applications "{{ benchmark_applications }}" \
+      --minimum-apply-seconds "{{ benchmark_minimum_apply_seconds }}" \
+      --max-applications "{{ benchmark_max_applications }}" \
+      --output-dir "{{ cuda_dir }}/benchmarks/results"
+
+benchmark-quick: parallel
+    #!/usr/bin/env bash
+    set -euo pipefail
+    launcher="{{ parallel_dir }}/build-config/mfem/mfem-prefix/bin/mpiexec"
+    dep_prefix="{{ dependency_prefix }}"
+    if [[ ! -x "$launcher" && -n "$dep_prefix" ]]; then
+      for candidate in "$dep_prefix/bin/mpiexec" "$dep_prefix/bin/mpirun"; do
+        [[ -x "$candidate" ]] && launcher="$candidate" && break
+      done
+    fi
+    if [[ ! -x "$launcher" ]]; then
+      launcher="$(command -v mpiexec || command -v mpirun || true)"
+    fi
+    [[ -n "$launcher" ]] || { echo "No MPI launcher was found." >&2; exit 2; }
+    if [[ -n "$dep_prefix" ]]; then
+      export PATH="$dep_prefix/bin:$PATH"
+      if [[ "$(uname -s)" == Darwin ]]; then
+        export DYLD_LIBRARY_PATH="$dep_prefix/lib:$dep_prefix/lib64:${DYLD_LIBRARY_PATH:-}"
+      else
+        export LD_LIBRARY_PATH="$dep_prefix/lib:$dep_prefix/lib64:${LD_LIBRARY_PATH:-}"
+      fi
+    fi
+    "{{ python_bin }}" "{{ project_root }}/tools/run_backend_benchmarks.py" \
+      --executable "{{ parallel_dir }}/benchmarks/mfem-backend-benchmark" \
+      --launcher "$launcher" \
+      --cases cpu,ceed-cpu,omp,mpi \
+      --omp-threads "{{ benchmark_omp_threads }}" \
+      --mpi-ranks "{{ benchmark_mpi_ranks }}" \
+      --trials 1 --mesh-n 16 --order 2 --applications 5 \
+      --minimum-apply-seconds 0 \
+      --output-dir "{{ parallel_dir }}/benchmarks/quick-results"
+
 install-basic: basic
     "{{ meson_bin }}" install -C "{{ basic_dir }}"
 

meson.build

@@ -30,6 +30,7 @@ subdir('build-check')
 subdir('build-config')
 subdir('include/meson_mfem_template')
 subdir('examples')
+subdir('benchmarks')
 subdir('web')
 subdir('build-python')
 subdir('tests')
@@ -52,6 +53,7 @@ summary(
     'Algoim': mfem_features.get('algoim'),
     'Python bindings': get_option('build_python'),
     'examples': get_option('build_examples'),
+    'benchmarks': get_option('build_benchmarks'),
     'tests': get_option('build_tests'),
   },
   section: 'meson-mfem-template ' + meson.project_version(),

meson_options.txt

@@ -2,6 +2,7 @@ option('allow_preinstalled', type: 'boolean', value: true, description: 'Allow o
 option('feature_profile', type: 'combo', choices: ['minimal', 'portable', 'full'], value: 'portable', description: 'Defaults for auto features. full is strict and requires all selected non-bundled TPLs in dependency_prefix.')
 option('build_python', type: 'boolean', value: false, description: 'Build the nanobind Python extension and install native assets inside the Python package.')
 option('build_examples', type: 'boolean', value: true, description: 'Build MFEM capability examples.')
+option('build_benchmarks', type: 'boolean', value: false, description: 'Build the native MPI-capable backend benchmark and register the benchmark suite.')
 option('build_tests', type: 'boolean', value: true, description: 'Register serial, parallel, and Python smoke tests when applicable.')
 option('install_mfem', type: 'boolean', value: true, description: 'Install the source-built MFEM bundle and its runtime dependencies.')
 option('dependency_prefix', type: 'string', value: '', description: 'Explicit prefix containing optional TPLs. Used even when allow_preinstalled=false.')

tests/meson.build

@@ -31,6 +31,16 @@ if get_option('build_tests') and get_option('build_examples')
     timeout: 120,
   )
 
+  if mfem_has_cuda
+    test(
+      'serial-poisson-cuda',
+      serial_example,
+      args: ['cuda'],
+      env: runtime_environment,
+      timeout: 120,
+    )
+  endif
+
   if mfem_has_mpi
     if mpi_launcher_from_dependency and not wheel_carries_native_bundle
       mpi_launcher_program = find_program(
@@ -57,8 +67,75 @@ if get_option('build_tests') and get_option('build_examples')
       env: runtime_environment,
       timeout: 180,
     )
+    if mfem_has_cuda
+      test(
+        'parallel-hypre-boomeramg-cuda',
+        python_build,
+        args: [
+          files('../tools/run_mpi_test.py'),
+          '--launcher', mpi_launcher,
+          '--processes', '2',
+          parallel_example,
+          'cuda',
+        ],
+        env: runtime_environment,
+        timeout: 180,
+      )
     endif
   endif
+endif
+
+if get_option('build_tests') and get_option('build_benchmarks')
+  benchmark_smoke_environment = environment()
+  benchmark_smoke_environment.set('OMP_NUM_THREADS', '1')
+  benchmark_smoke_environment.set('OMP_DYNAMIC', 'FALSE')
+  if (
+    mfem_runtime_prefix != '' and dependency_prefix != '' and
+    not wheel_carries_native_bundle
+  )
+    benchmark_smoke_environment.prepend('PATH', dependency_prefix / 'bin')
+    if host_machine.system() == 'darwin'
+      benchmark_smoke_environment.prepend('DYLD_LIBRARY_PATH', dependency_prefix / 'lib64')
+      benchmark_smoke_environment.prepend('DYLD_LIBRARY_PATH', dependency_prefix / 'lib')
+    elif host_machine.system() != 'windows' and not is_wasm
+      benchmark_smoke_environment.prepend('LD_LIBRARY_PATH', dependency_prefix / 'lib64')
+      benchmark_smoke_environment.prepend('LD_LIBRARY_PATH', dependency_prefix / 'lib')
+    endif
+  endif
+  if mfem_runtime_prefix != ''
+    benchmark_smoke_environment.prepend('PATH', mfem_runtime_prefix / 'bin')
+    if host_machine.system() == 'darwin'
+      benchmark_smoke_environment.prepend('DYLD_LIBRARY_PATH', mfem_runtime_prefix / 'lib64')
+      benchmark_smoke_environment.prepend('DYLD_LIBRARY_PATH', mfem_runtime_prefix / 'lib')
+    elif host_machine.system() != 'windows'
+      benchmark_smoke_environment.prepend('LD_LIBRARY_PATH', mfem_runtime_prefix / 'lib64')
+      benchmark_smoke_environment.prepend('LD_LIBRARY_PATH', mfem_runtime_prefix / 'lib')
+    endif
+  endif
+  test(
+    'backend-benchmark-smoke',
+    python_build,
+    args: [
+      files('../tools/run_mpi_test.py'),
+      '--launcher', benchmark_mpi_launcher,
+      '--processes', '1',
+      '--',
+      benchmark_executable,
+      '--device', 'cpu',
+      '--mesh-n', '4',
+      '--order', '2',
+      '--applications', '2',
+      '--minimum-apply-seconds', '0',
+      '--warmup-applications', '1',
+      '--relative-tolerance', '1e-6',
+      '--max-iterations', '200',
+      '--solve',
+    ],
+    depends: benchmark_executable,
+    env: benchmark_smoke_environment,
+    timeout: 180,
+  )
+endif
 
 if get_option('build_tests') and get_option('build_python')
   python_test_environment = environment()

tools/build_mfem_bundle.py

@@ -589,15 +589,7 @@ def build_zlib(
         build.mkdir(parents=True, exist_ok=True)
         cmake_configure(args, args.zlib_source, build, options, env)
         run(
-            [
-                args.cmake,
-                "--build",
-                build,
-                "--target",
-                "zlibstatic",
-                "--parallel",
-                str(args.jobs),
-            ],
+            [args.cmake, "--build", build, "--parallel", str(args.jobs)],
             env=env,
         )
         run([args.cmake, "--install", build], env=env)
@@ -658,7 +650,7 @@ def build_hypre(
     options = common_cmake_args(
         args,
         build_type,
-        c_compiler=mpicc if features["mpi"] else args.cc,
+        c_compiler=args.cc,
     ) + [
         f"-DHYPRE_ENABLE_MPI={'ON' if features['mpi'] else 'OFF'}",
         f"-DHYPRE_ENABLE_OPENMP={'ON' if features['openmp'] else 'OFF'}",
@@ -668,6 +660,12 @@ def build_hypre(
         "-DHYPRE_BUILD_EXAMPLES=OFF",
         "-DHYPRE_BUILD_TESTS=OFF",
     ]
+    if features["mpi"]:
+        # Keep CMake's compiler identity on the host compiler and let FindMPI
+        # interrogate the wrapper.  Treating mpicc itself as CMAKE_C_COMPILER
+        # makes MPI's include path look implicit, so CUDA translation units
+        # compiled by nvcc do not receive mpi.h.
+        options.append(f"-DMPI_C_COMPILER={mpicc}")
     if features["cuda"]:
         options.extend(
             [
@@ -1336,7 +1334,10 @@ def build_mfem(
             f"-DGSLIB_DIR={bundle_root(args, 'include/gslib/gslib.h')}"
         )
     if features["zlib"]:
-        options.append(f"-DZLIB_ROOT={bundle_root(args, 'include/zlib.h')}")
+        zlib_root = bundle_root(args, "include/zlib.h")
+        options.append(f"-DZLIB_ROOT={zlib_root}")
+        if args.host_system == "emscripten":
+            options.append(f"-DZLIB_LIBRARY={zlib_root / 'lib' / 'libz.a'}")
     if features["sundials"]:
         options.append(
             f"-DSUNDIALS_DIR={bundle_root(args, 'include/sundials/sundials_config.h')}"

tools/run_backend_benchmarks.py

@@ -0,0 +1,667 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import argparse
+import csv
+import datetime as dt
+import json
+import os
+import pathlib
+import platform
+import random
+import shutil
+import statistics
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from typing import Any
+
+
+JSON_PREFIX = "MFEM_BENCHMARK_JSON "
+
+
+@dataclass(frozen=True)
+class Case:
+    name: str
+    device: str
+    ranks: int
+    threads: int
+
+
+def positive_int(value: str) -> int:
+    parsed = int(value)
+    if parsed < 1:
+        raise argparse.ArgumentTypeError("must be at least one")
+    return parsed
+
+
+def integer_list(value: str) -> list[int]:
+    try:
+        parsed = [int(item.strip()) for item in value.split(",") if item.strip()]
+    except ValueError as error:
+        raise argparse.ArgumentTypeError("expected comma-separated integers") from error
+    if not parsed or any(item < 1 for item in parsed):
+        raise argparse.ArgumentTypeError("all entries must be positive")
+    return parsed
+
+
+def command_output(command: list[str]) -> str | None:
+    try:
+        result = subprocess.run(
+            command,
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    if result.returncode != 0:
+        return None
+    output = result.stdout.strip()
+    return output if output else None
+
+
+def resolve_program(value: str) -> str | None:
+    candidate = pathlib.Path(value)
+    if candidate.is_file():
+        return str(candidate.resolve())
+    discovered = shutil.which(value)
+    return str(pathlib.Path(discovered).resolve()) if discovered else None
+
+
+def compact_cpu_list(cpus: list[int]) -> str:
+    if not cpus:
+        return ""
+    groups: list[str] = []
+    start = previous = cpus[0]
+    for current in cpus[1:]:
+        if current == previous + 1:
+            previous = current
+            continue
+        groups.append(str(start) if start == previous else f"{start}-{previous}")
+        start = previous = current
+    groups.append(str(start) if start == previous else f"{start}-{previous}")
+    return ",".join(groups)
+
+
+def affinity_prefix(case: Case, setting: str) -> tuple[list[str], str | None]:
+    if setting == "none" or shutil.which("taskset") is None:
+        return [], None
+    if setting != "auto":
+        return ["taskset", "-c", setting], setting
+    if not hasattr(os, "sched_getaffinity"):
+        return [], None
+    available = sorted(os.sched_getaffinity(0))
+    count = min(len(available), max(case.ranks, case.threads))
+    chosen = compact_cpu_list(available[:count])
+    return (["taskset", "-c", chosen] if chosen else []), chosen or None
+
+
+def expand_cases(args: argparse.Namespace) -> list[Case]:
+    requested = [item.strip() for item in args.cases.split(",") if item.strip()]
+    valid = {"cpu", "ceed-cpu", "omp", "mpi", "cuda", "ceed-cuda"}
+    unknown = sorted(set(requested) - valid)
+    if unknown:
+        raise ValueError(f"unknown benchmark case(s): {', '.join(unknown)}")
+    cases: list[Case] = []
+    for requested_case in requested:
+        if requested_case == "cpu":
+            cases.append(Case("cpu-1r-1t", "cpu", 1, 1))
+        elif requested_case == "ceed-cpu":
+            cases.append(Case("ceed-cpu-1r-1t", args.ceed_cpu_device, 1, 1))
+        elif requested_case == "omp":
+            cases.extend(
+                Case(f"omp-1r-{threads}t", "omp", 1, threads)
+                for threads in args.omp_threads
+            )
+        elif requested_case == "mpi":
+            cases.extend(
+                Case(f"mpi-{ranks}r-1t", "cpu", ranks, 1)
+                for ranks in args.mpi_ranks
+            )
+        elif requested_case == "cuda":
+            cases.append(Case("cuda-1r-1t", "cuda", 1, 1))
+        elif requested_case == "ceed-cuda":
+            cases.append(Case("ceed-cuda-1r-1t", args.ceed_cuda_device, 1, 1))
+    deduplicated: list[Case] = []
+    for case in cases:
+        if case not in deduplicated:
+            deduplicated.append(case)
+    return deduplicated
+
+
+def run_case(
+    args: argparse.Namespace, case: Case, trial: int, warmup_trial: bool
+) -> dict[str, Any]:
+    prefix, affinity = affinity_prefix(case, args.affinity)
+    command = prefix + [args.launcher, "-n", str(case.ranks)]
+    command.extend(args.launcher_arg)
+    command.extend(
+        [
+            args.executable,
+            "--device",
+            case.device,
+            "--mesh-n",
+            str(args.mesh_n),
+            "--order",
+            str(args.order),
+            "--applications",
+            str(args.applications),
+            "--max-applications",
+            str(args.max_applications),
+            "--minimum-apply-seconds",
+            str(args.minimum_apply_seconds),
+            "--warmup-applications",
+            str(args.warmup_applications),
+            "--relative-tolerance",
+            str(args.relative_tolerance),
+            "--max-iterations",
+            str(args.max_iterations),
+            "--trial",
+            str(trial),
+        ]
+    )
+    if args.solve:
+        command.append("--solve")
+    environment = os.environ.copy()
+    environment.update(
+        {
+            "OMP_NUM_THREADS": str(case.threads),
+            "OMP_DYNAMIC": "FALSE",
+            "OMP_PROC_BIND": "close",
+            "OMP_PLACES": "cores",
+            "FI_PROVIDER": environment.get("FI_PROVIDER", "sockets"),
+        }
+    )
+    environment.setdefault("OMPI_ALLOW_RUN_AS_ROOT", "1")
+    environment.setdefault("OMPI_ALLOW_RUN_AS_ROOT_CONFIRM", "1")
+    selected_interface = args.network_interface
+    if selected_interface == "local":
+        selected_interface = "lo0" if sys.platform == "darwin" else "lo"
+    if selected_interface == "auto":
+        environment.pop("FI_SOCKETS_IFACE", None)
+        environment.pop("HYDRA_IFACE", None)
+        environment.pop("MPICH_INTERFACE_HOSTNAME", None)
+    else:
+        environment["FI_SOCKETS_IFACE"] = selected_interface
+        environment["HYDRA_IFACE"] = selected_interface
+        environment.pop("MPICH_INTERFACE_HOSTNAME", None)
+    if sys.platform == "darwin" and args.network_interface == "local":
+        environment["MPICH_INTERFACE_HOSTNAME"] = "127.0.0.1"
+    if case.device.startswith("cuda") or case.device.startswith("ceed-cuda"):
+        if args.gpu_aware_mpi:
+            environment["MFEM_GPU_AWARE_MPI"] = "1"
+        else:
+            environment.pop("MFEM_GPU_AWARE_MPI", None)
+        if args.cuda_visible_devices is not None:
+            environment["CUDA_VISIBLE_DEVICES"] = args.cuda_visible_devices
+
+    started = time.perf_counter()
+    completed = subprocess.run(
+        command,
+        check=False,
+        capture_output=True,
+        text=True,
+        env=environment,
+        timeout=args.timeout,
+    )
+    wall_seconds = time.perf_counter() - started
+    combined_output = "\n".join((completed.stdout, completed.stderr))
+    records = [
+        json.loads(line[len(JSON_PREFIX) :])
+        for line in combined_output.splitlines()
+        if line.startswith(JSON_PREFIX)
+    ]
+    if completed.returncode != 0 or len(records) != 1:
+        tail = "\n".join(combined_output.splitlines()[-30:])
+        raise RuntimeError(
+            f"{case.name} trial {trial} failed with exit code "
+            f"{completed.returncode}; JSON records={len(records)}\n{tail}"
+        )
+    record = records[0]
+    record.update(
+        {
+            "case": case.name,
+            "requested_device": case.device,
+            "runner_trial": trial,
+            "discarded_warmup_trial": warmup_trial,
+            "wall_seconds": wall_seconds,
+            "cpu_affinity": affinity,
+            "network_interface": environment.get("HYDRA_IFACE"),
+            "cuda_visible_devices": environment.get("CUDA_VISIBLE_DEVICES"),
+            "gpu_aware_mpi": environment.get("MFEM_GPU_AWARE_MPI") == "1",
+            "command": command,
+        }
+    )
+    if not record.get("valid", False):
+        raise RuntimeError(f"{case.name} trial {trial} failed numerical validation")
+    return record
+
+
+def median_absolute_deviation(values: list[float]) -> float:
+    center = statistics.median(values)
+    return statistics.median(abs(value - center) for value in values)
+
+
+def validate_cross_backend(
+    records: list[dict[str, Any]], relative_tolerance: float
+) -> dict[str, float]:
+    measured = [row for row in records if not row["discarded_warmup_trial"]]
+    baseline = [row for row in measured if row["case"] == "cpu-1r-1t"]
+    if not baseline:
+        return {}
+    validation_fields = ["probe_norm"]
+    if measured and all(row.get("solve_ran", False) for row in measured):
+        validation_fields.append("solution_norm")
+    references = {
+        field: statistics.median(float(row[field]) for row in baseline)
+        for field in validation_fields
+    }
+    maximum_errors = {field: 0.0 for field in references}
+    for row in measured:
+        for field, reference in references.items():
+            relative_error = abs(float(row[field]) - reference) / max(
+                abs(reference), 1.0e-300
+            )
+            maximum_errors[field] = max(maximum_errors[field], relative_error)
+            if relative_error > relative_tolerance:
+                raise RuntimeError(
+                    f"{row['case']} disagrees with CPU for {field}: relative "
+                    f"error {relative_error:.3e} > {relative_tolerance:.3e}"
+                )
+    return maximum_errors
+
+
+def checkpoint_results(
+    output_dir: pathlib.Path,
+    metadata: dict[str, Any],
+    records: list[dict[str, Any]],
+) -> None:
+    output_dir.mkdir(parents=True, exist_ok=True)
+    destination = output_dir / "partial-results.json"
+    temporary = output_dir / "partial-results.json.tmp"
+    temporary.write_text(
+        json.dumps({"schema_version": 1, "metadata": metadata, "raw": records}, indent=2)
+        + "\n",
+        encoding="utf-8",
+    )
+    temporary.replace(destination)
+
+
+def summarize(cases: list[Case], records: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    summaries: list[dict[str, Any]] = []
+    for case in cases:
+        rows = [
+            row
+            for row in records
+            if row["case"] == case.name and not row["discarded_warmup_trial"]
+        ]
+        if not rows:
+            continue
+        throughput = [float(row["apply_mdof_per_second"]) for row in rows]
+        apply_seconds = [float(row["apply_seconds"]) for row in rows]
+        solve_seconds = [float(row["solve_seconds"]) for row in rows]
+        total_seconds = [float(row["total_seconds"]) for row in rows]
+        timed_applications = [int(row["applications"]) for row in rows]
+        summaries.append(
+            {
+                "case": case.name,
+                "device": case.device,
+                "ranks": case.ranks,
+                "threads": case.threads,
+                "trials": len(rows),
+                "global_true_dofs": rows[0]["global_true_dofs"],
+                "requested_applications": rows[0]["requested_applications"],
+                "applications_min": min(timed_applications),
+                "applications_max": max(timed_applications),
+                "apply_mdof_per_second_median": statistics.median(throughput),
+                "apply_mdof_per_second_mad": median_absolute_deviation(throughput),
+                "apply_mdof_per_second_min": min(throughput),
+                "apply_mdof_per_second_max": max(throughput),
+                "apply_seconds_median": statistics.median(apply_seconds),
+                "solve_seconds_median": statistics.median(solve_seconds),
+                "total_seconds_median": statistics.median(total_seconds),
+                "cg_iterations": sorted({row["cg_iterations"] for row in rows}),
+                "solve_ran": all(bool(row["solve_ran"]) for row in rows),
+                "verified_relative_residual_max": max(
+                    float(row["verified_relative_residual"]) for row in rows
+                ),
+            }
+        )
+    baseline = next((row for row in summaries if row["case"] == "cpu-1r-1t"), None)
+    baseline_rate = (
+        float(baseline["apply_mdof_per_second_median"]) if baseline else None
+    )
+    for row in summaries:
+        row["speedup_vs_cpu"] = (
+            float(row["apply_mdof_per_second_median"]) / baseline_rate
+            if baseline_rate
+            else None
+        )
+    return summaries
+
+
+def markdown_table(summaries: list[dict[str, Any]]) -> str:
+    lines = [
+        "| Case | Device | Ranks | Threads | Timed apps | Apply window (s) | Apply MDoF/s, median [range] | MAD | Speedup | Solve (s) | CG iters |",
+        "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|",
+    ]
+    for row in summaries:
+        speedup = row["speedup_vs_cpu"]
+        speedup_text = f"{speedup:.2f}x" if speedup is not None else "n/a"
+        solve_text = f"{row['solve_seconds_median']:.4f}" if row["solve_ran"] else "—"
+        iteration_text = (
+            ",".join(str(value) for value in row["cg_iterations"])
+            if row["solve_ran"]
+            else "—"
+        )
+        applications_text = (
+            str(row["applications_min"])
+            if row["applications_min"] == row["applications_max"]
+            else f"{row['applications_min']}-{row['applications_max']}"
+        )
+        lines.append(
+            f"| {row['case']} | `{row['device']}` | {row['ranks']} | "
+            f"{row['threads']} | {applications_text} | "
+            f"{row['apply_seconds_median']:.3f} | "
+            f"{row['apply_mdof_per_second_median']:.2f} "
+            f"[{row['apply_mdof_per_second_min']:.2f}, "
+            f"{row['apply_mdof_per_second_max']:.2f}] | "
+            f"{row['apply_mdof_per_second_mad']:.2f} | "
+            f"{speedup_text} | {solve_text} | {iteration_text} |"
+        )
+    return "\n".join(lines) + "\n"
+
+
+def collect_metadata(args: argparse.Namespace) -> dict[str, Any]:
+    executable = pathlib.Path(args.executable).resolve()
+    source_root = pathlib.Path(__file__).resolve().parents[1]
+    return {
+        "timestamp_utc": dt.datetime.now(dt.timezone.utc).isoformat(),
+        "hostname": platform.node(),
+        "platform": platform.platform(),
+        "python": sys.version,
+        "cpu_count": os.cpu_count(),
+        "lscpu": command_output(["lscpu"]) if shutil.which("lscpu") else None,
+        "executable": str(executable),
+        "launcher": str(pathlib.Path(args.launcher).resolve()),
+        "launcher_version": command_output([args.launcher, "--version"]),
+        "compiler": command_output(["c++", "--version"]),
+        "nvidia_smi": command_output(
+            [
+                "nvidia-smi",
+                "--query-gpu=name,driver_version,temperature.gpu,pstate,clocks.sm,power.draw",
+                "--format=csv,noheader",
+            ]
+        )
+        if shutil.which("nvidia-smi")
+        else None,
+        "git_commit": command_output(
+            ["git", "-C", str(source_root), "rev-parse", "HEAD"]
+        ),
+        "git_dirty": bool(
+            command_output(["git", "-C", str(source_root), "status", "--porcelain"])
+        ),
+        "parameters": {
+            "mesh_n": args.mesh_n,
+            "order": args.order,
+            "applications": args.applications,
+            "max_applications": args.max_applications,
+            "minimum_apply_seconds": args.minimum_apply_seconds,
+            "warmup_applications": args.warmup_applications,
+            "trials": args.trials,
+            "discarded_warmup_trials": args.warmup_trials,
+            "relative_tolerance": args.relative_tolerance,
+            "max_iterations": args.max_iterations,
+            "solve": args.solve,
+            "affinity": args.affinity,
+            "network_interface": args.network_interface,
+            "cuda_visible_devices": args.cuda_visible_devices,
+            "gpu_aware_mpi": args.gpu_aware_mpi,
+            "cross_backend_relative_tolerance": args.cross_backend_relative_tolerance,
+        },
+    }
+
+
+def write_results(
+    output_dir: pathlib.Path,
+    metadata: dict[str, Any],
+    records: list[dict[str, Any]],
+    summaries: list[dict[str, Any]],
+) -> None:
+    output_dir.mkdir(parents=True, exist_ok=True)
+    (output_dir / "results.json").write_text(
+        json.dumps(
+            {"schema_version": 1, "metadata": metadata, "raw": records, "summary": summaries},
+            indent=2,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    fieldnames = [
+        "case",
+        "device",
+        "ranks",
+        "threads",
+        "trials",
+        "global_true_dofs",
+        "requested_applications",
+        "applications_min",
+        "applications_max",
+        "apply_mdof_per_second_median",
+        "apply_mdof_per_second_mad",
+        "apply_mdof_per_second_min",
+        "apply_mdof_per_second_max",
+        "speedup_vs_cpu",
+        "apply_seconds_median",
+        "solve_seconds_median",
+        "solve_ran",
+        "total_seconds_median",
+        "cg_iterations",
+        "verified_relative_residual_max",
+    ]
+    with (output_dir / "results.csv").open("w", newline="", encoding="utf-8") as handle:
+        writer = csv.DictWriter(handle, fieldnames=fieldnames)
+        writer.writeheader()
+        for summary in summaries:
+            writer.writerow(summary)
+    first = records[0]
+    if first["minimum_apply_seconds"] > 0.0:
+        timing_description = (
+            "Application counts were selected with a "
+            f"{first['minimum_apply_seconds']:.3g}-second calibration window, then "
+            "measured in a fresh timing window."
+        )
+    else:
+        timing_description = (
+            "The requested application count was measured directly in one timing "
+            "window."
+        )
+    report = "# MFEM backend benchmark\n\n"
+    report += (
+        f"Host: `{metadata['hostname']}`  \n"
+        f"UTC timestamp: `{metadata['timestamp_utc']}`  \n"
+        f"Workload: {first['global_elements']:,} 3D hex elements, H1 order "
+        f"{first['order']}, {first['global_true_dofs']:,} global true DoFs, "
+        f"at least {first['requested_applications']} timed operator applications "
+        f"per trial. {timing_description}\n\n"
+    )
+    report += markdown_table(summaries)
+    report += (
+        "\nThe table reports medians from fresh processes. Operator throughput uses "
+        "the same global partial-assembly diffusion operator and true-DOF field in "
+        "every case. MPI timings use the slowest rank.\n"
+    )
+    (output_dir / "results.md").write_text(report, encoding="utf-8")
+
+
+def parse_arguments() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--executable", required=True)
+    parser.add_argument("--launcher", required=True)
+    parser.add_argument(
+        "--cases",
+        default="cpu,ceed-cpu,omp,mpi,cuda,ceed-cuda",
+        help="Comma-separated subset of cpu, ceed-cpu, omp, mpi, cuda, ceed-cuda.",
+    )
+    parser.add_argument("--omp-threads", type=integer_list, default=[8])
+    parser.add_argument("--mpi-ranks", type=integer_list, default=[4])
+    parser.add_argument("--trials", type=positive_int, default=5)
+    parser.add_argument("--warmup-trials", type=int, default=0)
+    parser.add_argument("--mesh-n", type=positive_int, default=48)
+    parser.add_argument("--order", type=positive_int, default=3)
+    parser.add_argument("--applications", type=positive_int, default=50)
+    parser.add_argument("--max-applications", type=positive_int, default=1_000_000)
+    parser.add_argument(
+        "--minimum-apply-seconds",
+        type=float,
+        default=1.0,
+        help=(
+            "Double the requested applications until a calibration window is this "
+            "long, then measure a fresh window."
+        ),
+    )
+    parser.add_argument("--warmup-applications", type=int, default=5)
+    parser.add_argument("--relative-tolerance", type=float, default=1.0e-6)
+    parser.add_argument("--max-iterations", type=positive_int, default=1000)
+    parser.add_argument(
+        "--solve",
+        action="store_true",
+        help="Also run and time the Jacobi-preconditioned CG validation solve.",
+    )
+    parser.add_argument("--timeout", type=positive_int, default=900)
+    parser.add_argument(
+        "--affinity",
+        default="auto",
+        help="CPU list for taskset, 'auto' for a compact set, or 'none'.",
+    )
+    parser.add_argument(
+        "--network-interface",
+        default="local",
+        help="'local' for platform loopback, 'auto' for routing, or an interface name.",
+    )
+    parser.add_argument(
+        "--cuda-visible-devices",
+        default=None,
+        help="Optional CUDA_VISIBLE_DEVICES value for GPU cases.",
+    )
+    parser.add_argument(
+        "--gpu-aware-mpi",
+        action="store_true",
+        help="Opt in only when the selected MPI implementation is CUDA-aware.",
+    )
+    parser.add_argument(
+        "--cross-backend-relative-tolerance",
+        type=float,
+        default=1.0e-5,
+        help="Maximum relative difference from CPU for probe and solution norms.",
+    )
+    parser.add_argument(
+        "--ceed-cpu-device",
+        default="ceed-cpu",
+        help="Full libCEED CPU resource or portable ceed-cpu alias.",
+    )
+    parser.add_argument(
+        "--ceed-cuda-device",
+        default="ceed-cuda",
+        help="Full libCEED CUDA resource or portable ceed-cuda alias.",
+    )
+    parser.add_argument(
+        "--launcher-arg",
+        action="append",
+        default=[],
+        help="Additional MPI launcher argument; repeat as needed.",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=pathlib.Path,
+        default=pathlib.Path("benchmark-results"),
+    )
+    args = parser.parse_args()
+    if args.warmup_trials < 0 or args.warmup_applications < 0:
+        parser.error("warm-up counts may not be negative")
+    if args.relative_tolerance <= 0.0 or args.cross_backend_relative_tolerance <= 0.0:
+        parser.error("tolerances must be positive")
+    if args.minimum_apply_seconds < 0.0:
+        parser.error("minimum apply seconds may not be negative")
+    if args.max_applications < args.applications:
+        parser.error("max applications must be at least the requested applications")
+    for attribute in ("executable", "launcher"):
+        value = getattr(args, attribute)
+        resolved = resolve_program(value)
+        if resolved is None:
+            parser.error(f"program not found: {value}")
+        setattr(args, attribute, resolved)
+    return args
+
+
+def main() -> int:
+    args = parse_arguments()
+    try:
+        cases = expand_cases(args)
+    except ValueError as error:
+        print(f"error: {error}", file=sys.stderr)
+        return 2
+    metadata = collect_metadata(args)
+    records: list[dict[str, Any]] = []
+
+    for warmup in range(args.warmup_trials):
+        for case in cases:
+            print(f"warm-up {warmup + 1}/{args.warmup_trials}: {case.name}", flush=True)
+            records.append(run_case(args, case, -(warmup + 1), True))
+            checkpoint_results(args.output_dir, metadata, records)
+
+    schedule = [(trial, case) for trial in range(1, args.trials + 1) for case in cases]
+    random.Random(20260905).shuffle(schedule)
+    for index, (trial, case) in enumerate(schedule, start=1):
+        print(
+            f"run {index}/{len(schedule)}: {case.name}, trial {trial}/{args.trials}",
+            flush=True,
+        )
+        records.append(run_case(args, case, trial, False))
+        checkpoint_results(args.output_dir, metadata, records)
+
+    summaries = summarize(cases, records)
+    workload_signatures = {
+        (
+            row["global_elements"],
+            row["global_true_dofs"],
+            row["mesh_n"],
+            row["order"],
+            row["requested_applications"],
+            row["minimum_apply_seconds"],
+        )
+        for row in records
+        if not row["discarded_warmup_trial"]
+    }
+    if len(workload_signatures) != 1:
+        raise RuntimeError(
+            f"cases did not use one fixed global workload: {workload_signatures}"
+        )
+    metadata["cross_backend_maximum_relative_errors"] = validate_cross_backend(
+        records, args.cross_backend_relative_tolerance
+    )
+    if shutil.which("nvidia-smi"):
+        metadata["nvidia_smi_after"] = command_output(
+            [
+                "nvidia-smi",
+                "--query-gpu=name,driver_version,temperature.gpu,pstate,clocks.sm,power.draw",
+                "--format=csv,noheader",
+            ]
+        )
+    write_results(args.output_dir, metadata, records, summaries)
+    table = markdown_table(summaries)
+    print("\n" + table)
+    print(f"JSON, CSV, and Markdown written to {args.output_dir.resolve()}")
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except (OSError, subprocess.TimeoutExpired, RuntimeError) as error:
+        print(f"benchmark failed: {error}", file=sys.stderr)
+        raise SystemExit(1)