fix(omp): upped CppAD max num threads to 512

Also added more explicit error handeling to ensure that users know what to do when the thread count exceeds the compiled maximum
2026-06-13 07:16:50 -04:00
parent 5ea884897d
commit 6bad4415b9
5 changed files with 160 additions and 25 deletions
--- a/build-config/cppad/meson.build
+++ b/build-config/cppad/meson.build
@@ -2,7 +2,7 @@ cppad_cmake_options = cmake.subproject_options()

 cppad_cmake_options.add_cmake_defines({
                                          'cppad_static_lib': 'true',
-                                          'cpp_mas_num_threads': '10',
+                                          'cppad_max_num_threads': '512',
                                          'cppad_debug_and_release': 'false',
                                          'include_doc': 'false',
                                          'CMAKE_POSITION_INDEPENDENT_CODE': true
--- a/src/include/gridfire/utils/gf_omp.h
+++ b/src/include/gridfire/utils/gf_omp.h
@@ -7,7 +7,18 @@
 #include <omp.h>

 namespace gridfire::omp {
-    static bool s_par_mode_initialized = false;
+    /**
+     * @brief Namespace containing utilities for initializing and managing parallel execution with OpenMP and CppAD.
+     *
+     * @note GF_PAR_INIT should be called at the start of your program regardless of if you run in parallel or serial
+     * mode. When GridFire has been compiled without openMP support, GF_PAR_INIT will simply log a message that you are not in parallel mode and return.
+     * However, if in the future you wish to relink against a version which has been compiled with parallel support
+     * missing GF_PAR_INIT may lead to a silent failure.
+     *
+     * @note An end user should only ever need to call the GF_PAR_INIT macro. i.e. never call any of the actual
+     * functions in this header directly.
+     */
+    inline bool s_par_mode_initialized = false;

    inline unsigned long get_thread_id() {
        return static_cast<unsigned long>(omp_get_thread_num());
@@ -18,19 +29,30 @@ namespace gridfire::omp {
    }

    inline void init_parallel_mode() {
-        if (s_par_mode_initialized) {
-            return; // Only initialize once
-        }
+        if (s_par_mode_initialized) return;
        [[maybe_unused]] quill::Logger* logger = fourdst::logging::LogManager::getInstance().getLogger("log");
-        LOG_INFO(logger, "Initializing OpenMP parallel mode with {} threads", static_cast<unsigned long>(omp_get_max_threads()));
-        CppAD::thread_alloc::parallel_setup(
-            static_cast<size_t>(omp_get_max_threads()), // Max threads
-            []() -> bool { return in_parallel(); },                              // Function to get thread ID
-            []() -> size_t { return get_thread_id(); }                                 // Function to check parallel state
-        );

+        auto n_threads = static_cast<size_t>(omp_get_max_threads());
+        if (n_threads > CPPAD_MAX_NUM_THREADS) {
+            LOG_CRITICAL(logger,
+                "OpenMP reports {} threads but CppAD was built with CPPAD_MAX_NUM_THREADS={}; "
+                "clamping OpenMP to {} threads.",
+                n_threads, static_cast<size_t>(CPPAD_MAX_NUM_THREADS),
+                static_cast<size_t>(CPPAD_MAX_NUM_THREADS));
+            throw std::runtime_error(std::format(
+                "OpenMP reports {} threads but CppAD was built with CPPAD_MAX_NUM_THREADS={}; clamping default OpenMP number of threads to {}. Rebuild CppAD with a higher CPPAD_MAX_NUM_THREADS if you need more threads. Alternative, set the environmental variable OMP_NUM_THREADS to a value less than or equal to {} to avoid this error.",
+                n_threads, static_cast<size_t>(CPPAD_MAX_NUM_THREADS),
+                static_cast<size_t>(CPPAD_MAX_NUM_THREADS),
+                static_cast<size_t>(CPPAD_MAX_NUM_THREADS)));
+        }
+
+        LOG_INFO(logger, "Initializing OpenMP parallel mode with {} threads", n_threads);
+        CppAD::thread_alloc::parallel_setup(
+            n_threads,
+            []() -> bool { return in_parallel(); },
+            []() -> size_t { return get_thread_id(); });
        CppAD::thread_alloc::hold_memory(true);
-        CppAD::CheckSimpleVector<double, std::vector<double>>(0, 1);
+        CppAD::parallel_ad<double>();
        s_par_mode_initialized = true;
    }
 }
--- a/src/include/gridfire/utils/macros.h
+++ b/src/include/gridfire/utils/macros.h
@@ -2,12 +2,27 @@


 #if defined(GF_USE_OPENMP)
+    #include <omp.h>
+    #include <algorithm>
+    #include "cppad/configure.hpp"
+
    #define GF_OMP_PRAGMA(x) _Pragma(#x)
-    #define GF_OMP(omp_args, extra) GF_OMP_PRAGMA(omp omp_args) extra
-    #define GF_OMP_MAX_THREADS omp_get_max_threads()
-    #define GF_OMP_THREAD_NUM omp_get_thread_num()
+
+    namespace gridfire::omp {
+        inline int capped_max_threads() {
+            return std::min<int>(omp_get_max_threads(),
+                                  static_cast<int>(CPPAD_MAX_NUM_THREADS));
+        }
+    }
+
+    #define GF_OMP_NUM_THREADS (gridfire::omp::capped_max_threads())
+    #define GF_OMP(omp_args, extra) \
+    GF_OMP_PRAGMA(omp omp_args num_threads(GF_OMP_NUM_THREADS)) extra
+
+    #define GF_OMP_MAX_THREADS (gridfire::omp::capped_max_threads())
+    #define GF_OMP_THREAD_NUM  omp_get_thread_num()
 #else
-    #define GF_OMP(_,fallback_args) fallback_args
+    #define GF_OMP(_, fallback_args) fallback_args
    #define GF_OMP_MAX_THREADS 1
    #define GF_OMP_THREAD_NUM 0
 #endif
--- a/utils/wheels/build-wheels-linux_aarch64.sh
+++ b/utils/wheels/build-wheels-linux_aarch64.sh
@@ -31,27 +31,73 @@ do
  docker run --rm \
    "${DOCKER_MOUNTS[@]}" \
    "${IMAGE}" \
-    /bin/bash -eux -c '
+    /bin/bash -uxo pipefail -c '
      cd /io/project

+      # ----------------------------------------------------------------
+      # Project identity: package name from pyproject.toml, version from
+      # meson (the version pip will stamp into the wheel). Used both for
+      # the skip-if-already-built check and the post-repair checks, so a
+      # stale wheel from an OLDER project version never causes a skip.
+      # ----------------------------------------------------------------
+      PKG="$(sed -n "s/^name *= *\"\(.*\)\"/\1/p" pyproject.toml | head -n1)"
+      PKG="${PKG//-/_}"   # wheel filename normalization
+
+      BOOT_PY=/opt/python/cp312-cp312/bin/python
+      "$BOOT_PY" -m pip install --quiet meson
+      VERSION="$("$BOOT_PY" -c "
+import json, subprocess, sys
+out = subprocess.check_output(
+    [sys.executable, \"-m\", \"mesonbuild.mesonmain\", \"introspect\",
+     \"meson.build\", \"--projectinfo\"])
+print(json.loads(out)[\"version\"])
+" 2>/dev/null || true)"
+      if [ -z "$VERSION" ]; then
+        # fallback: literal version in project()
+        VERSION="$(grep -oE "version *: *.[0-9][0-9a-zA-Z.+-]*" meson.build | head -n1 | grep -oE "[0-9][0-9a-zA-Z.+-]*" || true)"
+      fi
+      if [ -z "$VERSION" ]; then
+        echo "ERROR: could not determine project version; refusing to guess for skip logic"
+        exit 1
+      fi
+      echo "➤ Building ${PKG} ${VERSION}"
+
+      # Does this project link against the fourdst wheel? Single source of
+      # truth: the pin in pyproject.toml.
      FOURDST_PIN="$(grep -oE "fourdst==[0-9][0-9a-zA-Z.]*" pyproject.toml | head -n1 || true)"

+      # If a local fourdst wheel dir was mounted, let pip (including the
+      # isolated build env) resolve fourdst from it.
      if [ -d /io/fourdst-wheels ]; then
        export PIP_FIND_LINKS=/io/fourdst-wheels
      fi

-      for PY in /opt/python/*/bin/python; do
+      build_one() {
+        # Runs the full build+repair for one interpreter. Returns nonzero
+        # on any failure; never exits the whole script (errexit is off in
+        # the caller around this function).
+        set -e
+        local PY="$1" PYTAG="$2"
+
        "$PY" -m pip install --upgrade pip setuptools wheel meson meson-python

+        # Build into a per-iteration temp dir so we repair exactly the
+        # wheel we just built.
+        local BUILD_WHEEL_DIR
        BUILD_WHEEL_DIR="$(mktemp -d)"
-        CC=clang CXX=clang++ "$PY" -m pip wheel . \
-          --no-deps  \
+        CC=clang CXX=clang++ "$PY" -m pip wheel . --no-deps \
          -w "$BUILD_WHEEL_DIR" -vv

+        local CURRENT_WHEEL
        CURRENT_WHEEL="$(find "$BUILD_WHEEL_DIR" -name "*.whl" | head -n1)"

        if [ -n "$FOURDST_PIN" ]; then
+          # Install fourdst for THIS interpreter so auditwheel can resolve
+          # the libraries it must NOT graft. Excluding them keeps fourdst a
+          # runtime dependency: grafting copies would break cross-package
+          # pybind11 type compatibility.
          "$PY" -m pip install --force-reinstall "$FOURDST_PIN"
+          local FOURDST_LIB_PATH
          FOURDST_LIB_PATH="$("$PY" -c "import fourdst, os; print(os.pathsep.join(fourdst.get_lib_dirs()))")"
          LD_LIBRARY_PATH="$FOURDST_LIB_PATH" auditwheel repair \
            --exclude "libcomposition.so*" \
@@ -60,18 +106,70 @@ do
            --exclude "libreflect_cpp.so*" \
            -w /io/wheels "$CURRENT_WHEEL"

-          REPAIRED="$(ls -t /io/wheels/*.whl | head -n1)"
+          # Post-repair sanity check on the wheel we just produced
+          local REPAIRED
+          REPAIRED="$(find /io/wheels -name "${PKG}-${VERSION}-${PYTAG}-*manylinux*.whl" | head -n1)"
+          if [ -z "$REPAIRED" ]; then
+            echo "ERROR: repaired wheel for ${PYTAG} not found after auditwheel"
+            return 1
+          fi
          if unzip -l "$REPAIRED" | grep -E "libcomposition|liblogging|libconst[^a-z]|libreflect_cpp"; then
            echo "ERROR: repaired wheel contains vendored fourdst libraries"
-            exit 1
+            rm -f "$REPAIRED"   # do not leave a poisoned wheel that would be skipped next run
+            return 1
          fi
        else
          auditwheel repair -w /io/wheels "$CURRENT_WHEEL"
        fi

        rm -rf "$BUILD_WHEEL_DIR"
+      }
+
+      FAILED_TAGS=""
+      SKIPPED_TAGS=""
+      BUILT_TAGS=""
+
+      for PY in /opt/python/*/bin/python; do
+        # /opt/python/<pythontag>-<abitag>/bin/python — the directory name
+        # is exactly the {python tag}-{abi tag} pair used in wheel filenames
+        PYTAG="$(basename "$(dirname "$(dirname "$PY")")")"
+
+        # ------------------------------------------------------------
+        # 1. Skip if a repaired wheel for THIS name+version+interpreter
+        #    already exists (a wheel from an older version will not match
+        #    because VERSION is part of the pattern).
+        # ------------------------------------------------------------
+        if compgen -G "/io/wheels/${PKG}-${VERSION}-${PYTAG}-*manylinux*.whl" > /dev/null; then
+          echo "➤ ${PYTAG}: wheel for ${PKG} ${VERSION} already present — skipping"
+          SKIPPED_TAGS="${SKIPPED_TAGS} ${PYTAG}"
+          continue
+        fi
+
+        # ------------------------------------------------------------
+        # 2. Build; on failure, record and continue with the next python
+        # ------------------------------------------------------------
+        echo "================================================================"
+        echo "➤ ${PYTAG}: building ${PKG} ${VERSION}"
+        echo "================================================================"
+        if ( build_one "$PY" "$PYTAG" ); then
+          BUILT_TAGS="${BUILT_TAGS} ${PYTAG}"
+        else
+          echo "✗ ${PYTAG}: BUILD FAILED — continuing with remaining versions"
+          FAILED_TAGS="${FAILED_TAGS} ${PYTAG}"
+        fi
      done

-      echo "Linux wheels ready in /io/wheels"
+      echo "================================================================"
+      echo "Summary for ${PKG} ${VERSION}:"
+      echo "  built:  ${BUILT_TAGS:- none}"
+      echo "  skipped:${SKIPPED_TAGS:- none}"
+      echo "  failed: ${FAILED_TAGS:- none}"
+      echo "================================================================"
+
+      if [ -n "$FAILED_TAGS" ]; then
+        echo "✗ Some builds failed:${FAILED_TAGS}"
+        exit 1
+      fi
+      echo "✅ Linux wheels ready in /io/wheels"
    '
 done
--- a/utils/wheels/build-wheels-linux_x86_64.sh
+++ b/utils/wheels/build-wheels-linux_x86_64.sh
@@ -45,7 +45,7 @@ do

        BUILD_WHEEL_DIR="$(mktemp -d)"
        CC=clang CXX=clang++ "$PY" -m pip wheel . \
-          --no-deps --config-settings=setup-args=-Dunity=on \
+          --no-deps  \
          -w "$BUILD_WHEEL_DIR" -vv

        CURRENT_WHEEL="$(find "$BUILD_WHEEL_DIR" -name "*.whl" | head -n1)"