build(python): gridfire uses fourdst wheel in python mode

2026-06-12 14:30:59 -04:00
parent 3b9a6eba5a
commit 61cd7359d4
16 changed files with 469 additions and 321 deletions
--- a/utils/wheels/build-wheels-linux_aarch64.sh
+++ b/utils/wheels/build-wheels-linux_aarch64.sh
@@ -1,12 +1,15 @@
 #!/usr/bin/env bash
 set -euo pipefail

-if [[ $# -ne 1 ]]; then
-  echo "Usage: $0 <git-repo-url>"
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 <git-repo-url> [fourdst-wheels-dir]"
+  echo "  fourdst-wheels-dir: optional local directory of fourdst wheels to"
+  echo "  install from instead of PyPI (for bootstrapping a new fourdst+gridfire pair)"
  exit 1
 fi

 REPO_URL="$1"
+LOCAL_FOURDST_WHEELS="${2:-}"
 WORK_DIR="$(pwd)"
 WHEEL_DIR="${WORK_DIR}/wheels_linux_aarch64"

@@ -17,21 +20,70 @@ TMPDIR="$(mktemp -d)"
 echo "➤ Cloning ${REPO_URL} → ${TMPDIR}/project"
 git clone "${REPO_URL}" "${TMPDIR}/project"

+DOCKER_MOUNTS=(-v "${WHEEL_DIR}":/io/wheels -v "${TMPDIR}/project":/io/project)
+if [[ -n "${LOCAL_FOURDST_WHEELS}" ]]; then
+  DOCKER_MOUNTS+=(-v "${LOCAL_FOURDST_WHEELS}":/io/fourdst-wheels)
+fi
+
 for IMAGE in \
  tboudreaux/manylinux_2_28_aarch64_boost_1_88_0:latest
 do
  docker run --rm \
-    -v "${WHEEL_DIR}":/io/wheels \
-    -v "${TMPDIR}/project":/io/project \
+    "${DOCKER_MOUNTS[@]}" \
    "${IMAGE}" \
    /bin/bash -eux -c '
      cd /io/project
+
+      # Does this project link against the fourdst wheel? Single source of
+      # truth: the pin in pyproject.toml.
+      FOURDST_PIN="$(grep -oE "fourdst==[0-9][0-9a-zA-Z.]*" pyproject.toml | head -n1 || true)"
+
+      # If a local fourdst wheel dir was mounted, let pip (including the
+      # isolated build env) resolve fourdst from it.
+      if [ -d /io/fourdst-wheels ]; then
+        export PIP_FIND_LINKS=/io/fourdst-wheels
+      fi
+
      for PY in /opt/python/*/bin/python; do
        "$PY" -m pip install --upgrade pip setuptools wheel meson meson-python
-        CC=clang CXX=clang++ "$PY" -m pip wheel . --config-settings=setup-args=-Dunity=on -w /io/wheels -vv
-        auditwheel repair /io/wheels/*.whl -w /io/wheels
+
+        # Build into a per-iteration temp dir so we repair exactly the
+        # wheel we just built (the old glob re-repaired every accumulated
+        # wheel on every loop iteration).
+        BUILD_WHEEL_DIR="$(mktemp -d)"
+        CC=clang CXX=clang++ "$PY" -m pip wheel . \
+          --config-settings=setup-args=-Dunity=on \
+          -w "$BUILD_WHEEL_DIR" -vv
+
+        CURRENT_WHEEL="$(find "$BUILD_WHEEL_DIR" -name "*.whl" | head -n1)"
+
+        if [ -n "$FOURDST_PIN" ]; then
+          # Install fourdst for THIS interpreter so auditwheel can resolve
+          # the libraries it must NOT graft. Excluding them keeps fourdst a
+          # runtime dependency: grafting copies would break cross-package
+          # pybind11 type compatibility.
+          "$PY" -m pip install --force-reinstall "$FOURDST_PIN"
+          FOURDST_LIB_PATH="$("$PY" -c "import fourdst, os; print(os.pathsep.join(fourdst.get_lib_dirs()))")"
+          LD_LIBRARY_PATH="$FOURDST_LIB_PATH" auditwheel repair \
+            --exclude "libcomposition.so*" \
+            --exclude "liblogging.so*" \
+            --exclude "libconst.so*" \
+            --exclude "libreflect_cpp.so*" \
+            -w /io/wheels "$CURRENT_WHEEL"
+
+          # Post-repair sanity check: no vendored fourdst libs.
+          REPAIRED="$(ls -t /io/wheels/*.whl | head -n1)"
+          if unzip -l "$REPAIRED" | grep -E "libcomposition|liblogging|libconst[^a-z]|libreflect_cpp"; then
+            echo "ERROR: repaired wheel contains vendored fourdst libraries"
+            exit 1
+          fi
+        else
+          auditwheel repair -w /io/wheels "$CURRENT_WHEEL"
+        fi
+
+        rm -rf "$BUILD_WHEEL_DIR"
      done

      echo "✅ Linux wheels ready in /io/wheels"
    '
-done
+done
--- a/utils/wheels/build-wheels-linux_x86_64.sh
+++ b/utils/wheels/build-wheels-linux_x86_64.sh
@@ -1,12 +1,15 @@
 #!/usr/bin/env bash
 set -euo pipefail

-if [[ $# -ne 1 ]]; then
-  echo "Usage: $0 <git-repo-url>"
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 <git-repo-url> [fourdst-wheels-dir]"
+  echo "  fourdst-wheels-dir: optional local directory of fourdst wheels to"
+  echo "  install from instead of PyPI (for bootstrapping a new fourdst+gridfire pair)"
  exit 1
 fi

 REPO_URL="$1"
+LOCAL_FOURDST_WHEELS="${2:-}"
 WORK_DIR="$(pwd)"
 WHEEL_DIR="${WORK_DIR}/wheels_linux_x86_64"

@@ -17,19 +20,68 @@ TMPDIR="$(mktemp -d)"
 echo "➤ Cloning ${REPO_URL} → ${TMPDIR}/project"
 git clone "${REPO_URL}" "${TMPDIR}/project"

+DOCKER_MOUNTS=(-v "${WHEEL_DIR}":/io/wheels -v "${TMPDIR}/project":/io/project)
+if [[ -n "${LOCAL_FOURDST_WHEELS}" ]]; then
+  DOCKER_MOUNTS+=(-v "${LOCAL_FOURDST_WHEELS}":/io/fourdst-wheels)
+fi
+
 for IMAGE in \
  tboudreaux/manylinux_2_28_x86_64_boost_1_88_0:latest
 do
  docker run --rm \
-    -v "${WHEEL_DIR}":/io/wheels \
-    -v "${TMPDIR}/project":/io/project \
+    "${DOCKER_MOUNTS[@]}" \
    "${IMAGE}" \
    /bin/bash -eux -c '
      cd /io/project
+
+      # Does this project link against the fourdst wheel? Single source of
+      # truth: the pin in pyproject.toml.
+      FOURDST_PIN="$(grep -oE "fourdst==[0-9][0-9a-zA-Z.]*" pyproject.toml | head -n1 || true)"
+
+      # If a local fourdst wheel dir was mounted, let pip (including the
+      # isolated build env) resolve fourdst from it.
+      if [ -d /io/fourdst-wheels ]; then
+        export PIP_FIND_LINKS=/io/fourdst-wheels
+      fi
+
      for PY in /opt/python/*/bin/python; do
        "$PY" -m pip install --upgrade pip setuptools wheel meson meson-python
-        CC=clang CXX=clang++ "$PY" -m pip wheel . --config-settings=setup-args=-Dunity=on -w /io/wheels -vv
-        auditwheel repair /io/wheels/*.whl -w /io/wheels
+
+        # Build into a per-iteration temp dir so we repair exactly the
+        # wheel we just built (the old glob re-repaired every accumulated
+        # wheel on every loop iteration).
+        BUILD_WHEEL_DIR="$(mktemp -d)"
+        CC=clang CXX=clang++ "$PY" -m pip wheel . \
+          --config-settings=setup-args=-Dunity=on \
+          -w "$BUILD_WHEEL_DIR" -vv
+
+        CURRENT_WHEEL="$(find "$BUILD_WHEEL_DIR" -name "*.whl" | head -n1)"
+
+        if [ -n "$FOURDST_PIN" ]; then
+          # Install fourdst for THIS interpreter so auditwheel can resolve
+          # the libraries it must NOT graft. Excluding them keeps fourdst a
+          # runtime dependency: grafting copies would break cross-package
+          # pybind11 type compatibility.
+          "$PY" -m pip install --force-reinstall "$FOURDST_PIN"
+          FOURDST_LIB_PATH="$("$PY" -c "import fourdst, os; print(os.pathsep.join(fourdst.get_lib_dirs()))")"
+          LD_LIBRARY_PATH="$FOURDST_LIB_PATH" auditwheel repair \
+            --exclude "libcomposition.so*" \
+            --exclude "liblogging.so*" \
+            --exclude "libconst.so*" \
+            --exclude "libreflect_cpp.so*" \
+            -w /io/wheels "$CURRENT_WHEEL"
+
+          # Post-repair sanity check: no vendored fourdst libs.
+          REPAIRED="$(ls -t /io/wheels/*.whl | head -n1)"
+          if unzip -l "$REPAIRED" | grep -E "libcomposition|liblogging|libconst[^a-z]|libreflect_cpp"; then
+            echo "ERROR: repaired wheel contains vendored fourdst libraries"
+            exit 1
+          fi
+        else
+          auditwheel repair -w /io/wheels "$CURRENT_WHEEL"
+        fi
+
+        rm -rf "$BUILD_WHEEL_DIR"
      done

      echo "✅ Linux wheels ready in /io/wheels"
--- a/utils/wheels/build-wheels-macos_aarch64.sh
+++ b/utils/wheels/build-wheels-macos_aarch64.sh
@@ -7,17 +7,19 @@ if [[ $(uname -m) != "arm64" ]]; then
  exit 1
 fi

-if [[ $# -ne 1 ]]; then
-  echo "Usage: $0 <git-repo-url>"
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 <git-repo-url> [fourdst-wheels-dir]"
+  echo "  fourdst-wheels-dir: optional local directory of fourdst wheels to"
+  echo "  install from instead of PyPI (for bootstrapping a new fourdst+gridfire pair)"
  exit 1
 fi

 # 2. Setup Directories
 REPO_URL="$1"
+LOCAL_FOURDST_WHEELS="${2:-}"
 WORK_DIR="$(pwd)"
 WHEEL_DIR="${WORK_DIR}/wheels_macos_aarch64_tmp"
 FINAL_WHEEL_DIR="${WORK_DIR}/wheels_macos_aarch64"
-RPATH_SCRIPT="${WORK_DIR}/../../build-python/fix_rpaths.py" # Assumes script is in this location relative to execution

 echo "➤ Creating wheel output directories"
 mkdir -p "${WHEEL_DIR}"
@@ -29,8 +31,17 @@ git clone --depth 1 "${REPO_URL}" "${TMPDIR}/project"
 cd "${TMPDIR}/project"

 # 3. Build Configuration
+# NOTE: must match the value used to build the fourdst wheels — the two
+# packages are one ABI unit.
 export MACOSX_DEPLOYMENT_TARGET=15.0

+# Does this project link against the fourdst wheel? Derive the pin from
+# pyproject.toml so there is a single source of truth.
+FOURDST_PIN="$(grep -oE 'fourdst==[0-9][0-9a-zA-Z.]*' pyproject.toml | head -n1 || true)"
+if [[ -n "${FOURDST_PIN}" ]]; then
+  echo "➤ Project depends on ${FOURDST_PIN}; wheel repair will exclude fourdst libraries"
+fi
+
 PYTHON_VERSIONS=("3.9.23" "3.10.18" "3.11.13" "3.12.11" "3.13.5" "3.13.5t" "3.14.0rc1" "3.14.0rc1t" 'pypy3.10-7.3.19' "pypy3.11-7.3.20")

 if ! command -v pyenv &> /dev/null; then
@@ -46,27 +57,60 @@ for PY_VERSION in "${PYTHON_VERSIONS[@]}"; do

    pyenv shell "${PY_VERSION}"
    PY="$(pyenv which python)"
-    
+
    echo "----------------------------------------------------------------"
    echo "➤ Building for $($PY --version) on macOS arm64"
    echo "----------------------------------------------------------------"

-    # Install build deps explicitly so we can skip build isolation
+    # Install build deps explicitly so we can skip build isolation.
+    # IMPORTANT: with --no-build-isolation, EVERYTHING in
+    # build-system.requires must be installed here by hand — including
+    # fourdst, otherwise the meson probe (`import fourdst`) fails.
    "$PY" -m pip install --upgrade pip setuptools wheel meson-python delocate
    "$PY" -m pip install meson==1.9.1

+    if [[ -n "${FOURDST_PIN}" ]]; then
+      if [[ -n "${LOCAL_FOURDST_WHEELS}" ]]; then
+        "$PY" -m pip install --force-reinstall \
+          --find-links "${LOCAL_FOURDST_WHEELS}" "${FOURDST_PIN}"
+      else
+        "$PY" -m pip install --force-reinstall "${FOURDST_PIN}"
+      fi
+    fi
+
    echo "➤ Building wheel with ccache enabled"
    echo "➤ Found meson version $(meson --version)"

-    # for every single build, saving significant I/O and network time.
-    CC="ccache clang" CXX="ccache clang++" "$PY" -m pip wheel . --no-build-isolation -w "${WHEEL_DIR}" -v
+    CC="ccache clang" CXX="ccache clang++" \
+      "$PY" -m pip wheel . --no-build-isolation -w "${WHEEL_DIR}" -v

    # We expect exactly one new wheel in the tmp dir per iteration
    CURRENT_WHEEL=$(find "${WHEEL_DIR}" -name "*.whl" | head -n 1)

    echo "➤ Repairing wheel with delocate"
-    # Delocate moves the repaired wheel to FINAL_WHEEL_DIR
-    delocate-wheel -w "${FINAL_WHEEL_DIR}" "$CURRENT_WHEEL"
+    if [[ -n "${FOURDST_PIN}" ]]; then
+      # Resolve @rpath references against the installed fourdst wheel,
+      # but EXCLUDE its libraries from being grafted into this wheel:
+      # they must stay a runtime dependency, or cross-package pybind11
+      # type compatibility breaks.
+      FOURDST_LIB_PATH="$("$PY" -c 'import fourdst, os; print(os.pathsep.join(fourdst.get_lib_dirs()))')"
+      DYLD_LIBRARY_PATH="${FOURDST_LIB_PATH}" \
+        delocate-wheel --require-archs arm64 \
+          -e composition -e logging -e const -e reflect_cpp \
+          -w "${FINAL_WHEEL_DIR}" -v "$CURRENT_WHEEL"
+    else
+      delocate-wheel --require-archs arm64 -w "${FINAL_WHEEL_DIR}" -v "$CURRENT_WHEEL"
+    fi
+
+    # Post-repair sanity check: import the wheel in a throwaway env and
+    # make sure no fourdst library snuck back in.
+    if [[ -n "${FOURDST_PIN}" ]]; then
+      REPAIRED_WHEEL=$(find "${FINAL_WHEEL_DIR}" -name "*.whl" -newer "$CURRENT_WHEEL" | head -n 1)
+      if [[ -n "${REPAIRED_WHEEL}" ]] && unzip -l "${REPAIRED_WHEEL}" | grep -E 'libcomposition|liblogging|libconst|libreflect_cpp' ; then
+        echo "ERROR: repaired wheel contains vendored fourdst libraries"
+        exit 1
+      fi
+    fi

    # Clean up the intermediate wheel from this iteration so it doesn't confuse the next
    rm "$CURRENT_WHEEL"
@@ -77,4 +121,4 @@ done
 rm -rf "${TMPDIR}"
 rm -rf "${WHEEL_DIR}"

-echo "✅ All builds complete. Artifacts in ${FINAL_WHEEL_DIR}"
+echo "✅ All builds complete. Artifacts in ${FINAL_WHEEL_DIR}"