tboudreaux/GridFire
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(wheels): system to continue if wheel build fails

Browse Source
This commit is contained in:
Emily Boudreaux
2026-06-13 10:49:16 -04:00
parent 6bad4415b9
commit 6f85eb6b2c
2 changed files with 79 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
utils/wheels/build-wheels-linux_aarch64.sh
View File

@@ -34,12 +34,6 @@ do
/bin/bash -uxo pipefail -c '
cd /io/project
# ----------------------------------------------------------------
# Project identity: package name from pyproject.toml, version from
# meson (the version pip will stamp into the wheel). Used both for
# the skip-if-already-built check and the post-repair checks, so a
# stale wheel from an OLDER project version never causes a skip.
# ----------------------------------------------------------------
PKG="$(sed -n "s/^name *= *\"\(.*\)\"/\1/p" pyproject.toml | head -n1)"
PKG="${PKG//-/_}" # wheel filename normalization
@@ -53,7 +47,6 @@ out = subprocess.check_output(
print(json.loads(out)[\"version\"])
" 2>/dev/null || true)"
if [ -z "$VERSION" ]; then
# fallback: literal version in project()
VERSION="$(grep -oE "version *: *.[0-9][0-9a-zA-Z.+-]*" meson.build | head -n1 | grep -oE "[0-9][0-9a-zA-Z.+-]*" || true)"
fi
if [ -z "$VERSION" ]; then
@@ -62,27 +55,18 @@ print(json.loads(out)[\"version\"])
fi
echo "➤ Building ${PKG} ${VERSION}"
# Does this project link against the fourdst wheel? Single source of
# truth: the pin in pyproject.toml.
FOURDST_PIN="$(grep -oE "fourdst==[0-9][0-9a-zA-Z.]*" pyproject.toml | head -n1 || true)"
# If a local fourdst wheel dir was mounted, let pip (including the
# isolated build env) resolve fourdst from it.
if [ -d /io/fourdst-wheels ]; then
export PIP_FIND_LINKS=/io/fourdst-wheels
fi
build_one() {
# Runs the full build+repair for one interpreter. Returns nonzero
# on any failure; never exits the whole script (errexit is off in
# the caller around this function).
set -e
local PY="$1" PYTAG="$2"
"$PY" -m pip install --upgrade pip setuptools wheel meson meson-python
# Build into a per-iteration temp dir so we repair exactly the
# wheel we just built.
local BUILD_WHEEL_DIR
BUILD_WHEEL_DIR="$(mktemp -d)"
CC=clang CXX=clang++ "$PY" -m pip wheel . --no-deps \
@@ -92,10 +76,6 @@ print(json.loads(out)[\"version\"])
CURRENT_WHEEL="$(find "$BUILD_WHEEL_DIR" -name "*.whl" | head -n1)"
if [ -n "$FOURDST_PIN" ]; then
# Install fourdst for THIS interpreter so auditwheel can resolve
# the libraries it must NOT graft. Excluding them keeps fourdst a
# runtime dependency: grafting copies would break cross-package
# pybind11 type compatibility.
"$PY" -m pip install --force-reinstall "$FOURDST_PIN"
local FOURDST_LIB_PATH
FOURDST_LIB_PATH="$("$PY" -c "import fourdst, os; print(os.pathsep.join(fourdst.get_lib_dirs()))")"
@@ -106,7 +86,6 @@ print(json.loads(out)[\"version\"])
--exclude "libreflect_cpp.so*" \
-w /io/wheels "$CURRENT_WHEEL"
# Post-repair sanity check on the wheel we just produced
local REPAIRED
REPAIRED="$(find /io/wheels -name "${PKG}-${VERSION}-${PYTAG}-*manylinux*.whl" | head -n1)"
if [ -z "$REPAIRED" ]; then
@@ -115,7 +94,7 @@ print(json.loads(out)[\"version\"])
fi
if unzip -l "$REPAIRED" | grep -E "libcomposition|liblogging|libconst[^a-z]|libreflect_cpp"; then
echo "ERROR: repaired wheel contains vendored fourdst libraries"
rm -f "$REPAIRED" # do not leave a poisoned wheel that would be skipped next run
rm -f "$REPAIRED"
return 1
fi
else
@@ -130,24 +109,14 @@ print(json.loads(out)[\"version\"])
BUILT_TAGS=""
for PY in /opt/python/*/bin/python; do
# /opt/python/<pythontag>-<abitag>/bin/python — the directory name
# is exactly the {python tag}-{abi tag} pair used in wheel filenames
PYTAG="$(basename "$(dirname "$(dirname "$PY")")")"
# ------------------------------------------------------------
# 1. Skip if a repaired wheel for THIS name+version+interpreter
# already exists (a wheel from an older version will not match
# because VERSION is part of the pattern).
# ------------------------------------------------------------
if compgen -G "/io/wheels/${PKG}-${VERSION}-${PYTAG}-*manylinux*.whl" > /dev/null; then
echo "➤ ${PYTAG}: wheel for ${PKG} ${VERSION} already present — skipping"
SKIPPED_TAGS="${SKIPPED_TAGS} ${PYTAG}"
continue
fi
# ------------------------------------------------------------
# 2. Build; on failure, record and continue with the next python
# ------------------------------------------------------------
echo "================================================================"
echo "➤ ${PYTAG}: building ${PKG} ${VERSION}"
echo "================================================================"
@@ -170,6 +139,6 @@ print(json.loads(out)[\"version\"])
echo "✗ Some builds failed:${FAILED_TAGS}"
exit 1
fi
echo "✅ Linux wheels ready in /io/wheels"
echo "Linux wheels ready in /io/wheels"
'
done

83
utils/wheels/build-wheels-linux_x86_64.sh
View File

@@ -31,27 +31,56 @@ do
docker run --rm \
"${DOCKER_MOUNTS[@]}" \
"${IMAGE}" \
/bin/bash -eux -c '
/bin/bash -uxo pipefail -c '
cd /io/project
PKG="$(sed -n "s/^name *= *\"\(.*\)\"/\1/p" pyproject.toml | head -n1)"
PKG="${PKG//-/_}" # wheel filename normalization
BOOT_PY=/opt/python/cp312-cp312/bin/python
"$BOOT_PY" -m pip install --quiet meson
VERSION="$("$BOOT_PY" -c "
import json, subprocess, sys
out = subprocess.check_output(
[sys.executable, \"-m\", \"mesonbuild.mesonmain\", \"introspect\",
\"meson.build\", \"--projectinfo\"])
print(json.loads(out)[\"version\"])
" 2>/dev/null || true)"
if [ -z "$VERSION" ]; then
# fallback: literal version in project()
VERSION="$(grep -oE "version *: *.[0-9][0-9a-zA-Z.+-]*" meson.build | head -n1 | grep -oE "[0-9][0-9a-zA-Z.+-]*" || true)"
fi
if [ -z "$VERSION" ]; then
echo "ERROR: could not determine project version; refusing to guess for skip logic"
exit 1
fi
echo "➤ Building ${PKG} ${VERSION}"
FOURDST_PIN="$(grep -oE "fourdst==[0-9][0-9a-zA-Z.]*" pyproject.toml | head -n1 || true)"
if [ -d /io/fourdst-wheels ]; then
export PIP_FIND_LINKS=/io/fourdst-wheels
fi
for PY in /opt/python/*/bin/python; do
build_one() {
set -e
local PY="$1" PYTAG="$2"
"$PY" -m pip install --upgrade pip setuptools wheel meson meson-python
# Build into a per-iteration temp dir so we repair exactly the
# wheel we just built.
local BUILD_WHEEL_DIR
BUILD_WHEEL_DIR="$(mktemp -d)"
CC=clang CXX=clang++ "$PY" -m pip wheel . \
--no-deps \
CC=clang CXX=clang++ "$PY" -m pip wheel . --no-deps \
-w "$BUILD_WHEEL_DIR" -vv
local CURRENT_WHEEL
CURRENT_WHEEL="$(find "$BUILD_WHEEL_DIR" -name "*.whl" | head -n1)"
if [ -n "$FOURDST_PIN" ]; then
"$PY" -m pip install --force-reinstall "$FOURDST_PIN"
local FOURDST_LIB_PATH
FOURDST_LIB_PATH="$("$PY" -c "import fourdst, os; print(os.pathsep.join(fourdst.get_lib_dirs()))")"
LD_LIBRARY_PATH="$FOURDST_LIB_PATH" auditwheel repair \
--exclude "libcomposition.so*" \
@@ -60,18 +89,60 @@ do
--exclude "libreflect_cpp.so*" \
-w /io/wheels "$CURRENT_WHEEL"
REPAIRED="$(ls -t /io/wheels/*.whl | head -n1)"
# Post-repair sanity check on the wheel we just produced
local REPAIRED
REPAIRED="$(find /io/wheels -name "${PKG}-${VERSION}-${PYTAG}-*manylinux*.whl" | head -n1)"
if [ -z "$REPAIRED" ]; then
echo "ERROR: repaired wheel for ${PYTAG} not found after auditwheel"
return 1
fi
if unzip -l "$REPAIRED" | grep -E "libcomposition|liblogging|libconst[^a-z]|libreflect_cpp"; then
echo "ERROR: repaired wheel contains vendored fourdst libraries"
exit 1
rm -f "$REPAIRED"
return 1
fi
else
auditwheel repair -w /io/wheels "$CURRENT_WHEEL"
fi
rm -rf "$BUILD_WHEEL_DIR"
}
FAILED_TAGS=""
SKIPPED_TAGS=""
BUILT_TAGS=""
for PY in /opt/python/*/bin/python; do
PYTAG="$(basename "$(dirname "$(dirname "$PY")")")"
if compgen -G "/io/wheels/${PKG}-${VERSION}-${PYTAG}-*manylinux*.whl" > /dev/null; then
echo "➤ ${PYTAG}: wheel for ${PKG} ${VERSION} already present — skipping"
SKIPPED_TAGS="${SKIPPED_TAGS} ${PYTAG}"
continue
fi
echo "================================================================"
echo "➤ ${PYTAG}: building ${PKG} ${VERSION}"
echo "================================================================"
if ( build_one "$PY" "$PYTAG" ); then
BUILT_TAGS="${BUILT_TAGS} ${PYTAG}"
else
echo "✗ ${PYTAG}: BUILD FAILED — continuing with remaining versions"
FAILED_TAGS="${FAILED_TAGS} ${PYTAG}"
fi
done
echo "================================================================"
echo "Summary for ${PKG} ${VERSION}:"
echo " built: ${BUILT_TAGS:- none}"
echo " skipped:${SKIPPED_TAGS:- none}"
echo " failed: ${FAILED_TAGS:- none}"
echo "================================================================"
if [ -n "$FAILED_TAGS" ]; then
echo "✗ Some builds failed:${FAILED_TAGS}"
exit 1
fi
echo "Linux wheels ready in /io/wheels"
'
done