tboudreaux/stroid
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

build(wheels): brought wheel build scripts from GridFire over

Browse Source
This commit is contained in:
Emily Boudreaux
2026-07-01 11:16:39 -04:00
parent 7f69f19273
commit edfcea6943
5 changed files with 498 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
utils/build-wheels-linux_aarch64.sh Executable file
View File

@@ -0,0 +1,144 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -lt 1 ]]; then
echo "Usage: $0 <git-repo-url> [fourdst-wheels-dir]"
echo " fourdst-wheels-dir: optional local directory of fourdst wheels to"
echo " install from instead of PyPI (for bootstrapping a new stroid wheel)"
exit 1
fi
REPO_URL="$1"
LOCAL_FOURDST_WHEELS="${2:-}"
WORK_DIR="$(pwd)"
WHEEL_DIR="${WORK_DIR}/wheels_linux_aarch64"
echo "➤ Creating wheel output directory at ${WHEEL_DIR}"
mkdir -p "${WHEEL_DIR}"
TMPDIR="$(mktemp -d)"
echo "➤ Cloning ${REPO_URL} → ${TMPDIR}/project"
git clone "${REPO_URL}" "${TMPDIR}/project"
DOCKER_MOUNTS=(-v "${WHEEL_DIR}":/io/wheels -v "${TMPDIR}/project":/io/project)
if [[ -n "${LOCAL_FOURDST_WHEELS}" ]]; then
DOCKER_MOUNTS+=(-v "${LOCAL_FOURDST_WHEELS}":/io/fourdst-wheels)
fi
for IMAGE in \
tboudreaux/manylinux_2_28_aarch64_boost_1_88_0:latest
do
docker run --rm \
"${DOCKER_MOUNTS[@]}" \
"${IMAGE}" \
/bin/bash -uxo pipefail -c '
cd /io/project
PKG="$(sed -n "s/^name *= *\"\(.*\)\"/\1/p" pyproject.toml | head -n1)"
PKG="${PKG//-/_}" # wheel filename normalization
BOOT_PY=/opt/python/cp312-cp312/bin/python
"$BOOT_PY" -m pip install --quiet meson
VERSION="$("$BOOT_PY" -c "
import json, subprocess, sys
out = subprocess.check_output(
[sys.executable, \"-m\", \"mesonbuild.mesonmain\", \"introspect\",
\"meson.build\", \"--projectinfo\"])
print(json.loads(out)[\"version\"])
" 2>/dev/null || true)"
if [ -z "$VERSION" ]; then
VERSION="$(grep -oE "version *: *.[0-9][0-9a-zA-Z.+-]*" meson.build | head -n1 | grep -oE "[0-9][0-9a-zA-Z.+-]*" || true)"
fi
if [ -z "$VERSION" ]; then
echo "ERROR: could not determine project version; refusing to guess for skip logic"
exit 1
fi
echo "➤ Building ${PKG} ${VERSION}"
FOURDST_PIN="$(grep -oE "fourdst==[0-9][0-9a-zA-Z.]*" pyproject.toml | head -n1 || true)"
if [ -d /io/fourdst-wheels ]; then
export PIP_FIND_LINKS=/io/fourdst-wheels
fi
build_one() {
set -e
local PY="$1" PYTAG="$2"
"$PY" -m pip install --upgrade pip setuptools wheel meson meson-python
local BUILD_WHEEL_DIR
BUILD_WHEEL_DIR="$(mktemp -d)"
CC=clang CXX=clang++ "$PY" -m pip wheel . --no-deps \
-w "$BUILD_WHEEL_DIR" -vv
local CURRENT_WHEEL
CURRENT_WHEEL="$(find "$BUILD_WHEEL_DIR" -name "*.whl" | head -n1)"
if [ -n "$FOURDST_PIN" ]; then
"$PY" -m pip install --force-reinstall "$FOURDST_PIN"
local FOURDST_LIB_PATH
FOURDST_LIB_PATH="$("$PY" -c "import fourdst, os; print(os.pathsep.join(fourdst.get_lib_dirs()))")"
LD_LIBRARY_PATH="$FOURDST_LIB_PATH" auditwheel repair \
--exclude "libcomposition.so*" \
--exclude "liblogging.so*" \
--exclude "libconst.so*" \
--exclude "libreflect_cpp.so*" \
-w /io/wheels "$CURRENT_WHEEL"
local REPAIRED
REPAIRED="$(find /io/wheels -name "${PKG}-${VERSION}-${PYTAG}-*manylinux*.whl" | head -n1)"
if [ -z "$REPAIRED" ]; then
echo "ERROR: repaired wheel for ${PYTAG} not found after auditwheel"
return 1
fi
if unzip -l "$REPAIRED" | grep -E "libcomposition|liblogging|libconst[^a-z]|libreflect_cpp"; then
echo "ERROR: repaired wheel contains vendored fourdst libraries"
rm -f "$REPAIRED"
return 1
fi
else
auditwheel repair -w /io/wheels "$CURRENT_WHEEL"
fi
rm -rf "$BUILD_WHEEL_DIR"
}
FAILED_TAGS=""
SKIPPED_TAGS=""
BUILT_TAGS=""
for PY in /opt/python/*/bin/python; do
PYTAG="$(basename "$(dirname "$(dirname "$PY")")")"
if compgen -G "/io/wheels/${PKG}-${VERSION}-${PYTAG}-*manylinux*.whl" > /dev/null; then
echo "➤ ${PYTAG}: wheel for ${PKG} ${VERSION} already present — skipping"
SKIPPED_TAGS="${SKIPPED_TAGS} ${PYTAG}"
continue
fi
echo "================================================================"
echo "➤ ${PYTAG}: building ${PKG} ${VERSION}"
echo "================================================================"
if ( build_one "$PY" "$PYTAG" ); then
BUILT_TAGS="${BUILT_TAGS} ${PYTAG}"
else
echo "✗ ${PYTAG}: BUILD FAILED — continuing with remaining versions"
FAILED_TAGS="${FAILED_TAGS} ${PYTAG}"
fi
done
echo "================================================================"
echo "Summary for ${PKG} ${VERSION}:"
echo " built: ${BUILT_TAGS:- none}"
echo " skipped:${SKIPPED_TAGS:- none}"
echo " failed: ${FAILED_TAGS:- none}"
echo "================================================================"
if [ -n "$FAILED_TAGS" ]; then
echo "✗ Some builds failed:${FAILED_TAGS}"
exit 1
fi
echo "Linux wheels ready in /io/wheels"
'
done

148
utils/build-wheels-linux_x86_64.sh Executable file
View File

@@ -0,0 +1,148 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -lt 1 ]]; then
echo "Usage: $0 <git-repo-url> [fourdst-wheels-dir]"
echo " fourdst-wheels-dir: optional local directory of fourdst wheels to"
echo " install from instead of PyPI (for bootstrapping a new stroid wheel)"
exit 1
fi
REPO_URL="$1"
LOCAL_FOURDST_WHEELS="${2:-}"
WORK_DIR="$(pwd)"
WHEEL_DIR="${WORK_DIR}/wheels_linux_x86_64"
echo "➤ Creating wheel output directory at ${WHEEL_DIR}"
mkdir -p "${WHEEL_DIR}"
TMPDIR="$(mktemp -d)"
echo "➤ Cloning ${REPO_URL} → ${TMPDIR}/project"
git clone "${REPO_URL}" "${TMPDIR}/project"
DOCKER_MOUNTS=(-v "${WHEEL_DIR}":/io/wheels -v "${TMPDIR}/project":/io/project)
if [[ -n "${LOCAL_FOURDST_WHEELS}" ]]; then
DOCKER_MOUNTS+=(-v "${LOCAL_FOURDST_WHEELS}":/io/fourdst-wheels)
fi
for IMAGE in \
tboudreaux/manylinux_2_28_x86_64_boost_1_88_0:latest
do
docker run --rm \
"${DOCKER_MOUNTS[@]}" \
"${IMAGE}" \
/bin/bash -uxo pipefail -c '
cd /io/project
PKG="$(sed -n "s/^name *= *\"\(.*\)\"/\1/p" pyproject.toml | head -n1)"
PKG="${PKG//-/_}" # wheel filename normalization
BOOT_PY=/opt/python/cp312-cp312/bin/python
"$BOOT_PY" -m pip install --quiet meson
VERSION="$("$BOOT_PY" -c "
import json, subprocess, sys
out = subprocess.check_output(
[sys.executable, \"-m\", \"mesonbuild.mesonmain\", \"introspect\",
\"meson.build\", \"--projectinfo\"])
print(json.loads(out)[\"version\"])
" 2>/dev/null || true)"
if [ -z "$VERSION" ]; then
# fallback: literal version in project()
VERSION="$(grep -oE "version *: *.[0-9][0-9a-zA-Z.+-]*" meson.build | head -n1 | grep -oE "[0-9][0-9a-zA-Z.+-]*" || true)"
fi
if [ -z "$VERSION" ]; then
echo "ERROR: could not determine project version; refusing to guess for skip logic"
exit 1
fi
echo "➤ Building ${PKG} ${VERSION}"
FOURDST_PIN="$(grep -oE "fourdst==[0-9][0-9a-zA-Z.]*" pyproject.toml | head -n1 || true)"
if [ -d /io/fourdst-wheels ]; then
export PIP_FIND_LINKS=/io/fourdst-wheels
fi
build_one() {
set -e
local PY="$1" PYTAG="$2"
"$PY" -m pip install --upgrade pip setuptools wheel meson meson-python
# Build into a per-iteration temp dir so we repair exactly the
# wheel we just built.
local BUILD_WHEEL_DIR
BUILD_WHEEL_DIR="$(mktemp -d)"
CC=clang CXX=clang++ "$PY" -m pip wheel . --no-deps \
-w "$BUILD_WHEEL_DIR" -vv
local CURRENT_WHEEL
CURRENT_WHEEL="$(find "$BUILD_WHEEL_DIR" -name "*.whl" | head -n1)"
if [ -n "$FOURDST_PIN" ]; then
"$PY" -m pip install --force-reinstall "$FOURDST_PIN"
local FOURDST_LIB_PATH
FOURDST_LIB_PATH="$("$PY" -c "import fourdst, os; print(os.pathsep.join(fourdst.get_lib_dirs()))")"
LD_LIBRARY_PATH="$FOURDST_LIB_PATH" auditwheel repair \
--exclude "libcomposition.so*" \
--exclude "liblogging.so*" \
--exclude "libconst.so*" \
--exclude "libreflect_cpp.so*" \
-w /io/wheels "$CURRENT_WHEEL"
# Post-repair sanity check on the wheel we just produced
local REPAIRED
REPAIRED="$(find /io/wheels -name "${PKG}-${VERSION}-${PYTAG}-*manylinux*.whl" | head -n1)"
if [ -z "$REPAIRED" ]; then
echo "ERROR: repaired wheel for ${PYTAG} not found after auditwheel"
return 1
fi
if unzip -l "$REPAIRED" | grep -E "libcomposition|liblogging|libconst[^a-z]|libreflect_cpp"; then
echo "ERROR: repaired wheel contains vendored fourdst libraries"
rm -f "$REPAIRED"
return 1
fi
else
auditwheel repair -w /io/wheels "$CURRENT_WHEEL"
fi
rm -rf "$BUILD_WHEEL_DIR"
}
FAILED_TAGS=""
SKIPPED_TAGS=""
BUILT_TAGS=""
for PY in /opt/python/*/bin/python; do
PYTAG="$(basename "$(dirname "$(dirname "$PY")")")"
if compgen -G "/io/wheels/${PKG}-${VERSION}-${PYTAG}-*manylinux*.whl" > /dev/null; then
echo "➤ ${PYTAG}: wheel for ${PKG} ${VERSION} already present — skipping"
SKIPPED_TAGS="${SKIPPED_TAGS} ${PYTAG}"
continue
fi
echo "================================================================"
echo "➤ ${PYTAG}: building ${PKG} ${VERSION}"
echo "================================================================"
if ( build_one "$PY" "$PYTAG" ); then
BUILT_TAGS="${BUILT_TAGS} ${PYTAG}"
else
echo "✗ ${PYTAG}: BUILD FAILED — continuing with remaining versions"
FAILED_TAGS="${FAILED_TAGS} ${PYTAG}"
fi
done
echo "================================================================"
echo "Summary for ${PKG} ${VERSION}:"
echo " built: ${BUILT_TAGS:- none}"
echo " skipped:${SKIPPED_TAGS:- none}"
echo " failed: ${FAILED_TAGS:- none}"
echo "================================================================"
if [ -n "$FAILED_TAGS" ]; then
echo "✗ Some builds failed:${FAILED_TAGS}"
exit 1
fi
echo "Linux wheels ready in /io/wheels"
'
done

164
utils/build-wheels-macos_aarch64.sh Executable file
View File

@@ -0,0 +1,164 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $(uname -m) != "arm64" ]]; then
echo "Error: This script is intended to run on an Apple Silicon (arm64) Mac."
exit 1
fi
if [[ $# -lt 1 ]]; then
echo "Usage: $0 <git-repo-url> [fourdst-wheels-dir]"
echo " fourdst-wheels-dir: optional local directory of fourdst wheels to"
echo " install from instead of PyPI (for bootstrapping a new stroid wheel)"
exit 1
fi
for TOOL in pyenv cmake git; do
if ! command -v "$TOOL" &> /dev/null; then
echo "Error: ${TOOL} not found."
exit 1
fi
done
REPO_URL="$1"
LOCAL_FOURDST_WHEELS="${2:-}"
WORK_DIR="$(pwd)"
WHEEL_DIR="${WORK_DIR}/wheels_macos_aarch64_tmp"
FINAL_WHEEL_DIR="${WORK_DIR}/wheels_macos_aarch64"
echo "➤ Creating wheel output directories"
mkdir -p "${WHEEL_DIR}"
mkdir -p "${FINAL_WHEEL_DIR}"
export MACOSX_DEPLOYMENT_TARGET=15.0
LLVM_VER="17.0.6"
LIBOMP_PREFIX="${WORK_DIR}/libomp-${MACOSX_DEPLOYMENT_TARGET}-${LLVM_VER}"
if [[ ! -f "${LIBOMP_PREFIX}/lib/libomp.dylib" ]]; then
echo "➤ Building libomp ${LLVM_VER} for macOS ${MACOSX_DEPLOYMENT_TARGET} → ${LIBOMP_PREFIX}"
LIBOMP_SRC_DIR="$(mktemp -d)"
pushd "${LIBOMP_SRC_DIR}" > /dev/null
curl -fLO "https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VER}/openmp-${LLVM_VER}.src.tar.xz"
curl -fLO "https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VER}/cmake-${LLVM_VER}.src.tar.xz"
tar xf "openmp-${LLVM_VER}.src.tar.xz"
tar xf "cmake-${LLVM_VER}.src.tar.xz"
mv "cmake-${LLVM_VER}.src" cmake
cmake -S "openmp-${LLVM_VER}.src" -B libomp-build \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_OSX_DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET}" \
-DCMAKE_OSX_ARCHITECTURES=arm64 \
-DCMAKE_INSTALL_PREFIX="${LIBOMP_PREFIX}" \
-DLIBOMP_INSTALL_ALIASES=OFF
cmake --build libomp-build -j "$(sysctl -n hw.ncpu)"
cmake --install libomp-build
popd > /dev/null
rm -rf "${LIBOMP_SRC_DIR}"
fi
LIBOMP_MINOS="$(otool -l "${LIBOMP_PREFIX}/lib/libomp.dylib" | awk '/minos/ {print $2; exit}')"
if [[ "${LIBOMP_MINOS}" != "${MACOSX_DEPLOYMENT_TARGET}"* ]]; then
echo "Error: cached libomp at ${LIBOMP_PREFIX} targets macOS ${LIBOMP_MINOS}," \
"expected ${MACOSX_DEPLOYMENT_TARGET}. Delete the directory and re-run."
exit 1
fi
echo "➤ Using libomp ${LLVM_VER} (min macOS ${LIBOMP_MINOS}) from ${LIBOMP_PREFIX}"
export CPPFLAGS="-I${LIBOMP_PREFIX}/include ${CPPFLAGS:-}"
export LDFLAGS="-L${LIBOMP_PREFIX}/lib ${LDFLAGS:-}"
export LIBRARY_PATH="${LIBOMP_PREFIX}/lib${LIBRARY_PATH:+:${LIBRARY_PATH}}"
TMPDIR="$(mktemp -d)"
echo "➤ Cloning ${REPO_URL} → ${TMPDIR}/project"
git clone --depth 1 "${REPO_URL}" "${TMPDIR}/project"
cd "${TMPDIR}/project"
FOURDST_PIN="$(grep -oE 'fourdst==[0-9][0-9a-zA-Z.]*' pyproject.toml | head -n1 || true)"
if [[ -n "${FOURDST_PIN}" ]]; then
echo "➤ Project depends on ${FOURDST_PIN}; wheel repair will exclude fourdst libraries"
fi
PYTHON_VERSIONS=("3.9.23" "3.10.18" "3.11.13" "3.12.11" "3.13.5" "3.14.0rc1" "3.14.0rc1t")
eval "$(pyenv init -)"
for PY_VERSION in "${PYTHON_VERSIONS[@]}"; do
(
set -e
pyenv shell "${PY_VERSION}"
PY="$(pyenv which python)"
echo "----------------------------------------------------------------"
echo "➤ Building for $($PY --version) on macOS arm64"
echo "----------------------------------------------------------------"
"$PY" -m pip install --upgrade pip setuptools wheel meson-python delocate
"$PY" -m pip install meson==1.9.1
if [[ -n "${FOURDST_PIN}" ]]; then
if [[ -n "${LOCAL_FOURDST_WHEELS}" ]]; then
"$PY" -m pip install --force-reinstall \
--find-links "${LOCAL_FOURDST_WHEELS}" "${FOURDST_PIN}"
else
"$PY" -m pip install --force-reinstall "${FOURDST_PIN}"
fi
fi
echo "➤ Building wheel with ccache enabled"
echo "➤ Found meson version $(meson --version)"
CC="ccache clang" CXX="ccache clang++" \
"$PY" -m pip wheel . --no-deps --no-build-isolation \
--config-settings=setup-args="-Dlibomp_prefix=${LIBOMP_PREFIX}" \
-w "${WHEEL_DIR}" -v
CURRENT_WHEEL=$(find "${WHEEL_DIR}" -name "*.whl" | head -n 1)
echo "➤ Repairing wheel with delocate"
DELOCATE_DYLD_PATH="${LIBOMP_PREFIX}/lib"
if [[ -n "${FOURDST_PIN}" ]]; then
FOURDST_LIB_PATH="$("$PY" -c 'import fourdst, os; print(os.pathsep.join(fourdst.get_lib_dirs()))')"
DELOCATE_DYLD_PATH="${FOURDST_LIB_PATH}:${DELOCATE_DYLD_PATH}"
DYLD_LIBRARY_PATH="${DELOCATE_DYLD_PATH}" \
delocate-wheel --require-archs arm64 \
-e composition -e logging -e const -e reflect_cpp \
-w "${FINAL_WHEEL_DIR}" -v "$CURRENT_WHEEL"
else
DYLD_LIBRARY_PATH="${DELOCATE_DYLD_PATH}" \
delocate-wheel --require-archs arm64 \
-w "${FINAL_WHEEL_DIR}" -v "$CURRENT_WHEEL"
fi
REPAIRED_WHEEL="${FINAL_WHEEL_DIR}/$(basename "$CURRENT_WHEEL")"
if [[ -n "${FOURDST_PIN}" ]]; then
if unzip -l "${REPAIRED_WHEEL}" | grep -E 'libcomposition|liblogging|libconst|libreflect_cpp'; then
echo "ERROR: repaired wheel contains vendored fourdst libraries"
exit 1
fi
fi
if unzip -l "${REPAIRED_WHEEL}" | grep -q 'libomp.dylib'; then
CHECK_DIR="$(mktemp -d)"
unzip -q "${REPAIRED_WHEEL}" '*/libomp.dylib' -d "${CHECK_DIR}" || true
BUNDLED_OMP="$(find "${CHECK_DIR}" -name 'libomp.dylib' | head -n1)"
BUNDLED_MINOS="$(otool -l "${BUNDLED_OMP}" | awk '/minos/ {print $2; exit}')"
rm -rf "${CHECK_DIR}"
if [[ "${BUNDLED_MINOS}" != "${MACOSX_DEPLOYMENT_TARGET}"* ]]; then
echo "ERROR: bundled libomp targets macOS ${BUNDLED_MINOS}, expected ${MACOSX_DEPLOYMENT_TARGET}"
exit 1
fi
echo "➤ Bundled libomp targets macOS ${BUNDLED_MINOS} ✓"
fi
rm "$CURRENT_WHEEL"
)
done
# Cleanup
rm -rf "${TMPDIR}"
rm -rf "${WHEEL_DIR}"
echo "✅ All builds complete. Artifacts in ${FINAL_WHEEL_DIR}"

14
utils/installPyEnvVersions.sh Executable file
View File

@@ -0,0 +1,14 @@
#!/bin/bash
pyenv install 3.8.20
pyenv install 3.9.23
pyenv install 3.10.18
pyenv install 3.11.13
pyenv install 3.12.11
pyenv install 3.13.5
pyenv install 3.13.5t
pyenv install 3.14.0rc1
pyenv install 3.14.0rc1t
pyenv install pypy3.10-7.3.19
pyenv install pypy3.11-7.3.20

28
utils/readme.md Normal file
View File

@@ -0,0 +1,28 @@
# Wheel Generation
This directory contains scripts to generate precompiled python wheels for GridFire
# Notes
- MacOS wheels can only be generated on macos
- aarch64 wheels can only be generated on aarch64 machines
- x86_64 wheels can only be generated on x86_64 machines
- linux wheels can be generated on any linux machine, but the target architecture must match the machine architecture
- Running each script will take **a very long time** (could be upwards of half of a day depending on your system) and will require roughly 2GB of disk space
- When generating MacOS wheels, you must have all the correct versions of python installed with `pyenv`. Run the script `utils/wheels/installPyEnvVersions.sh` to install the correct versions of python.
# Usage
Once you know you are on the correct machine, run the script for your desired architecture and operating system. For example, to generate a macos x86_64 wheel, run:
```bash
./build-wheels-macos-aarch64.sh https://github.com/4D-STAR/GridFire
```
Once you have all the wheels generated (which will likely require multiple systems), copy all the wheels into a single
directory (lets assume its called `wheels` and in the root of the directory) and then run (from the root of the repository):
```bash
python -m pip install --upgrade build
python -m build --sdist --outdir wheels
twine upload wheels/*
```
Thie will also take a while (it needs to upload all the wheels to PyPI) but will result in all the wheels being uploaded to PyPI.