diff --git a/utils/build-wheels-linux_aarch64.sh b/utils/build-wheels-linux_aarch64.sh new file mode 100755 index 0000000..ae32d60 --- /dev/null +++ b/utils/build-wheels-linux_aarch64.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 [fourdst-wheels-dir]" + echo " fourdst-wheels-dir: optional local directory of fourdst wheels to" + echo " install from instead of PyPI (for bootstrapping a new stroid wheel)" + exit 1 +fi + +REPO_URL="$1" +LOCAL_FOURDST_WHEELS="${2:-}" +WORK_DIR="$(pwd)" +WHEEL_DIR="${WORK_DIR}/wheels_linux_aarch64" + +echo "➤ Creating wheel output directory at ${WHEEL_DIR}" +mkdir -p "${WHEEL_DIR}" + +TMPDIR="$(mktemp -d)" +echo "➤ Cloning ${REPO_URL} → ${TMPDIR}/project" +git clone "${REPO_URL}" "${TMPDIR}/project" + +DOCKER_MOUNTS=(-v "${WHEEL_DIR}":/io/wheels -v "${TMPDIR}/project":/io/project) +if [[ -n "${LOCAL_FOURDST_WHEELS}" ]]; then + DOCKER_MOUNTS+=(-v "${LOCAL_FOURDST_WHEELS}":/io/fourdst-wheels) +fi + +for IMAGE in \ + tboudreaux/manylinux_2_28_aarch64_boost_1_88_0:latest +do + docker run --rm \ + "${DOCKER_MOUNTS[@]}" \ + "${IMAGE}" \ + /bin/bash -uxo pipefail -c ' + cd /io/project + + PKG="$(sed -n "s/^name *= *\"\(.*\)\"/\1/p" pyproject.toml | head -n1)" + PKG="${PKG//-/_}" # wheel filename normalization + + BOOT_PY=/opt/python/cp312-cp312/bin/python + "$BOOT_PY" -m pip install --quiet meson + VERSION="$("$BOOT_PY" -c " +import json, subprocess, sys +out = subprocess.check_output( + [sys.executable, \"-m\", \"mesonbuild.mesonmain\", \"introspect\", + \"meson.build\", \"--projectinfo\"]) +print(json.loads(out)[\"version\"]) +" 2>/dev/null || true)" + if [ -z "$VERSION" ]; then + VERSION="$(grep -oE "version *: *.[0-9][0-9a-zA-Z.+-]*" meson.build | head -n1 | grep -oE "[0-9][0-9a-zA-Z.+-]*" || true)" + fi + if [ -z "$VERSION" ]; then + echo "ERROR: could not determine project version; refusing to guess for skip logic" + exit 1 + fi + echo "➤ Building ${PKG} ${VERSION}" + + FOURDST_PIN="$(grep -oE "fourdst==[0-9][0-9a-zA-Z.]*" pyproject.toml | head -n1 || true)" + + if [ -d /io/fourdst-wheels ]; then + export PIP_FIND_LINKS=/io/fourdst-wheels + fi + + build_one() { + set -e + local PY="$1" PYTAG="$2" + + "$PY" -m pip install --upgrade pip setuptools wheel meson meson-python + + local BUILD_WHEEL_DIR + BUILD_WHEEL_DIR="$(mktemp -d)" + CC=clang CXX=clang++ "$PY" -m pip wheel . --no-deps \ + -w "$BUILD_WHEEL_DIR" -vv + + local CURRENT_WHEEL + CURRENT_WHEEL="$(find "$BUILD_WHEEL_DIR" -name "*.whl" | head -n1)" + + if [ -n "$FOURDST_PIN" ]; then + "$PY" -m pip install --force-reinstall "$FOURDST_PIN" + local FOURDST_LIB_PATH + FOURDST_LIB_PATH="$("$PY" -c "import fourdst, os; print(os.pathsep.join(fourdst.get_lib_dirs()))")" + LD_LIBRARY_PATH="$FOURDST_LIB_PATH" auditwheel repair \ + --exclude "libcomposition.so*" \ + --exclude "liblogging.so*" \ + --exclude "libconst.so*" \ + --exclude "libreflect_cpp.so*" \ + -w /io/wheels "$CURRENT_WHEEL" + + local REPAIRED + REPAIRED="$(find /io/wheels -name "${PKG}-${VERSION}-${PYTAG}-*manylinux*.whl" | head -n1)" + if [ -z "$REPAIRED" ]; then + echo "ERROR: repaired wheel for ${PYTAG} not found after auditwheel" + return 1 + fi + if unzip -l "$REPAIRED" | grep -E "libcomposition|liblogging|libconst[^a-z]|libreflect_cpp"; then + echo "ERROR: repaired wheel contains vendored fourdst libraries" + rm -f "$REPAIRED" + return 1 + fi + else + auditwheel repair -w /io/wheels "$CURRENT_WHEEL" + fi + + rm -rf "$BUILD_WHEEL_DIR" + } + + FAILED_TAGS="" + SKIPPED_TAGS="" + BUILT_TAGS="" + + for PY in /opt/python/*/bin/python; do + PYTAG="$(basename "$(dirname "$(dirname "$PY")")")" + + if compgen -G "/io/wheels/${PKG}-${VERSION}-${PYTAG}-*manylinux*.whl" > /dev/null; then + echo "➤ ${PYTAG}: wheel for ${PKG} ${VERSION} already present — skipping" + SKIPPED_TAGS="${SKIPPED_TAGS} ${PYTAG}" + continue + fi + + echo "================================================================" + echo "➤ ${PYTAG}: building ${PKG} ${VERSION}" + echo "================================================================" + if ( build_one "$PY" "$PYTAG" ); then + BUILT_TAGS="${BUILT_TAGS} ${PYTAG}" + else + echo "✗ ${PYTAG}: BUILD FAILED — continuing with remaining versions" + FAILED_TAGS="${FAILED_TAGS} ${PYTAG}" + fi + done + + echo "================================================================" + echo "Summary for ${PKG} ${VERSION}:" + echo " built: ${BUILT_TAGS:- none}" + echo " skipped:${SKIPPED_TAGS:- none}" + echo " failed: ${FAILED_TAGS:- none}" + echo "================================================================" + + if [ -n "$FAILED_TAGS" ]; then + echo "✗ Some builds failed:${FAILED_TAGS}" + exit 1 + fi + echo "Linux wheels ready in /io/wheels" + ' +done \ No newline at end of file diff --git a/utils/build-wheels-linux_x86_64.sh b/utils/build-wheels-linux_x86_64.sh new file mode 100755 index 0000000..26fd426 --- /dev/null +++ b/utils/build-wheels-linux_x86_64.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 [fourdst-wheels-dir]" + echo " fourdst-wheels-dir: optional local directory of fourdst wheels to" + echo " install from instead of PyPI (for bootstrapping a new stroid wheel)" + exit 1 +fi + +REPO_URL="$1" +LOCAL_FOURDST_WHEELS="${2:-}" +WORK_DIR="$(pwd)" +WHEEL_DIR="${WORK_DIR}/wheels_linux_x86_64" + +echo "➤ Creating wheel output directory at ${WHEEL_DIR}" +mkdir -p "${WHEEL_DIR}" + +TMPDIR="$(mktemp -d)" +echo "➤ Cloning ${REPO_URL} → ${TMPDIR}/project" +git clone "${REPO_URL}" "${TMPDIR}/project" + +DOCKER_MOUNTS=(-v "${WHEEL_DIR}":/io/wheels -v "${TMPDIR}/project":/io/project) +if [[ -n "${LOCAL_FOURDST_WHEELS}" ]]; then + DOCKER_MOUNTS+=(-v "${LOCAL_FOURDST_WHEELS}":/io/fourdst-wheels) +fi + +for IMAGE in \ + tboudreaux/manylinux_2_28_x86_64_boost_1_88_0:latest +do + docker run --rm \ + "${DOCKER_MOUNTS[@]}" \ + "${IMAGE}" \ + /bin/bash -uxo pipefail -c ' + cd /io/project + + PKG="$(sed -n "s/^name *= *\"\(.*\)\"/\1/p" pyproject.toml | head -n1)" + PKG="${PKG//-/_}" # wheel filename normalization + + BOOT_PY=/opt/python/cp312-cp312/bin/python + "$BOOT_PY" -m pip install --quiet meson + VERSION="$("$BOOT_PY" -c " +import json, subprocess, sys +out = subprocess.check_output( + [sys.executable, \"-m\", \"mesonbuild.mesonmain\", \"introspect\", + \"meson.build\", \"--projectinfo\"]) +print(json.loads(out)[\"version\"]) +" 2>/dev/null || true)" + if [ -z "$VERSION" ]; then + # fallback: literal version in project() + VERSION="$(grep -oE "version *: *.[0-9][0-9a-zA-Z.+-]*" meson.build | head -n1 | grep -oE "[0-9][0-9a-zA-Z.+-]*" || true)" + fi + if [ -z "$VERSION" ]; then + echo "ERROR: could not determine project version; refusing to guess for skip logic" + exit 1 + fi + echo "➤ Building ${PKG} ${VERSION}" + + FOURDST_PIN="$(grep -oE "fourdst==[0-9][0-9a-zA-Z.]*" pyproject.toml | head -n1 || true)" + + if [ -d /io/fourdst-wheels ]; then + export PIP_FIND_LINKS=/io/fourdst-wheels + fi + + build_one() { + set -e + local PY="$1" PYTAG="$2" + + "$PY" -m pip install --upgrade pip setuptools wheel meson meson-python + + # Build into a per-iteration temp dir so we repair exactly the + # wheel we just built. + local BUILD_WHEEL_DIR + BUILD_WHEEL_DIR="$(mktemp -d)" + CC=clang CXX=clang++ "$PY" -m pip wheel . --no-deps \ + -w "$BUILD_WHEEL_DIR" -vv + + local CURRENT_WHEEL + CURRENT_WHEEL="$(find "$BUILD_WHEEL_DIR" -name "*.whl" | head -n1)" + + if [ -n "$FOURDST_PIN" ]; then + "$PY" -m pip install --force-reinstall "$FOURDST_PIN" + local FOURDST_LIB_PATH + FOURDST_LIB_PATH="$("$PY" -c "import fourdst, os; print(os.pathsep.join(fourdst.get_lib_dirs()))")" + LD_LIBRARY_PATH="$FOURDST_LIB_PATH" auditwheel repair \ + --exclude "libcomposition.so*" \ + --exclude "liblogging.so*" \ + --exclude "libconst.so*" \ + --exclude "libreflect_cpp.so*" \ + -w /io/wheels "$CURRENT_WHEEL" + + # Post-repair sanity check on the wheel we just produced + local REPAIRED + REPAIRED="$(find /io/wheels -name "${PKG}-${VERSION}-${PYTAG}-*manylinux*.whl" | head -n1)" + if [ -z "$REPAIRED" ]; then + echo "ERROR: repaired wheel for ${PYTAG} not found after auditwheel" + return 1 + fi + if unzip -l "$REPAIRED" | grep -E "libcomposition|liblogging|libconst[^a-z]|libreflect_cpp"; then + echo "ERROR: repaired wheel contains vendored fourdst libraries" + rm -f "$REPAIRED" + return 1 + fi + else + auditwheel repair -w /io/wheels "$CURRENT_WHEEL" + fi + + rm -rf "$BUILD_WHEEL_DIR" + } + + FAILED_TAGS="" + SKIPPED_TAGS="" + BUILT_TAGS="" + + for PY in /opt/python/*/bin/python; do + PYTAG="$(basename "$(dirname "$(dirname "$PY")")")" + + if compgen -G "/io/wheels/${PKG}-${VERSION}-${PYTAG}-*manylinux*.whl" > /dev/null; then + echo "➤ ${PYTAG}: wheel for ${PKG} ${VERSION} already present — skipping" + SKIPPED_TAGS="${SKIPPED_TAGS} ${PYTAG}" + continue + fi + + echo "================================================================" + echo "➤ ${PYTAG}: building ${PKG} ${VERSION}" + echo "================================================================" + if ( build_one "$PY" "$PYTAG" ); then + BUILT_TAGS="${BUILT_TAGS} ${PYTAG}" + else + echo "✗ ${PYTAG}: BUILD FAILED — continuing with remaining versions" + FAILED_TAGS="${FAILED_TAGS} ${PYTAG}" + fi + done + + echo "================================================================" + echo "Summary for ${PKG} ${VERSION}:" + echo " built: ${BUILT_TAGS:- none}" + echo " skipped:${SKIPPED_TAGS:- none}" + echo " failed: ${FAILED_TAGS:- none}" + echo "================================================================" + + if [ -n "$FAILED_TAGS" ]; then + echo "✗ Some builds failed:${FAILED_TAGS}" + exit 1 + fi + echo "Linux wheels ready in /io/wheels" + ' +done \ No newline at end of file diff --git a/utils/build-wheels-macos_aarch64.sh b/utils/build-wheels-macos_aarch64.sh new file mode 100755 index 0000000..1d8b8d9 --- /dev/null +++ b/utils/build-wheels-macos_aarch64.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $(uname -m) != "arm64" ]]; then + echo "Error: This script is intended to run on an Apple Silicon (arm64) Mac." + exit 1 +fi + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 [fourdst-wheels-dir]" + echo " fourdst-wheels-dir: optional local directory of fourdst wheels to" + echo " install from instead of PyPI (for bootstrapping a new stroid wheel)" + exit 1 +fi + +for TOOL in pyenv cmake git; do + if ! command -v "$TOOL" &> /dev/null; then + echo "Error: ${TOOL} not found." + exit 1 + fi +done + +REPO_URL="$1" +LOCAL_FOURDST_WHEELS="${2:-}" +WORK_DIR="$(pwd)" +WHEEL_DIR="${WORK_DIR}/wheels_macos_aarch64_tmp" +FINAL_WHEEL_DIR="${WORK_DIR}/wheels_macos_aarch64" + +echo "➤ Creating wheel output directories" +mkdir -p "${WHEEL_DIR}" +mkdir -p "${FINAL_WHEEL_DIR}" + +export MACOSX_DEPLOYMENT_TARGET=15.0 + +LLVM_VER="17.0.6" +LIBOMP_PREFIX="${WORK_DIR}/libomp-${MACOSX_DEPLOYMENT_TARGET}-${LLVM_VER}" + +if [[ ! -f "${LIBOMP_PREFIX}/lib/libomp.dylib" ]]; then + echo "➤ Building libomp ${LLVM_VER} for macOS ${MACOSX_DEPLOYMENT_TARGET} → ${LIBOMP_PREFIX}" + LIBOMP_SRC_DIR="$(mktemp -d)" + pushd "${LIBOMP_SRC_DIR}" > /dev/null + + curl -fLO "https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VER}/openmp-${LLVM_VER}.src.tar.xz" + curl -fLO "https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VER}/cmake-${LLVM_VER}.src.tar.xz" + tar xf "openmp-${LLVM_VER}.src.tar.xz" + tar xf "cmake-${LLVM_VER}.src.tar.xz" + mv "cmake-${LLVM_VER}.src" cmake + + cmake -S "openmp-${LLVM_VER}.src" -B libomp-build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET}" \ + -DCMAKE_OSX_ARCHITECTURES=arm64 \ + -DCMAKE_INSTALL_PREFIX="${LIBOMP_PREFIX}" \ + -DLIBOMP_INSTALL_ALIASES=OFF + cmake --build libomp-build -j "$(sysctl -n hw.ncpu)" + cmake --install libomp-build + + popd > /dev/null + rm -rf "${LIBOMP_SRC_DIR}" +fi + +LIBOMP_MINOS="$(otool -l "${LIBOMP_PREFIX}/lib/libomp.dylib" | awk '/minos/ {print $2; exit}')" +if [[ "${LIBOMP_MINOS}" != "${MACOSX_DEPLOYMENT_TARGET}"* ]]; then + echo "Error: cached libomp at ${LIBOMP_PREFIX} targets macOS ${LIBOMP_MINOS}," \ + "expected ${MACOSX_DEPLOYMENT_TARGET}. Delete the directory and re-run." + exit 1 +fi +echo "➤ Using libomp ${LLVM_VER} (min macOS ${LIBOMP_MINOS}) from ${LIBOMP_PREFIX}" + +export CPPFLAGS="-I${LIBOMP_PREFIX}/include ${CPPFLAGS:-}" +export LDFLAGS="-L${LIBOMP_PREFIX}/lib ${LDFLAGS:-}" +export LIBRARY_PATH="${LIBOMP_PREFIX}/lib${LIBRARY_PATH:+:${LIBRARY_PATH}}" + +TMPDIR="$(mktemp -d)" +echo "➤ Cloning ${REPO_URL} → ${TMPDIR}/project" +git clone --depth 1 "${REPO_URL}" "${TMPDIR}/project" +cd "${TMPDIR}/project" + +FOURDST_PIN="$(grep -oE 'fourdst==[0-9][0-9a-zA-Z.]*' pyproject.toml | head -n1 || true)" +if [[ -n "${FOURDST_PIN}" ]]; then + echo "➤ Project depends on ${FOURDST_PIN}; wheel repair will exclude fourdst libraries" +fi + +PYTHON_VERSIONS=("3.9.23" "3.10.18" "3.11.13" "3.12.11" "3.13.5" "3.14.0rc1" "3.14.0rc1t") + +eval "$(pyenv init -)" + +for PY_VERSION in "${PYTHON_VERSIONS[@]}"; do + ( + set -e + + pyenv shell "${PY_VERSION}" + PY="$(pyenv which python)" + + echo "----------------------------------------------------------------" + echo "➤ Building for $($PY --version) on macOS arm64" + echo "----------------------------------------------------------------" + + "$PY" -m pip install --upgrade pip setuptools wheel meson-python delocate + "$PY" -m pip install meson==1.9.1 + + if [[ -n "${FOURDST_PIN}" ]]; then + if [[ -n "${LOCAL_FOURDST_WHEELS}" ]]; then + "$PY" -m pip install --force-reinstall \ + --find-links "${LOCAL_FOURDST_WHEELS}" "${FOURDST_PIN}" + else + "$PY" -m pip install --force-reinstall "${FOURDST_PIN}" + fi + fi + + echo "➤ Building wheel with ccache enabled" + echo "➤ Found meson version $(meson --version)" + + CC="ccache clang" CXX="ccache clang++" \ + "$PY" -m pip wheel . --no-deps --no-build-isolation \ + --config-settings=setup-args="-Dlibomp_prefix=${LIBOMP_PREFIX}" \ + -w "${WHEEL_DIR}" -v + CURRENT_WHEEL=$(find "${WHEEL_DIR}" -name "*.whl" | head -n 1) + + echo "➤ Repairing wheel with delocate" + DELOCATE_DYLD_PATH="${LIBOMP_PREFIX}/lib" + if [[ -n "${FOURDST_PIN}" ]]; then + FOURDST_LIB_PATH="$("$PY" -c 'import fourdst, os; print(os.pathsep.join(fourdst.get_lib_dirs()))')" + DELOCATE_DYLD_PATH="${FOURDST_LIB_PATH}:${DELOCATE_DYLD_PATH}" + DYLD_LIBRARY_PATH="${DELOCATE_DYLD_PATH}" \ + delocate-wheel --require-archs arm64 \ + -e composition -e logging -e const -e reflect_cpp \ + -w "${FINAL_WHEEL_DIR}" -v "$CURRENT_WHEEL" + else + DYLD_LIBRARY_PATH="${DELOCATE_DYLD_PATH}" \ + delocate-wheel --require-archs arm64 \ + -w "${FINAL_WHEEL_DIR}" -v "$CURRENT_WHEEL" + fi + + REPAIRED_WHEEL="${FINAL_WHEEL_DIR}/$(basename "$CURRENT_WHEEL")" + + if [[ -n "${FOURDST_PIN}" ]]; then + if unzip -l "${REPAIRED_WHEEL}" | grep -E 'libcomposition|liblogging|libconst|libreflect_cpp'; then + echo "ERROR: repaired wheel contains vendored fourdst libraries" + exit 1 + fi + fi + if unzip -l "${REPAIRED_WHEEL}" | grep -q 'libomp.dylib'; then + CHECK_DIR="$(mktemp -d)" + unzip -q "${REPAIRED_WHEEL}" '*/libomp.dylib' -d "${CHECK_DIR}" || true + BUNDLED_OMP="$(find "${CHECK_DIR}" -name 'libomp.dylib' | head -n1)" + BUNDLED_MINOS="$(otool -l "${BUNDLED_OMP}" | awk '/minos/ {print $2; exit}')" + rm -rf "${CHECK_DIR}" + if [[ "${BUNDLED_MINOS}" != "${MACOSX_DEPLOYMENT_TARGET}"* ]]; then + echo "ERROR: bundled libomp targets macOS ${BUNDLED_MINOS}, expected ${MACOSX_DEPLOYMENT_TARGET}" + exit 1 + fi + echo "➤ Bundled libomp targets macOS ${BUNDLED_MINOS} ✓" + fi + + rm "$CURRENT_WHEEL" + ) +done + +# Cleanup +rm -rf "${TMPDIR}" +rm -rf "${WHEEL_DIR}" + +echo "✅ All builds complete. Artifacts in ${FINAL_WHEEL_DIR}" \ No newline at end of file diff --git a/utils/installPyEnvVersions.sh b/utils/installPyEnvVersions.sh new file mode 100755 index 0000000..bdbe5ae --- /dev/null +++ b/utils/installPyEnvVersions.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +pyenv install 3.8.20 +pyenv install 3.9.23 +pyenv install 3.10.18 +pyenv install 3.11.13 +pyenv install 3.12.11 +pyenv install 3.13.5 +pyenv install 3.13.5t +pyenv install 3.14.0rc1 +pyenv install 3.14.0rc1t +pyenv install pypy3.10-7.3.19 +pyenv install pypy3.11-7.3.20 + diff --git a/utils/readme.md b/utils/readme.md new file mode 100644 index 0000000..b0c3d11 --- /dev/null +++ b/utils/readme.md @@ -0,0 +1,28 @@ +# Wheel Generation +This directory contains scripts to generate precompiled python wheels for GridFire + +# Notes +- MacOS wheels can only be generated on macos +- aarch64 wheels can only be generated on aarch64 machines +- x86_64 wheels can only be generated on x86_64 machines +- linux wheels can be generated on any linux machine, but the target architecture must match the machine architecture +- Running each script will take **a very long time** (could be upwards of half of a day depending on your system) and will require roughly 2GB of disk space +- When generating MacOS wheels, you must have all the correct versions of python installed with `pyenv`. Run the script `utils/wheels/installPyEnvVersions.sh` to install the correct versions of python. + +# Usage +Once you know you are on the correct machine, run the script for your desired architecture and operating system. For example, to generate a macos x86_64 wheel, run: + +```bash +./build-wheels-macos-aarch64.sh https://github.com/4D-STAR/GridFire +``` + +Once you have all the wheels generated (which will likely require multiple systems), copy all the wheels into a single +directory (lets assume its called `wheels` and in the root of the directory) and then run (from the root of the repository): + +```bash +python -m pip install --upgrade build +python -m build --sdist --outdir wheels +twine upload wheels/* +``` + +Thie will also take a while (it needs to upload all the wheels to PyPI) but will result in all the wheels being uploaded to PyPI. \ No newline at end of file