diff --git a/.gitignore b/.gitignore index 5cb49e9..c79978e 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,5 @@ output/ scratch/ node_modules/ + +*.whl diff --git a/utils/wheels/build-wheels-linux_aarch64.sh b/utils/wheels/build-wheels-linux_aarch64.sh new file mode 100755 index 0000000..5bdc607 --- /dev/null +++ b/utils/wheels/build-wheels-linux_aarch64.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Must be run on an aarch64 Linux host (uses docker so arm macos is fine so long as as the daemon is running) + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " + echo "Example: $0 https://github.com/4D-STAR/fourdst" + exit 1 +fi + +REPO_URL="$1" +WORK_DIR="$(pwd)" +WHEEL_DIR="${WORK_DIR}/wheels_linux_aarch64" + +echo "➤ Creating wheel output directory at ${WHEEL_DIR}" +mkdir -p "${WHEEL_DIR}" + +TMPDIR="$(mktemp -d)" +echo "➤ Cloning ${REPO_URL} → ${TMPDIR}/project" +git clone --depth 1 "${REPO_URL}" "${TMPDIR}/project" + +IMAGE="tboudreaux/manylinux_2_28_aarch64_boost_1_88_0:latest" + +docker run --rm \ + -v "${WHEEL_DIR}":/io/wheels \ + -v "${TMPDIR}/project":/io/project \ + "${IMAGE}" \ + /bin/bash -eux -c ' + cd /io/project + RAW=/tmp/raw_wheels + + for PY in /opt/python/*/bin/python; do + "$PY" -m pip install --upgrade pip + + rm -rf "$RAW"; mkdir -p "$RAW" + + CC=clang CXX=clang++ "$PY" -m pip wheel . \ + --no-deps \ + -w "$RAW" -vv + + for whl in "$RAW"/*.whl; do + auditwheel repair "$whl" -w /io/wheels + done + done + + echo "Linux aarch64 wheels ready in /io/wheels" + ' + +echo "Done. Repaired wheels in ${WHEEL_DIR}" +rm -rf "${TMPDIR}" diff --git a/utils/wheels/build-wheels-linux_x86_64.sh b/utils/wheels/build-wheels-linux_x86_64.sh new file mode 100755 index 0000000..574cbe3 --- /dev/null +++ b/utils/wheels/build-wheels-linux_x86_64.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " + echo "Example: $0 https://github.com/4D-STAR/fourdst" + exit 1 +fi + +REPO_URL="$1" +WORK_DIR="$(pwd)" +WHEEL_DIR="${WORK_DIR}/wheels_linux_x86_64" + +echo "➤ Creating wheel output directory at ${WHEEL_DIR}" +mkdir -p "${WHEEL_DIR}" + +TMPDIR="$(mktemp -d)" +echo "➤ Cloning ${REPO_URL} → ${TMPDIR}/project" +git clone --depth 1 "${REPO_URL}" "${TMPDIR}/project" + +IMAGE="tboudreaux/manylinux_2_28_x86_64_boost_1_88_0:latest" + +docker run --rm \ + -v "${WHEEL_DIR}":/io/wheels \ + -v "${TMPDIR}/project":/io/project \ + "${IMAGE}" \ + /bin/bash -eux -c ' + cd /io/project + RAW=/tmp/raw_wheels + + for PY in /opt/python/*/bin/python; do + "$PY" -m pip install --upgrade pip + + rm -rf "$RAW"; mkdir -p "$RAW" + + CC=clang CXX=clang++ "$PY" -m pip wheel . \ + --no-deps \ + --config-settings=setup-args=-Dunity=on \ + -w "$RAW" -vv + + # Repair only the freshly built wheel into the shared output dir. + for whl in "$RAW"/*.whl; do + auditwheel repair "$whl" -w /io/wheels + done + done + + echo "Linux x86_64 wheels ready in /io/wheels" + ' + +echo "Done. Repaired wheels in ${WHEEL_DIR}" +rm -rf "${TMPDIR}" diff --git a/utils/wheels/build-wheels-macos_aarch64.sh b/utils/wheels/build-wheels-macos_aarch64.sh new file mode 100755 index 0000000..0ee1087 --- /dev/null +++ b/utils/wheels/build-wheels-macos_aarch64.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Must be run on an Apple Silicon (arm64) Mac. + +if [[ $(uname -m) != "arm64" ]]; then + echo "Error: This script is intended to run on an Apple Silicon (arm64) Mac." + exit 1 +fi + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " + echo "Example: $0 https://github.com/4D-STAR/fourdst" + exit 1 +fi + +REPO_URL="$1" +WORK_DIR="$(pwd)" +WHEEL_DIR="${WORK_DIR}/wheels_macos_aarch64_tmp" +FINAL_WHEEL_DIR="${WORK_DIR}/wheels_macos_aarch64" + +echo "➤ Creating wheel output directories" +mkdir -p "${WHEEL_DIR}" +mkdir -p "${FINAL_WHEEL_DIR}" + +TMPDIR="$(mktemp -d)" +echo "➤ Cloning ${REPO_URL} → ${TMPDIR}/project" +git clone --depth 1 "${REPO_URL}" "${TMPDIR}/project" +cd "${TMPDIR}/project" + +export MACOSX_DEPLOYMENT_TARGET=15.0 + +PYTHON_VERSIONS=("3.9.23" "3.10.18" "3.11.13" "3.12.11" "3.13.5" "3.13.5t" "3.14.0rc1" "3.14.0rc1t" 'pypy3.10-7.3.19' "pypy3.11-7.3.20") + +if ! command -v pyenv &> /dev/null; then + echo "Error: pyenv not found. Please install it to manage Python versions." + echo " Then run installPyEnvVersions.sh to install the interpreters above." + exit 1 +fi +eval "$(pyenv init -)" + +for PY_VERSION in "${PYTHON_VERSIONS[@]}"; do + ( + set -e + + pyenv shell "${PY_VERSION}" + PY="$(pyenv which python)" + + echo "----------------------------------------------------------------" + echo "➤ Building for $($PY --version) on macOS arm64" + echo "----------------------------------------------------------------" + + "$PY" -m pip install --upgrade pip + "$PY" -m pip install "meson>=1.9.1,<1.10" "meson-python>=0.19,<0.20" "pybind11>=2.10" delocate + + echo "➤ Building wheel with ccache enabled" + echo "➤ Found meson version $(meson --version)" + + CC="ccache clang" CXX="ccache clang++" "$PY" -m pip wheel . \ + --no-deps --no-build-isolation -w "${WHEEL_DIR}" -v + + CURRENT_WHEEL=$(find "${WHEEL_DIR}" -name "*.whl" | head -n 1) + + echo "➤ Repairing wheel with delocate" + delocate-wheel -w "${FINAL_WHEEL_DIR}" "$CURRENT_WHEEL" + + rm "$CURRENT_WHEEL" + ) +done + +rm -rf "${TMPDIR}" +rm -rf "${WHEEL_DIR}" + +echo "➤ All builds complete. Artifacts in ${FINAL_WHEEL_DIR}" diff --git a/utils/wheels/installPyEnvVersions.sh b/utils/wheels/installPyEnvVersions.sh new file mode 100755 index 0000000..f381669 --- /dev/null +++ b/utils/wheels/installPyEnvVersions.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Install every interpreter the macOS arm64 build matrix iterates over. +# Run once before build-wheels-macos_aarch64.sh. + +pyenv install -s 3.9.23 +pyenv install -s 3.10.18 +pyenv install -s 3.11.13 +pyenv install -s 3.12.11 +pyenv install -s 3.13.5 +pyenv install -s 3.13.5t +pyenv install -s 3.14.0rc1 +pyenv install -s 3.14.0rc1t +pyenv install -s pypy3.10-7.3.19 +pyenv install -s pypy3.11-7.3.20 diff --git a/utils/wheels/readme.md b/utils/wheels/readme.md new file mode 100644 index 0000000..37ff016 --- /dev/null +++ b/utils/wheels/readme.md @@ -0,0 +1,42 @@ +# Wheel Generation + +This directory contains scripts to generate precompiled Python wheels for **fourdst**. + +## Notes + +- macOS wheels can only be generated on macOS. +- aarch64 wheels can only be generated on an aarch64 machine. +- x86_64 wheels can only be generated on an x86_64 machine. +- Linux wheels can be generated on any Linux machine, but the target architecture must match the host architecture (Docker runs natively, there is no emulation here). +- Running each script takes **a very long time** (potentially most of a day, depending on the machine) and needs roughly 2 GB of disk space. +- For the macOS build you must have all the listed Python versions installed via `pyenv`. Run `installPyEnvVersions.sh` first to install them. +- The old duplicate-RPATH workaround (`repair_wheel_macos.sh` + `fix_rpaths.py`) is **no longer needed** — the meson-python bug that caused it has been fixed, so the macOS script repairs with a plain `delocate-wheel` pass. Those two files can be deleted. + +## Usage + +Once you are on the correct machine, run the script for your target platform, passing the repository URL. For example, to build the macOS arm64 wheels: + +```bash +./build-wheels-macos_aarch64.sh https://github.com/4D-STAR/fourdst +``` + +For Linux: + +```bash +./build-wheels-linux_x86_64.sh https://github.com/4D-STAR/fourdst # on an x86_64 host +./build-wheels-linux_aarch64.sh https://github.com/4D-STAR/fourdst # on an aarch64 host +``` + +Each script writes its repaired, redistributable wheels to a per-platform directory (e.g. `wheels_macos_aarch64/`, `wheels_linux_x86_64/`). + +## Publishing + +Once every platform's wheels are generated (which generally requires multiple machines), copy them all into a single directory — assume it is called `wheels/` at the repository root — then, from the repository root: + +```bash +python -m pip install --upgrade build twine +python -m build --sdist --outdir wheels # adds the source distribution +twine upload wheels/* +``` + +This uploads every wheel plus the sdist to PyPI (also slow, since it has to upload all of them).