Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 29 additions & 10 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ on:
push:
branches: [master]

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
build:
name: build (${{ matrix.platform }})
Expand Down Expand Up @@ -34,24 +38,39 @@ jobs:
echo "Selected: $XCODE"
sudo xcode-select -s "$XCODE"
fi
- uses: actions/cache@v6
- uses: actions/cache/restore@v6
id: native-cache
with:
key: native-${{ matrix.platform }}-${{ github.run_id }}
restore-keys: native-${{ matrix.platform }}-
path: .native-cache
- uses: actions/cache/restore@v6
id: ccache
with:
key: ccache-${{ matrix.platform }}-${{ github.run_id }}
restore-keys: ccache-${{ matrix.platform }}-
path: .ccache
- uses: actions/cache@v6
with:
key: downloads-${{ matrix.platform }}-${{ hashFiles('gcc-arm-none-eabi/build.sh', 'git-lfs/build.sh') }}
path: |
.uv-cache
*/*/toolchain
*/*/bin
- name: Build wheels
id: build
env:
MANYLINUX: ${{ runner.os == 'Linux' && '1' || '0' }}
UV_CACHE_DIR: ${{ github.workspace }}/.uv-cache
BUILD_SH_REUSE_MANYLINUX_ARTIFACTS: "1"
run: ./build.sh
DEPENDENCIES_NATIVE_CACHE: .native-cache
run: |
./build.sh
[[ ! -e .native-cache/.changed ]] || echo "native-cache-changed=1" >> "$GITHUB_OUTPUT"
# only when a package was rebuilt; restored packages are already in the cache
- uses: actions/cache/save@v6
if: steps.build.outputs.native-cache-changed
with:
key: ${{ steps.native-cache.outputs.cache-primary-key }}
path: .native-cache
# PR caches can't be shared with other branches, so only master's are worth saving
- uses: actions/cache/save@v6
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
with:
key: ${{ steps.ccache.outputs.cache-primary-key }}
path: .ccache
- uses: actions/upload-artifact@v7
with:
name: wheels-${{ matrix.platform }}
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,6 @@ acados/acados/acados_template/
acados/casadi/
acados/casadi-venv/
acados/casadi-wheel/

# CI cache of native build outputs (build_native_cache.py)
.native-cache/
16 changes: 11 additions & 5 deletions acados/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ fi

ACADOS_FLAGS=(
-DACADOS_WITH_QPOASES=ON
-DBLASFEO_EXAMPLES=OFF
-UBLASFEO_TARGET
-DBLASFEO_TARGET="$BLAS_TARGET"
-DACADOS_INSTALL_DIR="$INSTALL_DIR"
Expand All @@ -47,14 +48,19 @@ if [[ "$OSTYPE" == "darwin"* ]]; then
)
fi

# clone/update source
# Fetch only the pinned source and the dependencies enabled above.
if [ ! -d "acados-src/.git" ]; then
rm -rf acados-src
git clone https://github.com/acados/acados.git acados-src
git init acados-src
git -C acados-src remote add origin https://github.com/acados/acados.git
fi
git -C acados-src fetch --all --tags
git -C acados-src checkout --force "$VERSION"
git -C acados-src submodule update --init --recursive --depth=1
if ! git -C acados-src cat-file -e "$VERSION^{commit}" 2>/dev/null; then
git -C acados-src fetch --depth=1 origin "$VERSION"
fi
git -C acados-src checkout --force --detach "$VERSION"
git -C acados-src submodule update --init --depth=1 -- \
external/blasfeo external/hpipm external/qpoases \
interfaces/acados_template/tera_renderer

# build acados
mkdir -p build
Expand Down
3 changes: 2 additions & 1 deletion acados/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ class BuildAcados(build_py):
def run(self):
pkg_dir = os.path.dirname(os.path.abspath(__file__))
build_script = os.path.join(pkg_dir, "build.sh")
subprocess.check_call(["bash", build_script], cwd=pkg_dir)
if "acados" not in os.environ.get("DEPENDENCIES_PREBUILT", "").split(","):
subprocess.check_call(["bash", build_script], cwd=pkg_dir)

super().run()

Expand Down
3 changes: 2 additions & 1 deletion bootstrap-icons/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ class BuildBootstrapIcons(build_py):
def run(self):
pkg_dir = os.path.dirname(os.path.abspath(__file__))
build_script = os.path.join(pkg_dir, "build.sh")
subprocess.check_call(["bash", build_script], cwd=pkg_dir)
if "bootstrap-icons" not in os.environ.get("DEPENDENCIES_PREBUILT", "").split(","):
subprocess.check_call(["bash", build_script], cwd=pkg_dir)

super().run()

Expand Down
57 changes: 42 additions & 15 deletions build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,24 +12,42 @@ fi
USE_MANYLINUX="${MANYLINUX:-0}"

if [[ -z "${BUILD_SH_IN_MANYLINUX:-}" ]] && ! command -v uv >/dev/null 2>&1; then
./setup.sh
if [[ "$USE_MANYLINUX" == "1" ]]; then
./setup.sh --uv-only
else
./setup.sh
fi
export PATH="$HOME/.local/bin:$PATH"
fi

if [[ "$USE_MANYLINUX" == "1" && -z "${BUILD_SH_IN_MANYLINUX:-}" ]]; then
UV_BIN="$(command -v uv)"
# pinned so the image (and everything cached against it) doesn't change under us daily
MANYLINUX_IMAGE="${MANYLINUX_IMAGE:-quay.io/pypa/manylinux_2_28_$(uname -m):2026.11.07-1}"
# a git worktree's .git points outside the repo, so mount both at their host paths
WORK=/work
GIT_MOUNT=()
GIT_COMMON_DIR="$(git rev-parse --path-format=absolute --git-common-dir)"
if [[ "$GIT_COMMON_DIR" != "$ROOT_DIR/.git" ]]; then
WORK="$ROOT_DIR"
GIT_MOUNT=(-v "$GIT_COMMON_DIR:$GIT_COMMON_DIR:ro")
fi
docker run --rm \
-e BUILD_SH_IN_MANYLINUX=1 \
-e DEPENDENCIES_NATIVE_CACHE="${DEPENDENCIES_NATIVE_CACHE:-}" \
-e DEPENDENCIES_BUILDER_ID="$MANYLINUX_IMAGE" \
-e BUILD_SH_REUSE_MANYLINUX_ARTIFACTS="${BUILD_SH_REUSE_MANYLINUX_ARTIFACTS:-}" \
-e BUILD_SH_HOST_UID="$(id -u)" \
-e BUILD_SH_HOST_GID="$(id -g)" \
-e HOME=/tmp \
-e UV_CACHE_DIR=/work/.uv-cache \
-e CCACHE_DIR=/work/.ccache \
-e UV_CACHE_DIR="$WORK/.uv-cache" \
-e CCACHE_DIR="$WORK/.ccache" \
-e UV_PYTHON=/opt/python/cp312-cp312/bin/python3 \
-v "$ROOT_DIR:/work" \
-v "$ROOT_DIR:$WORK" \
-v "$UV_BIN:/usr/local/bin/uv:ro" \
-w /work \
"quay.io/pypa/manylinux_2_28_$(uname -m)" \
"${GIT_MOUNT[@]}" \
-w "$WORK" \
"$MANYLINUX_IMAGE" \
bash build.sh
exit 0
fi
Expand All @@ -40,23 +58,28 @@ if [[ -n "${BUILD_SH_IN_MANYLINUX:-}" ]]; then
# cached *-src repos may be owned by the host runner user; tell git to trust them
git config --global --add safe.directory '*'

./setup.sh

if [[ -z "${BUILD_SH_REUSE_MANYLINUX_ARTIFACTS:-}" ]]; then
for toml in */pyproject.toml; do
pkg="${toml%/pyproject.toml}"
module="${pkg//-/_}"
rm -rf "$pkg/$module/install" "$pkg/$module/toolchain" "$pkg/$module/bin"
done
fi
fi

# Install all manylinux system deps upfront — individual build.sh scripts
# run in parallel via uv, so concurrent dnf calls would race on the cache.
if command -v dnf &>/dev/null; then
dnf install -y -q \
libX11-devel libXcursor-devel libXrandr-devel libXinerama-devel libXi-devel \
mesa-libGL-devel mesa-libEGL-devel \
wayland-devel wayland-protocols-devel libxkbcommon-devel
# CI only: reuse native build outputs of packages whose inputs haven't changed
unset DEPENDENCIES_PREBUILT
if [[ -n "${DEPENDENCIES_NATIVE_CACHE:-}" ]]; then
DEPENDENCIES_PREBUILT="$(python3 build_native_cache.py restore "$DEPENDENCIES_NATIVE_CACHE")"
export DEPENDENCIES_PREBUILT
fi

# build deps are only needed if something actually gets built
if [[ -n "${BUILD_SH_IN_MANYLINUX:-}" ]]; then
PACKAGES=(*/pyproject.toml)
IFS=, read -ra PREBUILT <<< "${DEPENDENCIES_PREBUILT:-}"
if [[ "${#PREBUILT[@]}" -ne "${#PACKAGES[@]}" ]]; then
./setup.sh
fi
fi

Expand Down Expand Up @@ -103,6 +126,10 @@ for toml in */pyproject.toml; do
"$VENV_DIR/bin/python" -c "import $module; $module.smoketest()" >/dev/null
done

if [[ -n "${DEPENDENCIES_NATIVE_CACHE:-}" ]]; then
python3 build_native_cache.py save "$DEPENDENCIES_NATIVE_CACHE"
fi

du -hs dist/* | sort -hr

echo
Expand Down
141 changes: 141 additions & 0 deletions build_native_cache.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""Per-package cache of native build outputs, used by CI.

Most changes touch one package, but every CI run used to rebuild all of them.
Each package gets a key hashing its own files, the root build scripts and the
builder environment. `restore` unpacks outputs whose key still matches and
prints those packages; their setup.py then skips build.sh via
DEPENDENCIES_PREBUILT. `save` archives outputs whose key changed and creates
CACHE_DIR/.changed so CI knows to upload a new cache.

Normal source builds (pip install git+...) never touch this.
"""
import hashlib
import json
import os
import platform
import shutil
import subprocess
import sys
import tarfile
import time
from pathlib import Path

# bump to invalidate every cached package
VERSION = 1

ROOT = Path(__file__).resolve().parent
COMMON_INPUTS = ["build.sh", "setup.sh", "build_native_cache.py"]


def run(*args):
try:
return subprocess.run(args, cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.strip()
except FileNotFoundError:
return ""


def builder_identity():
info = [VERSION, platform.system(), platform.machine(), sys.version, os.environ.get("DEPENDENCIES_BUILDER_ID", ""),
run("cc", "--version"), run("c++", "--version"), run("cmake", "--version")]
# on Linux the pinned manylinux image (DEPENDENCIES_BUILDER_ID) plus setup.sh
# pin the toolchain; restore runs before setup.sh's dnf install, so don't look at rpms
if platform.system() == "Darwin":
info += [run("sw_vers", "-productVersion"), run("xcrun", "--show-sdk-version"),
run("brew", "list", "--versions", "nasm", "pkg-config", "ccache", "autoconf", "automake", "libtool")]
return json.dumps(info)


def packages():
return sorted(p.parent.name for p in ROOT.glob("*/pyproject.toml"))


def outputs(pkg):
module = pkg.replace("-", "_")
paths = [f"{pkg}/{module}/toolchain" if pkg == "gcc-arm-none-eabi" else f"{pkg}/{module}/install"]
if pkg == "acados":
paths += ["acados/casadi", "acados/acados/acados_template"]
elif pkg == "raylib":
paths += [str(p.relative_to(ROOT)) for p in sorted((ROOT / "raylib/raylib").glob("_raylib_cffi*.so"))]
paths += ["raylib/raylib_nb"]
return paths


def keys():
builder = builder_identity()
result = {}
for pkg in packages():
h = hashlib.sha256(builder.encode())
if pkg == "acados" and platform.machine() == "x86_64":
# BLASFEO's X64_AUTOMATIC target is picked from the build machine's CPU
h.update(run("grep", "-m1", "^flags", "/proc/cpuinfo").encode())
files = subprocess.check_output(["git", "ls-files", "--cached", "--others", "--exclude-standard", "--", pkg],
cwd=ROOT, text=True).splitlines()
for name in sorted(set(files + COMMON_INPUTS)):
path = ROOT / name
h.update(name.encode() + b"\0" + (path.read_bytes() if path.is_file() else b"<missing>") + b"\0")
result[pkg] = h.hexdigest()
return result


def remove(path):
if path.is_dir() and not path.is_symlink():
shutil.rmtree(path)
elif path.exists() or path.is_symlink():
path.unlink()


def restore(cache):
cache.mkdir(parents=True, exist_ok=True)
remove(cache / ".changed")
current = keys()
(cache / "keys.json").write_text(json.dumps(current, indent=2))
hits = []
for pkg, key in current.items():
start = time.monotonic()
archive, stamp = cache / f"{pkg}.tar", cache / f"{pkg}.key"
hit = archive.is_file() and stamp.is_file() and stamp.read_text() == key
if hit:
for out in outputs(pkg):
remove(ROOT / out)
with tarfile.open(archive) as tar:
if hasattr(tarfile, "data_filter"):
tar.extractall(ROOT, filter="data")
else:
tar.extractall(ROOT)
hits.append(pkg)
print(f"native cache: {pkg} {'HIT' if hit else 'MISS'} ({time.monotonic() - start:.2f}s)", file=sys.stderr)
print(",".join(hits))


def save(cache):
current = json.loads((cache / "keys.json").read_text())
for pkg, key in current.items():
stamp = cache / f"{pkg}.key"
if stamp.is_file() and stamp.read_text() == key:
continue
tmp = cache / f"{pkg}.tar.tmp"
with tarfile.open(tmp, "w") as tar:
for out in outputs(pkg):
if not (ROOT / out).exists():
raise RuntimeError(f"{pkg}: missing build output {out}")
tar.add(ROOT / out, arcname=out)
tmp.replace(cache / f"{pkg}.tar")
stamp.write_text(key)
(cache / ".changed").touch()
print(f"native cache: saved {pkg}", file=sys.stderr)
# drop packages that no longer exist
for stale in cache.glob("*.key"):
if stale.stem not in current:
remove(stale)
remove(cache / f"{stale.stem}.tar")
(cache / ".changed").touch()


if __name__ == "__main__":
if len(sys.argv) != 3 or sys.argv[1] not in ("restore", "save"):
raise SystemExit("usage: build_native_cache.py restore|save CACHE_DIR")
cache_dir = Path(sys.argv[2])
if not cache_dir.is_absolute():
cache_dir = ROOT / cache_dir
{"restore": restore, "save": save}[sys.argv[1]](cache_dir)
3 changes: 2 additions & 1 deletion capnproto/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ class BuildCapnproto(build_py):
def run(self):
pkg_dir = os.path.dirname(os.path.abspath(__file__))
build_script = os.path.join(pkg_dir, "build.sh")
subprocess.check_call(["bash", build_script], cwd=pkg_dir)
if "capnproto" not in os.environ.get("DEPENDENCIES_PREBUILT", "").split(","):
subprocess.check_call(["bash", build_script], cwd=pkg_dir)

super().run()

Expand Down
3 changes: 2 additions & 1 deletion cppcheck/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ class BuildCppcheck(build_py):
def run(self):
pkg_dir = os.path.dirname(os.path.abspath(__file__))
build_script = os.path.join(pkg_dir, "build.sh")
subprocess.check_call(["bash", build_script], cwd=pkg_dir)
if "cppcheck" not in os.environ.get("DEPENDENCIES_PREBUILT", "").split(","):
subprocess.check_call(["bash", build_script], cwd=pkg_dir)

super().run()

Expand Down
3 changes: 2 additions & 1 deletion eigen/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ class BuildEigen(build_py):
def run(self):
pkg_dir = os.path.dirname(os.path.abspath(__file__))
build_script = os.path.join(pkg_dir, "build.sh")
subprocess.check_call(["bash", build_script], cwd=pkg_dir)
if "eigen" not in os.environ.get("DEPENDENCIES_PREBUILT", "").split(","):
subprocess.check_call(["bash", build_script], cwd=pkg_dir)

super().run()

Expand Down
Loading
Loading