From 346b2c2e8e3c8f4492a3a28fbcef043f06135943 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 1 Sep 2026 12:25:00 -0400 Subject: [PATCH] feat: bundle the CMake source tarball in the sdist The sdist now includes archive-cache/cmake-.tar.gz, so a from-source wheel build from the sdist needs no network. Building the sdist itself downloads the tarball. Archives now default to archive-cache/ in the source tree for both the CMake superbuild and the Python bootstrap, so both find the bundled file. Assisted-by: ClaudeCode:claude-fable-5 --- .github/workflows/build.yml | 12 +++++- CMakeLists.txt | 2 +- _build_backend/backend.py | 79 ++++++++++++++++++++++++++----------- docs/building.rst | 21 +++++++--- 4 files changed, 82 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e3cdd1edb..d0fde27c5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -134,6 +134,9 @@ jobs: - name: Build SDist run: pipx run build --sdist + - name: Check bundled CMake source archive + run: tar tzf dist/*.tar.gz | grep -E '/archive-cache/cmake-[0-9.]+\.tar\.gz$' + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: cibw-sdist @@ -171,9 +174,14 @@ jobs: name: cibw-sdist path: dist - - name: Install SDist + # Dead proxy blocks the CMake download; PyPI stays reachable for build deps. + - name: Install SDist (offline, from bundled source) env: - CMAKE_ARGS: "-DBUILD_CMAKE_FROM_SOURCE:BOOL=OFF" + CMAKE_ARGS: "-DBUILD_CMAKE_FROM_SOURCE:BOOL=ON" + CMAKE_BUILD_PARALLEL_LEVEL: "4" + http_proxy: "http://127.0.0.1:9" + https_proxy: "http://127.0.0.1:9" + no_proxy: "pypi.org,files.pythonhosted.org" run: | uv pip install dist/*.tar.gz rm -rf dist diff --git a/CMakeLists.txt b/CMakeLists.txt index d041301d2..b6d6a80f3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,7 @@ if(CMakePythonDistributions_SUPERBUILD) set(RUN_CMAKE_TEST_EXCLUDE "BootstrapTest" CACHE STRING "CMake test suite exclusion regex") - set(CMakePythonDistributions_ARCHIVE_DOWNLOAD_DIR "${CMAKE_BINARY_DIR}" + set(CMakePythonDistributions_ARCHIVE_DOWNLOAD_DIR "${CMAKE_SOURCE_DIR}/archive-cache" CACHE PATH "Directory where to download archives" ) diff --git a/_build_backend/backend.py b/_build_backend/backend.py index 45602887c..03dce5784 100644 --- a/_build_backend/backend.py +++ b/_build_backend/backend.py @@ -1,15 +1,19 @@ from __future__ import annotations import os +from pathlib import Path from scikit_build_core import build as _orig +# Archives are downloaded here (gitignored). The sdist bundles the unix source +# tarball in this directory so a build from the sdist needs no network. +_ARCHIVE_DIR = Path("archive-cache") + if hasattr(_orig, "prepare_metadata_for_build_editable"): prepare_metadata_for_build_editable = _orig.prepare_metadata_for_build_editable if hasattr(_orig, "prepare_metadata_for_build_wheel"): prepare_metadata_for_build_wheel = _orig.prepare_metadata_for_build_wheel build_editable = _orig.build_editable -build_sdist = _orig.build_sdist get_requires_for_build_editable = _orig.get_requires_for_build_editable get_requires_for_build_sdist = _orig.get_requires_for_build_sdist @@ -46,24 +50,66 @@ def get_requires_for_build_wheel( return packages -def _bootstrap_build(temp_path: str, config_settings: dict[str, list[str] | str] | None = None) -> str: +def _fetch_archive(kind: str, archive_dir: Path) -> Path: + """ + Return the path to the ``kind`` archive listed in ``CMakeUrls.cmake``, + downloading it into ``archive_dir`` if it is not already there. The SHA256 + is always verified. + """ import hashlib - import platform import re + import urllib.request + + cmake_urls = Path("CMakeUrls.cmake").read_text() + archive_url = re.findall(rf'set\({kind}_url\s+"(?P.*)"\)$', cmake_urls, flags=re.MULTILINE)[0] + archive_sha256 = re.findall(rf'set\({kind}_sha256\s+"(?P.*)"\)$', cmake_urls, flags=re.MULTILINE)[0] + + archive_name = archive_url.rsplit("/", maxsplit=1)[1] + archive_path = archive_dir / archive_name + if not archive_path.exists(): + archive_dir.mkdir(parents=True, exist_ok=True) + with urllib.request.urlopen(archive_url) as response: + archive_path.write_bytes(response.read()) + + sha256 = hashlib.sha256(archive_path.read_bytes()).hexdigest() + if archive_sha256.lower() != sha256.lower(): + msg = f"Invalid sha256 for {archive_url!r}. Expected {archive_sha256!r}, got {sha256!r}" + raise ValueError(msg) + + return archive_path + + +def build_sdist( + sdist_directory: str, + config_settings: dict[str, list[str] | str] | None = None, +) -> str: + archive_path = _fetch_archive("unix_source", _ARCHIVE_DIR) + + settings: dict[str, list[str] | str] = dict(config_settings or {}) + include = settings.get("sdist.include", []) + if isinstance(include, str): + include = include.split(";") + settings["sdist.include"] = [*include, archive_path.as_posix()] + return _orig.build_sdist(sdist_directory, settings) + + +def _bootstrap_build(temp_path: str, config_settings: dict[str, list[str] | str] | None = None) -> str: + import platform import shutil import subprocess import tarfile - import urllib.request import zipfile - from pathlib import Path env = os.environ.copy() temp_path_ = Path(temp_path) - archive_dir = temp_path_ + archive_dir = _ARCHIVE_DIR if config_settings: - archive_dir = Path(config_settings.get("cmake.define.CMakePythonDistributions_ARCHIVE_DOWNLOAD_DIR", archive_dir)) - archive_dir.mkdir(parents=True, exist_ok=True) + archive_dir_setting = config_settings.get("cmake.define.CMakePythonDistributions_ARCHIVE_DOWNLOAD_DIR") + if isinstance(archive_dir_setting, list): + archive_dir_setting = archive_dir_setting[-1] + if archive_dir_setting: + archive_dir = Path(archive_dir_setting) if os.name == "posix": if "MAKE" not in env: @@ -92,21 +138,8 @@ def _bootstrap_build(temp_path: str, config_settings: dict[str, list[str] | str] raise ValueError(msg) kind = kinds[machine] - - cmake_urls = Path("CMakeUrls.cmake").read_text() - archive_url = re.findall(rf'set\({kind}_url\s+"(?P.*)"\)$', cmake_urls, flags=re.MULTILINE)[0] - archive_sha256 = re.findall(rf'set\({kind}_sha256\s+"(?P.*)"\)$', cmake_urls, flags=re.MULTILINE)[0] - - archive_name = archive_url.rsplit("/", maxsplit=1)[1] - archive_path = archive_dir / archive_name - if not archive_path.exists(): - with urllib.request.urlopen(archive_url) as response: - archive_path.write_bytes(response.read()) - - sha256 = hashlib.sha256(archive_path.read_bytes()).hexdigest() - if archive_sha256.lower() != sha256.lower(): - msg = f"Invalid sha256 for {archive_url!r}. Expected {archive_sha256!r}, got {sha256!r}" - raise ValueError(msg) + archive_path = _fetch_archive(kind, archive_dir) + archive_name = archive_path.name if os.name == "posix": assert archive_name.endswith(".tar.gz") diff --git a/docs/building.rst b/docs/building.rst index 47a0b42ab..c57e2dbd4 100644 --- a/docs/building.rst +++ b/docs/building.rst @@ -43,9 +43,17 @@ wheels on PyPI are produced with `cibuildwheel `_ Source distribution (sdist) --------------------------- -The source distribution contains only this project's sources; the CMake source -or binary archive listed in ``CMakeUrls.cmake`` is downloaded when the wheel is -built. The source distribution is generated using the following command:: +The source distribution contains this project's sources plus the CMake source +tarball listed in ``CMakeUrls.cmake``, bundled as +``archive-cache/cmake-.tar.gz``. Building the sdist downloads that +tarball, so it needs network access; building a wheel from the sdist with +``BUILD_CMAKE_FROM_SOURCE=ON`` (the default on Linux) does not. + +Two paths still download at wheel-build time: ``BUILD_CMAKE_FROM_SOURCE=OFF`` +(the default on macOS and Windows) fetches a prebuilt binary archive, and a +Windows source build fetches the ``.zip`` source archive. + +The source distribution is generated using the following command:: python -m build --sdist @@ -101,9 +109,10 @@ in two ways. Caching downloads ^^^^^^^^^^^^^^^^^ -To avoid the re-download of CMake sources and/or binary packages, passing the -option ``-Ccmake.define.CMakePythonDistributions_ARCHIVE_DOWNLOAD_DIR=/path/to/cache`` -enables successive builds to re-use existing archives instead of re-downloading them. +Archives are downloaded into ``archive-cache/`` in the source tree (gitignored) +and re-used by later builds when their hash matches. To use a different +location, pass +``-Ccmake.define.CMakePythonDistributions_ARCHIVE_DOWNLOAD_DIR=/path/to/cache``. Re-using the build tree ^^^^^^^^^^^^^^^^^^^^^^^