diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d04b68ac..1d6ded33 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,8 +12,8 @@ on: description: >- Dry run: build and verify every wheel/sdist/wasm artifact across the full release matrix without publishing to PyPI. Use this to validate - the cross-compile matrix (musllinux, win-arm64, wasm, ...) before a - real version-tag release. Defaults to true so a manual dispatch + the runtime-verified release matrix before a real version-tag + release. Defaults to true so a manual dispatch never accidentally publishes. type: boolean default: true @@ -26,11 +26,9 @@ permissions: jobs: # One wheel per platform (the C-ABI core is CPython-version-independent, so # each entry produces a single py3-none- wheel). Coverage matches - # what the native core can target: Linux glibc + musl across x86-64/aarch64/ - # armv7, macOS x86-64/arm64, and Windows x86/x64/arm64. Linux targets are - # cross-compiled with cargo-zigbuild (musl support + a low manylinux_2_17 - # glibc floor without QEMU); the hook then packs the prebuilt core and stamps - # the platform tag (XY_CARGO_TARGET / XY_WHEEL_PLATFORM). + # what the release workflow can install, load, and exercise on its runners: + # Linux x86-64, macOS arm64, and Windows x86-64. Additional cross-compiled + # targets must not be added until CI has a matching runner or emulator. wheels: name: Wheel ${{ matrix.plat }} runs-on: ${{ matrix.os }} @@ -38,30 +36,12 @@ jobs: fail-fast: false matrix: include: - # Linux glibc (manylinux_2_17 floor) — cross-compiled with zig. + # Linux glibc (manylinux_2_17 floor), built with zig. - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu, zigtarget: x86_64-unknown-linux-gnu.2.17, plat: manylinux_2_17_x86_64, zig: true, native: true } - - { os: ubuntu-latest, target: aarch64-unknown-linux-gnu, zigtarget: aarch64-unknown-linux-gnu.2.17, plat: manylinux_2_17_aarch64, zig: true, native: false } - - { os: ubuntu-latest, target: armv7-unknown-linux-gnueabihf, zigtarget: armv7-unknown-linux-gnueabihf.2.17, plat: manylinux_2_17_armv7l, zig: true, native: false } - # Linux musl / Alpine. crt-static is musl's default, and rustc - # silently *drops* cdylib output under it (a warning, not an error) - # since a fully-static binary can't also be a shared library — - # -C target-feature=-crt-static switches to dynamic linking against - # musl libc so a real cdylib gets produced. - - { os: ubuntu-latest, target: x86_64-unknown-linux-musl, zigtarget: x86_64-unknown-linux-musl, plat: musllinux_1_2_x86_64, zig: true, native: false, rustflags: "-C target-feature=-crt-static" } - - { os: ubuntu-latest, target: aarch64-unknown-linux-musl, zigtarget: aarch64-unknown-linux-musl, plat: musllinux_1_2_aarch64, zig: true, native: false, rustflags: "-C target-feature=-crt-static" } - - { os: ubuntu-latest, target: armv7-unknown-linux-musleabihf, zigtarget: armv7-unknown-linux-musleabihf, plat: musllinux_1_2_armv7l, zig: true, native: false, rustflags: "-C target-feature=-crt-static" } - # macOS. Both build on the arm64 runner (macos-14): Apple Silicon - # runners are plentiful, while the last Intel runner (macos-13) is - # deprecated and frequently unschedulable. The Apple toolchain - # cross-links x86_64 Mach-O natively, so the Intel wheel is a cross - # build (native: false -> content-verified, not import-smoked, since - # an x86_64 lib can't be imported on the arm64 runner). - - { os: macos-14, target: x86_64-apple-darwin, plat: macosx_10_12_x86_64, zig: false, native: false, deployment: "10.12" } + # macOS arm64, the native architecture of macos-14. - { os: macos-14, target: aarch64-apple-darwin, plat: macosx_11_0_arm64, zig: false, native: true, deployment: "11.0" } - # Windows x64 (native), x86 and arm64 (cross-compiled). + # Windows x64. - { os: windows-latest, target: x86_64-pc-windows-msvc, plat: win_amd64, zig: false, native: true } - - { os: windows-latest, target: i686-pc-windows-msvc, plat: win32, zig: false, native: false } - - { os: windows-latest, target: aarch64-pc-windows-msvc, plat: win_arm64, zig: false, native: false } steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -109,7 +89,7 @@ jobs: shell: bash run: | whl=$(ls dist/*.whl) - python scripts/verify_wheel.py "$whl" --expect-native + python scripts/verify_wheel.py "$whl" --expect-native --expect-platform "${{ matrix.plat }}" --require-symbol xy_abi_version --require-linkage - name: Install-size budget (<= 15 MB) shell: bash run: | @@ -118,15 +98,14 @@ jobs: echo "wheel size: $size bytes ($whl)" test "$size" -le 15728640 - name: Verify the wheel installs and loads the native core - # Only host-arch wheels can be imported on the runner; cross-compiled - # arches (aarch64/armv7/win-arm64/win32) are content-verified instead. - if: matrix.native + # Every published wheel is built for the runner architecture and must + # pass an import plus representative native-kernel call. shell: bash run: | uv venv smoke uv pip install -p smoke dist/*.whl numpy anywidget "reflex>=0.9.6" - ./smoke/bin/python -c "import importlib.metadata as m, reflex_xy, xy.kernels as k; assert k.BACKEND=='native', k.BACKEND; assert reflex_xy.__version__ == m.version('xy'); print('native', k.__file__)" \ - || ./smoke/Scripts/python.exe -c "import importlib.metadata as m, reflex_xy, xy.kernels as k; assert k.BACKEND=='native', k.BACKEND; assert reflex_xy.__version__ == m.version('xy'); print('native')" + if [ -f ./smoke/bin/python ]; then py=./smoke/bin/python; else py=./smoke/Scripts/python.exe; fi + "$py" scripts/wheel_smoke.py - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: dist-${{ matrix.plat }} diff --git a/scripts/verify_ci_workflow.py b/scripts/verify_ci_workflow.py index 2fe0cfd4..27ed3a71 100644 --- a/scripts/verify_ci_workflow.py +++ b/scripts/verify_ci_workflow.py @@ -81,6 +81,13 @@ def _matrix_include_entries(job_text: str) -> list[dict[str, str]]: break else: continue + if current is not None and item.startswith("{") and item.endswith("}"): + inline = item[1:-1] + for match in re.finditer( + r"([A-Za-z0-9_-]+):\s*(\"[^\"]*\"|'[^']*'|[^,}]+)", inline + ): + current[match.group(1)] = match.group(2).strip() + continue match = re.fullmatch(r"([A-Za-z0-9_-]+):\s*(.*?)", item) if match and current is not None: current[match.group(1)] = match.group(2) @@ -561,6 +568,20 @@ def _step_run_lines(step_block: str) -> list[str]: return [] +def _require_step_run_contains( + errors: list[str], job_text: str, step: str, description: str, *needles: str +) -> None: + """Require commands in a named step's active ``run`` value only.""" + block = _named_step_blocks(job_text).get(step) + if block is None: + errors.append(f"missing required CI step {step!r}") + return + run_text = "\n".join(_step_run_lines(block)) + missing = _missing_needles(run_text, needles) + if missing: + errors.append(f"CI step {step!r} missing {description}: {missing}") + + def _require_step_runs_exactly( errors: list[str], job_text: str, step: str, description: str, *commands: str ) -> None: @@ -1251,7 +1272,7 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str jobs, "wheels", "release", - "cross-platform wheel matrix (glibc+musl, macOS, Windows), verification, and upload", + "runtime-verified wheel matrix (Linux, macOS, Windows), verification, and upload", "dtolnay/rust-toolchain@", "astral-sh/setup-uv@", "actions/setup-node@", @@ -1261,19 +1282,42 @@ def validate_release_workflow(path: Path = DEFAULT_RELEASE_WORKFLOW) -> list[str "uv build --wheel", "XY_REQUIRE_CARGO", "XY_WHEEL_PLATFORM", - "musllinux_1_2_x86_64", - "win_arm64", + "manylinux_2_17_x86_64", + "macosx_11_0_arm64", + "win_amd64", "scripts/verify_wheel.py", "--expect-native", + "--expect-platform", + "--require-symbol xy_abi_version", + "--require-linkage", "Install-size budget (<= 15 MB)", '"reflex>=0.9.6"', - "import importlib.metadata as m, reflex_xy", - "assert reflex_xy.__version__ == m.version('xy')", - "assert k.BACKEND=='native'", "actions/upload-artifact@", "dist/*.whl", ) wheels_job = jobs.get("wheels", "") + _require_step_run_contains( + errors, + wheels_job, + "Verify the wheel installs and loads the native core", + "native wheel smoke command", + '"$py" scripts/wheel_smoke.py', + ) + _require_step_run_contains( + errors, + wheels_job, + "Verify the wheel installs and loads the native core", + "native wheel smoke command", + '"$py" scripts/wheel_smoke.py', + ) + matrix_entries = _matrix_include_entries(wheels_job) + if not matrix_entries or any( + entry.get("native", "").strip().strip("\"'").lower() != "true" + for entry in matrix_entries + ): + errors.append( + "release wheels job must not publish a target without an install/load smoke" + ) if "continue-on-error:" in wheels_job: errors.append( "release wheels job must block publishing when any native wheel build or " diff --git a/scripts/verify_wheel.py b/scripts/verify_wheel.py index 609bc45c..dacb7204 100644 --- a/scripts/verify_wheel.py +++ b/scripts/verify_wheel.py @@ -14,6 +14,7 @@ import csv import hashlib import re +import struct import sys import zipfile from dataclasses import dataclass @@ -79,6 +80,538 @@ class WheelInfo: tags: list[str] +@dataclass(frozen=True) +class NativeBinaryInfo: + """Architecture facts read from a packaged native library header.""" + + format: str + machine: str + bits: int + exported_symbols: frozenset[str] = frozenset() + + +_ELF_MACHINES = {40: "arm", 62: "x86_64", 183: "aarch64"} +_PE_MACHINES = {0x14C: "x86", 0x8664: "x86_64", 0xAA64: "aarch64"} +_MACHO_CPU_TYPES = {0x01000007: "x86_64", 0x0100000C: "aarch64"} +_WHEEL_TARGETS = { + "manylinux_2_17_x86_64": ("ELF", "x86_64", 64), + "manylinux_2_17_aarch64": ("ELF", "aarch64", 64), + "manylinux_2_17_armv7l": ("ELF", "arm", 32), + "musllinux_1_2_x86_64": ("ELF", "x86_64", 64), + "musllinux_1_2_aarch64": ("ELF", "aarch64", 64), + "musllinux_1_2_armv7l": ("ELF", "arm", 32), + "macosx_10_12_x86_64": ("Mach-O", "x86_64", 64), + "macosx_11_0_arm64": ("Mach-O", "aarch64", 64), + "win_amd64": ("PE", "x86_64", 64), + "win32": ("PE", "x86", 32), + "win_arm64": ("PE", "aarch64", 64), +} +_ELF_ALLOWED_GLIBC = {"libc.so.6", "libm.so.6", "libgcc_s.so.1", "ld-linux-x86-64.so.2"} +_ELF_ALLOWED_MUSL_PREFIXES = ("libc.musl-", "ld-musl-", "libgcc_s.so.") +_PE_ALLOWED_IMPORTS = { + "api-ms-win-core", + "api-ms-win-crt", + "kernel32.dll", + "msvcrt.dll", + "ucrtbase.dll", + "vcruntime140.dll", +} + + +def _inspect_native_binary(name: str, data: bytes) -> NativeBinaryInfo: + """Read format, machine, and bitness without platform-specific tools.""" + if data[:4] == b"\x7fELF": + if len(data) < 20: + raise AssertionError(f"{name} has a truncated ELF header") + elf_class, endian = data[4], data[5] + if elf_class not in {1, 2} or endian not in {1, 2}: + raise AssertionError(f"{name} has an invalid ELF class or byte order") + prefix = "<" if endian == 1 else ">" + machine = struct.unpack_from(prefix + "H", data, 18)[0] + if machine not in _ELF_MACHINES: + raise AssertionError(f"{name} has unsupported ELF machine {machine}") + bits = 32 if elf_class == 1 else 64 + symbols = _elf_exported_symbols(name, data, prefix, bits) + return NativeBinaryInfo("ELF", _ELF_MACHINES[machine], bits, symbols) + + if len(data) >= 8 and data[:4] in { + b"\xfe\xed\xfa\xce", + b"\xce\xfa\xed\xfe", + b"\xfe\xed\xfa\xcf", + b"\xcf\xfa\xed\xfe", + }: + prefix = ">" if data[:4] in {b"\xfe\xed\xfa\xce", b"\xfe\xed\xfa\xcf"} else "<" + cputype = struct.unpack_from(prefix + "I", data, 4)[0] + if cputype not in _MACHO_CPU_TYPES: + raise AssertionError(f"{name} has unsupported Mach-O CPU type {cputype}") + bits = 64 if data[:4] in {b"\xfe\xed\xfa\xcf", b"\xcf\xfa\xed\xfe"} else 32 + symbols = _macho_exported_symbols(name, data, prefix, bits) + return NativeBinaryInfo("Mach-O", _MACHO_CPU_TYPES[cputype], bits, symbols) + + if len(data) >= 0x40 and data[:2] == b"MZ": + pe_offset = struct.unpack_from(" len(data) or data[pe_offset : pe_offset + 4] != b"PE\0\0": + raise AssertionError(f"{name} has an invalid PE header") + machine = struct.unpack_from(" frozenset[str]: + """Read defined global/weak names from an ELF dynamic symbol table.""" + if len(data) < (64 if bits == 64 else 52): + return frozenset() + if bits == 64: + section_offset, section_count = ( + struct.unpack_from(prefix + "QI", data, 40)[0], + struct.unpack_from(prefix + "H", data, 60)[0], + ) + section_size, _section_name_index = ( + struct.unpack_from(prefix + "H", data, 58)[0], + struct.unpack_from(prefix + "H", data, 62)[0], + ) + section_header_size = section_size + section_fields = (24, 32, 40, 56) + else: + section_offset = struct.unpack_from(prefix + "I", data, 32)[0] + section_count = struct.unpack_from(prefix + "H", data, 48)[0] + section_header_size = struct.unpack_from(prefix + "H", data, 46)[0] + _section_name_index = struct.unpack_from(prefix + "H", data, 50)[0] + section_fields = (16, 20, 24, 36) + if not section_offset or not section_count or not section_header_size: + return frozenset() + + sections: list[tuple[int, int, int, int, int]] = [] + for index in range(section_count): + offset = section_offset + index * section_header_size + if offset + section_header_size > len(data): + raise AssertionError(f"{name} has a truncated ELF section table") + section_type = struct.unpack_from(prefix + "I", data, offset + 4)[0] + section_file_offset = struct.unpack_from( + prefix + ("Q" if bits == 64 else "I"), data, offset + section_fields[0] + )[0] + section_length = struct.unpack_from( + prefix + ("Q" if bits == 64 else "I"), data, offset + section_fields[1] + )[0] + section_link = struct.unpack_from(prefix + "I", data, offset + section_fields[2])[0] + section_entry_size = struct.unpack_from( + prefix + ("Q" if bits == 64 else "I"), data, offset + section_fields[3] + )[0] + sections.append( + ( + section_type, + section_file_offset, + section_length, + section_link, + section_entry_size, + ) + ) + + dynamic_index = next((i for i, section in enumerate(sections) if section[0] == 11), None) + if dynamic_index is None: + return frozenset() + _, symbol_offset, symbol_length, string_index, entry_size = sections[dynamic_index] + if string_index >= len(sections): + raise AssertionError(f"{name} has an invalid ELF dynamic string-table link") + _, string_offset, string_length, _, _ = sections[string_index] + symbol_size = 24 if bits == 64 else 16 + entry_size = entry_size or symbol_size + if entry_size < symbol_size or symbol_offset + symbol_length > len(data): + raise AssertionError(f"{name} has an invalid ELF dynamic symbol table") + if string_offset + string_length > len(data): + raise AssertionError(f"{name} has a truncated ELF dynamic string table") + strings = data[string_offset : string_offset + string_length] + exported: set[str] = set() + symbol_end = symbol_offset + symbol_length - (symbol_length % entry_size) + for offset in range(symbol_offset, symbol_end, entry_size): + if bits == 64: + string_name, info, section_index = ( + struct.unpack_from(prefix + "I", data, offset)[0], + data[offset + 4], + struct.unpack_from(prefix + "H", data, offset + 6)[0], + ) + else: + string_name = struct.unpack_from(prefix + "I", data, offset)[0] + info = data[offset + 12] + section_index = struct.unpack_from(prefix + "H", data, offset + 14)[0] + if section_index == 0 or info >> 4 not in {1, 2} or string_name >= len(strings): + continue + end = strings.find(b"\0", string_name) + if end > string_name: + exported.add(strings[string_name:end].decode("utf-8", errors="replace")) + return frozenset(exported) + + +def _macho_exported_symbols(name: str, data: bytes, prefix: str, bits: int) -> frozenset[str]: + """Read externally defined names from a thin Mach-O symbol table.""" + header_size = 32 if bits == 64 else 28 + if len(data) < header_size: + return frozenset() + ncommands = struct.unpack_from(prefix + "I", data, 16)[0] + command_offset = header_size + symbol_table: tuple[int, int, int, int] | None = None + for _ in range(ncommands): + if command_offset + 8 > len(data): + raise AssertionError(f"{name} has a truncated Mach-O load-command table") + command, command_size = struct.unpack_from(prefix + "II", data, command_offset) + if command_size < 8 or command_offset + command_size > len(data): + raise AssertionError(f"{name} has an invalid Mach-O load command") + if command == 2 and command_size >= 24: # LC_SYMTAB + symbol_table = struct.unpack_from(prefix + "IIII", data, command_offset + 8) + break + command_offset += command_size + if symbol_table is None: + return frozenset() + symbol_offset, symbol_count, string_offset, string_size = symbol_table + entry_size = 16 if bits == 64 else 12 + if string_offset + string_size > len(data): + raise AssertionError(f"{name} has a truncated Mach-O string table") + strings = data[string_offset : string_offset + string_size] + exported: set[str] = set() + for index in range(symbol_count): + offset = symbol_offset + index * entry_size + if offset + entry_size > len(data): + raise AssertionError(f"{name} has a truncated Mach-O symbol table") + string_name = struct.unpack_from(prefix + "I", data, offset)[0] + symbol_type = data[offset + 4] + if not symbol_type & 0x01 or symbol_type & 0x0E == 0x00 or string_name >= len(strings): + continue + end = strings.find(b"\0", string_name) + if end > string_name: + exported.add(strings[string_name:end].decode("utf-8", errors="replace").lstrip("_")) + return frozenset(exported) + + +def _pe_exported_symbols( + name: str, data: bytes, pe_offset: int, optional_magic: int, bits: int +) -> frozenset[str]: + """Read names from the PE export directory.""" + optional_offset = pe_offset + 24 + data_directory_offset = optional_offset + (96 if optional_magic == 0x10B else 112) + if data_directory_offset + 8 > len(data): + return frozenset() + export_rva, export_size = struct.unpack_from(" len(data): + raise AssertionError(f"{name} has a truncated PE section table") + virtual_size, virtual_address, raw_size, raw_offset = struct.unpack_from( + " int: + for virtual_address, size, raw_offset, _raw_size in sections: + if virtual_address <= rva < virtual_address + size: + return raw_offset + (rva - virtual_address) + raise AssertionError(f"{name} has an export RVA outside its PE sections") + + export_offset = file_offset(export_rva) + if export_offset + 40 > len(data): + raise AssertionError(f"{name} has a truncated PE export directory") + number_of_names = struct.unpack_from(" len(data): + raise AssertionError(f"{name} has a truncated PE export-name table") + string_offset = file_offset(struct.unpack_from(" string_offset: + exported.add(data[string_offset:end].decode("ascii", errors="replace")) + return frozenset(exported) + + +def _macho_linkage( + name: str, data: bytes, prefix: str, bits: int +) -> tuple[tuple[str, ...], tuple[int, int] | None]: + """Read loaded dylibs and the minimum macOS version from Mach-O commands.""" + header_size = 32 if bits == 64 else 28 + if len(data) < header_size: + raise AssertionError(f"{name} has a truncated Mach-O header") + ncommands = struct.unpack_from(prefix + "I", data, 16)[0] + command_offset = header_size + dependencies: list[str] = [] + minimum: tuple[int, int] | None = None + dylib_commands = {0xC, 0x18 | 0x80000000, 0x1F | 0x80000000} + for _ in range(ncommands): + if command_offset + 8 > len(data): + raise AssertionError(f"{name} has a truncated Mach-O load-command table") + command, command_size = struct.unpack_from(prefix + "II", data, command_offset) + if command_size < 8 or command_offset + command_size > len(data): + raise AssertionError(f"{name} has an invalid Mach-O load command") + if command in dylib_commands and command_size >= 24: + name_offset = struct.unpack_from(prefix + "I", data, command_offset + 8)[0] + start = command_offset + name_offset + end = data.find(b"\0", start, command_offset + command_size) + if end > start: + dependencies.append(data[start:end].decode("utf-8", errors="replace")) + elif command == 0x32 and command_size >= 16: # LC_BUILD_VERSION + version = struct.unpack_from(prefix + "I", data, command_offset + 12)[0] + minimum = (version >> 16, (version >> 8) & 0xFF) + elif command == 0x24 and command_size >= 16: # LC_VERSION_MIN_MACOSX + version = struct.unpack_from(prefix + "I", data, command_offset + 8)[0] + minimum = (version >> 16, (version >> 8) & 0xFF) + command_offset += command_size + return tuple(dependencies), minimum + + +def _pe_imports(name: str, data: bytes, pe_offset: int, optional_magic: int) -> tuple[str, ...]: + """Read DLL names from the PE import directory.""" + optional_offset = pe_offset + 24 + data_directory_offset = optional_offset + (104 if optional_magic == 0x10B else 120) + if data_directory_offset + 8 > len(data): + return () + import_rva, import_size = struct.unpack_from(" len(data): + raise AssertionError(f"{name} has a truncated PE section table") + virtual_size, virtual_address, raw_size, raw_offset = struct.unpack_from( + " int: + for virtual_address, size, raw_offset in sections: + if virtual_address <= rva < virtual_address + size: + return raw_offset + rva - virtual_address + raise AssertionError(f"{name} has an import RVA outside its PE sections") + + import_offset = file_offset(import_rva) + imports: list[str] = [] + for offset in range(import_offset, import_offset + import_size, 20): + if offset + 20 > len(data): + raise AssertionError(f"{name} has a truncated PE import directory") + original_thunk, timestamp, forwarder, name_rva, first_thunk = struct.unpack_from( + " None: + expected = _WHEEL_TARGETS.get(platform) + if expected is None: + raise AssertionError( + f"no native binary contract is defined for wheel platform {platform!r}" + ) + actual = _inspect_native_binary(name, data) + if (actual.format, actual.machine, actual.bits) != expected: + raise AssertionError( + f"{name} identifies as {actual.format}/{actual.machine}/{actual.bits}-bit, " + f"expected {expected[0]}/{expected[1]}/{expected[2]}-bit for {platform}" + ) + + +def _require_exported_symbols(name: str, data: bytes, required: set[str]) -> None: + """Require ABI symbols from the binary's export table.""" + info = _inspect_native_binary(name, data) + missing = sorted(required - info.exported_symbols) + if missing: + raise AssertionError(f"{name} is missing exported ABI symbols: {missing}") + + +def _elf_linkage(name: str, data: bytes, prefix: str, bits: int) -> tuple[str, tuple[str, ...]]: + """Return the ELF interpreter and DT_NEEDED names.""" + if bits == 64: + if len(data) < 64: + raise AssertionError(f"{name} has a truncated ELF header") + program_offset = struct.unpack_from(prefix + "Q", data, 32)[0] + program_entry_size = struct.unpack_from(prefix + "H", data, 54)[0] + program_count = struct.unpack_from(prefix + "H", data, 56)[0] + program_fields = (8, 16, 32) + dynamic_entry_size = 16 + else: + if len(data) < 52: + raise AssertionError(f"{name} has a truncated ELF header") + program_offset = struct.unpack_from(prefix + "I", data, 28)[0] + program_entry_size = struct.unpack_from(prefix + "H", data, 42)[0] + program_count = struct.unpack_from(prefix + "H", data, 44)[0] + program_fields = (4, 8, 16) + dynamic_entry_size = 8 + if not program_offset or not program_entry_size or not program_count: + raise AssertionError(f"{name} has no ELF program headers") + + load_segments: list[tuple[int, int, int, int]] = [] + interpreter: str | None = None + dynamic: tuple[int, int] | None = None + number_size = "Q" if bits == 64 else "I" + for index in range(program_count): + offset = program_offset + index * program_entry_size + if offset + program_entry_size > len(data): + raise AssertionError(f"{name} has a truncated ELF program-header table") + program_type = struct.unpack_from(prefix + "I", data, offset)[0] + file_offset = struct.unpack_from(prefix + number_size, data, offset + program_fields[0])[0] + virtual_address = struct.unpack_from( + prefix + number_size, data, offset + program_fields[1] + )[0] + file_size = struct.unpack_from(prefix + number_size, data, offset + program_fields[2])[0] + if program_type == 1: + load_segments.append((virtual_address, file_offset, file_size, file_size)) + elif program_type == 3: + raw = data[file_offset : file_offset + file_size] + interpreter = raw.split(b"\0", 1)[0].decode("utf-8", errors="replace") + elif program_type == 2: + dynamic = (file_offset, file_size) + if dynamic is None: + return interpreter or "", () + + def virtual_to_file(address: int) -> int: + for virtual_address, file_offset, file_size, _ in load_segments: + if virtual_address <= address < virtual_address + file_size: + return file_offset + address - virtual_address + raise AssertionError(f"{name} has an ELF dynamic string table outside load segments") + + dynamic_offset, dynamic_size = dynamic + needed: list[int] = [] + string_address = string_size = None + for offset in range(dynamic_offset, dynamic_offset + dynamic_size, dynamic_entry_size): + if offset + dynamic_entry_size > len(data): + raise AssertionError(f"{name} has a truncated ELF dynamic section") + tag = struct.unpack_from(prefix + number_size, data, offset)[0] + value = struct.unpack_from(prefix + number_size, data, offset + (8 if bits == 64 else 4))[0] + if tag == 0: + break + if tag == 1: + needed.append(value) + elif tag == 5: + string_address = value + elif tag == 10: + string_size = value + if string_address is None or string_size is None: + raise AssertionError(f"{name} has DT_NEEDED entries but no complete dynamic string table") + string_offset = virtual_to_file(string_address) + strings = data[string_offset : string_offset + string_size] + if len(strings) != string_size: + raise AssertionError(f"{name} has a truncated ELF dynamic string table") + dependencies = [] + for string_index in needed: + if string_index >= len(strings): + raise AssertionError(f"{name} has an invalid DT_NEEDED string index") + end = strings.find(b"\0", string_index) + if end < string_index: + raise AssertionError(f"{name} has an unterminated DT_NEEDED name") + dependencies.append(strings[string_index:end].decode("utf-8", errors="replace")) + return interpreter or "", tuple(dependencies) + + +def _require_elf_linkage(name: str, data: bytes, platform: str) -> None: + """Validate libc family and dependency policy for Linux wheel targets.""" + if data[:4] != b"\x7fELF": + raise AssertionError(f"{name} is not an ELF binary for Linux linkage validation") + endian = data[5] + prefix = "<" if endian == 1 else ">" if endian == 2 else "" + if not prefix: + raise AssertionError(f"{name} has an invalid ELF byte order") + bits = 32 if data[4] == 1 else 64 if data[4] == 2 else 0 + if not bits: + raise AssertionError(f"{name} has an invalid ELF class") + interpreter, dependencies = _elf_linkage(name, data, prefix, bits) + if platform.startswith("manylinux_"): + if interpreter and "ld-linux" not in interpreter: + raise AssertionError(f"{name} is not linked against glibc: interpreter={interpreter!r}") + if not interpreter and "libc.so.6" not in dependencies: + raise AssertionError(f"{name} has no detectable glibc linkage") + unexpected = sorted(set(dependencies) - _ELF_ALLOWED_GLIBC) + if unexpected: + raise AssertionError(f"{name} has unsupported glibc dependencies: {unexpected}") + versions = { + (int(match.group(1)), int(match.group(2))) + for match in re.finditer(rb"GLIBC_(\d+)\.(\d+)", data) + } + if versions and max(versions) > (2, 17): + version = ".".join(map(str, max(versions))) + raise AssertionError(f"{name} requires glibc {version}, above manylinux_2_17") + elif platform.startswith("musllinux_"): + if "musl" not in interpreter: + raise AssertionError(f"{name} is not linked against musl: interpreter={interpreter!r}") + unexpected = sorted( + dependency + for dependency in dependencies + if not dependency.startswith(_ELF_ALLOWED_MUSL_PREFIXES) + ) + if unexpected: + raise AssertionError(f"{name} has unsupported musl dependencies: {unexpected}") + + +def _require_macho_linkage(name: str, data: bytes, platform: str) -> None: + magic = data[:4] + if magic not in { + b"\xfe\xed\xfa\xce", + b"\xce\xfa\xed\xfe", + b"\xfe\xed\xfa\xcf", + b"\xcf\xfa\xed\xfe", + }: + raise AssertionError(f"{name} is not a Mach-O binary for macOS linkage validation") + prefix = ">" if magic in {b"\xfe\xed\xfa\xce", b"\xfe\xed\xfa\xcf"} else "<" + bits = 64 if magic in {b"\xfe\xed\xfa\xcf", b"\xcf\xfa\xed\xfe"} else 32 + dependencies, minimum = _macho_linkage(name, data, prefix, bits) + unexpected = sorted( + dependency + for dependency in dependencies + if not dependency.startswith(("/usr/lib/", "/System/Library/", "@rpath/", "@loader_path/")) + ) + if unexpected: + raise AssertionError(f"{name} has unsupported macOS dependencies: {unexpected}") + expected_floor = (11, 0) if platform.endswith("arm64") else (10, 12) + if minimum is None: + raise AssertionError(f"{name} has no macOS deployment target") + if minimum > expected_floor: + raise AssertionError( + f"{name} targets macOS {minimum[0]}.{minimum[1]}, above {expected_floor[0]}.{expected_floor[1]}" + ) + + +def _require_pe_linkage(name: str, data: bytes) -> None: + if len(data) < 0x40 or data[:2] != b"MZ": + raise AssertionError(f"{name} is not a PE binary for Windows linkage validation") + pe_offset = struct.unpack_from(" len(data) or data[pe_offset : pe_offset + 4] != b"PE\0\0": + raise AssertionError(f"{name} has an invalid PE header") + optional_magic = struct.unpack_from(" str: matches = [n for n in names if n.endswith(f".dist-info/{filename}")] if len(matches) != 1: @@ -289,7 +822,18 @@ def _require_record(zf: zipfile.ZipFile, names: set[str]) -> None: ) -def verify_wheel(path: Path, *, expect_native: Optional[bool]) -> None: +def verify_wheel( + path: Path, + *, + expect_native: Optional[bool], + expect_platform: Optional[str] = None, + required_symbols: Optional[set[str]] = None, + require_linkage: bool = False, +) -> None: + if require_linkage and (expect_native is not True or expect_platform is None): + raise AssertionError( + "linkage validation requires an expected native wheel platform tag" + ) with zipfile.ZipFile(path) as zf: _require_unique_archive_members(zf.infolist()) names = set(zf.namelist()) @@ -398,6 +942,24 @@ def verify_wheel(path: Path, *, expect_native: Optional[bool]) -> None: raise AssertionError("native wheel must set Root-Is-Purelib: false") if any(tag == "py3-none-any" for tag in wheel.tags): raise AssertionError(f"native wheel must not use a pure tag: {wheel.tags}") + if expect_platform is not None or required_symbols: + with zipfile.ZipFile(path) as zf: + native_data = zf.read(native_libs[0]) + if expect_platform is not None: + _require_native_target(native_libs[0], native_data, expect_platform) + if required_symbols: + _require_exported_symbols(native_libs[0], native_data, required_symbols) + if require_linkage: + if expect_platform.startswith(("manylinux_", "musllinux_")): + _require_elf_linkage(native_libs[0], native_data, expect_platform) + elif expect_platform.startswith("macosx_"): + _require_macho_linkage(native_libs[0], native_data, expect_platform) + elif expect_platform.startswith("win_") or expect_platform == "win32": + _require_pe_linkage(native_libs[0], native_data) + else: + raise AssertionError( + f"linkage inspection is not implemented for wheel platform {expect_platform!r}" + ) elif expect_native is False: if native_libs: raise AssertionError( @@ -421,12 +983,33 @@ def main(argv: Optional[list[str]] = None) -> int: group = parser.add_mutually_exclusive_group() group.add_argument("--expect-native", action="store_true") group.add_argument("--expect-pure", action="store_true") + parser.add_argument( + "--expect-platform", + help="validate the native library header against this wheel platform tag", + ) + parser.add_argument( + "--require-symbol", + action="append", + default=[], + help="require an exported native ABI symbol (repeatable)", + ) + parser.add_argument( + "--require-linkage", + action="store_true", + help="validate native dynamic linkage against --expect-platform", + ) args = parser.parse_args(argv) expect_native = True if args.expect_native else False if args.expect_pure else None try: - verify_wheel(args.wheel, expect_native=expect_native) - except (AssertionError, KeyError, zipfile.BadZipFile) as e: + verify_wheel( + args.wheel, + expect_native=expect_native, + expect_platform=args.expect_platform, + required_symbols=set(args.require_symbol), + require_linkage=args.require_linkage, + ) + except (AssertionError, KeyError, struct.error, zipfile.BadZipFile) as e: print(f"wheel verification failed for {args.wheel}: {e}", file=sys.stderr) return 1 print(f"wheel verification OK: {args.wheel}") diff --git a/scripts/wheel_smoke.py b/scripts/wheel_smoke.py new file mode 100644 index 00000000..30b2e809 --- /dev/null +++ b/scripts/wheel_smoke.py @@ -0,0 +1,23 @@ +"""Run the representative native-kernel smoke test against an installed wheel.""" + +import importlib.metadata as metadata + +import numpy as np + +import reflex_xy +import xy.kernels as kernels + + +def main() -> None: + if kernels.BACKEND != "native": + raise RuntimeError(f"expected native backend, got {kernels.BACKEND!r}") + if reflex_xy.__version__ != metadata.version("xy"): + raise RuntimeError("installed reflex_xy and xy versions do not match") + codes, unique = kernels.factorize_fixed(np.asarray(["a", "b", "a"], dtype="S1")) + if codes.tolist() != [0, 1, 0] or unique.tolist() != [0, 1]: + raise RuntimeError(f"unexpected factorize_fixed result: {codes}, {unique}") + print("native", kernels.__file__) + + +if __name__ == "__main__": + main() diff --git a/spec/design-dossier.md b/spec/design-dossier.md index cb5cdaa9..91fef87e 100644 --- a/spec/design-dossier.md +++ b/spec/design-dossier.md @@ -1184,7 +1184,7 @@ hits a source build requiring a Rust toolchain — an instant adoption cliff. from Python with `ctypes`. There is no CPython extension ABI at all, so one `py3-none-` wheel covers every supported Python version on that platform without PyO3 or `abi3`. Wheel matrix in CI, release-blocking: manylinux - (x86_64 + aarch64), macOS (arm64 + x86_64), Windows x86_64. A missing native wheel + (x86_64), macOS (arm64), Windows x86_64. A missing native wheel is a release failure, not an end-user surprise. 2. **The JS/WebGL2 render client as bundled static assets** inside the same wheel — versioned, no CDN dependency (notebooks are often airgapped; §23's CSP rules @@ -1898,8 +1898,8 @@ detail or honesty. ### F1 — Packaging & distribution is unspecified. For Python-only, it's the highest risk. [Critical] **Failure scenario.** `pip install ` must deliver three separately-hard things: -(a) the **native Rust core** as prebuilt wheels across the matrix — manylinux (x86_64 + -aarch64), macOS (arm64 + x86_64), Windows — via a plain C-ABI `cdylib` built by +(a) the **native Rust core** as prebuilt wheels across the runtime-verified matrix — +manylinux x86_64, macOS arm64, and Windows x86_64 — via a plain C-ABI `cdylib` built by Hatchling, so the Python-version cross-product disappears without PyO3 or `abi3`; (b) the compiled **JS/WebGL2 render client** as bundled static assets; (c) a **notebook front-end integration** that injects that client diff --git a/spec/process/production-readiness.md b/spec/process/production-readiness.md index 0a2f00e7..1c86b178 100644 --- a/spec/process/production-readiness.md +++ b/spec/process/production-readiness.md @@ -90,7 +90,7 @@ These must pass before publishing. | Step tier update | A decimated `step` chart keeps its risers after a synthetic kernel `tier_update` replaces the vertex buffers | `python scripts/step_tier_smoke.py ` | | Dashboard reliability | Attempts 10/20/50/60 charts, hard-gates the 10-chart row as loss-free and nonblank, retains partial larger rows, and applies the production shader-cache oracle to a complete, fully nonblank, loss-free 60-chart row | `python benchmarks/bench_dashboard.py --chart-counts 10,20,50,60 --chromium --json dashboard-smoke.json` then `python scripts/verify_benchmark_report.py dashboard-smoke.json --kind dashboard-browser` | | sdist | Build-input-only source archive contains the `xy` and bundled `reflex_xy` packages, JSX/render-client bundles, complete JS/Rust build sources, and `PKG-INFO` version/dependencies (including `Provides-Extra: reflex` and `reflex>=0.9.6` under that marker) matching the archive's own `xy-` root; repository-only material, duplicate/unsafe members, native binaries, and generated junk are absent | `python scripts/verify_sdist.py dist/*.tar.gz` | -| Native wheel | Platform wheel contains package-only `xy` and `reflex_xy` files, exactly one native library, the JSX wrapper but no duplicate render client, `METADATA` version/base dependencies/`reflex` extra matching the wheel's own filename and `.dist-info`, complete hash-checked `RECORD`, public export-surface markers, matching filename/`WHEEL` tags, and is tagged non-pure | `python scripts/verify_wheel.py dist/*.whl --expect-native` | +| Native wheel | Platform wheel contains package-only `xy` and `reflex_xy` files, exactly one native library, the JSX wrapper but no duplicate render client, `METADATA` version/base dependencies/`reflex` extra matching the wheel's own filename and `.dist-info`, complete hash-checked `RECORD`, public export-surface markers, matching filename/`WHEEL` tags, native architecture/ABI/linkage matching its platform tag, and is tagged non-pure | `python scripts/verify_wheel.py dist/*.whl --expect-native --expect-platform --require-symbol xy_abi_version --require-linkage` | | Fallback wheel | No-toolchain wheel contains package-only `xy` and `reflex_xy` files, `METADATA` version/base dependencies/`reflex` extra matching the wheel's own filename and `.dist-info`, complete hash-checked `RECORD`, public export-surface markers, matching filename/`WHEEL` tags, is pure, and contains no native library | `python scripts/verify_wheel.py dist/*.whl --expect-pure` | | Wheel size | Platform wheel remains small enough for notebook installs | CI budget: 15 MB | | Benchmark artifact | JSON benchmark reports carry schema, environment, categories, row status, and finite non-negative metrics; native reports must declare the native backend | `python scripts/verify_benchmark_report.py benchmark.json --kind scatter-vs`; repeat for line, install, core-2D, pyplot-vs-matplotlib, native, interaction, dashboard, and workflow artifacts | @@ -359,13 +359,13 @@ Before tagging a release: - Before the first release after a change to the wheel matrix (new target, cross-compile toolchain, or tagging scheme), manually run the release workflow (`workflow_dispatch`, `dry_run` defaults to `true`) and confirm - every leg of the cross-compile matrix — including the newer aarch64/armv7/ - musllinux/win-arm64 targets and the wasm job — actually builds, since a - target added to the matrix but never exercised in CI is unverified, not - working. -- Confirm CI built and verified native wheels for Linux glibc and musl/Alpine - (x86-64, aarch64, armv7), macOS (x86-64, Apple Silicon), and Windows (x86, x64, - arm64). + every published native target — Linux x86-64, macOS arm64, and Windows x64 — + plus the separate PyEmscripten wheel actually builds and passes its runtime + gate. A target added to the matrix but never exercised in CI is unverified, + not working. +- Confirm CI built and verified native wheels for Linux x86-64, macOS arm64, and + Windows x64. Cross-compiled targets remain out of the published matrix until + matching runtime verification is available. - Confirm the Pyodide/Emscripten wheel passes its runtime load gate, not only its structural wheel check. The tested toolchain is Rust 1.97.0 with `panic=abort`, Emscripten 5.0.3, cibuildwheel 4.1.0, the PEP 783 @@ -387,6 +387,7 @@ Before tagging a release: docs, tests, scripts, benchmarks, examples, native binaries, and generated caches. - Confirm each platform wheel passes `scripts/verify_wheel.py --expect-native` + with its platform tag, required `xy_abi_version` export, and linkage policy, and its install smoke loads `xy.kernels.BACKEND == "native"`. Confirm the fallback `py3-none-any` wheel passes `--expect-pure` and fails compute with the documented native-core error. Wheel diff --git a/tests/test_verify_ci_workflow.py b/tests/test_verify_ci_workflow.py index b2dd88d9..b0849a27 100644 --- a/tests/test_verify_ci_workflow.py +++ b/tests/test_verify_ci_workflow.py @@ -1304,7 +1304,10 @@ def test_release_workflow_rejects_missing_native_wheel_verifier(tmp_path: Path) workflow = Path(".github/workflows/release.yml").read_text(encoding="utf-8") path = tmp_path / "release.yml" path.write_text( - workflow.replace(' python scripts/verify_wheel.py "$whl" --expect-native\n', ""), + workflow.replace( + ' python scripts/verify_wheel.py "$whl" --expect-native --expect-platform "${{ matrix.plat }}" --require-symbol xy_abi_version --require-linkage\n', + "", + ), encoding="utf-8", ) diff --git a/tests/test_verify_wheel.py b/tests/test_verify_wheel.py index e4b8fadc..393708c9 100644 --- a/tests/test_verify_wheel.py +++ b/tests/test_verify_wheel.py @@ -3,6 +3,7 @@ import base64 import hashlib import importlib.util +import struct import sys import zipfile from pathlib import Path @@ -104,6 +105,138 @@ def _load_verify_module(): verify_wheel = _load_verify_module() +@pytest.mark.parametrize( + ("platform", "binary"), + [ + ( + "manylinux_2_17_x86_64", + b"\x7fELF" + bytes([2, 1, 1, 0]) + bytes(10) + struct.pack(" None: + verify_wheel._require_native_target("native", binary, platform) + + +def test_native_binary_header_rejects_wrong_architecture() -> None: + binary = b"\x7fELF" + bytes([2, 1, 1, 0]) + bytes(10) + struct.pack(" None: + binary = b"\x7fELF" + bytes([2, 1, 1, 0]) + bytes(10) + struct.pack(" None: + data = bytearray(1024) + data[:4] = b"\x7fELF" + data[4:8] = bytes([2, 1, 1, 0]) + struct.pack_into(" None: + data = bytearray(48) + data[:4] = b"\xcf\xfa\xed\xfe" + struct.pack_into(" None: + with pytest.raises(AssertionError, match="requires an expected native wheel platform"): + verify_wheel.verify_wheel( + Path("missing.whl"), expect_native=True, require_linkage=True + ) + + +def test_linkage_validation_requires_native_wheel() -> None: + with pytest.raises(AssertionError, match="requires an expected native wheel platform"): + verify_wheel.verify_wheel( + Path("missing.whl"), + expect_native=None, + expect_platform="win_amd64", + require_linkage=True, + ) + + +def _elf_linkage_fixture(interpreter: bytes, dependency: bytes, version: bytes = b"") -> bytes: + data = bytearray(512) + data[:4] = b"\x7fELF" + data[4:8] = bytes([2, 1, 1, 0]) + struct.pack_into(" None: + binary = _elf_linkage_fixture(b"/lib64/ld-linux-x86-64.so.2\0", b"libc.so.6", b"GLIBC_2.17") + + verify_wheel._require_elf_linkage("native", binary, "manylinux_2_17_x86_64") + + with pytest.raises(AssertionError, match="above manylinux_2_17"): + verify_wheel._require_elf_linkage( + "native", + _elf_linkage_fixture(b"/lib64/ld-linux-x86-64.so.2\0", b"libc.so.6", b"GLIBC_2.28"), + "manylinux_2_17_x86_64", + ) + + def _record_hash(data: bytes) -> str: digest = hashlib.sha256(data).digest() return base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii")