From 63cc85af35ed47d8c1d2addd5c51847b612ac725 Mon Sep 17 00:00:00 2001 From: "yanan.zhangyn" Date: Fri, 4 Sep 2026 16:34:51 +0800 Subject: [PATCH] fix(studio): make scheduler requirements uv compatible --- .../studio_release_server/offline_runtime.py | 101 +++++++------ frontend/service/studio_scheduler/deploy.py | 14 +- tests/cli/test_studio_offline_runtime.py | 138 ++++++++++++++++-- .../studio_scheduler/test_scheduler_deploy.py | 16 +- 4 files changed, 204 insertions(+), 65 deletions(-) diff --git a/frontend/service/studio_release_server/offline_runtime.py b/frontend/service/studio_release_server/offline_runtime.py index 471abf5e3..54b51b0e2 100644 --- a/frontend/service/studio_release_server/offline_runtime.py +++ b/frontend/service/studio_release_server/offline_runtime.py @@ -193,17 +193,13 @@ def build_studio_offline_runtime( if not staged_veadk.is_file(): raise ValueError("Studio offline wheelhouse is incomplete.") _pin_runtime_lock_to_wheelhouse(runtime_lock, wheelhouse, staged_veadk) - requirements = ( - "--no-index\n" - f"--find-links ./{STUDIO_RUNTIME_WHEELHOUSE}\n" - "--require-hashes\n" - f"-r ./{STUDIO_RUNTIME_LOCK}\n" - f"./{STUDIO_RUNTIME_WHEELHOUSE}/{staged_veadk.name} " - f"--hash=sha256:{_sha256(staged_veadk)}\n" + requirements = build_studio_offline_requirements( + wheelhouse, + wheel_prefix=f"./{STUDIO_RUNTIME_WHEELHOUSE}/", ) _verify_offline_resolution( package_dir, - staged_veadk, + requirements, uv=uv, environment=build_environment, ) @@ -306,60 +302,73 @@ def _pin_runtime_lock_to_wheelhouse( runtime_lock.write_text("\n".join(locked_lines) + "\n", encoding="utf-8") +def build_studio_offline_requirements( + wheelhouse: Path, + *, + wheel_prefix: str, +) -> str: + """Return a hash-pinned contract without relative index URLs.""" + if ( + not wheel_prefix.startswith("./") + or ".." in Path(wheel_prefix).parts + or "\n" in wheel_prefix + or "\r" in wheel_prefix + ): + raise ValueError("Studio wheel prefix is invalid.") + wheels = sorted(wheelhouse.glob("*.whl")) + if not wheels: + raise ValueError("Studio offline wheelhouse is empty.") + + distributions: set[str] = set() + lines = ["--no-index", "--require-hashes"] + for wheel in wheels: + try: + name, _version, _build, _tags = parse_wheel_filename(wheel.name) + except InvalidWheelFilename as error: + raise ValueError("Studio wheelhouse contains an invalid wheel.") from error + distribution = canonicalize_name(name) + if distribution in distributions: + raise ValueError("Studio wheelhouse contains duplicate distributions.") + distributions.add(distribution) + lines.append(f"{wheel_prefix}{wheel.name} --hash=sha256:{_sha256(wheel)}") + return "\n".join(lines) + "\n" + + def _verify_offline_resolution( package_dir: Path, - veadk_wheel: Path, + requirements: str, *, uv: str, environment: Mapping[str, str], ) -> None: - """Resolve the final bundle once with networking disabled before release.""" + """Resolve the exact stdin contract used by the dependency installer.""" with tempfile.TemporaryDirectory(prefix="veadk_studio_verify_") as tmp: - workspace = Path(tmp) - verification_requirements = workspace / "requirements.txt" - verification_requirements.write_text( - f"--find-links {(package_dir / STUDIO_RUNTIME_WHEELHOUSE).resolve().as_uri()}\n" - "--require-hashes\n" - f"-r {(package_dir / STUDIO_RUNTIME_LOCK).resolve()}\n" - f"{veadk_wheel.resolve().as_uri()} " - f"--hash=sha256:{_sha256(veadk_wheel)}\n", - encoding="utf-8", - ) - resolved = workspace / "resolved" + resolved = Path(tmp) / "resolved" resolved.mkdir() command = [ uv, - "tool", - "run", - "--from", - f"pip=={_PIP_VERSION}", "pip", - "download", - "--disable-pip-version-check", + "install", + "--dry-run", + "--offline", "--no-index", - "--only-binary=:all:", + "--require-hashes", + "--no-python-downloads", + "--python-version", + _PYTHON_VERSION, + "--python-platform", + "x86_64-manylinux_2_28", + "--target", + str(resolved), + "--requirements", + "-", ] - for platform_name in _LINUX_PLATFORMS: - command.extend(("--platform", platform_name)) - command.extend( - ( - "--implementation", - "cp", - "--python-version", - _PYTHON_VERSION, - "--abi", - _PYTHON_ABI, - "--dest", - str(resolved), - "--requirement", - str(verification_requirements), - ) - ) _run( command, cwd=package_dir, environment=environment, failure="Studio offline wheelhouse failed isolated resolution.", + stdin=requirements, ) @@ -377,6 +386,7 @@ def _run( cwd: Path, environment: Mapping[str, str], failure: str, + stdin: str | None = None, ) -> None: try: subprocess.run( @@ -384,7 +394,9 @@ def _run( cwd=cwd, env=dict(environment), check=True, + input=stdin, stdout=subprocess.DEVNULL, + text=stdin is not None, ) except subprocess.CalledProcessError as error: raise ValueError(f"{failure} Exit code: {error.returncode}.") from error @@ -393,5 +405,6 @@ def _run( __all__ = [ "STUDIO_RUNTIME_LOCK", "STUDIO_RUNTIME_WHEELHOUSE", + "build_studio_offline_requirements", "build_studio_offline_runtime", ] diff --git a/frontend/service/studio_scheduler/deploy.py b/frontend/service/studio_scheduler/deploy.py index ad5c19c93..2476f43da 100644 --- a/frontend/service/studio_scheduler/deploy.py +++ b/frontend/service/studio_scheduler/deploy.py @@ -32,6 +32,7 @@ from frontend.service.studio_release_server.offline_runtime import ( STUDIO_RUNTIME_LOCK, STUDIO_RUNTIME_WHEELHOUSE, + build_studio_offline_requirements, ) from .diagnostics import sanitize_diagnostic @@ -260,7 +261,6 @@ def _stage_package(package_root: Path, destination: Path) -> None: requirements = package_root / "requirements.txt" if not requirements.is_file(): raise ValueError("Studio scheduler package is missing requirements.txt") - shutil.copy2(requirements, destination / requirements.name) runtime_lock = package_root / STUDIO_RUNTIME_LOCK if runtime_lock.is_file(): @@ -268,7 +268,17 @@ def _stage_package(package_root: Path, destination: Path) -> None: wheelhouse = package_root / STUDIO_RUNTIME_WHEELHOUSE if wheelhouse.is_dir(): - shutil.copytree(wheelhouse, destination / wheelhouse.name) + for wheel in wheelhouse.glob("*.whl"): + shutil.copy2(wheel, destination / wheel.name) + (destination / requirements.name).write_text( + build_studio_offline_requirements( + destination, + wheel_prefix="./", + ), + encoding="utf-8", + ) + else: + shutil.copy2(requirements, destination / requirements.name) # Preserve compatibility with packages produced before the offline # wheelhouse layout was introduced. diff --git a/tests/cli/test_studio_offline_runtime.py b/tests/cli/test_studio_offline_runtime.py index 6e555bc9d..f1163e69b 100644 --- a/tests/cli/test_studio_offline_runtime.py +++ b/tests/cli/test_studio_offline_runtime.py @@ -13,13 +13,34 @@ # limitations under the License. from pathlib import Path +import shutil import subprocess +from zipfile import ZIP_DEFLATED, ZipFile import pytest from frontend.service.studio_release_server import offline_runtime +def _write_pure_python_wheel(path: Path, *, name: str, version: str) -> None: + distribution = name.replace("-", "_") + dist_info = f"{distribution}-{version}.dist-info" + with ZipFile(path, "w", ZIP_DEFLATED) as wheel: + wheel.writestr(f"{distribution}/__init__.py", "") + wheel.writestr( + f"{dist_info}/METADATA", + f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n", + ) + wheel.writestr( + f"{dist_info}/WHEEL", + "Wheel-Version: 1.0\n" + "Generator: veadk-test\n" + "Root-Is-Purelib: true\n" + "Tag: py3-none-any\n", + ) + wheel.writestr(f"{dist_info}/RECORD", "") + + def test_lock_check_environment_uses_canonical_pypi() -> None: environment = offline_runtime._lock_check_environment( { @@ -101,16 +122,65 @@ def reject_lock(command: list[str], **_kwargs: object) -> None: assert not (tmp_path / "package").exists() +def test_build_offline_requirements_rejects_empty_wheelhouse( + tmp_path: Path, +) -> None: + with pytest.raises(ValueError, match="wheelhouse is empty"): + offline_runtime.build_studio_offline_requirements( + tmp_path, + wheel_prefix="./wheelhouse/", + ) + + +def test_build_offline_requirements_rejects_duplicate_distribution( + tmp_path: Path, +) -> None: + (tmp_path / "example_pkg-1.0-py3-none-any.whl").write_bytes(b"one") + (tmp_path / "example_pkg-2.0-py3-none-any.whl").write_bytes(b"two") + + with pytest.raises(ValueError, match="duplicate distributions"): + offline_runtime.build_studio_offline_requirements( + tmp_path, + wheel_prefix="./wheelhouse/", + ) + + +@pytest.mark.parametrize( + "wheel_prefix", + ( + "wheelhouse/", + "../wheelhouse/", + "./../wheelhouse/", + "./wheelhouse/\n--index-url https://example.invalid/", + "./wheelhouse/\r--index-url https://example.invalid/", + ), +) +def test_build_offline_requirements_rejects_unsafe_prefix( + tmp_path: Path, + wheel_prefix: str, +) -> None: + (tmp_path / "example_pkg-1.0-py3-none-any.whl").write_bytes(b"wheel") + + with pytest.raises(ValueError, match="wheel prefix is invalid"): + offline_runtime.build_studio_offline_requirements( + tmp_path, + wheel_prefix=wheel_prefix, + ) + + def test_build_offline_runtime_creates_local_only_contract( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: + original_run = subprocess.run + uv = shutil.which("uv") + assert uv is not None source_root = tmp_path / "source" source_root.mkdir() (source_root / "uv.lock").write_text("lock", encoding="utf-8") package_dir = tmp_path / "package" veadk_wheel = tmp_path / "veadk_python-1.0-py3-none-any.whl" - veadk_wheel.write_bytes(b"veadk") + _write_pure_python_wheel(veadk_wheel, name="veadk-python", version="1.0") source_archive = tmp_path / "tos-1.0.tar.gz" source_archive.write_bytes(b"source") commands: list[list[str]] = [] @@ -125,12 +195,28 @@ def fake_run(command: list[str], **_kwargs: object) -> subprocess.CompletedProce ) elif "wheel" in command: output = Path(command[command.index("--wheel-dir") + 1]) - (output / "tos-1.0-py3-none-any.whl").write_bytes(b"pure") + _write_pure_python_wheel( + output / "tos-1.0-py3-none-any.whl", + name="tos", + version="1.0", + ) elif "download" in command: output = Path(command[command.index("--dest") + 1]) - (output / "dependency-1-py3-none-any.whl").write_bytes(b"wheel") - (output / "linux_only-2-py3-none-any.whl").write_bytes(b"wheel") - (output / "tos-1-py3-none-any.whl").write_bytes(b"pure") + _write_pure_python_wheel( + output / "dependency-1-py3-none-any.whl", + name="dependency", + version="1", + ) + _write_pure_python_wheel( + output / "linux_only-2-py3-none-any.whl", + name="linux-only", + version="2", + ) + _write_pure_python_wheel( + output / "tos-1-py3-none-any.whl", + name="tos", + version="1", + ) return subprocess.CompletedProcess(command, 0) monkeypatch.setattr( @@ -146,27 +232,49 @@ def fake_run(command: list[str], **_kwargs: object) -> subprocess.CompletedProce environment={"PATH": "/usr/bin"}, ) - assert requirements == ( - "--no-index\n" - "--find-links ./wheelhouse\n" - "--require-hashes\n" - "-r ./studio-runtime.lock\n" - "./wheelhouse/veadk_python-1.0-py3-none-any.whl " - "--hash=sha256:62ee185cf74a591e7d1b3d2dbe3f389f92c893966e2fd39a3b92c19fc7fcd9c1\n" + staged_veadk = package_dir / "wheelhouse" / veadk_wheel.name + expected_wheels = sorted((package_dir / "wheelhouse").glob("*.whl")) + assert requirements == "--no-index\n--require-hashes\n" + "".join( + f"./wheelhouse/{wheel.name} --hash=sha256:{offline_runtime._sha256(wheel)}\n" + for wheel in expected_wheels ) + assert "--find-links" not in requirements + assert "-r " not in requirements runtime_lock = (package_dir / "studio-runtime.lock").read_text(encoding="utf-8") assert runtime_lock.count("--hash=sha256:") == 3 assert "dependency==1 --hash=sha256:" in runtime_lock assert "linux-only==2 --hash=sha256:" in runtime_lock assert "tos==1 --hash=sha256:" in runtime_lock - assert (package_dir / "wheelhouse" / veadk_wheel.name).read_bytes() == b"veadk" + assert staged_veadk.is_file() assert commands[0][1:] == ["lock", "--check"] download = next(command for command in commands if "download" in command) assert "--only-binary=:all:" in download assert "manylinux_2_17_x86_64" in download assert "--no-index" not in download - verification = [command for command in commands if "download" in command][-1] - assert "--no-index" in verification + verification = commands[-1] + assert verification[1:3] == ["pip", "install"] + assert verification[-2:] == ["--requirements", "-"] + + platform_parse = original_run( + [ + uv, + "pip", + "install", + "--dry-run", + "--no-deps", + "--no-python-downloads", + "--target", + str(tmp_path / "platform-target"), + "--requirements", + "-", + ], + cwd=package_dir, + input=requirements, + text=True, + capture_output=True, + check=False, + ) + assert platform_parse.returncode == 0, platform_parse.stderr def test_build_offline_runtime_rejects_native_source_wheel( diff --git a/tests/frontend/service/studio_scheduler/test_scheduler_deploy.py b/tests/frontend/service/studio_scheduler/test_scheduler_deploy.py index ddc2da3fa..4572b9d7c 100644 --- a/tests/frontend/service/studio_scheduler/test_scheduler_deploy.py +++ b/tests/frontend/service/studio_scheduler/test_scheduler_deploy.py @@ -14,6 +14,7 @@ from __future__ import annotations +from hashlib import sha256 from pathlib import Path from types import SimpleNamespace from typing import Any @@ -206,11 +207,18 @@ def test_stage_package_preserves_offline_runtime_dependencies( _stage_package(package_root, destination) - assert (destination / "requirements.txt").is_file() + expected_hash = sha256(b"dependency").hexdigest() + assert (destination / "requirements.txt").read_text(encoding="utf-8") == ( + "--no-index\n" + "--require-hashes\n" + "./fastapi-1.0-py3-none-any.whl " + f"--hash=sha256:{expected_hash}\n" + ) assert (destination / "studio-runtime.lock").is_file() - assert ( - destination / "wheelhouse" / "fastapi-1.0-py3-none-any.whl" - ).read_bytes() == b"dependency" + assert not (destination / "wheelhouse").exists() + assert (destination / "fastapi-1.0-py3-none-any.whl").read_bytes() == ( + b"dependency" + ) assert (destination / "veadk_python-1.0-py3-none-any.whl").read_bytes() == b"legacy"