Skip to content

fix(studio): make scheduler requirements compatible with VeFaaS uv #1060

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yaozheng-fang merged 1 commit into from
Sep 4, 2026
Merged

fix(studio): make scheduler requirements compatible with VeFaaS uv #1060

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 57 additions & 44 deletions frontend/service/studio_release_server/offline_runtime.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,17 +193,13 @@ def build_studio_offline_runtime(
if not staged_veadk.is_file():
raise ValueError("Studio offline wheelhouse is incomplete.")
_pin_runtime_lock_to_wheelhouse(runtime_lock, wheelhouse, staged_veadk)
requirements = (
"--no-index\n"
f"--find-links ./{STUDIO_RUNTIME_WHEELHOUSE}\n"
"--require-hashes\n"
f"-r ./{STUDIO_RUNTIME_LOCK}\n"
f"./{STUDIO_RUNTIME_WHEELHOUSE}/{staged_veadk.name} "
f"--hash=sha256:{_sha256(staged_veadk)}\n"
requirements = build_studio_offline_requirements(
wheelhouse,
wheel_prefix=f"./{STUDIO_RUNTIME_WHEELHOUSE}/",
)
_verify_offline_resolution(
package_dir,
staged_veadk,
requirements,
uv=uv,
environment=build_environment,
)
Expand Down Expand Up @@ -306,60 +302,73 @@ def _pin_runtime_lock_to_wheelhouse(
runtime_lock.write_text("\n".join(locked_lines) + "\n", encoding="utf-8")


def build_studio_offline_requirements(
wheelhouse: Path,
*,
wheel_prefix: str,
) -> str:
"""Return a hash-pinned contract without relative index URLs."""
if (
not wheel_prefix.startswith("./")
or ".." in Path(wheel_prefix).parts
or "\n" in wheel_prefix
or "\r" in wheel_prefix
):
raise ValueError("Studio wheel prefix is invalid.")
wheels = sorted(wheelhouse.glob("*.whl"))
if not wheels:
raise ValueError("Studio offline wheelhouse is empty.")

distributions: set[str] = set()
lines = ["--no-index", "--require-hashes"]
for wheel in wheels:
try:
name, _version, _build, _tags = parse_wheel_filename(wheel.name)
except InvalidWheelFilename as error:
raise ValueError("Studio wheelhouse contains an invalid wheel.") from error
distribution = canonicalize_name(name)
if distribution in distributions:
raise ValueError("Studio wheelhouse contains duplicate distributions.")
distributions.add(distribution)
lines.append(f"{wheel_prefix}{wheel.name} --hash=sha256:{_sha256(wheel)}")
return "\n".join(lines) + "\n"


def _verify_offline_resolution(
package_dir: Path,
veadk_wheel: Path,
requirements: str,
*,
uv: str,
environment: Mapping[str, str],
) -> None:
"""Resolve the final bundle once with networking disabled before release."""
"""Resolve the exact stdin contract used by the dependency installer."""
with tempfile.TemporaryDirectory(prefix="veadk_studio_verify_") as tmp:
workspace = Path(tmp)
verification_requirements = workspace / "requirements.txt"
verification_requirements.write_text(
f"--find-links {(package_dir / STUDIO_RUNTIME_WHEELHOUSE).resolve().as_uri()}\n"
"--require-hashes\n"
f"-r {(package_dir / STUDIO_RUNTIME_LOCK).resolve()}\n"
f"{veadk_wheel.resolve().as_uri()} "
f"--hash=sha256:{_sha256(veadk_wheel)}\n",
encoding="utf-8",
)
resolved = workspace / "resolved"
resolved = Path(tmp) / "resolved"
resolved.mkdir()
command = [
uv,
"tool",
"run",
"--from",
f"pip=={_PIP_VERSION}",
"pip",
"download",
"--disable-pip-version-check",
"install",
"--dry-run",
"--offline",
"--no-index",
"--only-binary=:all:",
"--require-hashes",
"--no-python-downloads",
"--python-version",
_PYTHON_VERSION,
"--python-platform",
"x86_64-manylinux_2_28",
"--target",
str(resolved),
"--requirements",
"-",
]
for platform_name in _LINUX_PLATFORMS:
command.extend(("--platform", platform_name))
command.extend(
(
"--implementation",
"cp",
"--python-version",
_PYTHON_VERSION,
"--abi",
_PYTHON_ABI,
"--dest",
str(resolved),
"--requirement",
str(verification_requirements),
)
)
_run(
command,
cwd=package_dir,
environment=environment,
failure="Studio offline wheelhouse failed isolated resolution.",
stdin=requirements,
)


Expand All @@ -377,14 +386,17 @@ def _run(
cwd: Path,
environment: Mapping[str, str],
failure: str,
stdin: str | None = None,
) -> None:
try:
subprocess.run(
command,
cwd=cwd,
env=dict(environment),
check=True,
input=stdin,
stdout=subprocess.DEVNULL,
text=stdin is not None,
)
except subprocess.CalledProcessError as error:
raise ValueError(f"{failure} Exit code: {error.returncode}.") from error
Expand All @@ -393,5 +405,6 @@ def _run(
__all__ = [
"STUDIO_RUNTIME_LOCK",
"STUDIO_RUNTIME_WHEELHOUSE",
"build_studio_offline_requirements",
"build_studio_offline_runtime",
]
14 changes: 12 additions & 2 deletions frontend/service/studio_scheduler/deploy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
from frontend.service.studio_release_server.offline_runtime import (
STUDIO_RUNTIME_LOCK,
STUDIO_RUNTIME_WHEELHOUSE,
build_studio_offline_requirements,
)

from .diagnostics import sanitize_diagnostic
Expand Down Expand Up @@ -260,15 +261,24 @@ def _stage_package(package_root: Path, destination: Path) -> None:
requirements = package_root / "requirements.txt"
if not requirements.is_file():
raise ValueError("Studio scheduler package is missing requirements.txt")
shutil.copy2(requirements, destination / requirements.name)

runtime_lock = package_root / STUDIO_RUNTIME_LOCK
if runtime_lock.is_file():
shutil.copy2(runtime_lock, destination / runtime_lock.name)

wheelhouse = package_root / STUDIO_RUNTIME_WHEELHOUSE
if wheelhouse.is_dir():
shutil.copytree(wheelhouse, destination / wheelhouse.name)
for wheel in wheelhouse.glob("*.whl"):
shutil.copy2(wheel, destination / wheel.name)
(destination / requirements.name).write_text(
build_studio_offline_requirements(
destination,
wheel_prefix="./",
),
encoding="utf-8",
)
else:
shutil.copy2(requirements, destination / requirements.name)

# Preserve compatibility with packages produced before the offline
# wheelhouse layout was introduced.
Expand Down
138 changes: 123 additions & 15 deletions tests/cli/test_studio_offline_runtime.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,34 @@
# limitations under the License.

from pathlib import Path
import shutil
import subprocess
from zipfile import ZIP_DEFLATED, ZipFile

import pytest

from frontend.service.studio_release_server import offline_runtime


def _write_pure_python_wheel(path: Path, *, name: str, version: str) -> None:
distribution = name.replace("-", "_")
dist_info = f"{distribution}-{version}.dist-info"
with ZipFile(path, "w", ZIP_DEFLATED) as wheel:
wheel.writestr(f"{distribution}/__init__.py", "")
wheel.writestr(
f"{dist_info}/METADATA",
f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n",
)
wheel.writestr(
f"{dist_info}/WHEEL",
"Wheel-Version: 1.0\n"
"Generator: veadk-test\n"
"Root-Is-Purelib: true\n"
"Tag: py3-none-any\n",
)
wheel.writestr(f"{dist_info}/RECORD", "")


def test_lock_check_environment_uses_canonical_pypi() -> None:
environment = offline_runtime._lock_check_environment(
{
Expand Down Expand Up @@ -101,16 +122,65 @@ def reject_lock(command: list[str], **_kwargs: object) -> None:
assert not (tmp_path / "package").exists()


def test_build_offline_requirements_rejects_empty_wheelhouse(
tmp_path: Path,
) -> None:
with pytest.raises(ValueError, match="wheelhouse is empty"):
offline_runtime.build_studio_offline_requirements(
tmp_path,
wheel_prefix="./wheelhouse/",
)


def test_build_offline_requirements_rejects_duplicate_distribution(
tmp_path: Path,
) -> None:
(tmp_path / "example_pkg-1.0-py3-none-any.whl").write_bytes(b"one")
(tmp_path / "example_pkg-2.0-py3-none-any.whl").write_bytes(b"two")

with pytest.raises(ValueError, match="duplicate distributions"):
offline_runtime.build_studio_offline_requirements(
tmp_path,
wheel_prefix="./wheelhouse/",
)


@pytest.mark.parametrize(
"wheel_prefix",
(
"wheelhouse/",
"../wheelhouse/",
"./../wheelhouse/",
"./wheelhouse/\n--index-url https://example.invalid/",
"./wheelhouse/\r--index-url https://example.invalid/",
),
)
def test_build_offline_requirements_rejects_unsafe_prefix(
tmp_path: Path,
wheel_prefix: str,
) -> None:
(tmp_path / "example_pkg-1.0-py3-none-any.whl").write_bytes(b"wheel")

with pytest.raises(ValueError, match="wheel prefix is invalid"):
offline_runtime.build_studio_offline_requirements(
tmp_path,
wheel_prefix=wheel_prefix,
)


def test_build_offline_runtime_creates_local_only_contract(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
original_run = subprocess.run
uv = shutil.which("uv")
assert uv is not None
source_root = tmp_path / "source"
source_root.mkdir()
(source_root / "uv.lock").write_text("lock", encoding="utf-8")
package_dir = tmp_path / "package"
veadk_wheel = tmp_path / "veadk_python-1.0-py3-none-any.whl"
veadk_wheel.write_bytes(b"veadk")
_write_pure_python_wheel(veadk_wheel, name="veadk-python", version="1.0")
source_archive = tmp_path / "tos-1.0.tar.gz"
source_archive.write_bytes(b"source")
commands: list[list[str]] = []
Expand All @@ -125,12 +195,28 @@ def fake_run(command: list[str], **_kwargs: object) -> subprocess.CompletedProce
)
elif "wheel" in command:
output = Path(command[command.index("--wheel-dir") + 1])
(output / "tos-1.0-py3-none-any.whl").write_bytes(b"pure")
_write_pure_python_wheel(
output / "tos-1.0-py3-none-any.whl",
name="tos",
version="1.0",
)
elif "download" in command:
output = Path(command[command.index("--dest") + 1])
(output / "dependency-1-py3-none-any.whl").write_bytes(b"wheel")
(output / "linux_only-2-py3-none-any.whl").write_bytes(b"wheel")
(output / "tos-1-py3-none-any.whl").write_bytes(b"pure")
_write_pure_python_wheel(
output / "dependency-1-py3-none-any.whl",
name="dependency",
version="1",
)
_write_pure_python_wheel(
output / "linux_only-2-py3-none-any.whl",
name="linux-only",
version="2",
)
_write_pure_python_wheel(
output / "tos-1-py3-none-any.whl",
name="tos",
version="1",
)
return subprocess.CompletedProcess(command, 0)

monkeypatch.setattr(
Expand All @@ -146,27 +232,49 @@ def fake_run(command: list[str], **_kwargs: object) -> subprocess.CompletedProce
environment={"PATH": "/usr/bin"},
)

assert requirements == (
"--no-index\n"
"--find-links ./wheelhouse\n"
"--require-hashes\n"
"-r ./studio-runtime.lock\n"
"./wheelhouse/veadk_python-1.0-py3-none-any.whl "
"--hash=sha256:62ee185cf74a591e7d1b3d2dbe3f389f92c893966e2fd39a3b92c19fc7fcd9c1\n"
staged_veadk = package_dir / "wheelhouse" / veadk_wheel.name
expected_wheels = sorted((package_dir / "wheelhouse").glob("*.whl"))
assert requirements == "--no-index\n--require-hashes\n" + "".join(
f"./wheelhouse/{wheel.name} --hash=sha256:{offline_runtime._sha256(wheel)}\n"
for wheel in expected_wheels
)
assert "--find-links" not in requirements
assert "-r " not in requirements
runtime_lock = (package_dir / "studio-runtime.lock").read_text(encoding="utf-8")
assert runtime_lock.count("--hash=sha256:") == 3
assert "dependency==1 --hash=sha256:" in runtime_lock
assert "linux-only==2 --hash=sha256:" in runtime_lock
assert "tos==1 --hash=sha256:" in runtime_lock
assert (package_dir / "wheelhouse" / veadk_wheel.name).read_bytes() == b"veadk"
assert staged_veadk.is_file()
assert commands[0][1:] == ["lock", "--check"]
download = next(command for command in commands if "download" in command)
assert "--only-binary=:all:" in download
assert "manylinux_2_17_x86_64" in download
assert "--no-index" not in download
verification = [command for command in commands if "download" in command][-1]
assert "--no-index" in verification
verification = commands[-1]
assert verification[1:3] == ["pip", "install"]
assert verification[-2:] == ["--requirements", "-"]

platform_parse = original_run(
[
uv,
"pip",
"install",
"--dry-run",
"--no-deps",
"--no-python-downloads",
"--target",
str(tmp_path / "platform-target"),
"--requirements",
"-",
],
cwd=package_dir,
input=requirements,
text=True,
capture_output=True,
check=False,
)
assert platform_parse.returncode == 0, platform_parse.stderr


def test_build_offline_runtime_rejects_native_source_wheel(
Expand Down
Loading
Loading