From c9fa497e61b6ba8086abf1deaf9fa131a1875f83 Mon Sep 17 00:00:00 2001 From: "yanan.zhangyn" Date: Fri, 4 Sep 2026 09:51:35 +0800 Subject: [PATCH 1/3] feat(studio): add provider-local thin releases --- .github/workflows/publish-studio-release.yaml | 7 + .../service/studio_release_server/builder.py | 9 + .../service/studio_release_server/deploy.py | 165 ++++ .../service/studio_release_server/models.py | 19 +- .../studio_release_server/publisher.py | 872 +++++++++++++++++- .../studio_release_server/requirements.txt | 1 + .../runtime-wheel-requirements.txt | 1 + tests/cli/test_studio_artifacts.py | 157 ++++ tests/cli/test_studio_companion.py | 83 +- tests/cli/test_studio_release.py | 131 ++- tests/cli/test_studio_self_update.py | 247 +++++ tests/test_studio_release_server.py | 820 +++++++++++++++- veadk/cli/agentkit_cli.py | 3 +- veadk/cli/studio_artifacts.py | 643 +++++++++++++ veadk/cli/studio_companion.py | 45 +- veadk/cli/studio_package.py | 12 +- veadk/cli/studio_release.py | 235 ++++- veadk/cli/studio_self_update.py | 68 +- 18 files changed, 3434 insertions(+), 84 deletions(-) create mode 100644 tests/cli/test_studio_artifacts.py create mode 100644 veadk/cli/studio_artifacts.py diff --git a/.github/workflows/publish-studio-release.yaml b/.github/workflows/publish-studio-release.yaml index 29746e5a0..965ce49a9 100644 --- a/.github/workflows/publish-studio-release.yaml +++ b/.github/workflows/publish-studio-release.yaml @@ -21,6 +21,11 @@ on: description: User-facing Studio release summary required: true type: string + thin_bundles: + description: Publish provider-local thin bundles (requires server opt-in) + required: true + default: false + type: boolean pull_request: paths: - '.github/workflows/publish-studio-release.yaml' @@ -254,6 +259,7 @@ jobs: RELEASE_SERVER_URL: ${{ secrets[matrix.url_secret] }} RELEASE_SERVER_API_KEY: ${{ secrets[matrix.key_secret] }} RELEASE_VERSION: ${{ needs.release-context.outputs.version }} + RELEASE_THIN_BUNDLES: ${{ inputs.thin_bundles }} steps: - name: Validate release server configuration @@ -303,6 +309,7 @@ jobs: "requestId": job_id, "version": os.environ["RELEASE_VERSION"], "changelog": [os.environ["STUDIO_CHANGELOG"]], + "thinBundle": os.environ["RELEASE_THIN_BUNDLES"].lower() == "true", } payload = json.dumps(payload_data).encode() request = urllib.request.Request( diff --git a/frontend/service/studio_release_server/builder.py b/frontend/service/studio_release_server/builder.py index 2a0956eda..9deaa1427 100644 --- a/frontend/service/studio_release_server/builder.py +++ b/frontend/service/studio_release_server/builder.py @@ -81,6 +81,7 @@ ) _NODE_HEAP_MEMORY_RATIO = 0.75 _DEFAULT_NODE_HEAP_MB = 4096 +_STUDIO_RELEASE_CONTRACT = "agentkit-cli-v1" logger = logging.getLogger(__name__) @@ -607,6 +608,10 @@ def _run_publisher( frontend_assets: Path | None, dependency_wheels: Path | None, ) -> None: + if request.thin_bundle and not self._settings.thin_releases: + raise RuntimeError( + "Studio thin releases are not enabled on this Release Server." + ) credentials = resolve_credentials(self._settings.provider) command = [ sys.executable, @@ -627,7 +632,11 @@ def _run_publisher( self._settings.provider, "--prefix", self._settings.release_prefix, + "--release-contract", + _STUDIO_RELEASE_CONTRACT, ] + if request.thin_bundle: + command.append("--thin") for item in request.changelog: command.extend(("--changelog", item)) if frontend_assets is not None: diff --git a/frontend/service/studio_release_server/deploy.py b/frontend/service/studio_release_server/deploy.py index fae25bab2..47d93a2d9 100644 --- a/frontend/service/studio_release_server/deploy.py +++ b/frontend/service/studio_release_server/deploy.py @@ -33,6 +33,11 @@ from typing import Any from veadk.cloud.cloud_agent_engine import CloudAgentEngine +from veadk.cli.studio_artifacts import ( + STUDIO_ARTIFACT_BUCKETS, + STUDIO_ARTIFACT_PREFIX, + STUDIO_ARTIFACT_REGIONS, +) from veadk.utils.cloud_provider import ( CloudProvider, default_region, @@ -54,6 +59,7 @@ _REPOSITORY = "volcengine/veadk-python" _ROLE_NAME = "VeADKStudioReleaseServerRole" _POLICY_NAME = "VeADKStudioReleaseServerPolicy" +_PUBLIC_ARTIFACT_POLICY_SID = "PublicReadStudioRuntimeArtifacts" _NODE_VERSION = "22.17.0" _NODE_ARCHIVE_NAME = f"node-v{_NODE_VERSION}-linux-x64.tar.xz" _NODE_ARCHIVE_URL = ( @@ -278,6 +284,7 @@ def _runtime_environment( bucket: str, provider: CloudProvider, region: str, + thin_bundles: bool = False, ) -> dict[str, str]: return { "STUDIO_RELEASE_SERVER_API_KEY": api_key, @@ -287,6 +294,7 @@ def _runtime_environment( "STUDIO_RELEASE_PREFIX": _RELEASE_PREFIX, "STUDIO_RELEASE_JOB_PREFIX": _JOB_PREFIX, "STUDIO_RELEASE_REPOSITORY": _REPOSITORY, + "STUDIO_RELEASE_THIN_BUNDLES": "true" if thin_bundles else "false", } @@ -425,6 +433,124 @@ def _ensure_release_bucket(client: Any, bucket: str) -> None: ) +def _public_artifact_policy(bucket: str) -> dict[str, object]: + """Allow anonymous reads only below the immutable artifact namespace.""" + + return { + "Statement": [ + { + "Sid": _PUBLIC_ARTIFACT_POLICY_SID, + "Effect": "Allow", + "Principal": "*", + "Action": ["tos:GetObject"], + "Resource": [f"trn:tos:::{bucket}/{STUDIO_ARTIFACT_PREFIX}/*"], + } + ] + } + + +def _is_tos_not_found(error: Exception) -> bool: + return isinstance(error, KeyError) or getattr(error, "status_code", None) == 404 + + +def _merge_public_artifact_policy( + current: object, + bucket: str, +) -> tuple[dict[str, object], bool]: + """Add only the policy statement owned by this deployer.""" + + if current is None: + policy: dict[str, object] = {} + elif isinstance(current, dict): + policy = dict(current) + else: + raise RuntimeError("Studio public artifact bucket policy is invalid.") + raw_statements = policy.get("Statement", []) + if isinstance(raw_statements, dict): + statements = [dict(raw_statements)] + elif isinstance(raw_statements, list) and all( + isinstance(item, dict) for item in raw_statements + ): + statements = [dict(item) for item in raw_statements] + else: + raise RuntimeError("Studio public artifact bucket policy is invalid.") + expected_policy = _public_artifact_policy(bucket) + expected_statements = expected_policy["Statement"] + if not isinstance(expected_statements, list) or not isinstance( + expected_statements[0], dict + ): + raise RuntimeError("Studio public artifact bucket policy is invalid.") + expected = dict(expected_statements[0]) + owned = [ + statement + for statement in statements + if statement.get("Sid") == _PUBLIC_ARTIFACT_POLICY_SID + ] + if len(owned) > 1 or (owned and owned[0] != expected): + raise RuntimeError("Studio public artifact bucket policy has a conflict.") + if owned: + policy["Statement"] = statements + return policy, False + statements.append(expected) + policy["Statement"] = statements + return policy, True + + +def _ensure_do_not_delete_tag(client: Any, bucket: str) -> None: + """Add the marker without replacing unrelated bucket tags.""" + + import tos + + try: + current = list( + getattr(client.get_bucket_tagging(bucket=bucket), "tag_set", []) or [] + ) + except Exception as error: + if not _is_tos_not_found(error): + raise + current = [] + if any(getattr(item, "key", None) == "note" for item in current): + return + client.put_bucket_tagging( + bucket=bucket, + tag_set=[*current, tos.models2.Tag(key="note", value="勿删")], + ) + + +def _ensure_public_artifact_bucket(client: Any, provider: CloudProvider) -> str: + """Create one isolated bucket and merge its prefix-only public-read policy.""" + + bucket = STUDIO_ARTIFACT_BUCKETS[provider] + buckets = list(getattr(client.list_buckets(), "buckets", []) or []) + if not any(getattr(item, "name", "") == bucket for item in buckets): + client.create_bucket(bucket=bucket) + _ensure_do_not_delete_tag(client, bucket) + try: + current = json.loads(client.get_bucket_policy(bucket=bucket).policy) + except Exception as error: + if not _is_tos_not_found(error): + raise RuntimeError( + "Studio public artifact bucket policy lookup failed." + ) from error + current = None + expected, changed = _merge_public_artifact_policy(current, bucket) + if changed: + client.put_bucket_policy( + bucket=bucket, + policy=json.dumps(expected, sort_keys=True), + ) + try: + actual = json.loads(client.get_bucket_policy(bucket=bucket).policy) + except Exception as error: + raise RuntimeError( + "Studio public artifact bucket policy verification failed." + ) from error + verified, _ = _merge_public_artifact_policy(actual, bucket) + if verified != expected: + raise RuntimeError("Studio public artifact bucket policy verification failed.") + return bucket + + def _release_function(service: Any, function_id: str) -> None: from volcenginesdkvefaas import GetReleaseStatusRequest, ReleaseRequest @@ -726,6 +852,7 @@ def _deploy( access_key: str, secret_key: str, session_token: str, + thin_bundles: bool = False, ) -> tuple[str, str, str]: """Create or update the Function and bind it to an existing gateway.""" engine = CloudAgentEngine( @@ -741,6 +868,7 @@ def _deploy( bucket=_release_bucket(provider), provider=provider, region=region, + thin_bundles=thin_bundles, ) with tempfile.TemporaryDirectory(prefix="studio_release_server_") as tmp: deployment_root = Path(tmp) @@ -829,6 +957,16 @@ def _parser() -> argparse.ArgumentParser: action="store_true", help="Deploy without changing repository secrets.", ) + parser.add_argument( + "--provision-public-artifacts-only", + action="store_true", + help="Provision only the isolated public runtime artifact bucket.", + ) + parser.add_argument( + "--enable-thin-bundles", + action="store_true", + help="Enable public-artifact thin Studio bundles (default: disabled).", + ) return parser @@ -846,6 +984,31 @@ def main() -> None: f"{credential_prefix}_ACCESS_KEY and {credential_prefix}_SECRET_KEY " "are required." ) + if args.provision_public_artifacts_only: + artifact_region = STUDIO_ARTIFACT_REGIONS[provider] + bucket = _ensure_public_artifact_bucket( + _tos_client( + access_key, + secret_key, + session_token, + provider=provider, + region=artifact_region, + ), + provider, + ) + print( + json.dumps( + { + "provider": provider, + "region": artifact_region, + "bucket": bucket, + "publicPrefix": STUDIO_ARTIFACT_PREFIX, + "provisioned": True, + }, + ensure_ascii=False, + ) + ) + return if not args.skip_github_secrets: _validate_github_secret_access() _ensure_release_bucket( @@ -874,6 +1037,7 @@ def main() -> None: access_key=access_key, secret_key=secret_key, session_token=session_token, + thin_bundles=args.enable_thin_bundles, ) _wait_for_health(endpoint, api_key) if not args.skip_github_secrets: @@ -890,6 +1054,7 @@ def main() -> None: "functionId": function_id, "endpoint": endpoint, "githubSecretsConfigured": not args.skip_github_secrets, + "thinBundlesEnabled": args.enable_thin_bundles, }, ensure_ascii=False, ) diff --git a/frontend/service/studio_release_server/models.py b/frontend/service/studio_release_server/models.py index b0c88ca05..fa643e976 100644 --- a/frontend/service/studio_release_server/models.py +++ b/frontend/service/studio_release_server/models.py @@ -21,7 +21,7 @@ from dataclasses import dataclass from typing import Literal, cast -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, StrictBool, field_validator _GIT_SHA_PATTERN = re.compile(r"^[0-9a-f]{40}$") _JOB_ID_PATTERN = re.compile(r"^[A-Za-z0-9_.-]{1,100}$") @@ -31,6 +31,18 @@ ReleaseProvider = Literal["volcengine", "byteplus"] +def _strict_env_bool(name: str, *, default: bool = False) -> bool: + value = os.getenv(name) + if value is None or not value.strip(): + return default + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + raise ValueError(f"{name} must be an explicit boolean value.") + + @dataclass(frozen=True) class ReleaseServerSettings: """Runtime settings injected into the VeFaaS Function.""" @@ -42,6 +54,7 @@ class ReleaseServerSettings: job_prefix: str repository: str provider: ReleaseProvider = "volcengine" + thin_releases: bool = False def __post_init__(self) -> None: if len(self.api_key) < 32: @@ -52,6 +65,8 @@ def __post_init__(self) -> None: raise ValueError("STUDIO_RELEASE_REGION is required.") if self.provider not in {"volcengine", "byteplus"}: raise ValueError("STUDIO_RELEASE_PROVIDER is invalid.") + if not isinstance(self.thin_releases, bool): + raise ValueError("STUDIO_RELEASE_THIN_BUNDLES must be boolean.") for value, name in ( (self.release_prefix, "STUDIO_RELEASE_PREFIX"), (self.job_prefix, "STUDIO_RELEASE_JOB_PREFIX"), @@ -84,6 +99,7 @@ def from_env(cls) -> ReleaseServerSettings: ReleaseProvider, os.getenv("STUDIO_RELEASE_PROVIDER", "volcengine").strip(), ), + thin_releases=_strict_env_bool("STUDIO_RELEASE_THIN_BUNDLES"), ) @@ -98,6 +114,7 @@ class ReleaseRequest(BaseModel): changelog: tuple[str, ...] = () source_key: str = Field(default="", alias="sourceKey") version: str = "" + thin_bundle: StrictBool = Field(default=False, alias="thinBundle") @field_validator("repository") @classmethod diff --git a/frontend/service/studio_release_server/publisher.py b/frontend/service/studio_release_server/publisher.py index 8716ac273..21a56b124 100644 --- a/frontend/service/studio_release_server/publisher.py +++ b/frontend/service/studio_release_server/publisher.py @@ -18,26 +18,31 @@ import argparse import hashlib +import importlib.util import json import os import re import shlex import shutil import subprocess +import sys import tempfile +import tomllib +import urllib.parse +import urllib.request import zipfile from collections.abc import Mapping from dataclasses import asdict, dataclass from datetime import datetime +from email.parser import BytesParser +from email.policy import default as email_policy from pathlib import Path -from typing import Any +from typing import Any, cast from zoneinfo import ZoneInfo if __package__: from .offline_runtime import build_studio_offline_runtime else: - import importlib.util - _offline_runtime_path = Path(__file__).with_name("offline_runtime.py") _offline_runtime_spec = importlib.util.spec_from_file_location( "veadk_studio_offline_runtime", @@ -58,6 +63,59 @@ _AGENTKIT_CLI_ARCHIVE_SHA256 = ( "4e76e32c60473b5037c331a7c74bb99b1c23b62eb8ce26379d3a8c41af38a64e" ) +_STUDIO_RELEASE_CONTRACT = "agentkit-cli-v1" +_STUDIO_RUNTIME_MANIFEST = "studio-runtime.json" +_PUBLIC_RUNTIME_LICENSES = frozenset( + { + "0BSD", + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "CC0-1.0", + "CNRI-Python", + "ISC", + "MIT", + "MIT-0", + "MIT-CMU", + "MPL-2.0", + "PSF-2.0", + "Python-2.0", + "Unicode-3.0", + "Unlicense", + "Zlib", + } +) +_PUBLIC_RUNTIME_LICENSE_CLASSIFIERS = { + "License :: OSI Approved :: Apache Software License": "Apache-2.0", + "License :: OSI Approved :: BSD License": "BSD-3-Clause", + "License :: OSI Approved :: ISC License (ISCL)": "ISC", + "License :: OSI Approved :: MIT License": "MIT", + "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)": "MPL-2.0", + "License :: OSI Approved :: Python Software Foundation License": "PSF-2.0", + "License :: OSI Approved :: The Unlicense (Unlicense)": "Unlicense", + "License :: Public Domain": "Unlicense", +} +_PUBLIC_RUNTIME_LEGACY_LICENSES = { + "apache 2 0": "Apache-2.0", + "apache license 2 0": "Apache-2.0", + "apache software license": "Apache-2.0", + "bsd": "BSD-3-Clause", + "bsd 2 clause": "BSD-2-Clause", + "bsd 3 clause": "BSD-3-Clause", + "isc": "ISC", + "mit": "MIT", + "mit license": "MIT", + "mpl 2 0": "MPL-2.0", + "mozilla public license 2 0": "MPL-2.0", + "psf": "PSF-2.0", + "python software foundation license": "PSF-2.0", + "the unlicense": "Unlicense", + "unlicense": "Unlicense", + "zlib": "Zlib", + "3 clause bsd license": "BSD-3-Clause", + "bsd public domain": "BSD-3-Clause OR Unlicense", + "bsd 3 clause apache 2 0 dependency licenses": "BSD-3-Clause OR Apache-2.0", +} class StudioPublisherError(ValueError): @@ -74,6 +132,9 @@ class StudioReleaseManifest: size: int created_at: str changelog: tuple[str, ...] = () + runtime_epoch: str = "" + thin_sha256: str = "" + thin_size: int = 0 def __post_init__(self) -> None: try: @@ -104,6 +165,15 @@ def __post_init__(self) -> None: not item.strip() or len(item) > 240 for item in self.changelog ): raise StudioPublisherError("Studio release changelog is invalid.") + thin_values = (self.runtime_epoch, self.thin_sha256, self.thin_size) + if any(thin_values): + if ( + not _SHA256_PATTERN.fullmatch(self.runtime_epoch) + or not _SHA256_PATTERN.fullmatch(self.thin_sha256) + or self.thin_size <= 0 + or self.thin_size > _MAX_STUDIO_BUNDLE_BYTES + ): + raise StudioPublisherError("Studio thin release metadata is invalid.") @classmethod def from_json(cls, payload: bytes | str) -> StudioReleaseManifest: @@ -116,6 +186,9 @@ def from_json(cls, payload: bytes | str) -> StudioReleaseManifest: size=int(raw["size"]), created_at=str(raw["createdAt"]), changelog=tuple(str(item) for item in raw.get("changelog", [])), + runtime_epoch=str(raw.get("runtimeEpoch", "")), + thin_sha256=str(raw.get("thinSha256", "")), + thin_size=int(raw.get("thinSize", 0) or 0), ) except (json.JSONDecodeError, KeyError, TypeError, ValueError) as error: raise StudioPublisherError("Studio release manifest is invalid.") from error @@ -130,6 +203,14 @@ def to_json(self) -> bytes: "createdAt": data["created_at"], "changelog": list(data["changelog"]), } + if self.runtime_epoch: + payload.update( + { + "runtimeEpoch": self.runtime_epoch, + "thinSha256": self.thin_sha256, + "thinSize": self.thin_size, + } + ) return (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode() @@ -148,6 +229,10 @@ def _manifest_key(prefix: str, version: str) -> str: return f"{prefix}/releases/{version}/manifest.json" +def _thin_bundle_key(prefix: str, version: str) -> str: + return f"{prefix}/releases/{version}/studio-bundle-thin.zip" + + def _latest_key(prefix: str) -> str: return f"{prefix}/latest.json" @@ -199,7 +284,15 @@ def __init__( region=region, ) - def publish(self, bundle: Path, manifest: StudioReleaseManifest) -> None: + def publish( + self, + bundle: Path, + manifest: StudioReleaseManifest, + *, + thin_bundle: Path | None = None, + ) -> None: + """Publish immutable content and repair an identical interrupted attempt.""" + content = bundle.read_bytes() if ( len(content) != manifest.size @@ -208,26 +301,48 @@ def publish(self, bundle: Path, manifest: StudioReleaseManifest) -> None: raise StudioPublisherError( "Studio release bundle does not match its manifest." ) + thin_content: bytes | None = None + if manifest.runtime_epoch: + if thin_bundle is None or not thin_bundle.is_file(): + raise StudioPublisherError("Studio thin release bundle is missing.") + thin_content = thin_bundle.read_bytes() + if ( + len(thin_content) != manifest.thin_size + or hashlib.sha256(thin_content).hexdigest() != manifest.thin_sha256 + ): + raise StudioPublisherError( + "Studio thin bundle does not match manifest." + ) + elif thin_bundle is not None: + raise StudioPublisherError("Studio full release has no thin bundle.") releases = self._existing_releases() - if any(item.version > manifest.version for item in releases): + same_version = [item for item in releases if item.version == manifest.version] + if same_version and same_version != [manifest]: + raise StudioPublisherError("Studio release version has a conflict.") + newer_exists = any(item.version > manifest.version for item in releases) + if newer_exists and not same_version: raise StudioPublisherError( "Studio release version must be newer than the published releases." ) manifest_bytes = manifest.to_json() - self._client.put_object( - bucket=self._bucket, + self._put_immutable( key=_bundle_key(self._prefix, manifest.version), content=content, content_type="application/zip", - forbid_overwrite=True, ) - self._client.put_object( - bucket=self._bucket, + if thin_content is not None: + self._put_immutable( + key=_thin_bundle_key(self._prefix, manifest.version), + content=thin_content, + content_type="application/zip", + ) + self._put_immutable( key=_manifest_key(self._prefix, manifest.version), content=manifest_bytes, content_type="application/json", - forbid_overwrite=True, ) + if newer_exists: + return releases = [item for item in releases if item.version != manifest.version] releases.append(manifest) releases.sort(key=lambda item: item.version, reverse=True) @@ -244,20 +359,81 @@ def publish(self, bundle: Path, manifest: StudioReleaseManifest) -> None: ) + "\n" ).encode() - self._client.put_object( - bucket=self._bucket, + self._put_mutable_verified( key=_catalog_key(self._prefix), content=catalog, content_type="application/json", ) - self._client.put_object( - bucket=self._bucket, + self._put_mutable_verified( key=_latest_key(self._prefix), content=manifest_bytes, content_type="application/json", ) + def _get_optional_object(self, key: str, max_bytes: int) -> bytes | None: + try: + response = self._client.get_object(bucket=self._bucket, key=key) + return _read_object(response, max_bytes) + except Exception as error: + if _is_not_found(error): + return None + raise StudioPublisherError( + "Studio release object lookup failed." + ) from error + + def _put_immutable(self, *, key: str, content: bytes, content_type: str) -> None: + existing = self._get_optional_object(key, _MAX_STUDIO_BUNDLE_BYTES) + if existing is not None: + if existing != content: + raise StudioPublisherError("Studio immutable release object conflicts.") + return + try: + self._client.put_object( + bucket=self._bucket, + key=key, + content=content, + content_type=content_type, + forbid_overwrite=True, + ) + except Exception as error: + existing = self._get_optional_object(key, _MAX_STUDIO_BUNDLE_BYTES) + if existing == content: + return + if existing is not None: + raise StudioPublisherError( + "Studio immutable release object conflicts." + ) from error + raise StudioPublisherError( + "Studio immutable release object upload failed." + ) from error + + def _put_mutable_verified( + self, + *, + key: str, + content: bytes, + content_type: str, + ) -> None: + if self._get_optional_object(key, _MAX_STUDIO_BUNDLE_BYTES) == content: + return + try: + self._client.put_object( + bucket=self._bucket, + key=key, + content=content, + content_type=content_type, + ) + except Exception as error: + if self._get_optional_object(key, _MAX_STUDIO_BUNDLE_BYTES) == content: + return + raise StudioPublisherError( + "Studio release pointer upload failed." + ) from error + if self._get_optional_object(key, _MAX_STUDIO_BUNDLE_BYTES) != content: + raise StudioPublisherError("Studio release pointer verification failed.") + def _existing_releases(self) -> list[StudioReleaseManifest]: + releases: list[StudioReleaseManifest] = [] try: response = self._client.get_object( bucket=self._bucket, @@ -273,7 +449,6 @@ def _existing_releases(self) -> list[StudioReleaseManifest]: StudioReleaseManifest.from_json(json.dumps(item)) for item in raw_releases ] - return sorted(releases, key=lambda item: item.version, reverse=True) except Exception as error: if not _is_not_found(error): raise @@ -282,11 +457,160 @@ def _existing_releases(self) -> list[StudioReleaseManifest]: bucket=self._bucket, key=_latest_key(self._prefix), ) - return [StudioReleaseManifest.from_json(_read_object(response, 64 * 1024))] + latest = StudioReleaseManifest.from_json(_read_object(response, 64 * 1024)) + except Exception as error: + if not _is_not_found(error): + raise + else: + matches = [item for item in releases if item.version == latest.version] + if matches and matches != [latest]: + raise StudioPublisherError( + "Studio release catalog and latest pointer conflict." + ) + if not matches: + releases.append(latest) + return sorted(releases, key=lambda item: item.version, reverse=True) + + +class StudioPublicArtifactStore: + """Publish content-addressed runtime files before a thin release is visible.""" + + def __init__( + self, + *, + contract: Any, + provider: str, + access_key: str, + secret_key: str, + session_token: str, + client: Any | None = None, + public_opener: Any | None = None, + ) -> None: + if provider not in {"volcengine", "byteplus"}: + raise StudioPublisherError("Studio artifact provider is invalid.") + import tos + + self._contract = contract + self._provider = provider + self._bucket = contract.STUDIO_ARTIFACT_BUCKETS[provider] + self._region = contract.STUDIO_ARTIFACT_REGIONS[provider] + domain = "bytepluses.com" if provider == "byteplus" else "volces.com" + self._client = client or tos.TosClientV2( + access_key, + secret_key, + security_token=session_token or None, + endpoint=f"tos-{self._region}.{domain}", + region=self._region, + ) + self._public_opener = public_opener or urllib.request.urlopen + + def publish(self, runtime_manifest: Any, artifact_dir: Path) -> tuple[int, int]: + """Upload missing artifacts, rejecting any immutable-key conflict.""" + + if runtime_manifest.provider != self._provider: + raise StudioPublisherError("Studio artifact provider is invalid.") + created = 0 + reused = 0 + for artifact in runtime_manifest.artifacts: + source = artifact_dir / artifact.filename + if ( + not source.is_file() + or source.stat().st_size != artifact.size + or _sha256_file(source) != artifact.sha256 + ): + raise StudioPublisherError("Studio public artifact is invalid.") + key = self._contract.studio_artifact_key( + artifact.sha256, + artifact.filename, + ) + head = self._head(key) + if head is not None: + self._validate_head(head, artifact) + self._verify_public(artifact) + reused += 1 + continue + try: + self._client.put_object_from_file( + bucket=self._bucket, + key=key, + file_path=str(source), + content_length=artifact.size, + content_sha256=artifact.sha256, + content_type=_artifact_content_type(artifact.filename), + meta={"sha256": artifact.sha256}, + forbid_overwrite=True, + ) + except Exception: + head = self._head(key) + if head is None: + raise StudioPublisherError( + "Studio public artifact upload failed." + ) from None + self._validate_head(head, artifact) + self._verify_public(artifact) + reused += 1 + continue + head = self._head(key) + if head is None: + raise StudioPublisherError( + "Studio public artifact upload could not be verified." + ) + self._validate_head(head, artifact) + self._verify_public(artifact) + created += 1 + return created, reused + + def _head(self, key: str) -> Any | None: + try: + return self._client.head_object(bucket=self._bucket, key=key) except Exception as error: if _is_not_found(error): - return [] - raise + return None + raise StudioPublisherError( + "Studio public artifact lookup failed." + ) from error + + @staticmethod + def _validate_head(head: Any, artifact: Any) -> None: + metadata = dict(getattr(head, "meta", None) or {}) + digest = str(metadata.get("sha256", "") or "").lower() + if ( + int(getattr(head, "content_length", 0) or 0) != artifact.size + or digest != artifact.sha256 + ): + raise StudioPublisherError("Studio public artifact key has a conflict.") + + def _verify_public(self, artifact: Any) -> None: + request = urllib.request.Request(artifact.url, method="HEAD") + try: + with self._public_opener(request, timeout=30) as response: + final_url = str(getattr(response, "geturl", lambda: artifact.url)()) + length = int(response.headers.get("Content-Length", 0) or 0) + status = int(getattr(response, "status", 200) or 200) + except Exception as error: + raise StudioPublisherError( + "Studio public artifact is not anonymously readable." + ) from error + if final_url != artifact.url or status != 200 or length != artifact.size: + raise StudioPublisherError( + "Studio public artifact anonymous-read verification failed." + ) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while chunk := source.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _artifact_content_type(filename: str) -> str: + if filename.endswith(".whl"): + return "application/zip" + if filename.endswith(".tar.gz"): + return "application/gzip" + return "application/octet-stream" def _validate_source_checkout(source_root: Path) -> None: @@ -448,6 +772,54 @@ def validate_studio_agentkit_cli_archive(artifacts: list[Path]) -> Path: return archive +def ensure_studio_bundle_agentkit_cli( + bundle: Path, + dependency_wheels: Path, +) -> None: + """Fail closed on a bad CLI and repair a missing archive before publish.""" + + try: + with zipfile.ZipFile(bundle, "r") as archive: + candidates = [ + name for name in archive.namelist() if name == _AGENTKIT_CLI_ARCHIVE + ] + if len(candidates) > 1: + raise StudioPublisherError( + "The Studio release contains duplicate AgentKit CLI archives." + ) + if candidates: + digest = hashlib.sha256(archive.read(candidates[0])).hexdigest() + if digest != _AGENTKIT_CLI_ARCHIVE_SHA256: + raise StudioPublisherError( + "The Studio release AgentKit CLI archive checksum is invalid." + ) + return + except (OSError, zipfile.BadZipFile) as error: + raise StudioPublisherError("Built Studio release bundle is invalid.") from error + + cli_archive = validate_studio_agentkit_cli_archive( + list(dependency_wheels.iterdir()) + ) + try: + with zipfile.ZipFile(bundle, "a") as archive: + archive.write( + cli_archive, + _AGENTKIT_CLI_ARCHIVE, + compress_type=zipfile.ZIP_STORED, + ) + except (OSError, zipfile.BadZipFile) as error: + raise StudioPublisherError( + "Could not add the AgentKit CLI archive to the Studio release." + ) from error + + with zipfile.ZipFile(bundle, "r") as archive: + digest = hashlib.sha256(archive.read(_AGENTKIT_CLI_ARCHIVE)).hexdigest() + if digest != _AGENTKIT_CLI_ARCHIVE_SHA256: + raise StudioPublisherError( + "The Studio release AgentKit CLI archive checksum is invalid." + ) + + def validate_studio_bundle_dependencies(package_dir: Path) -> Path: """Validate the local VeADK/CLI dependency pair in an extracted bundle.""" requirements_path = package_dir / "requirements.txt" @@ -457,6 +829,61 @@ def validate_studio_bundle_dependencies(package_dir: Path) -> Path: raise StudioPublisherError( "Studio release requirements are unavailable." ) from error + runtime_manifest_path = package_dir / _STUDIO_RUNTIME_MANIFEST + if runtime_manifest_path.is_file(): + try: + from veadk.cli.studio_artifacts import StudioRuntimeManifest + + runtime_manifest = StudioRuntimeManifest.from_json( + runtime_manifest_path.read_bytes() + ) + except (ImportError, OSError, ValueError) as error: + raise StudioPublisherError("Studio runtime manifest is invalid.") from error + cli_artifact = runtime_manifest.agentkit_cli() + remote_veadk_wheels = [ + item + for item in runtime_manifest.artifacts + if item.kind == "wheel" + and item.filename.startswith(("veadk_python-", "veadk-python-")) + ] + local_veadk_wheels = sorted(package_dir.glob("veadk*.whl")) + expected_requirements = runtime_manifest.remote_requirements() + if len(local_veadk_wheels) == 1: + local_veadk = local_veadk_wheels[0] + expected_requirements += ( + f"./{local_veadk.name} --hash=sha256:{_sha256_file(local_veadk)}\n" + ) + bundled_wheelhouse = package_dir / "bundled-wheelhouse" + bundled_files = ( + sorted(path for path in bundled_wheelhouse.iterdir() if path.is_file()) + if bundled_wheelhouse.is_dir() + else [] + ) + bundled_valid = len(bundled_files) == len(runtime_manifest.bundled_artifacts) + if bundled_valid: + expected_bundled = { + item.filename: item for item in runtime_manifest.bundled_artifacts + } + bundled_valid = all( + path.name in expected_bundled + and path.stat().st_size == expected_bundled[path.name].size + and _sha256_file(path) == expected_bundled[path.name].sha256 + for path in bundled_files + ) + if ( + cli_artifact.filename != _AGENTKIT_CLI_ARCHIVE + or cli_artifact.sha256 != _AGENTKIT_CLI_ARCHIVE_SHA256 + or remote_veadk_wheels + or len(local_veadk_wheels) != 1 + or requirements_path.read_text(encoding="utf-8") != expected_requirements + or (package_dir / "wheelhouse").exists() + or (package_dir / _AGENTKIT_CLI_ARCHIVE).exists() + or not bundled_valid + ): + raise StudioPublisherError( + "Studio thin release dependency contract is invalid." + ) + return runtime_manifest_path local_wheels: list[Path] = [] for raw_line in lines: line = raw_line.strip() @@ -539,7 +966,15 @@ def _build_local_requirements( raise StudioPublisherError(str(error)) from error -def _studio_run_script() -> str: +def _studio_run_script(*, thin: bool = False) -> str: + companion = ( + "python3 -m veadk.cli.studio_companion " + f'--runtime-manifest "$ROOT_DIR/{_STUDIO_RUNTIME_MANIFEST}" ' + '--provider "${CLOUD_PROVIDER:-${AGENTKIT_CLOUD_PROVIDER:-volcengine}}"\n' + if thin + else "python3 -m veadk.cli.studio_companion " + f'--archive "$ROOT_DIR/{_AGENTKIT_CLI_ARCHIVE}"\n' + ) return ( "#!/bin/bash\n" "set -ex\n" @@ -549,8 +984,7 @@ def _studio_run_script() -> str: "HOST=0.0.0.0\n" "PORT=${_FAAS_RUNTIME_PORT:-8000}\n" 'export PYTHONPATH="./site-packages${PYTHONPATH:+:$PYTHONPATH}"\n' - "python3 -m veadk.cli.studio_companion " - f'--archive "$ROOT_DIR/{_AGENTKIT_CLI_ARCHIVE}"\n' + f"{companion}" "exec python3 -m veadk.cli.cli studio " '--provider "${CLOUD_PROVIDER:-${AGENTKIT_CLOUD_PROVIDER:-volcengine}}" ' "--auth-mode frontend " @@ -558,6 +992,331 @@ def _studio_run_script() -> str: ) +def _load_studio_artifact_contract(source_root: Path) -> Any: + contract_path = source_root / "veadk" / "cli" / "studio_artifacts.py" + if not contract_path.is_file(): + raise StudioPublisherError("Studio artifact contract is missing.") + module_name = "veadk_studio_artifact_contract" + spec = importlib.util.spec_from_file_location(module_name, contract_path) + if spec is None or spec.loader is None: + raise StudioPublisherError("Studio artifact contract is unavailable.") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + except Exception as error: + raise StudioPublisherError("Studio artifact contract is invalid.") from error + return module + + +def stage_studio_thin_runtime( + source_root: Path, + package_dir: Path, + output_dir: Path, + *, + provider: str, +) -> tuple[str, Path]: + """Replace local runtime payloads with one exact public artifact manifest.""" + + contract = _load_studio_artifact_contract(source_root) + wheelhouse = package_dir / "wheelhouse" + wheels = sorted(wheelhouse.glob("*.whl")) + runtime_veadk_wheels = [ + path + for path in wheels + if path.name.startswith(("veadk_python-", "veadk-python-")) + ] + dependency_wheels = [path for path in wheels if path not in runtime_veadk_wheels] + local_veadk_wheels = sorted(package_dir.glob("veadk*.whl")) + if len(runtime_veadk_wheels) == 1 and not local_veadk_wheels: + local_veadk = package_dir / runtime_veadk_wheels[0].name + shutil.copy2(runtime_veadk_wheels[0], local_veadk) + local_veadk_wheels = [local_veadk] + cli_archive = package_dir / _AGENTKIT_CLI_ARCHIVE + if ( + not dependency_wheels + or len(runtime_veadk_wheels) != 1 + or len(local_veadk_wheels) != 1 + or _sha256_file(runtime_veadk_wheels[0]) != _sha256_file(local_veadk_wheels[0]) + or not cli_archive.is_file() + ): + raise StudioPublisherError("Studio offline runtime is incomplete.") + validate_studio_agentkit_cli_archive([cli_archive]) + public_wheels, bundled_wheels = partition_public_runtime_wheels( + source_root, + dependency_wheels, + ) + artifacts = tuple( + contract.StudioArtifact.from_path( + path, + provider=provider, + kind="wheel", + ) + for path in public_wheels + ) + ( + contract.StudioArtifact.from_path( + cli_archive, + provider=provider, + kind="agentkit-cli", + ), + ) + bundled_artifacts = tuple( + contract.StudioBundledArtifact.from_path(path) for path in bundled_wheels + ) + runtime_manifest = contract.StudioRuntimeManifest.create( + provider, + artifacts, + bundled_artifacts, + ) + artifact_dir = output_dir / f"runtime-artifacts-{runtime_manifest.runtime_epoch}" + artifact_dir.mkdir(parents=True, exist_ok=False) + for path in (*public_wheels, cli_archive): + shutil.copy2(path, artifact_dir / path.name) + if bundled_wheels: + bundled_wheelhouse = package_dir / "bundled-wheelhouse" + bundled_wheelhouse.mkdir() + for path in bundled_wheels: + shutil.copy2(path, bundled_wheelhouse / path.name) + manifest_content = runtime_manifest.to_json() + (package_dir / _STUDIO_RUNTIME_MANIFEST).write_bytes(manifest_content) + ( + output_dir / f"runtime-manifest-{runtime_manifest.runtime_epoch}.json" + ).write_bytes(manifest_content) + (package_dir / "requirements.txt").write_text( + runtime_manifest.remote_requirements() + + f"./{local_veadk_wheels[0].name} " + + f"--hash=sha256:{_sha256_file(local_veadk_wheels[0])}\n", + encoding="utf-8", + ) + shutil.rmtree(wheelhouse) + cli_archive.unlink() + (package_dir / "run.sh").write_text( + _studio_run_script(thin=True), + encoding="utf-8", + newline="\n", + ) + return runtime_manifest.runtime_epoch, artifact_dir + + +def partition_public_runtime_wheels( + source_root: Path, + wheels: list[Path], +) -> tuple[list[Path], list[Path]]: + """Keep non-approved wheels private instead of weakening the public gate.""" + + public: list[Path] = [] + bundled: list[Path] = [] + for wheel in wheels: + try: + validate_public_runtime_provenance(source_root, [wheel]) + except StudioPublisherError as error: + if not any( + marker in str(error) + for marker in ( + "non-PyPI wheel", + "has no locked wheel", + "license is not allowlisted", + ) + ): + raise + bundled.append(wheel) + else: + public.append(wheel) + return public, bundled + + +def validate_public_runtime_provenance( + source_root: Path, + wheels: list[Path], +) -> None: + """Allow public publication only for the local project and PyPI lock entries.""" + + from packaging.utils import canonicalize_name, parse_wheel_filename + + lock_path = source_root / "uv.lock" + try: + lock = tomllib.loads(lock_path.read_text(encoding="utf-8")) + packages = lock["package"] + except (OSError, KeyError, TypeError, tomllib.TOMLDecodeError) as error: + raise StudioPublisherError( + "Studio public artifact provenance is invalid." + ) from error + allowed: dict[str, set[tuple[str, int, str]]] = { + canonicalize_name("veadk-python"): set() + } + for package in packages: + if not isinstance(package, dict): + raise StudioPublisherError("Studio public artifact provenance is invalid.") + name = package.get("name") + source = package.get("source") + if not isinstance(name, str) or not isinstance(source, dict): + raise StudioPublisherError("Studio public artifact provenance is invalid.") + if source.get("registry") == "https://pypi.org/simple": + identities = allowed.setdefault(canonicalize_name(name), set()) + locked_wheels = package.get("wheels", []) + if not isinstance(locked_wheels, list): + raise StudioPublisherError( + "Studio public artifact provenance is invalid." + ) + for wheel in locked_wheels: + if not isinstance(wheel, dict): + raise StudioPublisherError( + "Studio public artifact provenance is invalid." + ) + url = wheel.get("url") + digest = wheel.get("hash") + size = wheel.get("size") + if ( + not isinstance(url, str) + or not isinstance(digest, str) + or not isinstance(size, int) + or size <= 0 + or not digest.startswith("sha256:") + or not _SHA256_PATTERN.fullmatch(digest.removeprefix("sha256:")) + ): + raise StudioPublisherError( + "Studio public artifact provenance is invalid." + ) + parsed = urllib.parse.urlsplit(url) + filename = urllib.parse.unquote(Path(parsed.path).name) + if ( + parsed.scheme != "https" + or parsed.hostname != "files.pythonhosted.org" + or parsed.port is not None + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + ): + raise StudioPublisherError( + "Studio public artifact provenance is invalid." + ) + identities.add((filename, size, digest.removeprefix("sha256:"))) + try: + published = {path: parse_wheel_filename(path.name)[0] for path in wheels} + except ValueError as error: + raise StudioPublisherError( + "Studio public artifact wheel is invalid." + ) from error + disallowed = sorted(str(name) for name in published.values() if name not in allowed) + if disallowed: + raise StudioPublisherError( + "Studio public runtime contains a non-PyPI wheel: " + ", ".join(disallowed) + ) + for path, name in published.items(): + identity = (path.name, path.stat().st_size, _sha256_file(path)) + if not allowed[name]: + raise StudioPublisherError( + f"Studio public runtime package has no locked wheel: {name}" + ) + if identity not in allowed[name]: + raise StudioPublisherError( + f"Studio public runtime wheel does not match uv.lock: {name}" + ) + license_rejections: list[str] = [] + for path, name in published.items(): + try: + _validate_public_wheel_license(path, str(name)) + except StudioPublisherError as error: + if "license is not allowlisted" not in str(error): + raise + license_rejections.append(str(name)) + if license_rejections: + raise StudioPublisherError( + "Studio public runtime wheel license is not allowlisted: " + + ", ".join(sorted(license_rejections)) + ) + + +def _validate_public_wheel_license(path: Path, expected_name: str) -> None: + """Require one explicit, allowlisted redistributable wheel license.""" + + from packaging.licenses import ( + InvalidLicenseExpression, + canonicalize_license_expression, + ) + from packaging.utils import canonicalize_name + + try: + with zipfile.ZipFile(path) as archive: + metadata_names = [ + name + for name in archive.namelist() + if name.count("/") == 1 and name.endswith(".dist-info/METADATA") + ] + if len(metadata_names) != 1: + raise StudioPublisherError( + "Studio public runtime wheel metadata is invalid." + ) + metadata = BytesParser(policy=cast(Any, email_policy)).parsebytes( + archive.read(metadata_names[0]) + ) + except (OSError, KeyError, zipfile.BadZipFile) as error: + raise StudioPublisherError( + "Studio public runtime wheel metadata is invalid." + ) from error + if canonicalize_name(str(metadata.get("Name", ""))) != canonicalize_name( + expected_name + ): + raise StudioPublisherError("Studio public runtime wheel metadata is invalid.") + + expression = str(metadata.get("License-Expression", "") or "").strip() + if not expression: + classifiers = [ + value.strip() + for value in metadata.get_all("Classifier", []) + if value.strip().startswith("License ::") + ] + mapped = { + _PUBLIC_RUNTIME_LICENSE_CLASSIFIERS[value] + for value in classifiers + if value in _PUBLIC_RUNTIME_LICENSE_CLASSIFIERS + } + unknown_classifiers = { + value + for value in classifiers + if value not in _PUBLIC_RUNTIME_LICENSE_CLASSIFIERS + and value != "License :: OSI Approved" + } + if mapped and not unknown_classifiers: + expression = " OR ".join(sorted(mapped)) + elif not classifiers: + raw_legacy = str(metadata.get("License", "") or "").strip() + if raw_legacy and len(raw_legacy) <= 200: + try: + expression = canonicalize_license_expression(raw_legacy) + except InvalidLicenseExpression: + expression = "" + legacy = re.sub( + r"[^a-z0-9]+", + " ", + raw_legacy.lower(), + ).strip() + expression = expression or _PUBLIC_RUNTIME_LEGACY_LICENSES.get( + legacy, + "", + ) + if not expression and legacy.startswith("apache license version 2 0"): + expression = "Apache-2.0" + if not expression and legacy.startswith("mit license copyright"): + expression = "MIT" + try: + normalized = canonicalize_license_expression(expression) + except InvalidLicenseExpression as error: + raise StudioPublisherError( + "Studio public runtime wheel license is not allowlisted." + ) from error + tokens = { + token + for token in re.findall(r"[A-Za-z0-9][A-Za-z0-9.+-]*", normalized) + if token not in {"AND", "OR", "WITH"} + } + if not tokens or not tokens.issubset(_PUBLIC_RUNTIME_LICENSES): + raise StudioPublisherError( + "Studio public runtime wheel license is not allowlisted." + ) + + def _zip_directory(source: Path, destination: Path) -> None: with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_DEFLATED) as archive: for path in sorted(source.rglob("*")): @@ -575,6 +1334,8 @@ def build_studio_release( frontend_assets: Path | None, dependency_wheels: Path, env: Mapping[str, str], + thin: bool = False, + provider: str = "volcengine", ) -> tuple[Path, StudioReleaseManifest]: """Build a release while treating VeADK only as source input.""" _validate_source_checkout(source_root) @@ -609,9 +1370,22 @@ def build_studio_release( newline="\n", ) (package_dir / "requirements.txt").write_text(requirements, encoding="utf-8") + runtime_epoch = "" bundle = output_dir / f"studio-bundle-{version}.zip" _zip_directory(package_dir, bundle) + thin_bundle: Path | None = None + if thin: + runtime_epoch, _artifact_dir = stage_studio_thin_runtime( + source_root, + package_dir, + output_dir, + provider=provider, + ) + thin_bundle = output_dir / f"studio-bundle-{version}-thin.zip" + _zip_directory(package_dir, thin_bundle) + ensure_studio_bundle_agentkit_cli(bundle, dependency_wheels) content = bundle.read_bytes() + thin_content = thin_bundle.read_bytes() if thin_bundle is not None else b"" manifest = StudioReleaseManifest( version=version, git_sha=git_sha, @@ -621,6 +1395,9 @@ def build_studio_release( timespec="seconds" ), changelog=changelog, + runtime_epoch=runtime_epoch, + thin_sha256=(hashlib.sha256(thin_content).hexdigest() if thin_content else ""), + thin_size=len(thin_content), ) (output_dir / f"manifest-{version}.json").write_bytes(manifest.to_json()) return bundle, manifest @@ -643,6 +1420,12 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--changelog", action="append", default=[]) parser.add_argument("--frontend-assets", type=Path) parser.add_argument("--dependency-wheels", type=Path, required=True) + parser.add_argument("--thin", action="store_true") + parser.add_argument( + "--release-contract", + choices=(_STUDIO_RELEASE_CONTRACT,), + required=True, + ) return parser @@ -657,18 +1440,50 @@ def main() -> None: frontend_assets=args.frontend_assets, dependency_wheels=args.dependency_wheels, env=os.environ, + thin=args.thin, + provider=args.provider, ) credential_prefix = "BYTEPLUS" if args.provider == "byteplus" else "VOLCENGINE" + access_key = os.getenv(f"{credential_prefix}_ACCESS_KEY", "") + secret_key = os.getenv(f"{credential_prefix}_SECRET_KEY", "") + session_token = os.getenv(f"{credential_prefix}_SESSION_TOKEN", "") + artifact_counts = (0, 0) + if args.thin: + contract = _load_studio_artifact_contract(args.source_root.resolve()) + runtime_manifest = contract.StudioRuntimeManifest.from_json( + ( + args.output_dir.resolve() + / f"runtime-manifest-{manifest.runtime_epoch}.json" + ).read_bytes() + ) + artifact_counts = StudioPublicArtifactStore( + contract=contract, + provider=args.provider, + access_key=access_key, + secret_key=secret_key, + session_token=session_token, + ).publish( + runtime_manifest, + args.output_dir.resolve() / f"runtime-artifacts-{manifest.runtime_epoch}", + ) store = StudioReleaseStore( bucket=args.bucket, region=args.region, - access_key=os.getenv(f"{credential_prefix}_ACCESS_KEY", ""), - secret_key=os.getenv(f"{credential_prefix}_SECRET_KEY", ""), - session_token=os.getenv(f"{credential_prefix}_SESSION_TOKEN", ""), + access_key=access_key, + secret_key=secret_key, + session_token=session_token, prefix=args.prefix, provider=args.provider, ) - store.publish(bundle, manifest) + store.publish( + bundle, + manifest, + thin_bundle=( + args.output_dir.resolve() / f"studio-bundle-{manifest.version}-thin.zip" + if args.thin + else None + ), + ) print( json.dumps( { @@ -676,6 +1491,11 @@ def main() -> None: "gitSha": manifest.git_sha, "sha256": manifest.sha256, "size": manifest.size, + "runtimeEpoch": manifest.runtime_epoch, + "thinSha256": manifest.thin_sha256, + "thinSize": manifest.thin_size, + "artifactsCreated": artifact_counts[0], + "artifactsReused": artifact_counts[1], } ) ) diff --git a/frontend/service/studio_release_server/requirements.txt b/frontend/service/studio_release_server/requirements.txt index d9500fa51..c8da84543 100644 --- a/frontend/service/studio_release_server/requirements.txt +++ b/frontend/service/studio_release_server/requirements.txt @@ -1,6 +1,7 @@ fastapi>=0.115,<1 filetype==1.2.0 httpx>=0.27,<1 +packaging==26.2 tos>=2.8.4,<3 uv>=0.8,<1 uvicorn>=0.34,<1 diff --git a/frontend/service/studio_release_server/runtime-wheel-requirements.txt b/frontend/service/studio_release_server/runtime-wheel-requirements.txt index 81bec76e0..b447f405c 100644 --- a/frontend/service/studio_release_server/runtime-wheel-requirements.txt +++ b/frontend/service/studio_release_server/runtime-wheel-requirements.txt @@ -1,6 +1,7 @@ fastapi>=0.115,<1 filetype==1.2.0 httpx>=0.27,<1 +packaging==26.2 Deprecated>=1.2.13,<2 pytz requests>=2.19.1,<3 diff --git a/tests/cli/test_studio_artifacts.py b/tests/cli/test_studio_artifacts.py new file mode 100644 index 000000000..3483bdd75 --- /dev/null +++ b/tests/cli/test_studio_artifacts.py @@ -0,0 +1,157 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. + +from __future__ import annotations + +import io +import json +from pathlib import Path + +import pytest + +from veadk.cli import studio_artifacts +from veadk.cli.studio_artifacts import ( + StudioArtifact, + StudioBundledArtifact, + StudioRuntimeManifest, + download_studio_artifact, +) + + +def _artifact( + tmp_path: Path, + filename: str, + content: bytes, + *, + provider: str = "volcengine", + kind: str = "wheel", +) -> StudioArtifact: + path = tmp_path / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + return StudioArtifact.from_path(path, provider=provider, kind=kind) # type: ignore[arg-type] + + +def _manifest(tmp_path: Path, provider: str = "volcengine") -> StudioRuntimeManifest: + return StudioRuntimeManifest.create( + provider=provider, # type: ignore[arg-type] + artifacts=( + _artifact( + tmp_path, "dependency-1.0-py3-none-any.whl", b"wheel", provider=provider + ), + _artifact( + tmp_path, + "agentkit-linux-x64.tar.gz", + b"cli", + provider=provider, + kind="agentkit-cli", + ), + ), + ) + + +def test_runtime_epoch_is_provider_independent(tmp_path: Path) -> None: + volcengine = _manifest(tmp_path / "volcengine", "volcengine") + byteplus = _manifest(tmp_path / "byteplus", "byteplus") + + assert volcengine.runtime_epoch == byteplus.runtime_epoch + assert volcengine.to_json() != byteplus.to_json() + assert StudioRuntimeManifest.from_json(volcengine.to_json()) == volcengine + + +def test_runtime_manifest_tracks_private_bundled_wheels(tmp_path: Path) -> None: + cli = _artifact( + tmp_path, + "agentkit-linux-x64.tar.gz", + b"cli", + kind="agentkit-cli", + ) + private_wheel = tmp_path / "private_dependency-1.0-py3-none-any.whl" + private_wheel.write_bytes(b"private") + manifest = StudioRuntimeManifest.create( + "volcengine", + (cli,), + (StudioBundledArtifact.from_path(private_wheel),), + ) + + restored = StudioRuntimeManifest.from_json(manifest.to_json()) + + assert restored == manifest + assert "./bundled-wheelhouse/private_dependency-1.0-py3-none-any.whl" in ( + manifest.remote_requirements() + ) + + +def test_manifest_generates_exact_remote_requirements(tmp_path: Path) -> None: + manifest = _manifest(tmp_path) + wheel = next(item for item in manifest.artifacts if item.kind == "wheel") + + assert manifest.remote_requirements() == ( + f"--no-index\n{wheel.url}#sha256={wheel.sha256}\n" + ) + assert manifest.agentkit_cli().kind == "agentkit-cli" + + +def test_manifest_requires_wheel_and_unique_cli(tmp_path: Path) -> None: + wheel = _artifact(tmp_path, "dependency.whl", b"wheel") + cli = _artifact( + tmp_path, + "agentkit-linux-x64.tar.gz", + b"cli", + kind="agentkit-cli", + ) + second_cli = _artifact(tmp_path, "other.tar.gz", b"other", kind="agentkit-cli") + + with pytest.raises(ValueError, match="incomplete"): + StudioRuntimeManifest.create("volcengine", (wheel,)) + with pytest.raises(ValueError, match="incomplete"): + StudioRuntimeManifest.create("volcengine", (wheel, cli, second_cli)) + + +def test_manifest_rejects_coerced_fields_and_unsafe_filenames(tmp_path: Path) -> None: + payload = json.loads(_manifest(tmp_path).to_json()) + payload["artifacts"][0]["size"] = str(payload["artifacts"][0]["size"]) + + with pytest.raises(ValueError, match="invalid"): + StudioRuntimeManifest.from_json(json.dumps(payload)) + with pytest.raises(ValueError, match="filename"): + _artifact(tmp_path, "unsafe\nname.whl", b"wheel") + + +def test_download_is_atomic_and_digest_checked( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + artifact = _artifact(tmp_path, "dependency.whl", b"verified") + + class _Response(io.BytesIO): + headers = {"Content-Length": str(artifact.size)} + + def __enter__(self) -> _Response: + return self + + def __exit__(self, *_args: object) -> None: + self.close() + + def geturl(self) -> str: + return artifact.url + + monkeypatch.setattr( + studio_artifacts.urllib.request, + "urlopen", + lambda *_args, **_kwargs: _Response(b"verified"), + ) + destination = tmp_path / "cache" / artifact.filename + + assert download_studio_artifact(artifact, destination) == destination + assert destination.read_bytes() == b"verified" + assert not list(destination.parent.glob("*.part")) + + monkeypatch.setattr( + studio_artifacts.urllib.request, + "urlopen", + lambda *_args, **_kwargs: _Response(b"tampered"), + ) + with pytest.raises(ValueError, match="checksum"): + download_studio_artifact(artifact, tmp_path / "bad.whl") diff --git a/tests/cli/test_studio_companion.py b/tests/cli/test_studio_companion.py index 79d2bef2b..6f2fdcbc2 100644 --- a/tests/cli/test_studio_companion.py +++ b/tests/cli/test_studio_companion.py @@ -26,6 +26,7 @@ import pytest from veadk.cli import agentkit_cli +from veadk.cli import studio_companion from veadk.cli.agentkit_cli import ( AGENTKIT_CLI_ENV, AGENTKIT_CLI_VERSION, @@ -37,7 +38,11 @@ install_agentkit_cli, resolve_agentkit_cli, ) -from veadk.cli.studio_companion import required_agentkit_cli_version +from veadk.cli.studio_artifacts import StudioArtifact, StudioRuntimeManifest +from veadk.cli.studio_companion import ( + required_agentkit_cli_version, + validate_installed_agentkit_cli, +) def _script(version: str = AGENTKIT_CLI_VERSION) -> bytes: @@ -106,6 +111,82 @@ def test_required_version_is_owned_by_veadk_not_distribution_metadata() -> None: assert required_agentkit_cli_version() == "0.52.14" +def test_companion_materializes_manifest_cli_for_provider( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + wheel = tmp_path / "dependency.whl" + wheel.write_bytes(b"wheel") + archive = tmp_path / "agentkit-linux-x64.tar.gz" + archive.write_bytes(b"cli") + manifest = StudioRuntimeManifest.create( + "byteplus", + ( + StudioArtifact.from_path(wheel, provider="byteplus", kind="wheel"), + StudioArtifact.from_path( + archive, + provider="byteplus", + kind="agentkit-cli", + ), + ), + ) + manifest_path = tmp_path / "studio-runtime.json" + manifest_path.write_bytes(manifest.to_json()) + captured: dict[str, object] = {} + + def _download(artifact: StudioArtifact, destination: Path) -> Path: + captured["artifact"] = artifact + captured["destination"] = destination + return archive + + def _resolve(**kwargs: object) -> Path: + captured.update(kwargs) + return tmp_path / "ak" + + monkeypatch.setattr(studio_companion, "download_studio_artifact", _download) + monkeypatch.setattr(studio_companion, "resolve_agentkit_cli", _resolve) + monkeypatch.setattr( + studio_companion, + "default_agentkit_cli_cache_root", + lambda: tmp_path / "cache", + ) + + assert ( + validate_installed_agentkit_cli( + runtime_manifest=manifest_path, + provider="byteplus", + ) + == AGENTKIT_CLI_VERSION + ) + assert captured["artifact"].provider == "byteplus" # type: ignore[union-attr] + assert captured["archive"] == archive + + +def test_companion_rejects_runtime_manifest_provider_mismatch(tmp_path: Path) -> None: + wheel = tmp_path / "dependency.whl" + wheel.write_bytes(b"wheel") + archive = tmp_path / "agentkit-linux-x64.tar.gz" + archive.write_bytes(b"cli") + manifest = StudioRuntimeManifest.create( + "volcengine", + ( + StudioArtifact.from_path(wheel, provider="volcengine", kind="wheel"), + StudioArtifact.from_path( + archive, + provider="volcengine", + kind="agentkit-cli", + ), + ), + ) + manifest_path = tmp_path / "studio-runtime.json" + manifest_path.write_bytes(manifest.to_json()) + + with pytest.raises(AgentKitCliError, match="provider"): + validate_installed_agentkit_cli( + runtime_manifest=manifest_path, + provider="byteplus", + ) + + def test_install_verified_archive_and_reuse_cache(tmp_path: Path) -> None: archive = tmp_path / "agentkit-linux-x64.tar.gz" artifact = _write_test_archive(archive) diff --git a/tests/cli/test_studio_release.py b/tests/cli/test_studio_release.py index 6a4289a19..d1a10620c 100644 --- a/tests/cli/test_studio_release.py +++ b/tests/cli/test_studio_release.py @@ -53,6 +53,7 @@ manifest_object_key, release_catalog_object_key, studio_release_region, + thin_bundle_object_key, ) from veadk.utils.cloud_provider import CloudProvider @@ -152,6 +153,26 @@ def test_manifest_round_trip_uses_public_field_names() -> None: assert StudioReleaseManifest.from_json(manifest.to_json()) == manifest +def test_thin_manifest_round_trip_includes_separate_thin_bundle() -> None: + manifest = StudioReleaseManifest( + version="20260724153046", + git_sha="a" * 40, + sha256="b" * 64, + size=100, + created_at="2026-07-24T15:30:46+08:00", + runtime_epoch="c" * 64, + thin_sha256="d" * 64, + thin_size=200, + ) + + payload = json.loads(manifest.to_json()) + + assert payload["runtimeEpoch"] == "c" * 64 + assert payload["thinSha256"] == "d" * 64 + assert payload["thinSize"] == 200 + assert StudioReleaseManifest.from_json(manifest.to_json()) == manifest + + def test_frontend_build_exposes_release_changelog_to_vite( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -219,6 +240,59 @@ def test_publish_moves_latest_pointer_after_immutable_objects(tmp_path: Path) -> assert store.release_catalog() == [manifest] +def test_publish_thin_release_preserves_full_legacy_bundle(tmp_path: Path) -> None: + content = b"full-offline" + thin_content = b"thin" + bundle = tmp_path / "full.zip" + bundle.write_bytes(content) + thin_bundle = tmp_path / "thin.zip" + thin_bundle.write_bytes(thin_content) + manifest = StudioReleaseManifest( + version="20260724153047", + git_sha="a" * 40, + sha256=hashlib.sha256(content).hexdigest(), + size=len(content), + created_at="2026-07-24T15:30:47+08:00", + runtime_epoch="c" * 64, + thin_sha256=hashlib.sha256(thin_content).hexdigest(), + thin_size=len(thin_content), + ) + client = _FakeTosClient() + store = _store(client) + + store.publish(bundle, manifest, thin_bundle=thin_bundle) + + thin_key = thin_bundle_object_key(store.prefix, manifest.version) + assert client.put_order[:3] == [ + bundle_object_key(store.prefix, manifest.version), + thin_key, + manifest_object_key(store.prefix, manifest.version), + ] + legacy_destination = tmp_path / "downloaded-full.zip" + store.download_bundle(manifest, legacy_destination) + assert legacy_destination.read_bytes() == content + thin_destination = tmp_path / "downloaded-thin.zip" + store.download_thin_bundle(manifest, thin_destination) + assert thin_destination.read_bytes() == thin_content + + +def test_bundle_download_is_atomic_on_checksum_failure(tmp_path: Path) -> None: + content = b"expected" + manifest = _manifest(content) + client = _FakeTosClient() + client.objects[ + ("studio-releases", bundle_object_key("veadk/studio/main", manifest.version)) + ] = b"corrupt!" + destination = tmp_path / "bundle.zip" + destination.write_bytes(b"previous") + + with pytest.raises(StudioReleaseError, match="checksum"): + _store(client).download_bundle(manifest, destination) + + assert destination.read_bytes() == b"previous" + assert not list(tmp_path.glob("*.part")) + + def test_publish_catalog_keeps_newest_release_first(tmp_path: Path) -> None: client = _FakeTosClient() store = _store(client) @@ -246,7 +320,7 @@ def test_publish_catalog_keeps_newest_release_first(tmp_path: Path) -> None: assert store.manifest(older.version) == older -def test_publish_does_not_replace_an_immutable_release(tmp_path: Path) -> None: +def test_publish_identical_release_is_idempotent(tmp_path: Path) -> None: content = b"complete-studio-bundle" bundle = tmp_path / "bundle.zip" bundle.write_bytes(content) @@ -255,10 +329,63 @@ def test_publish_does_not_replace_an_immutable_release(tmp_path: Path) -> None: store = _store(client) store.publish(bundle, manifest) + put_order = list(client.put_order) + + store.publish(bundle, manifest) + + assert client.put_order == put_order + + +@pytest.mark.parametrize("failed_pointer", ["releases.json", "latest.json"]) +def test_publish_repairs_interrupted_pointer_update( + tmp_path: Path, + failed_pointer: str, +) -> None: + content = b"complete-studio-bundle" + bundle = tmp_path / "bundle.zip" + bundle.write_bytes(content) + manifest = _manifest(content) + + class _FailOnceClient(_FakeTosClient): + failed = False + + def put_object(self, **kwargs: Any) -> None: + if kwargs["key"].endswith(failed_pointer) and not self.failed: + self.failed = True + raise OSError("injected ambiguous write failure") + super().put_object(**kwargs) + + client = _FailOnceClient() + store = _store(client) + + with pytest.raises(StudioReleaseError, match="pointer upload failed"): + store.publish(bundle, manifest) + + store.publish(bundle, manifest) + + assert store.latest_manifest() == manifest + assert store.release_catalog() == [manifest] + - with pytest.raises(FileExistsError): +def test_publish_rejects_conflicting_partial_immutable_object(tmp_path: Path) -> None: + content = b"complete-studio-bundle" + bundle = tmp_path / "bundle.zip" + bundle.write_bytes(content) + manifest = _manifest(content) + client = _FakeTosClient() + store = _store(client) + client.objects[ + (store.bucket, bundle_object_key(store.prefix, manifest.version)) + ] = b"conflicting-content" + + with pytest.raises(StudioReleaseError, match="immutable release object conflicts"): store.publish(bundle, manifest) + assert ( + store.bucket, + latest_manifest_object_key(store.prefix), + ) not in client.objects + def test_publish_does_not_move_latest_pointer_to_an_older_release( tmp_path: Path, diff --git a/tests/cli/test_studio_self_update.py b/tests/cli/test_studio_self_update.py index 101fa8e85..605aa8cf5 100644 --- a/tests/cli/test_studio_self_update.py +++ b/tests/cli/test_studio_self_update.py @@ -27,6 +27,7 @@ from frontend.service.studio_release_server import publisher from veadk.cli.frontend_branding import SiteLogo +from veadk.cli.studio_artifacts import StudioArtifact, StudioRuntimeManifest from veadk.cli.studio_dependencies import STUDIO_AGENTKIT_CLI_ARTIFACT from veadk.cli.studio_release import ( BYTEPLUS_STUDIO_RELEASE_REGION, @@ -299,6 +300,252 @@ def test_online_update_preserves_preloaded_cli_archive_for_each_provider( assert f"--provider {provider}" in run_script +def test_thin_update_uses_public_runtime_when_all_artifacts_are_reachable( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + wheel = tmp_path / "dependency.whl" + wheel.write_bytes(b"wheel") + cli = tmp_path / "agentkit-linux-x64.tar.gz" + cli.write_bytes(b"pinned-cli") + runtime = StudioRuntimeManifest.create( + "volcengine", + ( + StudioArtifact.from_path(wheel, provider="volcengine", kind="wheel"), + StudioArtifact.from_path( + cli, + provider="volcengine", + kind="agentkit-cli", + ), + ), + ) + release = StudioReleaseManifest( + version="20260724153046", + git_sha="a" * 40, + sha256="b" * 64, + size=1, + created_at="2026-07-24T15:30:46+08:00", + runtime_epoch=runtime.runtime_epoch, + thin_sha256="c" * 64, + thin_size=1, + ) + + class _Store: + def download_thin_bundle( + self, + _release: StudioReleaseManifest, + destination: Path, + ) -> None: + destination.write_bytes(b"thin") + + probed: list[str] = [] + monkeypatch.setattr( + "veadk.cli.studio_self_update.probe_studio_artifact", + lambda artifact: probed.append(artifact.filename), + ) + updater = StudioSelfUpdater( + settings=_settings(), + credential_resolver=lambda: ("ak", "sk", "token"), + branding_logo=None, + ) + monkeypatch.setattr( + "veadk.cli.studio_self_update.extract_studio_bundle", + lambda _archive, destination: ( + destination.mkdir(), + (destination / "studio-runtime.json").write_bytes(runtime.to_json()), + ), + ) + + selected = updater._download_runtime_package( + _Store(), # type: ignore[arg-type] + release, + tmp_path, + ) + + assert selected == tmp_path / "package-thin" + assert sorted(probed) == ["agentkit-linux-x64.tar.gz", "dependency.whl"] + + +def test_thin_update_falls_back_to_legacy_full_bundle( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + wheel = tmp_path / "dependency.whl" + wheel.write_bytes(b"wheel") + cli = tmp_path / "agentkit-linux-x64.tar.gz" + cli.write_bytes(b"cli") + runtime = StudioRuntimeManifest.create( + "volcengine", + ( + StudioArtifact.from_path(wheel, provider="volcengine", kind="wheel"), + StudioArtifact.from_path( + cli, + provider="volcengine", + kind="agentkit-cli", + ), + ), + ) + full_source = tmp_path / "full-source.zip" + _bundle(full_source) + full_content = full_source.read_bytes() + release = StudioReleaseManifest( + version="20260724153047", + git_sha="a" * 40, + sha256=hashlib.sha256(full_content).hexdigest(), + size=len(full_content), + created_at="2026-07-24T15:30:47+08:00", + runtime_epoch=runtime.runtime_epoch, + thin_sha256="c" * 64, + thin_size=1, + ) + + class _Store: + def download_thin_bundle( + self, + _release: StudioReleaseManifest, + _destination: Path, + ) -> None: + raise StudioReleaseError("thin unavailable") + + def download_bundle( + self, + _release: StudioReleaseManifest, + destination: Path, + ) -> None: + destination.write_bytes(full_content) + + monkeypatch.setattr( + "veadk.cli.studio_self_update.probe_studio_artifact", + lambda _artifact: (_ for _ in ()).throw(ValueError("unavailable")), + ) + updater = StudioSelfUpdater( + settings=_settings(), + credential_resolver=lambda: ("ak", "sk", "token"), + branding_logo=None, + ) + + selected = updater._download_runtime_package( + _Store(), # type: ignore[arg-type] + release, + tmp_path, + ) + + assert selected == tmp_path / "package" + assert (selected / "agentkit-linux-x64.tar.gz").read_bytes() == b"pinned-cli" + + +def test_submit_thin_release_falls_back_to_legacy_full_bundle_end_to_end( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + wheel = tmp_path / "dependency.whl" + wheel.write_bytes(b"wheel") + cli = tmp_path / "agentkit-linux-x64.tar.gz" + cli.write_bytes(b"pinned-cli") + runtime = StudioRuntimeManifest.create( + "volcengine", + ( + StudioArtifact.from_path(wheel, provider="volcengine", kind="wheel"), + StudioArtifact.from_path( + cli, + provider="volcengine", + kind="agentkit-cli", + ), + ), + ) + thin_base = tmp_path / "thin-base.zip" + _bundle(thin_base, include_cli_archive=False) + thin_archive = tmp_path / "thin.zip" + with ( + zipfile.ZipFile(thin_base) as source, + zipfile.ZipFile(thin_archive, "w") as archive, + ): + wheel_name = next( + item.filename + for item in source.infolist() + if item.filename.endswith(".whl") + ) + wheel_content = source.read(wheel_name) + for item in source.infolist(): + if item.filename != "requirements.txt": + archive.writestr(item, source.read(item.filename)) + archive.writestr( + "requirements.txt", + runtime.remote_requirements() + + f"./{wheel_name} --hash=sha256:{hashlib.sha256(wheel_content).hexdigest()}\n", + ) + archive.writestr("studio-runtime.json", runtime.to_json()) + full_archive = tmp_path / "full.zip" + _bundle(full_archive) + with zipfile.ZipFile(full_archive, "a") as archive: + archive.writestr("full-marker", "selected") + thin_content = thin_archive.read_bytes() + full_content = full_archive.read_bytes() + manifest = StudioReleaseManifest( + version="20260724153048", + git_sha="a" * 40, + sha256=hashlib.sha256(full_content).hexdigest(), + size=len(full_content), + created_at="2026-07-24T15:30:48+08:00", + runtime_epoch=runtime.runtime_epoch, + thin_sha256=hashlib.sha256(thin_content).hexdigest(), + thin_size=len(thin_content), + ) + captured: dict[str, Any] = {} + + class _Store: + def latest_manifest(self) -> StudioReleaseManifest: + return manifest + + def download_bundle( + self, _release: StudioReleaseManifest, destination: Path + ) -> None: + captured["full_downloaded"] = True + destination.write_bytes(full_content) + + def download_thin_bundle( + self, _release: StudioReleaseManifest, destination: Path + ) -> None: + captured["thin_downloaded"] = True + destination.write_bytes(thin_content) + + class _VeFaaS: + def __init__(self, **_kwargs: str) -> None: + self.client = object() + + def submit_application_code_bundle_update(self, **kwargs: Any) -> None: + package = Path(kwargs["path"]) + captured["full_selected"] = (package / "full-marker").read_text() + + updater = StudioSelfUpdater( + settings=_settings(), + credential_resolver=lambda: ("ak", "sk", "token"), + branding_logo=None, + ) + monkeypatch.setattr(updater, "_store", lambda *_args: _Store()) + monkeypatch.setattr( + "veadk.cli.studio_self_update.probe_studio_artifact", + lambda _artifact: (_ for _ in ()).throw(ValueError("unavailable")), + ) + monkeypatch.setattr("veadk.integrations.ve_faas.ve_faas.VeFaaS", _VeFaaS) + monkeypatch.setattr( + "frontend.server.studio_update_resources.reconcile_studio_update_resources", + lambda **_kwargs: {}, + ) + monkeypatch.setattr( + "frontend.service.studio_scheduler.deploy.deploy_scheduler_for_studio_update", + lambda *_args, **_kwargs: ("", "", "", "", "scheduler"), + ) + monkeypatch.setenv("VEADK_STUDIO_RELEASE_VERSION", "bundled") + + assert updater.submit_latest() == manifest + assert captured == { + "thin_downloaded": True, + "full_downloaded": True, + "full_selected": "selected", + } + + def test_self_update_preserves_deployed_branding_logo(tmp_path: Path) -> None: package = tmp_path / "package" package.mkdir() diff --git a/tests/test_studio_release_server.py b/tests/test_studio_release_server.py index 1a8bf973f..46da95707 100644 --- a/tests/test_studio_release_server.py +++ b/tests/test_studio_release_server.py @@ -56,6 +56,47 @@ ) +def _write_test_wheel( + path: Path, + *, + name: str, + version: str, + license_expression: str = "MIT", + marker: str = "", +) -> None: + """Write a minimal deterministic wheel with auditable license metadata.""" + + distribution = name.replace("-", "_") + metadata = ( + "Metadata-Version: 2.4\n" + f"Name: {name}\n" + f"Version: {version}\n" + f"License-Expression: {license_expression}\n" + ) + with zipfile.ZipFile(path, "w") as archive: + for filename, content in ( + (f"{distribution}/__init__.py", marker), + (f"{distribution}-{version}.dist-info/METADATA", metadata), + ): + info = zipfile.ZipInfo(filename, date_time=(2025, 1, 1, 0, 0, 0)) + info.external_attr = 0o644 << 16 + archive.writestr(info, content) + + +def _pypi_lock(*items: tuple[str, str, Path]) -> str: + return "".join( + "[[package]]\n" + f'name = "{name}"\n' + f'version = "{version}"\n' + 'source = { registry = "https://pypi.org/simple" }\n' + 'wheels = [{ url = "https://files.pythonhosted.org/packages/' + f'{wheel.name}", hash = "sha256:' + f'{hashlib.sha256(wheel.read_bytes()).hexdigest()}", size = ' + f"{wheel.stat().st_size} }}]\n" + for name, version, wheel in items + ) + + class _InlineExecutor(Executor): def submit(self, function: Any, *args: Any, **kwargs: Any) -> Future[None]: future: Future[None] = Future() @@ -175,7 +216,11 @@ def materialize( return (wheel,) -def _settings(*, provider: str = "volcengine") -> ReleaseServerSettings: +def _settings( + *, + provider: str = "volcengine", + thin_releases: bool = False, +) -> ReleaseServerSettings: return ReleaseServerSettings( api_key="release-key-with-at-least-thirty-two-characters", bucket="veadk-studio", @@ -184,6 +229,7 @@ def _settings(*, provider: str = "volcengine") -> ReleaseServerSettings: job_prefix="veadk/studio/release-server/jobs", repository="volcengine/veadk-python", provider=provider, # type: ignore[arg-type] + thin_releases=thin_releases, ) @@ -205,6 +251,7 @@ def test_release_request_accepts_one_shared_version_for_all_providers() -> None: ) assert request.version == "20260828123045" + assert request.thin_bundle is False with pytest.raises(ValueError, match="YYYYMMDDHHMMSS"): ReleaseRequest( repository="volcengine/veadk-python", @@ -214,6 +261,24 @@ def test_release_request_accepts_one_shared_version_for_all_providers() -> None: ) +def test_release_request_accepts_explicit_thin_bundle_opt_in() -> None: + request = ReleaseRequest( + repository="volcengine/veadk-python", + gitSha="a" * 40, + requestId="thin-release", + thinBundle=True, + ) + + assert request.thin_bundle is True + with pytest.raises(ValueError): + ReleaseRequest( + repository="volcengine/veadk-python", + gitSha="a" * 40, + requestId="coerced-thin-release", + thinBundle="true", # type: ignore[arg-type] + ) + + def test_release_server_settings_load_byteplus_provider( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -228,6 +293,17 @@ def test_release_server_settings_load_byteplus_provider( assert settings.region == "ap-southeast-1" +def test_release_server_settings_reject_ambiguous_thin_opt_in( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("STUDIO_RELEASE_SERVER_API_KEY", "x" * 32) + monkeypatch.setenv("STUDIO_RELEASE_BUCKET", "veadk-studio") + monkeypatch.setenv("STUDIO_RELEASE_THIN_BUNDLES", "enabled") + + with pytest.raises(ValueError, match="explicit boolean"): + ReleaseServerSettings.from_env() + + def _service() -> ReleaseService: settings = _settings() source_store = _MemorySourceStore() @@ -466,7 +542,7 @@ def test_builder_prefers_domestic_source_and_node_mirrors() -> None: def test_builder_passes_only_publisher_runtime_environment( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - request = _request() + request = _request().model_copy(update={"thin_bundle": True}) captured: dict[str, Any] = {} monkeypatch.setattr( @@ -490,7 +566,7 @@ def _run(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[Any]: "--trace-warnings --max-old-space-size=1024", ) monkeypatch.setattr(release_builder, "_node_heap_limit_mb", lambda: 24_576) - builder = StudioReleaseBuilder(_settings()) + builder = StudioReleaseBuilder(_settings(thin_releases=True)) builder._run_publisher( request=request, @@ -510,10 +586,32 @@ def _run(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[Any]: "--trace-warnings --max-old-space-size=24576" ) assert captured["command"][1].endswith("studio_release_server/publisher.py") + assert ( + captured["command"][captured["command"].index("--release-contract") + 1] + == "agentkit-cli-v1" + ) + assert "--thin" in captured["command"] assert "veadk.cli.studio_release" not in captured["command"] assert str(tmp_path) not in captured["env"].get("PYTHONPATH", "").split(os.pathsep) +def test_builder_rejects_thin_request_without_server_opt_in(tmp_path: Path) -> None: + request = _request().model_copy(update={"thin_bundle": True}) + builder = StudioReleaseBuilder(_settings(thin_releases=False)) + + with pytest.raises(RuntimeError, match="not enabled"): + builder._run_publisher( + request=request, + source_root=tmp_path, + output_dir=tmp_path / "dist", + version="20260805170000", + node_bin=None, + uv=Path("/bin/uv"), + frontend_assets=None, + dependency_wheels=tmp_path, + ) + + def test_byteplus_builder_uses_local_tos_endpoint_and_credentials( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -551,6 +649,7 @@ def _run(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[Any]: ) assert captured["env"]["BYTEPLUS_ACCESS_KEY"] == "byteplus-ak" assert captured["env"]["BYTEPLUS_SECRET_KEY"] == "byteplus-sk" + assert "--thin" not in captured["command"] def test_byteplus_runtime_store_uses_byteplus_tos_endpoint( @@ -697,7 +796,11 @@ def test_standalone_publisher_builds_bundle_from_source_files( (frontend_assets / "index.html").write_text("studio", encoding="utf-8") dependency_wheels = tmp_path / "dependencies" dependency_wheels.mkdir() - (dependency_wheels / "dependency-1.0-py3-none-any.whl").write_bytes(b"wheel") + _write_test_wheel( + dependency_wheels / "six-1.17.0-py2.py3-none-any.whl", + name="six", + version="1.17.0", + ) cli_archive = dependency_wheels / "agentkit-linux-x64.tar.gz" cli_archive.write_bytes(b"pinned-cli") monkeypatch.setattr( @@ -717,7 +820,8 @@ def _run(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[Any] wheel.writestr(name, "") wheel.writestr( "veadk_python-1.0.0.dist-info/METADATA", - "Metadata-Version: 2.1\nName: veadk-python\nVersion: 1.0.0\n", + "Metadata-Version: 2.4\nName: veadk-python\nVersion: 1.0.0\n" + "License-Expression: Apache-2.0\n", ) return subprocess.CompletedProcess(command, 0) @@ -732,6 +836,11 @@ def _offline_runtime( wheelhouse.mkdir() target = wheelhouse / veadk_wheel.name target.write_bytes(veadk_wheel.read_bytes()) + _write_test_wheel( + wheelhouse / "six-1.17.0-py2.py3-none-any.whl", + name="six", + version="1.17.0", + ) (package_dir / "studio-runtime.lock").write_text( "dependency==1.0\n", encoding="utf-8", @@ -782,6 +891,570 @@ def _offline_runtime( assert manifest.git_sha == "a" * 40 assert manifest.sha256 == hashlib.sha256(bundle.read_bytes()).hexdigest() + monkeypatch.setattr( + release_publisher, + "validate_public_runtime_provenance", + lambda _source_root, _wheels: None, + ) + thin_output = tmp_path / "thin-output" + full_bundle, thin_manifest = release_publisher.build_studio_release( + source_root=Path(__file__).parents[1], + output_dir=thin_output, + version="20260805190001", + git_sha="a" * 40, + changelog=("发布 Studio 瘦包",), + frontend_assets=frontend_assets, + dependency_wheels=dependency_wheels, + env={"PATH": os.environ["PATH"]}, + thin=True, + provider="volcengine", + ) + thin_bundle = thin_output / "studio-bundle-20260805190001-thin.zip" + assert thin_manifest.runtime_epoch + assert thin_manifest.sha256 == hashlib.sha256(full_bundle.read_bytes()).hexdigest() + assert thin_manifest.size == full_bundle.stat().st_size + assert thin_manifest.thin_size == thin_bundle.stat().st_size + assert ( + thin_manifest.thin_sha256 + == hashlib.sha256(thin_bundle.read_bytes()).hexdigest() + ) + with zipfile.ZipFile(thin_bundle) as archive: + names = archive.namelist() + assert "studio-runtime.json" in names + assert "agentkit-linux-x64.tar.gz" not in names + assert not any(name.startswith("wheelhouse/") for name in names) + assert b"--runtime-manifest" in archive.read("run.sh") + with zipfile.ZipFile(full_bundle) as archive: + assert archive.read("agentkit-linux-x64.tar.gz") == b"pinned-cli" + assert any(name.startswith("wheelhouse/") for name in archive.namelist()) + extracted = tmp_path / "thin-extracted" + with zipfile.ZipFile(thin_bundle) as archive: + archive.extractall(extracted) + assert release_publisher.validate_studio_bundle_dependencies(extracted) == ( + extracted / "studio-runtime.json" + ) + + +def test_publisher_repairs_missing_agentkit_cli_before_manifest( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + dependency_wheels = tmp_path / "dependencies" + dependency_wheels.mkdir() + cli_archive = dependency_wheels / "agentkit-linux-x64.tar.gz" + cli_archive.write_bytes(b"pinned-cli") + monkeypatch.setattr( + release_publisher, + "_AGENTKIT_CLI_ARCHIVE_SHA256", + hashlib.sha256(cli_archive.read_bytes()).hexdigest(), + ) + bundle = tmp_path / "studio-bundle.zip" + with zipfile.ZipFile(bundle, "w") as archive: + archive.writestr("run.sh", "") + + release_publisher.ensure_studio_bundle_agentkit_cli( + bundle, + dependency_wheels, + ) + + with zipfile.ZipFile(bundle) as archive: + assert archive.read("agentkit-linux-x64.tar.gz") == b"pinned-cli" + + +def test_publisher_rejects_bad_agentkit_cli_in_final_bundle( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + dependency_wheels = tmp_path / "dependencies" + dependency_wheels.mkdir() + monkeypatch.setattr( + release_publisher, + "_AGENTKIT_CLI_ARCHIVE_SHA256", + hashlib.sha256(b"pinned-cli").hexdigest(), + ) + bundle = tmp_path / "studio-bundle.zip" + with zipfile.ZipFile(bundle, "w") as archive: + archive.writestr("agentkit-linux-x64.tar.gz", b"wrong-cli") + + with pytest.raises( + release_publisher.StudioPublisherError, + match="checksum is invalid", + ): + release_publisher.ensure_studio_bundle_agentkit_cli( + bundle, + dependency_wheels, + ) + + +def test_standalone_release_store_reuses_identical_immutable_objects( + tmp_path: Path, +) -> None: + content = b"full-bundle" + bundle = tmp_path / "bundle.zip" + bundle.write_bytes(content) + manifest = release_publisher.StudioReleaseManifest( + version="20260805190100", + git_sha="a" * 40, + sha256=hashlib.sha256(content).hexdigest(), + size=len(content), + created_at="2026-08-05T19:01:00+08:00", + ) + + class _Client: + def __init__(self) -> None: + self.objects: dict[tuple[str, str], bytes] = {} + self.puts = 0 + + def get_object(self, *, bucket: str, key: str) -> list[bytes]: + return [self.objects[(bucket, key)]] + + def put_object(self, **kwargs: Any) -> None: + identity = (kwargs["bucket"], kwargs["key"]) + if kwargs.get("forbid_overwrite") and identity in self.objects: + raise FileExistsError(kwargs["key"]) + self.objects[identity] = kwargs["content"] + self.puts += 1 + + client = _Client() + store = release_publisher.StudioReleaseStore( + bucket="studio-releases", + region="cn-beijing", + access_key="ak", + secret_key="sk", + session_token="", + prefix="veadk/studio/main", + ) + store._client = client + + store.publish(bundle, manifest) + first_puts = client.puts + store.publish(bundle, manifest) + + assert client.puts == first_puts + + +@pytest.mark.parametrize("provider", ["volcengine", "byteplus"]) +def test_publisher_stages_provider_local_thin_runtime( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + provider: str, +) -> None: + source_root = tmp_path / "source" + contract = source_root / "veadk" / "cli" / "studio_artifacts.py" + contract.parent.mkdir(parents=True) + shutil.copy2(Path(__file__).parents[1] / "veadk/cli/studio_artifacts.py", contract) + (source_root / "uv.lock").write_text( + '[[package]]\nname = "dependency"\nversion = "1.0"\n' + 'source = { registry = "https://pypi.org/simple" }\n', + encoding="utf-8", + ) + package_dir = tmp_path / "package" + wheelhouse = package_dir / "wheelhouse" + wheelhouse.mkdir(parents=True) + dependency_wheel = wheelhouse / "dependency-1.0-py3-none-any.whl" + _write_test_wheel( + dependency_wheel, + name="dependency", + version="1.0", + ) + (source_root / "uv.lock").write_text( + _pypi_lock(("dependency", "1.0", dependency_wheel)), + encoding="utf-8", + ) + veadk_wheel = wheelhouse / "veadk_python-1.0.0-py3-none-any.whl" + _write_test_wheel( + veadk_wheel, + name="veadk-python", + version="1.0.0", + license_expression="Apache-2.0", + ) + cli_archive = package_dir / "agentkit-linux-x64.tar.gz" + cli_archive.write_bytes(b"cli") + (package_dir / "requirements.txt").write_text("local\n", encoding="utf-8") + (package_dir / "run.sh").write_text("local\n", encoding="utf-8") + monkeypatch.setattr( + release_publisher, + "_AGENTKIT_CLI_ARCHIVE_SHA256", + hashlib.sha256(cli_archive.read_bytes()).hexdigest(), + ) + + epoch, artifact_dir = release_publisher.stage_studio_thin_runtime( + source_root, + package_dir, + tmp_path / "output", + provider=provider, + ) + + manifest = json.loads((package_dir / "studio-runtime.json").read_text()) + assert manifest["runtimeEpoch"] == epoch + assert manifest["provider"] == provider + assert len(list(artifact_dir.iterdir())) == 2 + assert not wheelhouse.exists() + assert not cli_archive.exists() + assert len(list(package_dir.glob("veadk*.whl"))) == 1 + assert ( + package_dir.joinpath("requirements.txt") + .read_text() + .startswith("--no-index\nhttps://") + ) + assert ( + "./veadk_python-1.0.0-py3-none-any.whl --hash=sha256:" + in (package_dir / "requirements.txt").read_text() + ) + assert "--runtime-manifest" in package_dir.joinpath("run.sh").read_text() + + +def test_public_artifact_store_uploads_once_and_reuses_by_digest( + tmp_path: Path, +) -> None: + from veadk.cli import studio_artifacts as contract + + wheel = tmp_path / "dependency.whl" + wheel.write_bytes(b"wheel") + cli = tmp_path / "agentkit-linux-x64.tar.gz" + cli.write_bytes(b"cli") + manifest = contract.StudioRuntimeManifest.create( + "volcengine", + ( + contract.StudioArtifact.from_path( + wheel, + provider="volcengine", + kind="wheel", + ), + contract.StudioArtifact.from_path( + cli, + provider="volcengine", + kind="agentkit-cli", + ), + ), + ) + + class _Client: + def __init__(self) -> None: + self.objects: dict[tuple[str, str], tuple[bytes, dict[str, str]]] = {} + self.puts = 0 + self.fail_file = "" + self.failed_once = False + + def head_object(self, *, bucket: str, key: str) -> SimpleNamespace: + if (bucket, key) not in self.objects: + raise _NotFoundError(key) + content, metadata = self.objects[(bucket, key)] + return SimpleNamespace(content_length=len(content), meta=metadata) + + def put_object_from_file(self, **kwargs: Any) -> None: + if ( + Path(kwargs["file_path"]).name == self.fail_file + and not self.failed_once + ): + self.failed_once = True + raise RuntimeError("injected upload failure") + content = Path(kwargs["file_path"]).read_bytes() + self.objects[(kwargs["bucket"], kwargs["key"])] = ( + content, + dict(kwargs["meta"]), + ) + self.puts += 1 + + client = _Client() + artifact_sizes = {item.url: item.size for item in manifest.artifacts} + + class _PublicResponse: + status = 200 + + def __init__(self, url: str) -> None: + self._url = url + self.headers = {"Content-Length": str(artifact_sizes[url])} + + def __enter__(self) -> _PublicResponse: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def geturl(self) -> str: + return self._url + + store = release_publisher.StudioPublicArtifactStore( + contract=contract, + provider="volcengine", + access_key="", + secret_key="", + session_token="", + client=client, + public_opener=lambda request, **_kwargs: _PublicResponse(request.full_url), + ) + + assert store.publish(manifest, tmp_path) == (2, 0) + assert store.publish(manifest, tmp_path) == (0, 2) + assert client.puts == 2 + + first_key = next(iter(client.objects)) + content, _metadata = client.objects[first_key] + client.objects[first_key] = (content, {"sha256": "0" * 64}) + with pytest.raises( + release_publisher.StudioPublisherError, + match="conflict", + ): + store.publish(manifest, tmp_path) + + retry_client = _Client() + retry_client.fail_file = manifest.artifacts[1].filename + retry_store = release_publisher.StudioPublicArtifactStore( + contract=contract, + provider="volcengine", + access_key="", + secret_key="", + session_token="", + client=retry_client, + public_opener=lambda request, **_kwargs: _PublicResponse(request.full_url), + ) + with pytest.raises( + release_publisher.StudioPublisherError, + match="upload failed", + ): + retry_store.publish(manifest, tmp_path) + assert len(retry_client.objects) == 1 + assert retry_store.publish(manifest, tmp_path) == (1, 1) + assert retry_client.puts == 2 + + +@pytest.mark.parametrize( + ("status", "length", "redirect"), + [ + (403, 5, False), + (200, 0, False), + (200, 5, True), + ], +) +def test_public_artifact_store_rejects_anonymous_head_failures( + tmp_path: Path, + status: int, + length: int, + redirect: bool, +) -> None: + from veadk.cli import studio_artifacts as contract + + wheel = tmp_path / "dependency.whl" + wheel.write_bytes(b"wheel") + artifact = contract.StudioArtifact.from_path( + wheel, + provider="volcengine", + kind="wheel", + ) + + class _Response: + headers = {"Content-Length": str(length)} + + def __enter__(self) -> _Response: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def geturl(self) -> str: + return "https://redirect.invalid/artifact" if redirect else artifact.url + + response = _Response() + response.status = status + store = release_publisher.StudioPublicArtifactStore( + contract=contract, + provider="volcengine", + access_key="", + secret_key="", + session_token="", + client=object(), + public_opener=lambda *_args, **_kwargs: response, + ) + + with pytest.raises( + release_publisher.StudioPublisherError, + match="anonymous-read verification failed", + ): + store._verify_public(artifact) + + +def test_public_runtime_rejects_wheel_without_public_pypi_provenance( + tmp_path: Path, +) -> None: + (tmp_path / "uv.lock").write_text( + '[[package]]\nname = "private-dependency"\nversion = "1.0"\n' + 'source = { git = "https://example.com/private.git" }\n', + encoding="utf-8", + ) + wheel = tmp_path / "private_dependency-1.0-py3-none-any.whl" + wheel.write_bytes(b"wheel") + + with pytest.raises( + release_publisher.StudioPublisherError, + match="non-PyPI", + ): + release_publisher.validate_public_runtime_provenance(tmp_path, [wheel]) + + +def test_public_runtime_requires_exact_locked_wheel_bytes(tmp_path: Path) -> None: + wheel = tmp_path / "dependency-1.0-py3-none-any.whl" + _write_test_wheel(wheel, name="dependency", version="1.0") + (tmp_path / "uv.lock").write_text( + _pypi_lock(("dependency", "1.0", wheel)), + encoding="utf-8", + ) + with wheel.open("ab") as output: + output.write(b"tampered") + + with pytest.raises( + release_publisher.StudioPublisherError, + match="does not match uv.lock", + ): + release_publisher.validate_public_runtime_provenance(tmp_path, [wheel]) + + +def test_source_built_wheel_stays_in_private_bundle(tmp_path: Path) -> None: + wheel = tmp_path / "dependency-1.0-py3-none-any.whl" + _write_test_wheel(wheel, name="dependency", version="1.0") + (tmp_path / "uv.lock").write_text( + '[[package]]\nname = "dependency"\nversion = "1.0"\n' + 'source = { registry = "https://pypi.org/simple" }\n' + 'sdist = { url = "https://files.pythonhosted.org/packages/dependency-1.0.tar.gz", ' + 'hash = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", ' + "size = 100 }\n", + encoding="utf-8", + ) + + public, bundled = release_publisher.partition_public_runtime_wheels( + tmp_path, + [wheel], + ) + + assert public == [] + assert bundled == [wheel] + + +def test_public_runtime_requires_allowlisted_wheel_license(tmp_path: Path) -> None: + wheel = tmp_path / "dependency-1.0-py3-none-any.whl" + _write_test_wheel( + wheel, + name="dependency", + version="1.0", + license_expression="LicenseRef-Proprietary", + ) + (tmp_path / "uv.lock").write_text( + _pypi_lock(("dependency", "1.0", wheel)), + encoding="utf-8", + ) + + with pytest.raises( + release_publisher.StudioPublisherError, + match="license is not allowlisted", + ): + release_publisher.validate_public_runtime_provenance(tmp_path, [wheel]) + + _write_test_wheel( + wheel, + name="dependency", + version="1.0", + license_expression="MIT OR Apache-2.0", + ) + (tmp_path / "uv.lock").write_text( + _pypi_lock(("dependency", "1.0", wheel)), + encoding="utf-8", + ) + release_publisher.validate_public_runtime_provenance(tmp_path, [wheel]) + + +def test_non_allowlisted_wheel_stays_in_private_bundle(tmp_path: Path) -> None: + public_wheel = tmp_path / "public_dependency-1.0-py3-none-any.whl" + private_wheel = tmp_path / "private_dependency-1.0-py3-none-any.whl" + _write_test_wheel(public_wheel, name="public-dependency", version="1.0") + _write_test_wheel( + private_wheel, + name="private-dependency", + version="1.0", + license_expression="LicenseRef-Proprietary", + ) + (tmp_path / "uv.lock").write_text( + _pypi_lock( + ("public-dependency", "1.0", public_wheel), + ("private-dependency", "1.0", private_wheel), + ), + encoding="utf-8", + ) + + public, bundled = release_publisher.partition_public_runtime_wheels( + tmp_path, + [private_wheel, public_wheel], + ) + + assert public == [public_wheel] + assert bundled == [private_wheel] + + +def test_runtime_epoch_reuses_dependencies_across_veadk_releases( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_root = tmp_path / "source" + contract = source_root / "veadk" / "cli" / "studio_artifacts.py" + contract.parent.mkdir(parents=True) + shutil.copy2(Path(__file__).parents[1] / "veadk/cli/studio_artifacts.py", contract) + (source_root / "uv.lock").write_text( + '[[package]]\nname = "dependency"\nversion = "1.0"\n' + 'source = { registry = "https://pypi.org/simple" }\n', + encoding="utf-8", + ) + cli_content = b"same-cli" + monkeypatch.setattr( + release_publisher, + "_AGENTKIT_CLI_ARCHIVE_SHA256", + hashlib.sha256(cli_content).hexdigest(), + ) + epochs: list[str] = [] + artifact_names: list[set[str]] = [] + for version, content in (("1.0.0", b"app-one"), ("1.0.1", b"app-two")): + package = tmp_path / f"package-{version}" + wheelhouse = package / "wheelhouse" + wheelhouse.mkdir(parents=True) + _write_test_wheel( + wheelhouse / "dependency-1.0-py3-none-any.whl", + name="dependency", + version="1.0", + ) + (source_root / "uv.lock").write_text( + _pypi_lock( + ( + "dependency", + "1.0", + wheelhouse / "dependency-1.0-py3-none-any.whl", + ) + ), + encoding="utf-8", + ) + veadk_name = f"veadk_python-{version}-py3-none-any.whl" + _write_test_wheel( + wheelhouse / veadk_name, + name="veadk-python", + version=version, + license_expression="Apache-2.0", + marker=content.decode(), + ) + shutil.copy2(wheelhouse / veadk_name, package / veadk_name) + (package / "agentkit-linux-x64.tar.gz").write_bytes(cli_content) + (package / "requirements.txt").write_text("local\n", encoding="utf-8") + (package / "run.sh").write_text("local\n", encoding="utf-8") + + epoch, artifacts = release_publisher.stage_studio_thin_runtime( + source_root, + package, + tmp_path / f"output-{version}", + provider="volcengine", + ) + epochs.append(epoch) + artifact_names.append({path.name for path in artifacts.iterdir()}) + + assert epochs[0] == epochs[1] + assert artifact_names == [ + {"dependency-1.0-py3-none-any.whl", "agentkit-linux-x64.tar.gz"}, + {"dependency-1.0-py3-none-any.whl", "agentkit-linux-x64.tar.gz"}, + ] + def test_standalone_publisher_stages_scheduler_backend(tmp_path: Path) -> None: source_root = tmp_path / "source" @@ -1292,6 +1965,23 @@ def test_release_server_runtime_environment_records_provider() -> None: assert environment["STUDIO_RELEASE_PROVIDER"] == "byteplus" assert environment["STUDIO_RELEASE_REGION"] == "ap-southeast-1" assert environment["STUDIO_RELEASE_BUCKET"] == "veadk-studio-byteplus" + assert environment["STUDIO_RELEASE_THIN_BUNDLES"] == "false" + + enabled = release_deploy._runtime_environment( + "x" * 32, + bucket="veadk-studio-byteplus", + provider="byteplus", + region="ap-southeast-1", + thin_bundles=True, + ) + assert enabled["STUDIO_RELEASE_THIN_BUNDLES"] == "true" + + +def test_release_server_deploy_parser_requires_explicit_thin_bundle_opt_in() -> None: + parser = release_deploy._parser() + + assert parser.parse_args([]).enable_thin_bundles is False + assert parser.parse_args(["--enable-thin-bundles"]).enable_thin_bundles is True def test_release_server_function_matches_production_resources(tmp_path: Path) -> None: @@ -1374,6 +2064,9 @@ def list_buckets(self) -> Any: def create_bucket(self, **kwargs: Any) -> None: captured["create"] = kwargs + def get_bucket_tagging(self, **_kwargs: Any) -> Any: + raise KeyError("no tags") + def put_bucket_tagging(self, **kwargs: Any) -> None: captured["tags"] = kwargs @@ -1386,6 +2079,121 @@ def put_bucket_tagging(self, **kwargs: Any) -> None: } +@pytest.mark.parametrize("provider", ["volcengine", "byteplus"]) +def test_public_artifact_bucket_exposes_only_immutable_prefix( + provider: str, +) -> None: + captured: dict[str, Any] = {} + + class _Client: + def list_buckets(self) -> Any: + return SimpleNamespace(buckets=[]) + + def create_bucket(self, **kwargs: Any) -> None: + captured["create"] = kwargs + + def get_bucket_tagging(self, **_kwargs: Any) -> Any: + raise KeyError("no tags") + + def put_bucket_tagging(self, **kwargs: Any) -> None: + captured["tags"] = kwargs + + def put_bucket_policy(self, **kwargs: Any) -> None: + captured["policy"] = kwargs + + def get_bucket_policy(self, **kwargs: Any) -> Any: + assert kwargs["bucket"] == captured["create"]["bucket"] + return SimpleNamespace(policy=captured["policy"]["policy"]) + + bucket = release_deploy._ensure_public_artifact_bucket( + _Client(), + provider, # type: ignore[arg-type] + ) + policy = json.loads(captured["policy"]["policy"]) + statement = policy["Statement"][0] + + assert captured["create"] == {"bucket": bucket} + assert statement["Principal"] == "*" + assert statement["Action"] == ["tos:GetObject"] + assert statement["Resource"] == [f"trn:tos:::{bucket}/veadk/studio/artifacts/v1/*"] + assert "release-server/jobs" not in captured["policy"]["policy"] + + +def test_public_artifact_bucket_preserves_existing_policy_and_tags() -> None: + bucket = "veadk-studio-public" + + class _Client: + def __init__(self) -> None: + self.policy: dict[str, object] = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ExistingPrivateAutomation", + "Effect": "Allow", + "Principal": {"Service": "internal"}, + "Action": ["tos:PutObject"], + "Resource": [f"trn:tos:::{bucket}/internal/*"], + } + ], + } + self.tags = [SimpleNamespace(key="owner", value="studio")] + self.policy_puts = 0 + self.tag_puts = 0 + + def list_buckets(self) -> Any: + return SimpleNamespace(buckets=[SimpleNamespace(name=bucket)]) + + def get_bucket_tagging(self, **_kwargs: Any) -> Any: + return SimpleNamespace(tag_set=self.tags) + + def put_bucket_tagging(self, **kwargs: Any) -> None: + self.tags = list(kwargs["tag_set"]) + self.tag_puts += 1 + + def get_bucket_policy(self, **_kwargs: Any) -> Any: + return SimpleNamespace(policy=json.dumps(self.policy)) + + def put_bucket_policy(self, **kwargs: Any) -> None: + self.policy = json.loads(kwargs["policy"]) + self.policy_puts += 1 + + client = _Client() + + assert release_deploy._ensure_public_artifact_bucket(client, "volcengine") == bucket + assert release_deploy._ensure_public_artifact_bucket(client, "volcengine") == bucket + + assert client.policy_puts == 1 + assert client.tag_puts == 1 + assert {item.key: item.value for item in client.tags} == { + "owner": "studio", + "note": "勿删", + } + statements = client.policy["Statement"] + assert isinstance(statements, list) + assert [item["Sid"] for item in statements] == [ + "ExistingPrivateAutomation", + "PublicReadStudioRuntimeArtifacts", + ] + + +def test_public_artifact_bucket_rejects_owned_policy_conflict() -> None: + with pytest.raises(RuntimeError, match="policy has a conflict"): + release_deploy._merge_public_artifact_policy( + { + "Statement": [ + { + "Sid": "PublicReadStudioRuntimeArtifacts", + "Effect": "Allow", + "Principal": "*", + "Action": ["tos:GetObject"], + "Resource": ["trn:tos:::wrong-bucket/*"], + } + ] + }, + "veadk-studio-public", + ) + + def test_release_workflow_publishes_one_version_to_both_providers() -> None: workflow = ( Path(__file__).parents[1] @@ -1398,3 +2206,5 @@ def test_release_workflow_publishes_one_version_to_both_providers() -> None: assert "provider: byteplus" in workflow assert "BYTEPLUS_STUDIO_RELEASE_SERVER_URL" in workflow assert '"version": os.environ["RELEASE_VERSION"]' in workflow + assert "thin_bundles:" in workflow + assert '"thinBundle": os.environ["RELEASE_THIN_BUNDLES"]' in workflow diff --git a/veadk/cli/agentkit_cli.py b/veadk/cli/agentkit_cli.py index 77e7be831..82b5a15ab 100644 --- a/veadk/cli/agentkit_cli.py +++ b/veadk/cli/agentkit_cli.py @@ -37,8 +37,9 @@ import zipfile from typing import Any - AGENTKIT_CLI_VERSION = "0.52.14" +# Direct bootstrap uses the one verified public upstream. Provider-local Studio +# bundles materialize this archive through their signed runtime manifest instead. AGENTKIT_CLI_RELEASE_HOST = "agentkit-cli.tos-cn-beijing.volces.com" AGENTKIT_CLI_RELEASE_BASE = f"https://{AGENTKIT_CLI_RELEASE_HOST}" AGENTKIT_CLI_ENV = "VEADK_AGENTKIT_CLI" diff --git a/veadk/cli/studio_artifacts.py b/veadk/cli/studio_artifacts.py new file mode 100644 index 000000000..b4b5c2a48 --- /dev/null +++ b/veadk/cli/studio_artifacts.py @@ -0,0 +1,643 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Immutable public artifacts used by thin Studio release bundles.""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path +import re +import tempfile +import urllib.error +import urllib.parse +import urllib.request +from typing import Literal, TypeAlias + +CloudProvider: TypeAlias = Literal["volcengine", "byteplus"] + +STUDIO_ARTIFACT_SCHEMA_VERSION = 2 +STUDIO_ARTIFACT_PREFIX = "veadk/studio/artifacts/v1" +STUDIO_BUNDLED_WHEELHOUSE = "bundled-wheelhouse" +STUDIO_RUNTIME_PLATFORM = "linux-x64" +STUDIO_RUNTIME_PYTHON_ABI = "cp312" +STUDIO_ARTIFACT_BUCKETS: dict[CloudProvider, str] = { + "volcengine": "veadk-studio-public", + "byteplus": "veadk-studio-byteplus-public", +} +STUDIO_ARTIFACT_REGIONS: dict[CloudProvider, str] = { + "volcengine": "cn-beijing", + "byteplus": "ap-southeast-1", +} + +_SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +_FILENAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+-]{0,254}$") +_MAX_ARTIFACT_BYTES = 512 * 1024 * 1024 +_MAX_RUNTIME_BYTES = 2 * 1024 * 1024 * 1024 +_DOWNLOAD_CHUNK_BYTES = 1024 * 1024 + + +def studio_artifact_host(provider: CloudProvider) -> str: + """Return the provider-local anonymous-read artifact host.""" + + bucket = STUDIO_ARTIFACT_BUCKETS[provider] + region = STUDIO_ARTIFACT_REGIONS[provider] + domain = "bytepluses.com" if provider == "byteplus" else "volces.com" + return f"{bucket}.tos-{region}.{domain}" + + +def studio_artifact_base_url(provider: CloudProvider) -> str: + """Return the immutable public artifact prefix for one provider.""" + + return f"https://{studio_artifact_host(provider)}/{STUDIO_ARTIFACT_PREFIX}" + + +def studio_artifact_key(sha256: str, filename: str) -> str: + """Return the content-addressed object key for one artifact.""" + + _validate_sha256(sha256) + _validate_filename(filename) + return f"{STUDIO_ARTIFACT_PREFIX}/{sha256}/{filename}" + + +def studio_artifact_url( + provider: CloudProvider, + sha256: str, + filename: str, +) -> str: + """Return a provider-local immutable public URL.""" + + return f"https://{studio_artifact_host(provider)}/{studio_artifact_key(sha256, filename)}" + + +@dataclass(frozen=True) +class StudioArtifact: + """One immutable file required by a Studio runtime epoch.""" + + provider: CloudProvider + kind: str + platform: str + python_abi: str + filename: str + url: str + size: int + sha256: str + + def __post_init__(self) -> None: + if self.provider not in {"volcengine", "byteplus"}: + raise ValueError("Studio artifact provider is invalid.") + if self.kind not in {"wheel", "agentkit-cli"}: + raise ValueError("Studio artifact kind is invalid.") + if self.platform != STUDIO_RUNTIME_PLATFORM: + raise ValueError("Studio artifact platform is invalid.") + if self.python_abi != STUDIO_RUNTIME_PYTHON_ABI: + raise ValueError("Studio artifact Python ABI is invalid.") + _validate_filename(self.filename) + _validate_sha256(self.sha256) + if self.size <= 0 or self.size > _MAX_ARTIFACT_BYTES: + raise ValueError("Studio artifact size is invalid.") + _validate_artifact_url(self.url, self.provider, self.sha256, self.filename) + + @classmethod + def from_path( + cls, + path: Path, + *, + provider: CloudProvider, + kind: str = "wheel", + ) -> StudioArtifact: + """Create exact metadata from an already-built local artifact.""" + + digest = _sha256(path) + return cls( + provider=provider, + kind=kind, + platform=STUDIO_RUNTIME_PLATFORM, + python_abi=STUDIO_RUNTIME_PYTHON_ABI, + filename=path.name, + url=studio_artifact_url(provider, digest, path.name), + size=path.stat().st_size, + sha256=digest, + ) + + def to_dict(self) -> dict[str, object]: + """Serialize using stable public field names.""" + + return { + "provider": self.provider, + "kind": self.kind, + "platform": self.platform, + "pythonAbi": self.python_abi, + "filename": self.filename, + "url": self.url, + "size": self.size, + "sha256": self.sha256, + } + + @classmethod + def from_dict(cls, payload: object) -> StudioArtifact: + """Parse fail-closed artifact metadata.""" + + if not isinstance(payload, dict) or set(payload) != { + "provider", + "kind", + "platform", + "pythonAbi", + "filename", + "url", + "size", + "sha256", + }: + raise ValueError("Studio artifact metadata is invalid.") + if ( + not all( + isinstance(payload[field], str) + for field in ( + "provider", + "kind", + "platform", + "pythonAbi", + "filename", + "url", + "sha256", + ) + ) + or not isinstance(payload["size"], int) + or isinstance(payload["size"], bool) + ): + raise ValueError("Studio artifact metadata is invalid.") + try: + return cls( + provider=payload["provider"], # type: ignore[arg-type] + kind=payload["kind"], + platform=payload["platform"], + python_abi=payload["pythonAbi"], + filename=payload["filename"], + url=payload["url"], + size=payload["size"], + sha256=payload["sha256"], + ) + except (KeyError, TypeError, ValueError) as error: + raise ValueError("Studio artifact metadata is invalid.") from error + + +@dataclass(frozen=True) +class StudioBundledArtifact: + """One exact dependency retained in the private Studio release bundle.""" + + kind: str + platform: str + python_abi: str + filename: str + size: int + sha256: str + + def __post_init__(self) -> None: + if self.kind != "wheel" or not self.filename.endswith(".whl"): + raise ValueError("Studio bundled artifact kind is invalid.") + if self.platform != STUDIO_RUNTIME_PLATFORM: + raise ValueError("Studio bundled artifact platform is invalid.") + if self.python_abi != STUDIO_RUNTIME_PYTHON_ABI: + raise ValueError("Studio bundled artifact Python ABI is invalid.") + _validate_filename(self.filename) + _validate_sha256(self.sha256) + if self.size <= 0 or self.size > _MAX_ARTIFACT_BYTES: + raise ValueError("Studio bundled artifact size is invalid.") + + @classmethod + def from_path(cls, path: Path) -> StudioBundledArtifact: + return cls( + kind="wheel", + platform=STUDIO_RUNTIME_PLATFORM, + python_abi=STUDIO_RUNTIME_PYTHON_ABI, + filename=path.name, + size=path.stat().st_size, + sha256=_sha256(path), + ) + + def to_dict(self) -> dict[str, object]: + return { + "kind": self.kind, + "platform": self.platform, + "pythonAbi": self.python_abi, + "filename": self.filename, + "size": self.size, + "sha256": self.sha256, + } + + @classmethod + def from_dict(cls, payload: object) -> StudioBundledArtifact: + if not isinstance(payload, dict) or set(payload) != { + "kind", + "platform", + "pythonAbi", + "filename", + "size", + "sha256", + }: + raise ValueError("Studio bundled artifact metadata is invalid.") + if ( + not all( + isinstance(payload[field], str) + for field in ( + "kind", + "platform", + "pythonAbi", + "filename", + "sha256", + ) + ) + or not isinstance(payload["size"], int) + or isinstance(payload["size"], bool) + ): + raise ValueError("Studio bundled artifact metadata is invalid.") + try: + return cls( + kind=payload["kind"], + platform=payload["platform"], + python_abi=payload["pythonAbi"], + filename=payload["filename"], + size=payload["size"], + sha256=payload["sha256"], + ) + except (KeyError, TypeError, ValueError) as error: + raise ValueError("Studio bundled artifact metadata is invalid.") from error + + +@dataclass(frozen=True) +class StudioRuntimeManifest: + """The exact public dependency set reused by ordinary Studio releases.""" + + provider: CloudProvider + runtime_epoch: str + artifacts: tuple[StudioArtifact, ...] + bundled_artifacts: tuple[StudioBundledArtifact, ...] = () + schema_version: int = STUDIO_ARTIFACT_SCHEMA_VERSION + platform: str = STUDIO_RUNTIME_PLATFORM + python_abi: str = STUDIO_RUNTIME_PYTHON_ABI + + def __post_init__(self) -> None: + if self.schema_version != STUDIO_ARTIFACT_SCHEMA_VERSION: + raise ValueError("Studio runtime manifest schema is unsupported.") + if self.provider not in {"volcengine", "byteplus"}: + raise ValueError("Studio runtime manifest provider is invalid.") + if self.platform != STUDIO_RUNTIME_PLATFORM: + raise ValueError("Studio runtime manifest platform is invalid.") + if self.python_abi != STUDIO_RUNTIME_PYTHON_ABI: + raise ValueError("Studio runtime manifest Python ABI is invalid.") + _validate_sha256(self.runtime_epoch) + if ( + not self.artifacts + or len(self.artifacts) + len(self.bundled_artifacts) > 512 + or sum(item.size for item in (*self.artifacts, *self.bundled_artifacts)) + > _MAX_RUNTIME_BYTES + ): + raise ValueError("Studio runtime manifest artifact count is invalid.") + filenames: set[str] = set() + wheel_count = 0 + cli_count = 0 + for artifact in self.artifacts: + if ( + artifact.provider != self.provider + or artifact.platform != self.platform + or artifact.python_abi != self.python_abi + or artifact.filename in filenames + ): + raise ValueError("Studio runtime manifest artifact is invalid.") + filenames.add(artifact.filename) + if artifact.kind == "wheel": + if not artifact.filename.endswith(".whl"): + raise ValueError("Studio runtime manifest wheel is invalid.") + wheel_count += 1 + elif artifact.kind == "agentkit-cli": + cli_count += 1 + for artifact in self.bundled_artifacts: + if ( + artifact.platform != self.platform + or artifact.python_abi != self.python_abi + or artifact.filename in filenames + ): + raise ValueError("Studio bundled runtime artifact is invalid.") + filenames.add(artifact.filename) + wheel_count += 1 + if wheel_count == 0 or cli_count != 1: + raise ValueError("Studio runtime manifest artifact set is incomplete.") + if self.runtime_epoch != runtime_epoch( + self.artifacts, + self.bundled_artifacts, + ): + raise ValueError("Studio runtime manifest epoch is invalid.") + + @classmethod + def create( + cls, + provider: CloudProvider, + artifacts: tuple[StudioArtifact, ...], + bundled_artifacts: tuple[StudioBundledArtifact, ...] = (), + ) -> StudioRuntimeManifest: + """Create a stable epoch from exact artifact bytes.""" + + return cls( + provider=provider, + runtime_epoch=runtime_epoch(artifacts, bundled_artifacts), + artifacts=artifacts, + bundled_artifacts=bundled_artifacts, + ) + + def to_json(self) -> bytes: + """Serialize deterministically for bundle signing and review.""" + + payload = { + "schemaVersion": self.schema_version, + "provider": self.provider, + "platform": self.platform, + "pythonAbi": self.python_abi, + "runtimeEpoch": self.runtime_epoch, + "artifacts": [item.to_dict() for item in self.artifacts], + "bundledArtifacts": [item.to_dict() for item in self.bundled_artifacts], + } + return (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode() + + def remote_requirements(self) -> str: + """Return a deterministic pip input containing only pinned public wheels.""" + + wheels = sorted( + (item for item in self.artifacts if item.kind == "wheel"), + key=lambda item: item.filename, + ) + return ( + "--no-index\n" + + "".join(f"{item.url}#sha256={item.sha256}\n" for item in wheels) + + "".join( + f"./{STUDIO_BUNDLED_WHEELHOUSE}/{item.filename} " + f"--hash=sha256:{item.sha256}\n" + for item in sorted( + self.bundled_artifacts, + key=lambda value: value.filename, + ) + ) + ) + + def agentkit_cli(self) -> StudioArtifact: + """Return the manifest's unique pinned AgentKit CLI artifact.""" + + return next(item for item in self.artifacts if item.kind == "agentkit-cli") + + @classmethod + def from_json(cls, content: bytes | str) -> StudioRuntimeManifest: + """Parse a bounded, exact manifest without accepting extra fields.""" + + if isinstance(content, bytes): + if len(content) > 1024 * 1024: + raise ValueError("Studio runtime manifest is too large.") + text = content.decode("utf-8") + else: + text = content + if len(text.encode()) > 1024 * 1024: + raise ValueError("Studio runtime manifest is too large.") + try: + payload = json.loads(text) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError("Studio runtime manifest is invalid.") from error + if not isinstance(payload, dict) or set(payload) != { + "schemaVersion", + "provider", + "platform", + "pythonAbi", + "runtimeEpoch", + "artifacts", + "bundledArtifacts", + }: + raise ValueError("Studio runtime manifest is invalid.") + raw_artifacts = payload.get("artifacts") + raw_bundled_artifacts = payload.get("bundledArtifacts") + if not isinstance(raw_artifacts, list) or not isinstance( + raw_bundled_artifacts, + list, + ): + raise ValueError("Studio runtime manifest is invalid.") + if ( + not isinstance(payload["schemaVersion"], int) + or isinstance(payload["schemaVersion"], bool) + or not all( + isinstance(payload[field], str) + for field in ( + "provider", + "platform", + "pythonAbi", + "runtimeEpoch", + ) + ) + ): + raise ValueError("Studio runtime manifest is invalid.") + try: + return cls( + schema_version=payload["schemaVersion"], + provider=payload["provider"], # type: ignore[arg-type] + platform=payload["platform"], + python_abi=payload["pythonAbi"], + runtime_epoch=payload["runtimeEpoch"], + artifacts=tuple( + StudioArtifact.from_dict(item) for item in raw_artifacts + ), + bundled_artifacts=tuple( + StudioBundledArtifact.from_dict(item) + for item in raw_bundled_artifacts + ), + ) + except (KeyError, TypeError, ValueError) as error: + raise ValueError("Studio runtime manifest is invalid.") from error + + +def runtime_epoch( + artifacts: tuple[StudioArtifact, ...], + bundled_artifacts: tuple[StudioBundledArtifact, ...] = (), +) -> str: + """Hash content identity while remaining independent of provider host.""" + + payload = [ + { + "kind": item.kind, + "platform": item.platform, + "pythonAbi": item.python_abi, + "filename": item.filename, + "size": item.size, + "sha256": item.sha256, + } + for item in sorted(artifacts, key=lambda value: (value.kind, value.filename)) + ] + payload.extend( + { + "kind": item.kind, + "platform": item.platform, + "pythonAbi": item.python_abi, + "filename": item.filename, + "size": item.size, + "sha256": item.sha256, + } + for item in sorted( + bundled_artifacts, + key=lambda value: (value.kind, value.filename), + ) + ) + return hashlib.sha256( + json.dumps(payload, separators=(",", ":"), sort_keys=True).encode() + ).hexdigest() + + +def download_studio_artifact( + artifact: StudioArtifact, + destination: Path, +) -> Path: + """Download one exact public object atomically and fail closed.""" + + _validate_artifact_url( + artifact.url, + artifact.provider, + artifact.sha256, + artifact.filename, + ) + if ( + destination.is_file() + and destination.stat().st_size == artifact.size + and _sha256(destination) == artifact.sha256 + ): + return destination + destination.parent.mkdir(parents=True, exist_ok=True) + temporary_handle = tempfile.NamedTemporaryFile( + mode="wb", + prefix=f".{destination.name}.{os.getpid()}.", + suffix=".part", + dir=destination.parent, + delete=False, + ) + temporary = Path(temporary_handle.name) + temporary_handle.close() + digest = hashlib.sha256() + size = 0 + try: + with urllib.request.urlopen(artifact.url, timeout=120) as response: + final_url = str(getattr(response, "geturl", lambda: artifact.url)()) + _validate_artifact_url( + final_url, + artifact.provider, + artifact.sha256, + artifact.filename, + ) + raw_length = response.headers.get("Content-Length") + if raw_length is not None and int(raw_length) != artifact.size: + raise ValueError("Studio artifact Content-Length is invalid.") + with temporary.open("wb") as output: + while chunk := response.read(_DOWNLOAD_CHUNK_BYTES): + size += len(chunk) + if size > artifact.size or size > _MAX_ARTIFACT_BYTES: + raise ValueError("Studio artifact size is invalid.") + output.write(chunk) + digest.update(chunk) + if size != artifact.size or digest.hexdigest() != artifact.sha256: + raise ValueError("Studio artifact checksum verification failed.") + os.replace(temporary, destination) + return destination + except (OSError, TimeoutError, urllib.error.URLError, ValueError): + temporary.unlink(missing_ok=True) + raise + + +def probe_studio_artifact(artifact: StudioArtifact) -> None: + """Verify anonymous provider-local access without downloading artifact bytes.""" + + _validate_artifact_url( + artifact.url, + artifact.provider, + artifact.sha256, + artifact.filename, + ) + request = urllib.request.Request(artifact.url, method="HEAD") + try: + with urllib.request.urlopen(request, timeout=30) as response: + final_url = str(getattr(response, "geturl", lambda: artifact.url)()) + _validate_artifact_url( + final_url, + artifact.provider, + artifact.sha256, + artifact.filename, + ) + if ( + int(getattr(response, "status", 200) or 200) != 200 + or int(response.headers.get("Content-Length", 0) or 0) != artifact.size + ): + raise ValueError("Studio artifact public metadata is invalid.") + except (OSError, TimeoutError, urllib.error.URLError, ValueError) as error: + raise ValueError("Studio artifact is not publicly available.") from error + + +def _validate_artifact_url( + url: str, + provider: CloudProvider, + sha256: str, + filename: str, +) -> None: + expected = studio_artifact_url(provider, sha256, filename) + parsed = urllib.parse.urlsplit(url) + if ( + url != expected + or parsed.scheme != "https" + or parsed.hostname != studio_artifact_host(provider) + or parsed.port is not None + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + ): + raise ValueError("Studio artifact URL is invalid.") + + +def _validate_filename(filename: str) -> None: + if not _FILENAME_PATTERN.fullmatch(filename) or Path(filename).name != filename: + raise ValueError("Studio artifact filename is invalid.") + + +def _validate_sha256(value: str) -> None: + if not _SHA256_PATTERN.fullmatch(value): + raise ValueError("Studio artifact SHA-256 is invalid.") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while chunk := source.read(_DOWNLOAD_CHUNK_BYTES): + digest.update(chunk) + return digest.hexdigest() + + +__all__ = [ + "STUDIO_ARTIFACT_BUCKETS", + "STUDIO_ARTIFACT_PREFIX", + "STUDIO_BUNDLED_WHEELHOUSE", + "STUDIO_RUNTIME_PLATFORM", + "STUDIO_RUNTIME_PYTHON_ABI", + "StudioArtifact", + "StudioBundledArtifact", + "StudioRuntimeManifest", + "download_studio_artifact", + "probe_studio_artifact", + "runtime_epoch", + "studio_artifact_base_url", + "studio_artifact_host", + "studio_artifact_key", + "studio_artifact_url", +] diff --git a/veadk/cli/studio_companion.py b/veadk/cli/studio_companion.py index a324c13bd..d36215466 100644 --- a/veadk/cli/studio_companion.py +++ b/veadk/cli/studio_companion.py @@ -18,12 +18,19 @@ import argparse from pathlib import Path +from typing import cast from veadk.cli.agentkit_cli import ( AGENTKIT_CLI_VERSION, AgentKitCliError, + default_agentkit_cli_cache_root, resolve_agentkit_cli, ) +from veadk.cli.studio_artifacts import ( + StudioRuntimeManifest, + download_studio_artifact, +) +from veadk.utils.cloud_provider import CloudProvider, normalize_cloud_provider StudioCompanionError = AgentKitCliError @@ -35,16 +42,44 @@ def required_agentkit_cli_version() -> str: return AGENTKIT_CLI_VERSION -def validate_installed_agentkit_cli(*, archive: Path | None = None) -> str: +def validate_installed_agentkit_cli( + *, + archive: Path | None = None, + runtime_manifest: Path | None = None, + provider: CloudProvider | str | None = None, +) -> str: """Resolve or install the exact CLI required by Studio.""" + if archive is not None and runtime_manifest is not None: + raise StudioCompanionError( + "Studio must use either a local CLI archive or a runtime manifest." + ) + resolved_provider = normalize_cloud_provider(provider) if provider else None + if runtime_manifest is not None: + try: + manifest = StudioRuntimeManifest.from_json(runtime_manifest.read_bytes()) + if resolved_provider is not None and manifest.provider != resolved_provider: + raise ValueError("Studio runtime manifest provider is invalid.") + artifact = manifest.agentkit_cli() + archive = download_studio_artifact( + artifact, + default_agentkit_cli_cache_root() + / "studio-artifacts" + / artifact.sha256 + / artifact.filename, + ) + except (OSError, ValueError) as error: + raise StudioCompanionError(str(error)) from error resolve_agentkit_cli(archive=archive) return AGENTKIT_CLI_VERSION def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--archive", type=Path) + source = parser.add_mutually_exclusive_group() + source.add_argument("--archive", type=Path) + source.add_argument("--runtime-manifest", type=Path) + parser.add_argument("--provider", choices=("volcengine", "byteplus")) return parser @@ -52,7 +87,11 @@ def main() -> None: """Fail the candidate Studio revision before serving if bootstrap fails.""" args = _parser().parse_args() - version = validate_installed_agentkit_cli(archive=args.archive) + version = validate_installed_agentkit_cli( + archive=args.archive, + runtime_manifest=args.runtime_manifest, + provider=cast(CloudProvider | None, args.provider), + ) print(f"AgentKit CLI {version} is ready.") diff --git a/veadk/cli/studio_package.py b/veadk/cli/studio_package.py index 411fc45b0..51475dbf4 100644 --- a/veadk/cli/studio_package.py +++ b/veadk/cli/studio_package.py @@ -56,6 +56,7 @@ def studio_run_script( site_logo_filename: str | None = None, *, provider: CloudProvider | None = DEFAULT_CLOUD_PROVIDER, + runtime_manifest_filename: str | None = None, ) -> str: """Return the authenticated VeFaaS entrypoint used by Studio.""" provider_argument = ( @@ -70,6 +71,14 @@ def studio_run_script( if site_logo_filename: command += f' --site-logo "$ROOT_DIR/{site_logo_filename}"' command += ' --host "$HOST" --port "$PORT"\n' + companion = ( + "python3 -m veadk.cli.studio_companion " + f'--runtime-manifest "$ROOT_DIR/{runtime_manifest_filename}" ' + f"--provider {provider_argument}\n" + if runtime_manifest_filename + else "python3 -m veadk.cli.studio_companion " + f'--archive "$ROOT_DIR/{STUDIO_AGENTKIT_CLI_ARTIFACT.filename}"\n' + ) return ( "#!/bin/bash\n" "set -ex\n" @@ -79,8 +88,7 @@ def studio_run_script( "HOST=0.0.0.0\n" "PORT=${_FAAS_RUNTIME_PORT:-8000}\n" 'export PYTHONPATH="./site-packages${PYTHONPATH:+:$PYTHONPATH}"\n' - "python3 -m veadk.cli.studio_companion " - f'--archive "$ROOT_DIR/{STUDIO_AGENTKIT_CLI_ARTIFACT.filename}"\n' + f"{companion}" f"{command}" ) diff --git a/veadk/cli/studio_release.py b/veadk/cli/studio_release.py index 89146d33f..2890a0b05 100644 --- a/veadk/cli/studio_release.py +++ b/veadk/cli/studio_release.py @@ -72,6 +72,9 @@ class StudioReleaseManifest: size: int created_at: str changelog: tuple[str, ...] = () + runtime_epoch: str = "" + thin_sha256: str = "" + thin_size: int = 0 def __post_init__(self) -> None: try: @@ -104,6 +107,15 @@ def __post_init__(self) -> None: not item.strip() or len(item) > 240 for item in self.changelog ): raise StudioReleaseError("Studio release changelog is invalid.") + thin_values = (self.runtime_epoch, self.thin_sha256, self.thin_size) + if any(thin_values): + if ( + not _SHA256_PATTERN.fullmatch(self.runtime_epoch) + or not _SHA256_PATTERN.fullmatch(self.thin_sha256) + or self.thin_size <= 0 + or self.thin_size > MAX_STUDIO_BUNDLE_BYTES + ): + raise StudioReleaseError("Studio thin release metadata is invalid.") @classmethod def from_json(cls, payload: bytes | str) -> StudioReleaseManifest: @@ -124,6 +136,9 @@ def from_json(cls, payload: bytes | str) -> StudioReleaseManifest: size=int(raw["size"]), created_at=str(raw["createdAt"]), changelog=tuple(str(item) for item in raw.get("changelog", [])), + runtime_epoch=str(raw.get("runtimeEpoch", "")), + thin_sha256=str(raw.get("thinSha256", "")), + thin_size=int(raw.get("thinSize", 0) or 0), ) except (KeyError, TypeError, ValueError) as error: raise StudioReleaseError( @@ -141,6 +156,14 @@ def to_json(self) -> bytes: "createdAt": data["created_at"], "changelog": list(data["changelog"]), } + if self.runtime_epoch: + payload.update( + { + "runtimeEpoch": self.runtime_epoch, + "thinSha256": self.thin_sha256, + "thinSize": self.thin_size, + } + ) return (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode() @@ -157,6 +180,14 @@ def bundle_object_key(prefix: str, version: str) -> str: return f"{normalize_release_prefix(prefix)}/releases/{version}/studio-bundle.zip" +def thin_bundle_object_key(prefix: str, version: str) -> str: + """Return the immutable thin bundle object key for capable updaters.""" + + return ( + f"{normalize_release_prefix(prefix)}/releases/{version}/studio-bundle-thin.zip" + ) + + def manifest_object_key(prefix: str, version: str) -> str: """Return the immutable object key for a release manifest.""" return f"{normalize_release_prefix(prefix)}/releases/{version}/manifest.json" @@ -254,33 +285,77 @@ def download_bundle( destination: Path, ) -> None: """Download one bundle and verify its exact size and digest.""" + self._download_bundle_object( + key=bundle_object_key(self.prefix, manifest.version), + destination=destination, + expected_size=manifest.size, + expected_sha256=manifest.sha256, + ) + + def download_thin_bundle( + self, + manifest: StudioReleaseManifest, + destination: Path, + ) -> None: + """Download the optional thin bundle used only by capable updaters.""" + + if not manifest.runtime_epoch: + raise StudioReleaseError("Studio release has no thin bundle.") + self._download_bundle_object( + key=thin_bundle_object_key(self.prefix, manifest.version), + destination=destination, + expected_size=manifest.thin_size, + expected_sha256=manifest.thin_sha256, + ) + + def _download_bundle_object( + self, + *, + key: str, + destination: Path, + expected_size: int, + expected_sha256: str, + ) -> None: response = self._client.get_object( bucket=self.bucket, - key=bundle_object_key(self.prefix, manifest.version), + key=key, ) destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(f".{destination.name}.{os.getpid()}.part") + temporary.unlink(missing_ok=True) digest = hashlib.sha256() size = 0 - with destination.open("wb") as output: - for chunk in response: - size += len(chunk) - if size > MAX_STUDIO_BUNDLE_BYTES or size > manifest.size: - raise StudioReleaseError( - "Studio release bundle exceeds its manifest size." - ) - digest.update(chunk) - output.write(chunk) - if size != manifest.size: - raise StudioReleaseError( - "Studio release bundle size does not match manifest." - ) - if digest.hexdigest() != manifest.sha256: - raise StudioReleaseError( - "Studio release bundle checksum does not match manifest." - ) + try: + with temporary.open("wb") as output: + for chunk in response: + size += len(chunk) + if size > MAX_STUDIO_BUNDLE_BYTES or size > expected_size: + raise StudioReleaseError( + "Studio release bundle exceeds its manifest size." + ) + digest.update(chunk) + output.write(chunk) + if size != expected_size: + raise StudioReleaseError( + "Studio release bundle size does not match manifest." + ) + if digest.hexdigest() != expected_sha256: + raise StudioReleaseError( + "Studio release bundle checksum does not match manifest." + ) + os.replace(temporary, destination) + except Exception: + temporary.unlink(missing_ok=True) + raise - def publish(self, bundle: Path, manifest: StudioReleaseManifest) -> None: - """Publish immutable objects, then move the latest pointer last.""" + def publish( + self, + bundle: Path, + manifest: StudioReleaseManifest, + *, + thin_bundle: Path | None = None, + ) -> None: + """Publish immutable objects and repair an identical interrupted attempt.""" if not bundle.is_file(): raise StudioReleaseError(f"Studio release bundle does not exist: {bundle}") content = bundle.read_bytes() @@ -292,26 +367,46 @@ def publish(self, bundle: Path, manifest: StudioReleaseManifest) -> None: raise StudioReleaseError( "Studio release bundle checksum does not match manifest." ) + thin_content: bytes | None = None + if manifest.runtime_epoch: + if thin_bundle is None or not thin_bundle.is_file(): + raise StudioReleaseError("Studio thin release bundle is missing.") + thin_content = thin_bundle.read_bytes() + if ( + len(thin_content) != manifest.thin_size + or hashlib.sha256(thin_content).hexdigest() != manifest.thin_sha256 + ): + raise StudioReleaseError("Studio thin bundle does not match manifest.") + elif thin_bundle is not None: + raise StudioReleaseError("Studio full release has no thin bundle.") manifest_bytes = manifest.to_json() releases = self._existing_releases() - if any(item.version > manifest.version for item in releases): + same_version = [item for item in releases if item.version == manifest.version] + if same_version and same_version != [manifest]: + raise StudioReleaseError("Studio release version has a conflict.") + newer_exists = any(item.version > manifest.version for item in releases) + if newer_exists and not same_version: raise StudioReleaseError( "Studio release version must be newer than the published releases." ) - self._client.put_object( - bucket=self.bucket, + self._put_immutable( key=bundle_object_key(self.prefix, manifest.version), content=content, content_type="application/zip", - forbid_overwrite=True, ) - self._client.put_object( - bucket=self.bucket, + if thin_content is not None: + self._put_immutable( + key=thin_bundle_object_key(self.prefix, manifest.version), + content=thin_content, + content_type="application/zip", + ) + self._put_immutable( key=manifest_object_key(self.prefix, manifest.version), content=manifest_bytes, content_type="application/json", - forbid_overwrite=True, ) + if newer_exists: + return releases = [item for item in releases if item.version != manifest.version] releases.append(manifest) releases.sort(key=lambda item: item.version, reverse=True) @@ -328,32 +423,98 @@ def publish(self, bundle: Path, manifest: StudioReleaseManifest) -> None: ) + "\n" ).encode() - self._client.put_object( - bucket=self.bucket, + self._put_mutable_verified( key=release_catalog_object_key(self.prefix), content=catalog_bytes, content_type="application/json", ) - self._client.put_object( - bucket=self.bucket, + self._put_mutable_verified( key=latest_manifest_object_key(self.prefix), content=manifest_bytes, content_type="application/json", ) + def _get_optional_object(self, key: str, max_bytes: int) -> bytes | None: + try: + response = self._client.get_object(bucket=self.bucket, key=key) + return _read_object(response, max_bytes) + except Exception as error: + if _is_not_found(error): + return None + raise StudioReleaseError("Studio release object lookup failed.") from error + + def _put_immutable(self, *, key: str, content: bytes, content_type: str) -> None: + existing = self._get_optional_object(key, MAX_STUDIO_BUNDLE_BYTES) + if existing is not None: + if existing != content: + raise StudioReleaseError("Studio immutable release object conflicts.") + return + try: + self._client.put_object( + bucket=self.bucket, + key=key, + content=content, + content_type=content_type, + forbid_overwrite=True, + ) + except Exception as error: + existing = self._get_optional_object(key, MAX_STUDIO_BUNDLE_BYTES) + if existing == content: + return + if existing is not None: + raise StudioReleaseError( + "Studio immutable release object conflicts." + ) from error + raise StudioReleaseError( + "Studio immutable release object upload failed." + ) from error + + def _put_mutable_verified( + self, + *, + key: str, + content: bytes, + content_type: str, + ) -> None: + if self._get_optional_object(key, MAX_STUDIO_BUNDLE_BYTES) == content: + return + try: + self._client.put_object( + bucket=self.bucket, + key=key, + content=content, + content_type=content_type, + ) + except Exception as error: + if self._get_optional_object(key, MAX_STUDIO_BUNDLE_BYTES) == content: + return + raise StudioReleaseError("Studio release pointer upload failed.") from error + if self._get_optional_object(key, MAX_STUDIO_BUNDLE_BYTES) != content: + raise StudioReleaseError("Studio release pointer verification failed.") + def _existing_releases(self) -> list[StudioReleaseManifest]: - """Load the catalog, seeding it from the legacy latest pointer.""" + """Merge the catalog and latest pointer without trusting either alone.""" + + releases: list[StudioReleaseManifest] = [] try: - return self.release_catalog() + releases = self.release_catalog() except Exception as error: if not _is_not_found(error): raise try: - return [self.latest_manifest()] + latest = self.latest_manifest() except Exception as error: - if _is_not_found(error): - return [] - raise + if not _is_not_found(error): + raise + else: + matches = [item for item in releases if item.version == latest.version] + if matches and matches != [latest]: + raise StudioReleaseError( + "Studio release catalog and latest pointer conflict." + ) + if not matches: + releases.append(latest) + return sorted(releases, key=lambda item: item.version, reverse=True) def build_studio_release( diff --git a/veadk/cli/studio_self_update.py b/veadk/cli/studio_self_update.py index dd5cb9ba3..6d329788a 100644 --- a/veadk/cli/studio_self_update.py +++ b/veadk/cli/studio_self_update.py @@ -37,6 +37,10 @@ from fastapi import HTTPException, Request from veadk.cli.frontend_branding import SiteLogo +from veadk.cli.studio_artifacts import ( + StudioRuntimeManifest, + probe_studio_artifact, +) from veadk.cli.studio_package import studio_run_script from veadk.cli.studio_release import ( DEFAULT_RELEASE_PREFIX, @@ -405,12 +409,12 @@ def submit_version(self, version: str | None) -> StudioReleaseManifest: raise StudioReleaseError("只能选择比当前版本新的 Studio 版本。") with tempfile.TemporaryDirectory(prefix="veadk_studio_self_update_") as tmp: workspace = Path(tmp) - archive = workspace / "studio-bundle.zip" - package_dir = workspace / "package" - self._set_progress("downloading", "正在下载并校验完整更新包") - store.download_bundle(manifest, archive) + package_dir = self._download_runtime_package( + store, + manifest, + workspace, + ) self._set_progress("preparing", "正在准备 VeFaaS Function 代码") - extract_studio_bundle(archive, package_dir) self._prepare_package(package_dir) from veadk.integrations.ve_faas.ve_faas import VeFaaS @@ -824,12 +828,64 @@ def _prepare_package(self, package_dir: Path) -> None: filename = f"site-logo.{self._branding_logo.extension}" (package_dir / filename).write_bytes(self._branding_logo.content) (package_dir / "run.sh").write_text( - studio_run_script(filename, provider=self._settings.provider), + studio_run_script( + filename, + provider=self._settings.provider, + runtime_manifest_filename=( + "studio-runtime.json" + if (package_dir / "studio-runtime.json").is_file() + else None + ), + ), encoding="utf-8", newline="\n", ) (package_dir / "run.sh").chmod(0o755) + def _download_runtime_package( + self, + store: StudioReleaseStore, + release: StudioReleaseManifest, + workspace: Path, + ) -> Path: + """Prefer a verified thin bundle while preserving the legacy full fallback.""" + + if release.runtime_epoch: + try: + self._set_progress("downloading", "正在下载并校验精简更新包") + thin_archive = workspace / "studio-bundle-thin.zip" + thin_package = workspace / "package-thin" + store.download_thin_bundle(release, thin_archive) + extract_studio_bundle(thin_archive, thin_package) + runtime = StudioRuntimeManifest.from_json( + (thin_package / "studio-runtime.json").read_bytes() + ) + if ( + runtime.provider != self._settings.provider + or runtime.runtime_epoch != release.runtime_epoch + ): + raise ValueError("Studio runtime manifest does not match release.") + for artifact in runtime.artifacts: + probe_studio_artifact(artifact) + return thin_package + except Exception: + logger.warning( + "Studio thin bundle is unavailable; using the full bundle" + ) + self._set_progress( + "downloading", + ( + "公共运行时制品不可用，正在切换完整离线更新包" + if release.runtime_epoch + else "正在下载并校验完整更新包" + ), + ) + archive = workspace / "studio-bundle.zip" + package = workspace / "package" + store.download_bundle(release, archive) + extract_studio_bundle(archive, package) + return package + def extract_studio_bundle(archive: Path, destination: Path) -> None: """Safely extract a bounded Studio bundle and validate its entrypoint.""" From 4a717ca79bc34a31c24effb58b15ced937bc0e85 Mon Sep 17 00:00:00 2001 From: "yanan.zhangyn" Date: Fri, 4 Sep 2026 09:56:00 +0800 Subject: [PATCH 2/3] fix(ci): complete artifact test license header --- tests/cli/test_studio_artifacts.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/cli/test_studio_artifacts.py b/tests/cli/test_studio_artifacts.py index 3483bdd75..ce313f766 100644 --- a/tests/cli/test_studio_artifacts.py +++ b/tests/cli/test_studio_artifacts.py @@ -2,6 +2,15 @@ # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from __future__ import annotations From 152d98571de4da689726fb1a773b54a59ed34e0f Mon Sep 17 00:00:00 2001 From: "yanan.zhangyn" Date: Fri, 4 Sep 2026 10:29:23 +0800 Subject: [PATCH 3/3] fix(studio): support thin publisher on Python 3.10 --- .../studio_release_server/publisher.py | 6 ++++- pyproject.toml | 1 + uv.lock | 26 ++++++++++--------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/frontend/service/studio_release_server/publisher.py b/frontend/service/studio_release_server/publisher.py index 21a56b124..3d8385708 100644 --- a/frontend/service/studio_release_server/publisher.py +++ b/frontend/service/studio_release_server/publisher.py @@ -27,7 +27,6 @@ import subprocess import sys import tempfile -import tomllib import urllib.parse import urllib.request import zipfile @@ -40,6 +39,11 @@ from typing import Any, cast from zoneinfo import ZoneInfo +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - Python 3.10 compatibility + import tomli as tomllib # pyright: ignore[reportMissingImports] + if __package__: from .offline_runtime import build_studio_offline_runtime else: diff --git a/pyproject.toml b/pyproject.toml index 5faa3be98..451616f33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ dependencies = [ "httpx>=0.27,<1", # Secure server-side webpage fetching for Studio knowledge imports "jsonschema>=4.23,<5", # Validate Studio BFF dynamic-tool arguments "trafilatura>=2.0,<2.1", # Extract webpage main content as Markdown for knowledge imports + "tomli>=2.0.1; python_version < '3.11'", # TOML parser for supported Python 3.10 ] [project.scripts] diff --git a/uv.lock b/uv.lock index 35d87a0ab..3cb43effb 100644 --- a/uv.lock +++ b/uv.lock @@ -4612,18 +4612,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, ] -[[package]] -name = "qrcode" -version = "8.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8f/b2/7fc2931bfae0af02d5f53b174e9cf701adbb35f39d69c2af63d4a39f81a9/qrcode-8.2.tar.gz", hash = "sha256:35c3f2a4172b33136ab9f6b3ef1c00260dd2f66f858f24d88418a015f446506c", size = 43317, upload-time = "2025-05-01T15:44:24.726Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/b8/d2d6d731733f51684bbf76bf34dab3b70a9148e8f2cef2bb544fccec681a/qrcode-8.2-py3-none-any.whl", hash = "sha256:16e64e0716c14960108e85d853062c9e8bba5ca8252c0b4d0231b9df4060ff4f", size = 45986, upload-time = "2025-05-01T15:44:22.781Z" }, -] - [[package]] name = "qdrant-client" version = "1.19.0" @@ -4643,6 +4631,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/3c/480c61cc8d5a3e76bb44e86231f408c93643e5498beadbbeb381ab55d02b/qdrant_client-1.19.0-py3-none-any.whl", hash = "sha256:13602a2b3478a95ecdf42f97b93d7f703b63a3361cd912a04495a33a5ac14121", size = 396157, upload-time = "2026-08-04T14:32:55.734Z" }, ] +[[package]] +name = "qrcode" +version = "8.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/b2/7fc2931bfae0af02d5f53b174e9cf701adbb35f39d69c2af63d4a39f81a9/qrcode-8.2.tar.gz", hash = "sha256:35c3f2a4172b33136ab9f6b3ef1c00260dd2f66f858f24d88418a015f446506c", size = 43317, upload-time = "2025-05-01T15:44:24.726Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/b8/d2d6d731733f51684bbf76bf34dab3b70a9148e8f2cef2bb544fccec681a/qrcode-8.2-py3-none-any.whl", hash = "sha256:16e64e0716c14960108e85d853062c9e8bba5ca8252c0b4d0231b9df4060ff4f", size = 45986, upload-time = "2025-05-01T15:44:22.781Z" }, +] + [[package]] name = "questionary" version = "2.1.1" @@ -5809,6 +5809,7 @@ dependencies = [ { name = "pyyaml" }, { name = "qrcode" }, { name = "sqlalchemy", extra = ["asyncio"] }, + { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "tos" }, { name = "trafilatura" }, { name = "trustedmcp" }, @@ -5947,6 +5948,7 @@ requires-dist = [ { name = "redis", marker = "extra == 'database'", specifier = ">=5.0" }, { name = "redis", marker = "extra == 'extensions'", specifier = ">=5.0" }, { name = "sqlalchemy", extras = ["asyncio"], specifier = ">=2,<3" }, + { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0.1" }, { name = "tos", specifier = ">=2.8.4" }, { name = "trafilatura", specifier = ">=2.0,<2.1" }, { name = "trustedmcp", specifier = "==0.0.5" },