Skip to content

Commit 03ef178

Browse files
Ronald Tseronaldtse
authored andcommitted
feat(imf): multi-part release assets — GitHub 2GiB cap + one-command publish
heb-diac-1.0 fp32 (2.59 GiB) exceeds GitHub's hard 2GiB per-asset cap. models.yaml entries may now carry parts: [{url, sha256, size}]; the registry streams parts in order, verifies each sha256 as it lands, and checks the assembled file against the whole-file sha256 — the cache contract is identical to single-file models. - scripts/split_release.py: split + per-part sha256 + models.yaml block - scripts/publish_model.py: validate (strict) -> split -> GH Release -> models.yaml entry -> release branch -> PR; idempotent re-runs - runtime registry: parts resolution + tests (assembly, corrupt-part rejection, verified cache hit)
1 parent 2836d60 commit 03ef178

4 files changed

Lines changed: 450 additions & 2 deletions

File tree

‎runtime/src/interscript_ml/registry.py‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,13 @@ class RegistryError(ValueError):
2929
"""The index cannot be fetched/parsed, or the id is unknown."""
3030

3131

32+
@dataclass(frozen=True)
33+
class Part:
34+
url: str
35+
sha256: str
36+
size: int
37+
38+
3239
@dataclass(frozen=True)
3340
class IndexEntry:
3441
id: str
@@ -38,6 +45,7 @@ class IndexEntry:
3845
size: int
3946
precision: str
4047
task: str
48+
parts: tuple[Part, ...] = ()
4149

4250

4351
def cache_dir() -> Path:
@@ -58,14 +66,19 @@ def load_index(index_url: str | None = None) -> dict[str, IndexEntry]:
5866
raise RegistryError("index must be a mapping with version: 1")
5967
entries: dict[str, IndexEntry] = {}
6068
for model_id, spec in raw.get("models", {}).items():
69+
parts = tuple(
70+
Part(url=part["url"], sha256=part["sha256"], size=int(part.get("size", 0)))
71+
for part in spec.get("parts", [])
72+
)
6173
entries[model_id] = IndexEntry(
6274
id=model_id,
6375
filename=spec["filename"],
64-
url=spec["url"],
76+
url=spec.get("url", ""),
6577
sha256=spec["sha256"],
6678
size=int(spec.get("size", 0)),
6779
precision=spec.get("precision", "fp32"),
6880
task=spec.get("task", ""),
81+
parts=parts,
6982
)
7083
return entries
7184

@@ -78,6 +91,32 @@ def _sha256_file(path: Path) -> str:
7891
return digest.hexdigest()
7992

8093

94+
def _open_channel(url: str):
95+
if url.startswith("file://"):
96+
return open(urlparse(url).path, "rb")
97+
return urllib.request.urlopen(url)
98+
99+
100+
def _download_parts(entry: IndexEntry, downloaded: Path) -> None:
101+
"""Stream parts into `downloaded` in index order, verifying each part's
102+
sha256 as it lands. Used when the artifact exceeds GitHub's 2 GiB
103+
per-asset cap; the assembled file is checked against entry.sha256 by
104+
the caller, so the cache contract is identical to single-file models."""
105+
with downloaded.open("ab") as out:
106+
for index, part in enumerate(entry.parts):
107+
digest = hashlib.sha256()
108+
with _open_channel(part.url) as remote:
109+
while chunk := remote.read(1024 * 1024):
110+
out.write(chunk)
111+
digest.update(chunk)
112+
actual = digest.hexdigest()
113+
if actual != part.sha256:
114+
raise RegistryError(
115+
f"part {index} of {entry.filename} sha256 mismatch: "
116+
f"got {actual}, index says {part.sha256}"
117+
)
118+
119+
81120
def resolve(model_id: str, index_url: str | None = None) -> Path:
82121
"""Return a verified local zip path for `model_id`, downloading and
83122
installing into the cache when needed. Never returns an unverified
@@ -96,7 +135,9 @@ def resolve(model_id: str, index_url: str | None = None) -> Path:
96135
fd, tmp_name = tempfile.mkstemp(dir=target.parent, suffix=".part")
97136
os.close(fd)
98137
downloaded = Path(tmp_name)
99-
if entry.url.startswith("file://"):
138+
if entry.parts:
139+
_download_parts(entry, downloaded)
140+
elif entry.url.startswith("file://"):
100141
source = Path(urlparse(entry.url).path)
101142
if not source.is_file():
102143
raise RegistryError(f"channel file missing: {source}")

‎runtime/tests/test_registry.py‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,3 +80,91 @@ def test_model_load_by_id(tmp_path: Path) -> None:
8080
assert isinstance(model.translate("he", max_len=4), str)
8181
finally:
8282
os.environ.pop("INTERSCRIPT_ML_CACHE", None)
83+
84+
85+
def test_resolve_parts_assembles_and_verifies(tmp_path: Path) -> None:
86+
import hashlib
87+
88+
zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip")
89+
blob = zip_path.read_bytes()
90+
part_a, part_b = blob[: len(blob) // 2 + 3], blob[len(blob) // 2 + 3 :]
91+
channel = tmp_path / "channel"
92+
(channel / "tiny.zip.part-00").write_bytes(part_a)
93+
(channel / "tiny.zip.part-01").write_bytes(part_b)
94+
index = {
95+
"version": 1,
96+
"models": {
97+
"tiny-1.0": {
98+
"task": "translit",
99+
"precision": "fp32",
100+
"filename": "tiny.zip",
101+
"sha256": hashlib.sha256(blob).hexdigest(),
102+
"size": len(blob),
103+
"parts": [
104+
{
105+
"url": f"file://{channel / 'tiny.zip.part-00'}",
106+
"sha256": hashlib.sha256(part_a).hexdigest(),
107+
"size": len(part_a),
108+
},
109+
{
110+
"url": f"file://{channel / 'tiny.zip.part-01'}",
111+
"sha256": hashlib.sha256(part_b).hexdigest(),
112+
"size": len(part_b),
113+
},
114+
],
115+
},
116+
},
117+
}
118+
index_path = tmp_path / "models.yaml"
119+
index_path.write_text(yaml.safe_dump(index), encoding="utf-8")
120+
cache = tmp_path / "cache"
121+
os.environ["INTERSCRIPT_ML_CACHE"] = str(cache)
122+
try:
123+
local = resolve("tiny-1.0", index_url=str(index_path))
124+
assert local == cache / "models" / "tiny-1.0" / "tiny.zip"
125+
assert local.read_bytes() == blob
126+
zip_path.unlink()
127+
(channel / "tiny.zip.part-00").unlink()
128+
assert resolve("tiny-1.0", index_url=str(index_path)) == local
129+
finally:
130+
os.environ.pop("INTERSCRIPT_ML_CACHE", None)
131+
132+
133+
def test_resolve_parts_rejects_corrupt_part(tmp_path: Path) -> None:
134+
import hashlib
135+
136+
zip_path = build_tiny_zip(tmp_path / "channel" / "tiny.zip")
137+
blob = zip_path.read_bytes()
138+
part_a, part_b = blob[:7], blob[7:]
139+
channel = tmp_path / "channel"
140+
(channel / "tiny.zip.part-00").write_bytes(part_a)
141+
(channel / "tiny.zip.part-01").write_bytes(part_b)
142+
index = {
143+
"version": 1,
144+
"models": {
145+
"tiny-1.0": {
146+
"filename": "tiny.zip",
147+
"sha256": hashlib.sha256(blob).hexdigest(),
148+
"parts": [
149+
{
150+
"url": f"file://{channel / 'tiny.zip.part-00'}",
151+
"sha256": "0" * 64,
152+
"size": len(part_a),
153+
},
154+
{
155+
"url": f"file://{channel / 'tiny.zip.part-01'}",
156+
"sha256": hashlib.sha256(part_b).hexdigest(),
157+
"size": len(part_b),
158+
},
159+
],
160+
},
161+
},
162+
}
163+
index_path = tmp_path / "models.yaml"
164+
index_path.write_text(yaml.safe_dump(index), encoding="utf-8")
165+
os.environ["INTERSCRIPT_ML_CACHE"] = str(tmp_path / "cache")
166+
try:
167+
with pytest.raises(RegistryError, match="part 0"):
168+
resolve("tiny-1.0", index_url=str(index_path))
169+
finally:
170+
os.environ.pop("INTERSCRIPT_ML_CACHE", None)

0 commit comments

Comments
 (0)