Skip to content

Commit b94f399

Browse files
author
Ronald Tse
committed
fix(distill): split label files on newline only
str.splitlines() fragments on U+0085/U+2028-class codepoints inside Arabic label text, shredding healthy files into mid-JSON fragments — the original '3 valid srcs' mystery. All label-file reads now split on plain \n (verified locally: 11,793 srcs, 0 bad after the fix).
1 parent 05bc2f0 commit b94f399

1 file changed

Lines changed: 3 additions & 3 deletions

File tree

‎src/gpu/modal_distill.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -781,7 +781,7 @@ def read_label_srcs() -> set[str]:
781781
got: set[str] = set()
782782
for line in teacher_labels_path.read_text(
783783
encoding="utf-8", errors="ignore"
784-
).splitlines():
784+
).split("\n"):
785785
if line.strip():
786786
try:
787787
got.add(json.loads(line)["src"])
@@ -914,7 +914,7 @@ def accept_label(src: str, label: str) -> None:
914914
teacher_labels.append((src, label))
915915

916916
if not fresh_rows:
917-
for line in teacher_labels_path.read_text(encoding="utf-8", errors="ignore").splitlines():
917+
for line in teacher_labels_path.read_text(encoding="utf-8", errors="ignore").split("\n"):
918918
if not line.strip():
919919
continue
920920
try:
@@ -1238,7 +1238,7 @@ def distill_microkimi(spec_id: str, epochs: int = 3, calib_batches: int = 64,
12381238
raise RuntimeError("microkimi expects pre-generated trusted labels")
12391239
teacher_labels = []
12401240
seen: set[str] = set()
1241-
for line in labels_file.read_text(encoding="utf-8", errors="ignore").splitlines():
1241+
for line in labels_file.read_text(encoding="utf-8", errors="ignore").split("\n"):
12421242
if not line.strip():
12431243
continue
12441244
try:

0 commit comments

Comments
 (0)