interscript-train/probe_morph_format.py at main · interscript/interscript-train · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
"""Probe: raw dictabert-morph predict() output shape under two transformers versions."""

from __future__ import annotations

import modal

img38 = (
    modal.Image.debian_slim(python_version="3.11")
    .pip_install("torch==2.5.1", "transformers==4.38.0", "huggingface_hub>=0.24")
)
img46 = (
    modal.Image.debian_slim(python_version="3.11")
    .pip_install("torch==2.5.1", "transformers==4.46.3", "huggingface_hub>=0.24")
)

app = modal.App("probe-morph-format")

SENTS = ["הכלב של רון רץ במהירות גדולה בפארק", "ובבית הספר למדנו תורה"]


def _probe():
    from transformers import AutoModel, AutoTokenizer

    m = "dicta-il/dictabert-morph"
    tok = AutoTokenizer.from_pretrained(m)
    model = AutoModel.from_pretrained(m, trust_remote_code=True)
    model.eval()
    res = model.predict(SENTS, tok)
    for r in res:
        print("TYPE:", type(r))
        keys = list(r.keys()) if isinstance(r, dict) else dir(r)
        print("KEYS:", keys)
        toks = r.get("tokens") if isinstance(r, dict) else getattr(r, "tokens", None)
        print("N_TOKS:", len(toks) if toks else None)
        print("TEXT_REPR:", repr(r.get("text") if isinstance(r, dict) else str(r))[:200])
        if toks:
            print("TOK0_REPR:", repr(toks[0])[:300])
        print("N_TEXT_SPLIT:", len((r.get("text") or "").split()) if isinstance(r, dict) else "?")
        print("=====")


@app.function(image=img38, secrets=[modal.Secret.from_name("huggingface")], timeout=10 * 60)
def probe_38():
    _probe()


@app.function(image=img46, secrets=[modal.Secret.from_name("huggingface")], timeout=10 * 60)
def probe_46():
    _probe()


@app.local_entrypoint()
def main():
    print("### transformers 4.38.0 ###")
    probe_38.remote()
    print("### transformers 4.46.3 ###")
    probe_46.remote()