diff --git a/benchmarks/typedpython/py/binary_trees.py b/benchmarks/typedpython/py/binary_trees.py index 9ab44c379..43e5ea17e 100644 --- a/benchmarks/typedpython/py/binary_trees.py +++ b/benchmarks/typedpython/py/binary_trees.py @@ -2,6 +2,7 @@ class Node: + __slots__ = ("left", "right") left: "Node | None" right: "Node | None" diff --git a/benchmarks/typedpython/run.py b/benchmarks/typedpython/run.py index 288cfd746..7ff2457ec 100644 --- a/benchmarks/typedpython/run.py +++ b/benchmarks/typedpython/run.py @@ -12,6 +12,10 @@ interpreter with the gate's dependencies, e.g. that module's .venv): each benchmark is copied with the `# typedpython: compiled` marker, compiled through `typedpython.pipeline`, and its `main()` is run from the built extension. Without the option the column reports n/a. + +binary-trees declares `__slots__` (TypedPython compiles fixed-layout classes only, SPEC N-11). The +speedup is reported against that CPython version; `CPython (no __slots__)` runs the same program +with the `__slots__` line removed, so what adding the slots changes in CPython is recorded too. """ import argparse import datetime @@ -40,7 +44,10 @@ "binary-trees": ("binary_trees.py", "binary_trees.mjs", "BinaryTrees", "binary_trees", "10", "16"), "wordfreq": ("wordfreq.py", "wordfreq.mjs", "Wordfreq", "wordfreq", "100000", "3000000"), } -COLUMNS = ["CPython", "Node", "Java", "Rust", "TypedPython @compiled"] +NOSLOTS = "CPython (no __slots__)" +COLUMNS = ["CPython", NOSLOTS, "Node", "Java", "Rust", "TypedPython @compiled"] +SLOTS_LINE = ' __slots__ = ("left", "right")\n' +NOSLOTS_BENCHMARKS = {"binary-trees"} NOT_BUILT = "n/a (not built)" @@ -99,6 +106,18 @@ def build_compiled(tp_dir: Path, tp_python: str) -> dict[str, tuple[str, list[st return built +def noslots_source(name: str) -> Path: + """The benchmark with its `__slots__` line removed, written under tp-build/noslots/.""" + py = BENCHMARKS[name][0] + source = (ROOT / "py" / py).read_text() + if SLOTS_LINE not in source: + sys.exit(f"{name}: expected the line {SLOTS_LINE.strip()!r} in py/{py}") + out = TP_BUILD / "noslots" / py + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(source.replace(SLOTS_LINE, "", 1)) + return out + + def commands(name: str, size: str, python: str, java_source: bool, compiled: dict[str, tuple[str, list[str]] | str] | None = None) -> dict[str, list[str]]: py, js, cls, rs, _, _ = BENCHMARKS[name] @@ -110,6 +129,8 @@ def commands(name: str, size: str, python: str, java_source: bool, "Java": java, "Rust": [str(TARGET / "release" / rs), size], } + if name in NOSLOTS_BENCHMARKS: + cmds[NOSLOTS] = [python, str(noslots_source(name)), size] entry = (compiled or {}).get(name) if isinstance(entry, tuple): cmds["TypedPython @compiled"] = [python, "-c", RUN_SNIPPET, Path(py).stem, entry[0], size] @@ -179,7 +200,8 @@ def measure(args: argparse.Namespace) -> int: print("| benchmark | size | " + " | ".join(COLUMNS) + " |") print("|---|---|" + "---|" * len(COLUMNS)) for name, row in results.items(): - cells = [f"{row[c]['median_s']:.3f} s" if isinstance(row.get(c), dict) else str(row.get(c, NOT_BUILT)) + cells = [f"{row[c]['median_s']:.3f} s" if isinstance(row.get(c), dict) + else str(row.get(c, "—" if c == NOSLOTS else NOT_BUILT)) for c in COLUMNS] # type: ignore[index] flag = "" if row["outputs_agree"] else " (OUTPUTS DIFFER)" print(f"| {name}{flag} | {row['size']} | " + " | ".join(cells) + " |") @@ -216,6 +238,7 @@ def interleaved(args: argparse.Namespace) -> int: size = args.size.get(name, spec[5]) cmds = commands(name, size, args.python, False, compiled) a_cmd, b_cmd = cmds["CPython"], cmds["TypedPython @compiled"] + c_cmd = cmds.get(NOSLOTS) pairs = [] for i in range(args.pairs): order = (a_cmd, b_cmd) if i % 2 == 0 else (b_cmd, a_cmd) # alternate which runs first @@ -225,11 +248,21 @@ def interleaved(args: argparse.Namespace) -> int: if o_a != o_b: print(f"{name}: OUTPUTS DIFFER in pair {i}: {o_a!r} vs {o_b!r}") return 1 - pairs.append({"cpython_s": t_a, "compiled_s": t_b, "ratio": t_a / t_b, "load": load}) + pair = {"cpython_s": t_a, "compiled_s": t_b, "ratio": t_a / t_b, "load": load} + if c_cmd is not None: + t_c, o_c = run_once(c_cmd) + if o_c != o_a: + print(f"{name}: OUTPUTS DIFFER in pair {i} (no __slots__): {o_c!r} vs {o_a!r}") + return 1 + pair.update(cpython_noslots_s=t_c, ratio_vs_noslots=t_c / t_b) + pairs.append(pair) ratios = [p["ratio"] for p in pairs] results[name] = {"size": size, "compiled_functions": entry[1], "pairs": pairs, "median_ratio": statistics.median(ratios), "min_ratio": min(ratios), "max_ratio": max(ratios)} + if c_cmd is not None: + results[name]["median_ratio_vs_noslots"] = statistics.median( + p["ratio_vs_noslots"] for p in pairs) print(f"measured {name}", file=sys.stderr) print() diff --git a/docs/SPEC.md b/docs/SPEC.md index bc7f48e5c..22a86aef6 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -483,8 +483,19 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. fill the slots directly (the same observable result as running that `__init__`); any other `__init__` is called as an object; - methods compile like functions with `self` guarded `type(self) is C`; anything else on the class stays - interpreted. - A class outside these rules stays interpreted with a reason. `Status: planned`. + interpreted; + - a function that writes a slot is never "closed" (no `entry_globals` snapshot): overwriting a slot can + drop the last reference to the old value, and its `__del__` is user code that can rebind globals. This + is stricter than the IR contract requires, on purpose; + - a change after init keeps CPython's results: every slot access checks `type(x) is C` and the + class's `tp_version_tag` against the one captured at init (a tag is never reused; 0 never matches), + and every `C(args)` checks that the module global still is the captured class (a module-dict + watcher, or a dict lookup per call when no watcher id is free). On a mismatch an impure function + takes the generic path (`getattr`/`setattr`/calling the current global) and a pure one deopts. + A class that defines `__getattr__`, `__getattribute__`, `__setattr__` or `__delattr__` is not + compiled. + A class outside these rules stays interpreted with a reason. Methods are not compiled yet. + `Status: implemented (module-level functions; methods planned)`. --- diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index f5c4f7b6d..6359ed9d4 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -55,6 +55,55 @@ `Tuple` evaluates each element into an owned temporary, builds the tuple with `tp_tuple` (which steals nothing), then releases the temporaries. +Fixed-layout classes (SPEC N-11, ir.ClassDecl). The module state holds, per ClassDecl, a strong +reference to the class object, a `cls_ok` flag and one captured `PyMemberDef *` per field. The exec +slot fills them AFTER the original source ran (`tp_class_capture`: a heap type made by `type`, base +object, no __dict__, every field a writable object slot made for that very class); a class that fails +any check is "not compiled" (`cls_ok = 0`) and is never touched through memory: + 1. every wrapper first tests `cls_ok` of every class its function uses *or reaches through C to C + Calls* (`_ModGen.uses`), else it takes the interpreted path (a static property of the module, so + callees need no check of their own); + 2. a `Param.cls` guard (`type(x) is C`, or `x is None` when optional) runs in the wrapper AND at the + top of the impl, before tp_enter_call and before anything is owned: the wrapper's copy serves + impure functions (whose impl may never return 1), the impl's copy serves C to C Calls, where a + failure is a deopt — so a Call of a function with a `Param.cls` parameter counts as a node that + can deopt (`_can_deopt`), exactly like a callee with entry_globals; + 3. FieldGet / FieldSet / New go through `tp_field_get` / `tp_field_set` / `tp_new_fixed` with the + captured descriptors; CheckExact deopts (return 1) on mismatch and is legal only in a pure + function; Is is a plain BOOL. + +Classes changed after init (the class, the module global or an instance's __class__). The entry +guards above only select the path; correctness never depends on them, because user code can change +a class between calls or, in an impure function, between entry and the access. Every access +re-proves what it relies on, right where CPython would look: + 4. FieldGet / FieldSet: `tp_class_current(obj, ...)` — `type(obj) is C` and C's tp_version_tag is + the tag captured at init (or re-proved by `tp_class_refresh`, which adopts a new tag when every + field is still the very member descriptor and __init__ the very function captured at init: + `Node.counter = 1` costs one refresh, not a slow path forever). One type compare and one tag + compare when nothing changed. + 5. New: `tp_class_global_ok` BEFORE the arguments are evaluated (CPython's LOAD_GLOBAL comes first: + `Node(cb(), b)` builds the class bound when Node was loaded), then `tp_class_current(NULL, ...)` + after them (type.__call__ sees __init__ as it is at the call). The global test is one compare + while a dict watcher on the module dict (`tp_globals_watch`, its id in the module state) has not + reported a change of a class-name key (`tp_globals_epoch`); without a free watcher id it is a + dict lookup per New (`watch_id1 == 0`, the conservative path). + 6. IsExact: `type(obj) is C` with C looked up as a global (`tp_class_global_ok`, then the current + binding on the slow path). + When a check fails: in an impure function, the slow path with CPython's semantics and no deopt + (FieldGet -> tp_getattr, FieldSet -> tp_setattr, New -> call what the global held when it was + loaded, IsExact -> compare with the current global); in a pure function, a deopt (return 1) — + running the property or __init__ there could repeat its effect if a later node deopted and the + call were redone. A pure function may thus return 1 even when `may_deopt` is False, so: + 7. an impure caller of a pure callee that uses classes, in a Call without `redo`, redoes that + callee interpreted on a 1 (`_redo_call`, any return type) — the callee deopted before running + any user code, so the redo is unobservable; the impure caller itself never deopts; + 8. an impure function with entry globals (closed: it runs no user code, ir.Function.entry_globals) + checks at entry, in the wrapper, that every class it reaches is current and every class global + it reads is still bound: then no check inside can fail, so no slow path can run user code that + rebinds an entry global after its snapshot. A failure there is a deopt before any effect. + `__typedpython_class_info__()` reports, per class, the captured and current tags, the slow-path + and refresh counters and whether the module dict is watched (introspection for tests). + Generated arithmetic is in separate C statements and the file sets `#pragma STDC FP_CONTRACT OFF` (cbuild also passes -ffp-contract=off): a fused multiply-add would round differently from CPython. """ @@ -156,6 +205,9 @@ def _exprs_of_stmt(s: ir.Stmt): elif isinstance(s, ir.StoreIndex): yield s.value yield s.index + elif isinstance(s, ir.FieldSet): + yield s.value + yield s.obj elif isinstance(s, ir.ExprStmt): yield s.value elif isinstance(s, ir.If): @@ -211,17 +263,54 @@ def _can_deopt(e: ir.Expr, functions: dict[str, ir.Function]) -> bool: return not e.proven # a proven op cannot deopt (overflow = SystemError) if isinstance(e, ir.UnaryOp) and e.op is ir.UnaryOpKind.NEG and e.operand.type is Type.I64: return not e.proven - if isinstance(e, ir.Unbox): + if isinstance(e, (ir.Unbox, ir.CheckExact)): return True if isinstance(e, ir.Call): if e.redo: return False # the callee's deopt is absorbed by its interpreted redo callee = functions.get(e.function) # a callee's entry-global guard runs at the call (`_FnGen.ev_Call`) and can fail there - return callee is not None and (callee.may_deopt or bool(callee.entry_globals)) + return callee is not None and (callee.may_deopt or bool(callee.entry_globals) + or _has_cls_param(callee)) return False +def _has_cls_param(f: ir.Function) -> bool: + """A Param.cls guard runs at every call of `f`, C to C included, and a failure is a deopt.""" + return any(isinstance(p, ir.Param) and p.cls is not None for p in f.params) + + +def _direct_classes(f: ir.Function) -> set[str]: + out = {p.cls for p in f.params if isinstance(p, ir.Param) and p.cls is not None} + for e in _all_exprs(f): + if isinstance(e, (ir.IsExact, ir.CheckExact, ir.FieldGet, ir.New)): + out.add(e.cls) + for s in _walk_stmts(f.body): + if isinstance(s, ir.FieldSet): + out.add(s.cls) + return out + + +def _param_cls_fail(obj_c: str, k: int, optional: bool) -> str: + """The C condition under which a `Param.cls` guard FAILS for the borrowed argument `obj_c` + (class slot `k` of the module state): not exactly the class, and not None when optional. + Emitted both in the wrapper and at the top of the impl (module docstring).""" + exact = f"tp_is_exact({obj_c}, (PyTypeObject *)tp_st->classes[{k}])" + return f"!({exact} || {obj_c} == Py_None)" if optional else f"!{exact}" + + +def _class_current_expr(obj_c: str, k: int, fields_c: str, n: int, descrs_c: str) -> str: + """The per-access check of FieldGet / FieldSet (`obj_c` a C object) and New (`obj_c` "NULL"): + 1 current, 0 slow path, -1 error (module docstring, rules 4 and 5).""" + return (f"tp_class_current({obj_c}, tp_st->classes[{k}], &tp_st->cls_rt[{k}], " + f"{fields_c}, {n}, {descrs_c})") + + +def _watch_event_stmt() -> str: + """The body of the module's dict-watcher callback (module docstring, rule 5).""" + return "return tp_globals_watch_event(event, key, tp_class_names);" + + def _is_effect(e: ir.Expr) -> bool: if isinstance(e, (ir.GetAttr, ir.CallObject, ir.Truth, ir.CompareObj, ir.ObjToFloat)): return True @@ -238,6 +327,75 @@ def __init__(self, module: ir.Module): raise CGenError("duplicate compiled function names") self.names: list[str] = [] self.kwtuples: list[tuple[str, ...]] = [] + self.classes = {cd.name: cd for cd in module.classes} + if len(self.classes) != len(module.classes): + raise CGenError("duplicate ClassDecl names") + self.class_index = {cd.name: i for i, cd in enumerate(module.classes)} + self.slot_base: dict[str, int] = {} + total = 0 + for cd in module.classes: + if len(set(cd.fields)) != len(cd.fields): + raise CGenError(f"class {cd.name}: duplicate field names") + self.slot_base[cd.name] = total + total += len(cd.fields) + self.n_slots = total + self._uses: dict[str, set[str]] = {} + + def cls(self, fname: str, name: str) -> ir.ClassDecl: + cd = self.classes.get(name) + if cd is None: + raise CGenError(f"{fname}: class {name!r} has no ClassDecl in the module") + return cd + + def slot(self, fname: str, cls: str, field: str) -> str: + cd = self.cls(fname, cls) + if field not in cd.fields: + raise CGenError(f"{fname}: class {cls} has no field {field!r}") + return f"tp_st->slots[{self.slot_base[cls] + cd.fields.index(field)}]" + + def uses(self, f: ir.Function) -> list[str]: + """Every class `f` uses or reaches through C to C Calls (module docstring, rule 1).""" + if not self._uses: + for g in self.module.functions: + self._uses[g.name] = _direct_classes(g) + changed = True + while changed: + changed = False + for g in self.module.functions: + for e in _all_exprs(g): + if isinstance(e, ir.Call) and e.function in self._uses: + extra = self._uses[e.function] - self._uses[g.name] + if extra: + self._uses[g.name] |= extra + changed = True + return sorted(self._uses[f.name], key=lambda n: self.class_index.get(n, -1)) + + def fields_c(self, cls: str) -> tuple[str, int, str]: + """(C fields array, field count, C descriptor-array pointer) of class `cls`.""" + cd = self.classes[cls] + k, base = self.class_index[cls], self.slot_base[cls] + return f"tp_fields_{k}", len(cd.fields), f"&tp_st->descrs[{base}]" + + def globals_read(self, f: ir.Function) -> list[str]: + """Classes whose module global `f` reads (New, IsExact) itself or through C to C Calls.""" + out: set[str] = set() + seen: set[str] = set() + todo = [f.name] + while todo: + g = self.functions[todo.pop()] + if g.name in seen: + continue + seen.add(g.name) + for e in _all_exprs(g): + if isinstance(e, (ir.New, ir.IsExact)): + out.add(e.cls) + if isinstance(e, ir.Call) and e.function in self.functions: + todo.append(e.function) + return sorted(out, key=lambda n: self.class_index.get(n, -1)) + + def watches_globals(self) -> bool: + return any(isinstance(e, (ir.New, ir.IsExact)) + for f in self.module.functions for e in _all_exprs(f)) def name_index(self, s: str) -> int: if s not in self.names: @@ -288,6 +446,12 @@ def generate(module: ir.Module, source_path: Path) -> str: w("typedef struct {") w(f" PyObject *names[{n_names + 1}];") w(f" PyObject *kwnames[{n_kw + 1}];") + w(f" PyObject *classes[{len(module.classes) + 1}]; /* strong refs; NULL when not compiled */") + w(f" int cls_ok[{len(module.classes) + 1}];") + w(f" PyMemberDef *slots[{mg.n_slots + 1}]; /* owned by the classes above */") + w(f" PyObject *descrs[{mg.n_slots + 1}]; /* strong: the member descriptor of each slot */") + w(f" tp_class_rt cls_rt[{len(module.classes) + 1}]; /* version-tag / global guard state */") + w(" int watch_id1; /* dict watcher id + 1 on the module dict; 0 = none */") w("} tp_state;") w("") w("static const char *const tp_name_strings[] = {") @@ -296,6 +460,20 @@ def generate(module: ir.Module, source_path: Path) -> str: w(" NULL") w("};") w("") + for cd in module.classes: + items = "".join(f"{c_string(x)}, " for x in cd.fields) + w(f"static const char *const tp_fields_{mg.class_index[cd.name]}[] = {{{items}NULL}};") + names = "".join(f"{c_string(cd.name)}, " for cd in module.classes) + w(f"static const char *const tp_class_names[] = {{{names}NULL}};") + w("") + if module.classes and mg.watches_globals(): + w("/* Dict watcher of the module dict (tp_globals_watch): a class-name key changed. */") + w("static int tp_watch_cb(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *value)") + w("{") + w(" (void)dict; (void)value;") + w(f" {_watch_event_stmt()}") + w("}") + w("") for f in module.functions: w(_impl_signature(f) + ";") w("") @@ -310,8 +488,10 @@ def generate(module: ir.Module, source_path: Path) -> str: w(" {NULL, NULL, 0, NULL}") w("};") w("") + if module.classes: + w(_class_info_function(mg)) w(_exec_function(mg)) - w(_TEMPLATE_STATE) + w(_state_functions(mg)) w("static PyModuleDef_Slot tp_slots[] = {") w(" {Py_mod_exec, (void *)tp_exec},") w(" {0, NULL}") @@ -337,9 +517,10 @@ def _check_function(f: ir.Function, functions: dict[str, ir.Function]) -> None: effects = [e for e in _all_exprs(f) if _is_effect(e)] # a store into a local array is not an effect (ir.py "Local arrays"); into a parameter it is stores = [s for s in _walk_stmts(f.body) - if isinstance(s, ir.StoreIndex) and f.locals.get(s.array) not in ir.ARRAYS] + if (isinstance(s, ir.StoreIndex) and f.locals.get(s.array) not in ir.ARRAYS) + or isinstance(s, ir.FieldSet)] if f.pure and (effects or stores): - what = type(effects[0]).__name__ if effects else "StoreIndex" + what = type(effects[0]).__name__ if effects else type(stores[0]).__name__ raise CGenError(f"{f.name}: marked pure but contains an effect ({what})") for e in _all_exprs(f): if isinstance(e, (ir.BinOp, ir.UnaryOp)) and e.proven: @@ -362,7 +543,7 @@ def _check_function(f: ir.Function, functions: dict[str, ir.Function]) -> None: if not callee.pure: raise CGenError(f"{f.name}: redo Call of impure {e.function!r} " "(redoing it could repeat an effect)") - elif (callee.may_deopt or callee.entry_globals) and not f.pure: + elif (callee.may_deopt or callee.entry_globals or _has_cls_param(callee)) and not f.pure: raise CGenError(f"{f.name}: Call of deopting {e.function!r} from an impure caller " "without redo") if f.entry_globals: @@ -375,6 +556,10 @@ def _check_function(f: ir.Function, functions: dict[str, ir.Function]) -> None: raise CGenError(f"{f.name}: array param {p.name} is stored into in a pure function") elif p.type not in _CT: raise CGenError(f"{f.name}: param {p.name} has type {p.type}") + elif p.cls is not None and p.type is not Type.OBJ: + raise CGenError(f"{f.name}: param {p.name} has a class guard but type {p.type}") + elif p.optional and p.cls is None: + raise CGenError(f"{f.name}: param {p.name} is optional without a class") def _is_closed(f: ir.Function, functions: dict[str, ir.Function], seen=None) -> bool: @@ -738,6 +923,10 @@ def ev_Call(self, e: ir.Call) -> str | None: self.emit("}") if e.redo: self._redo_call(callee, e.args, values, t) + elif not self.f.pure and callee.pure and self.mg.uses(callee): + # a pure callee using classes returns 1 when a class check fails (module docstring, + # rule 7): redo it interpreted; this impure caller must not deopt + self._redo_call(callee, e.args, values, t) self._own_release(*owned) self.emit("if (tp_s != 0) { tp_rc = tp_s; goto tp_exit; }") return t @@ -792,15 +981,147 @@ def _redo_call(self, callee: ir.Function, arg_exprs, values: list[str], t: str | if rt is Type.NONE: self.emit(f"tp_s = ({rr} == Py_None) ? 0 : 1;") what = "None" + elif rt is Type.OBJ: + self.emit(f"{t} = {rr}; {rr} = NULL; tp_s = 0;") # moved: the temp owns it now + what = "an object" else: self.emit(f"tp_s = {_UNBOX[rt]}({rr}, &{t});") - what = {Type.F64: "a float", Type.BOOL: "True or False"}[rt] + what = {Type.F64: "a float", Type.BOOL: "True or False", Type.I64: "an int in i64"}[rt] self._own_release(rr) msg = c_string(f"typedpython: the interpreted redo of {callee.name} did not return {what}") self.emit(f"if (tp_s != 0) {{ PyErr_SetString(PyExc_SystemError, {msg}); tp_rc = -1; goto tp_exit; }}") self.depth -= 1 self.emit("}") + def _obj_operand(self, e: ir.Expr, what: str) -> str: + if e.type is not Type.OBJ: + raise CGenError(f"{self.f.name}: {what} on {e.type}") + return self.ev(e) + + def ev_Is(self, e: ir.Is) -> str: + a = self._obj_operand(e.left, "Is") + b = self._obj_operand(e.right, "Is") + t = self.tmp("int") + self.emit(f"{t} = ({a} {'!=' if e.negate else '=='} {b});") + self._own_release(a, b) + return t + + def _current(self, obj_c: str, cls: str) -> str: + k = self._class(cls) + fields_c, n, descrs_c = self.mg.fields_c(cls) + return _class_current_expr(obj_c, k, fields_c, n, descrs_c) + + def _global_ok(self, cls: str) -> str: + k = self._class(cls) + nk = self.mg.name_index(cls) + return (f"tp_class_global_ok(tp_dict, tp_st->names[{nk}], tp_st->classes[{k}], " + f"&tp_st->cls_rt[{k}], tp_st->watch_id1 > 0)") + + def _slow(self, cls: str) -> None: + """Count a failed per-access check; in a pure function it is a deopt (module docstring).""" + self.emit(f"tp_st->cls_rt[{self._class(cls)}].slow++;") + if self.f.pure: + self.emit("tp_rc = 1; goto tp_exit;") + + def ev_IsExact(self, e: ir.IsExact) -> str: + k = self._class(e.cls) + o = self._obj_operand(e.obj, "IsExact") + t = self.tmp("int") + self.emit(f"tp_s = {self._global_ok(e.cls)};") + self.emit("if (tp_s < 0) { tp_rc = -1; goto tp_exit; }") + self.emit("if (tp_s) {") + self.emit(f" {t} = tp_is_exact({o}, (PyTypeObject *)tp_st->classes[{k}]);") + self.emit("} else {") + self.depth += 1 + self._slow(e.cls) + if not self.f.pure: + g = self._own_new(f"tp_global(tp_dict, tp_st->names[{self.mg.name_index(e.cls)}])") + self.emit(f"{t} = tp_is_exact({o}, (PyTypeObject *){g});") + self._own_release(g) + self.depth -= 1 + self.emit("}") + self._own_release(o) + return t + + def ev_CheckExact(self, e: ir.CheckExact) -> str: + k = self._class(e.cls) + o = self._obj_operand(e.obj, "CheckExact") + # a deopt: legal only in a pure function (_check_function); the exit block releases `o` + self.emit(f"if (!tp_is_exact({o}, (PyTypeObject *)tp_st->classes[{k}])) {{ tp_rc = 1; goto tp_exit; }}") + return o + + def ev_FieldGet(self, e: ir.FieldGet) -> str: + slot = self.mg.slot(self.f.name, e.cls, e.field) + o = self._obj_operand(e.obj, "FieldGet") + t = self.tmp("PyObject *") + self.emit(f"tp_s = {self._current(o, e.cls)};") + self.emit("if (tp_s < 0) { tp_rc = -1; goto tp_exit; }") + self.emit("if (tp_s) {") + self.emit(f" {t} = tp_field_get({o}, {slot});") + self.emit("} else {") + self.depth += 1 + self._slow(e.cls) + if not self.f.pure: # LOAD_ATTR, whatever the class is now + self.emit(f"{t} = tp_getattr({o}, tp_st->names[{self.mg.name_index(e.field)}]);") + self.depth -= 1 + self.emit("}") + self._own_release(o) + self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def ev_New(self, e: ir.New) -> str: + cd = self.mg.cls(self.f.name, e.cls) + if not cd.trivial_init: + raise CGenError(f"{self.f.name}: New of class {e.cls}, whose __init__ is not trivial") + if len(e.args) != len(cd.fields): + raise CGenError(f"{self.f.name}: New {e.cls} with {len(e.args)} args for " + f"{len(cd.fields)} fields") + k = self._class(e.cls) + # LOAD_GLOBAL C comes before the arguments (module docstring, rule 5) + gk, g = self.tmp("int"), None + self.emit(f"tp_s = {self._global_ok(e.cls)};") + self.emit("if (tp_s < 0) { tp_rc = -1; goto tp_exit; }") + self.emit(f"{gk} = tp_s;") + self.emit(f"if (!{gk}) {{") + self.depth += 1 + self._slow(e.cls) + if not self.f.pure: + g = self.tmp("PyObject *") + self.emit(f"{g} = tp_global(tp_dict, tp_st->names[{self.mg.name_index(e.cls)}]);") + self.emit(f"if ({g} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + self.depth -= 1 + self.emit("}") + items = [self._obj_operand(a, "New argument") for a in e.args] + t = self.tmp("PyObject *") + base = self.mg.slot_base[e.cls] + av = f"{{ PyObject *tp_av[{len(items)}] = {{{', '.join(items)}}};" if items else "{ PyObject **tp_av = NULL;" + self.emit(f"tp_s = {gk} ? {self._current('NULL', e.cls)} : 0;") + self.emit("if (tp_s < 0) { tp_rc = -1; goto tp_exit; }") + self.emit(av) + self.emit(" if (tp_s) {") + self.emit(f" {t} = tp_new_fixed((PyTypeObject *)tp_st->classes[{k}], " + f"{f'&tp_st->slots[{base}]' if items else 'NULL'}, tp_av, {len(items)});") + self.emit(" } else {") + self.depth += 1 + if self.f.pure: + self.emit(f"if ({gk}) {{ tp_st->cls_rt[{k}].slow++; }}") + self.emit("tp_rc = 1; goto tp_exit;") # (g is NULL here: pure never loads it) + else: + self.emit(f"if ({gk}) tp_st->cls_rt[{k}].slow++;") + # call what the global held when it was loaded: the class itself, or the rebinding + self.emit(f" {t} = tp_call({gk} ? tp_st->classes[{k}] : {g}, tp_av, {len(items)}, NULL);") + self.depth -= 1 + self.emit(" } }") + self._own_release(*items) # the slots took their own references + if g is not None: + self._own_release(g) + self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def _class(self, name: str) -> int: + self.mg.cls(self.f.name, name) + return self.mg.class_index[name] + def ev_Global(self, e: ir.Global) -> str: k = self.mg.name_index(e.name) return self._own_new(f"tp_global(tp_dict, tp_st->names[{k}])") @@ -965,6 +1286,23 @@ def st_StoreIndex(self, s: ir.StoreIndex) -> None: if not local: # a local array has no list to write back to self.emit(f"{ptr}->dirty[{slot}] = 1;") + def st_FieldSet(self, s: ir.FieldSet) -> None: + slot = self.mg.slot(self.f.name, s.cls, s.field) + v = self._obj_operand(s.value, "FieldSet value") # CPython: value, then the object + o = self._obj_operand(s.obj, "FieldSet") + self.emit(f"tp_s = {self._current(o, s.cls)};") + self.emit("if (tp_s < 0) { tp_rc = -1; goto tp_exit; }") + self.emit("if (tp_s) {") + self.emit(f" tp_s = tp_field_set({o}, {slot}, {v});") + self.emit("} else {") + self.depth += 1 + self._slow(s.cls) # FieldSet is never in a pure function + self.emit(f"tp_s = tp_setattr({o}, tp_st->names[{self.mg.name_index(s.field)}], {v});") + self.depth -= 1 + self.emit("}") + self._own_release(o, v) + self.emit("if (tp_s < 0) { tp_rc = -1; goto tp_exit; }") + def st_ExprStmt(self, s: ir.ExprStmt) -> None: v = self.ev(s.value) if v is None: @@ -1100,6 +1438,10 @@ def generate(self) -> str: + "".join(f" (void){_ident('l', n)};" for n in self.local_arrays)) # Depth guard (issue #57): nothing is owned yet, so a refusal returns directly and counts # nothing; every other exit goes through tp_exit, which leaves exactly once. + for p in f.params: + if isinstance(p, ir.Param) and p.cls is not None: + k = self._class(p.cls) + out.append(f" if ({_param_cls_fail(_ident('p', p.name), k, p.optional)}) return 1;") out.append(" if (tp_enter_call() != 0) return -1;") out += prologue out += self.lines @@ -1140,6 +1482,28 @@ def wrapper(self) -> str: L.append(" (void)tp_x;") # arity / keywords -> the interpreted function raises CPython's own TypeError L.append(f" if (tp_nargs != {n} || (tp_kwnames != NULL && PyTuple_GET_SIZE(tp_kwnames) != 0)) goto tp_deopt;") + # fixed-layout classes: not compiled -> interpreted; Param.cls guards (before any effect) + classes = self.mg.uses(f) + cls_params = [(i, p) for i, p in enumerate(f.params) + if isinstance(p, ir.Param) and p.cls is not None] + for cname in classes: + self.mg.cls(f.name, cname) + if classes: + L.insert(3, " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);") + L.append(" if (!(" + " && ".join(f"tp_st->cls_ok[{self.mg.class_index[n]}]" + for n in classes) + ")) goto tp_deopt;") + for i, p in cls_params: + L.append(f" if ({_param_cls_fail(f'tp_args[{i}]', self._class(p.cls), p.optional)}) goto tp_deopt;") + if classes and f.entry_globals and not f.pure: + # module docstring, rule 8: a closed impure function must not reach a slow path + for cname in classes: + L.append(f" tp_s = {self._current('NULL', cname)};") + L.append(" if (tp_s < 0) goto tp_error;") + L.append(" if (!tp_s) goto tp_deopt;") + for cname in self.mg.globals_read(f): + L.append(f" tp_s = {self._global_ok(cname)};") + L.append(" if (tp_s < 0) goto tp_error;") + L.append(" if (!tp_s) goto tp_deopt;") # scalar guards: exact type (and i64 range) for i, p in enumerate(f.params): if not isinstance(p, ir.ArrayParam) and p.type in _UNBOX: @@ -1158,7 +1522,8 @@ def wrapper(self) -> str: gnames = [] if f.entry_globals: L.insert(3, " PyObject *tp_dict = PyModule_GetDict(tp_module); /* borrowed */") - L.insert(3, " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);") + if not classes: + L.insert(3, " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);") for j, g in enumerate(f.entry_globals): # entry read of a module global: exact type or deopt (still before any effect) k = self.mg.name_index(g.name) @@ -1249,29 +1614,80 @@ def wrapper(self) -> str: } ''' -_TEMPLATE_STATE = r''' -static int tp_traverse(PyObject *tp_module, visitproc visit, void *arg) -{ - tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module); - size_t i; - if (tp_st == NULL) return 0; - for (i = 0; i < sizeof(tp_st->names) / sizeof(tp_st->names[0]); i++) Py_VISIT(tp_st->names[i]); - for (i = 0; i < sizeof(tp_st->kwnames) / sizeof(tp_st->kwnames[0]); i++) Py_VISIT(tp_st->kwnames[i]); - return 0; -} +def _state_functions(mg: _ModGen) -> str: + """tp_traverse / tp_clear / tp_free of the module state. tp_clear also drops the dict watcher + (module docstring, rule 5) while the module dict still exists: CPython calls m_clear and m_free + before it releases md_dict (moduleobject.c module_clear / module_dealloc).""" + L = [ + "static int tp_traverse(PyObject *tp_module, visitproc visit, void *arg)", + "{", + " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);", + " size_t i;", + " if (tp_st == NULL) return 0;", + " for (i = 0; i < sizeof(tp_st->names) / sizeof(tp_st->names[0]); i++) Py_VISIT(tp_st->names[i]);", + " for (i = 0; i < sizeof(tp_st->kwnames) / sizeof(tp_st->kwnames[0]); i++) Py_VISIT(tp_st->kwnames[i]);", + " for (i = 0; i < sizeof(tp_st->classes) / sizeof(tp_st->classes[0]); i++) Py_VISIT(tp_st->classes[i]);", + " for (i = 0; i < sizeof(tp_st->descrs) / sizeof(tp_st->descrs[0]); i++) Py_VISIT(tp_st->descrs[i]);", + " for (i = 0; i < sizeof(tp_st->cls_rt) / sizeof(tp_st->cls_rt[0]); i++) Py_VISIT(tp_st->cls_rt[i].init);", + " return 0;", + "}", + "", + "static int tp_clear(PyObject *tp_module)", + "{", + " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);", + " size_t i;", + " if (tp_st == NULL) return 0;", + " if (tp_st->watch_id1 > 0) tp_globals_unwatch(PyModule_GetDict(tp_module), &tp_st->watch_id1);", + " for (i = 0; i < sizeof(tp_st->names) / sizeof(tp_st->names[0]); i++) tp_release(&tp_st->names[i]);", + " for (i = 0; i < sizeof(tp_st->kwnames) / sizeof(tp_st->kwnames[0]); i++) tp_release(&tp_st->kwnames[i]);", + " for (i = 0; i < sizeof(tp_st->cls_ok) / sizeof(tp_st->cls_ok[0]); i++) tp_st->cls_ok[i] = 0;", + " for (i = 0; i < sizeof(tp_st->classes) / sizeof(tp_st->classes[0]); i++) tp_release(&tp_st->classes[i]);", + " for (i = 0; i < sizeof(tp_st->cls_rt) / sizeof(tp_st->cls_rt[0]); i++) tp_class_rt_clear(&tp_st->cls_rt[i], NULL, 0);", + " for (i = 0; i < sizeof(tp_st->descrs) / sizeof(tp_st->descrs[0]); i++) tp_release(&tp_st->descrs[i]);", + " return 0;", + "}", + "", + "static void tp_free(void *tp_module) { (void)tp_clear((PyObject *)tp_module); }", + "", + ] + return "\n".join(L) -static int tp_clear(PyObject *tp_module) -{ - tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module); - size_t i; - if (tp_st == NULL) return 0; - for (i = 0; i < sizeof(tp_st->names) / sizeof(tp_st->names[0]); i++) tp_release(&tp_st->names[i]); - for (i = 0; i < sizeof(tp_st->kwnames) / sizeof(tp_st->kwnames[0]); i++) tp_release(&tp_st->kwnames[i]); - return 0; -} -static void tp_free(void *tp_module) { (void)tp_clear((PyObject *)tp_module); } -''' +def _class_info_function(mg: _ModGen) -> str: + """`__typedpython_class_info__()`: {class name: {compiled, captured_tag, tag, current, slow, + refreshed, watched}} — introspection of the guards (module docstring).""" + L = [ + "static PyObject *tp_class_info(PyObject *tp_module, PyObject *tp_unused)", + "{", + " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);", + " PyObject *tp_out = PyDict_New(), *tp_d;", + " PyObject *tp_c;", + " (void)tp_unused;", + " if (tp_out == NULL) return NULL;", + ] + for cd in mg.module.classes: + k = mg.class_index[cd.name] + L += [ + f" tp_c = tp_st->classes[{k}];", + " tp_d = Py_BuildValue(\"{s:O,s:I,s:I,s:O,s:n,s:n,s:O}\",", + f" \"compiled\", tp_st->cls_ok[{k}] ? Py_True : Py_False,", + f" \"captured_tag\", tp_st->cls_rt[{k}].tag,", + " \"tag\", tp_class_tag(tp_c),", + f" \"current\", (tp_c != NULL && tp_class_tag(tp_c) == tp_st->cls_rt[{k}].tag) ? Py_True : Py_False,", + f" \"slow\", tp_st->cls_rt[{k}].slow, \"refreshed\", tp_st->cls_rt[{k}].refreshed,", + " \"watched\", tp_st->watch_id1 > 0 ? Py_True : Py_False);", + f" if (tp_d == NULL || PyDict_SetItemString(tp_out, {c_string(cd.name)}, tp_d) < 0) {{", + " Py_XDECREF(tp_d); Py_DECREF(tp_out); return NULL; }", + " Py_DECREF(tp_d);", + ] + L += [ + " return tp_out;", + "}", + "", + "static PyMethodDef tp_class_info_def = {\"__typedpython_class_info__\", tp_class_info, METH_NOARGS, NULL};", + "", + ] + return "\n".join(L) def _exec_function(mg: _ModGen) -> str: @@ -1282,7 +1698,7 @@ def _exec_function(mg: _ModGen) -> str: " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);", " PyObject *tp_dict = PyModule_GetDict(tp_module); /* borrowed: owned by the module */", " PyObject *tp_code = NULL, *tp_res = NULL, *tp_table = NULL, *tp_zero = NULL;", - " PyObject *tp_modname = NULL, *tp_f = NULL, *tp_cf = NULL;", + " PyObject *tp_modname = NULL, *tp_f = NULL, *tp_cf = NULL, *tp_c = NULL;", " int tp_rc = -1, tp_k;", " size_t i;", " for (i = 0; tp_name_strings[i] != NULL; i++) {", @@ -1310,6 +1726,31 @@ def _exec_function(mg: _ModGen) -> str: " if (tp_res == NULL) goto tp_done;", " tp_table = PyDict_New();", " if (tp_table == NULL || PyDict_SetItemString(tp_dict, \"__typedpython_interpreted__\", tp_table) < 0) goto tp_done;", + ] + for cd in mg.module.classes: + j = mg.class_index[cd.name] + L += [ + f" tp_k = PyDict_GetItemStringRef(tp_dict, {c_string(cd.name)}, &tp_c);", + " if (tp_k < 0) goto tp_done;", + " if (tp_k == 1) {", + f" tp_k = tp_class_capture(tp_c, tp_fields_{j}, {len(cd.fields)}, " + f"&tp_st->slots[{mg.slot_base[cd.name]}], &tp_st->descrs[{mg.slot_base[cd.name]}], " + f"&tp_st->cls_rt[{j}]);", + " if (tp_k < 0) goto tp_done;", + f" if (tp_k == 1) {{ tp_st->classes[{j}] = tp_c; tp_c = NULL; tp_st->cls_ok[{j}] = 1; }}", + " }", + " tp_release(&tp_c);", + ] + if mg.module.classes: + if mg.watches_globals(): + # after the source ran: from here a rebinding of a class name moves tp_globals_epoch + L.append(" tp_globals_watch(tp_dict, tp_watch_cb, &tp_st->watch_id1);") + L += [ + " tp_cf = PyCFunction_NewEx(&tp_class_info_def, tp_module, NULL);", + " if (tp_cf == NULL || PyDict_SetItemString(tp_dict, \"__typedpython_class_info__\", tp_cf) < 0) goto tp_done;", + " tp_release(&tp_cf);", + ] + L += [ " tp_zero = PyLong_FromLong(0);", " if (tp_zero == NULL || PyDict_SetItemString(tp_dict, \"__typedpython_deopts__\", tp_zero) < 0) goto tp_done;", " tp_modname = PyModule_GetNameObject(tp_module);", @@ -1330,7 +1771,7 @@ def _exec_function(mg: _ModGen) -> str: " tp_rc = 0;", "tp_done:", " tp_release(&tp_code); tp_release(&tp_res); tp_release(&tp_table); tp_release(&tp_zero);", - " tp_release(&tp_modname); tp_release(&tp_f); tp_release(&tp_cf);", + " tp_release(&tp_modname); tp_release(&tp_f); tp_release(&tp_cf); tp_release(&tp_c);", " return tp_rc;", "}", "", diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index 8f78ce128..fa86512c0 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -24,6 +24,19 @@ so they count as deopting. An impure caller of a pure callee that may deopt and returns F64, BOOL or NONE emits `Call(redo=True)`: only the callee is redone on a deopt. +Fixed-layout classes (SPEC N-11, `ir.ClassDecl`). A module-level class (in a marked module, or +decorated `@compiled`) becomes a `ClassDecl` only when its layout is fixed in CPython too; see +`_class_decl` for the rules. A class that fails them has no `ClassDecl`, an entry in +`Module.skipped` under the class name with the reason, and the functions using it lower as before +(`GetAttr`, `CallObject`). Over a `ClassDecl`: `None` is `Const(None, OBJ)`, `is`/`is not` is +`ir.Is`, a parameter annotated `C` / `C | None` / `Optional[C]` is an OBJ `Param(cls=C, optional)`, +`obj.field` is a `FieldGet` (exact type proved: a non-optional never-rebound parameter, or a local +only ever bound to `C(...)` built as `New`; otherwise `CheckExact` in a pure function, and the +function is skipped in an impure one), `obj.field = v` is a `FieldSet` (an effect; only on a proved +object), and `C(args)` is `New` when `ClassDecl.trivial_init` (arguments in `__slots__` order) else +`CallObject(Global(C))`. `trivial_init` requires the i-th `__init__` parameter to be stored into +the i-th slot (ir.ClassDecl: "`__init__(self, *fields-in-order)`"); other orders stay an object call. + Entry globals (ir.Function.entry_globals). A module-level name assigned once at top level, of type float/int/bool (annotated, or literals and arithmetic of such names), never rebound or deleted in the module, and read in a *closed* function (one that runs no user code, decided after lowering) @@ -112,6 +125,7 @@ class _ModuleInfo: bindings: dict[str, list[tuple[int, str]]] # module-level name -> [(line, how)] stdlib_math: bool scalar_globals: dict[str, Type] = field(default_factory=dict) # entry-global candidates + classes: dict[str, ir.ClassDecl] = field(default_factory=dict) # compiled fixed-layout classes def bound(self, name: str) -> bool: return name in self.bindings @@ -135,6 +149,9 @@ def lower(path: Path) -> ir.Module: info = _module_info(tree) skipped: dict[str, str] = {} + class_decls, class_skips = _classes(tree, source, info) + info.classes = class_decls + skipped.update(class_skips) signatures: dict[str, _Signature] = {} for node, reason in _targets(tree, source, info): if reason is not None: @@ -142,7 +159,7 @@ def lower(path: Path) -> ir.Module: continue assert isinstance(node, ast.FunctionDef) try: - signatures[node.name] = _signature(node) + signatures[node.name] = _signature(node, info.classes) except Skip as e: skipped[node.name] = e.reason @@ -173,7 +190,7 @@ def lower(path: Path) -> ir.Module: else: skipped[name] = result functions.sort(key=lambda f: f.source_line) - return ir.Module(path.stem, tuple(functions), skipped) + return ir.Module(path.stem, tuple(functions), skipped, tuple(class_decls.values())) def _module_info(tree: ast.Module) -> _ModuleInfo: @@ -265,11 +282,172 @@ def _binding_targets(stmt: ast.stmt) -> list[ast.expr]: return [] +# --- fixed-layout classes (SPEC N-11) -------------------------------------------------------------- + +# A user override of any of these runs user code on a slot read, write or construction, which the +# direct slot access (FieldGet/FieldSet/New) would skip. +SLOT_ACCESS_DUNDERS = ("__getattribute__", "__getattr__", "__setattr__", "__delattr__", "__new__") + + +def _whole_module(source: str) -> bool: + return next((line.strip() for line in source.splitlines() if line.strip()), "") == MARKER + + +def _classes( + tree: ast.Module, source: str, info: _ModuleInfo, +) -> tuple[dict[str, ir.ClassDecl], dict[str, str]]: + whole = _whole_module(source) + decls: dict[str, ir.ClassDecl] = {} + skipped: dict[str, str] = {} + for node in tree.body: + if not isinstance(node, ast.ClassDef): + continue + marked = any(isinstance(d, ast.Name) and d.id == "compiled" for d in node.decorator_list) + if not (whole or marked): + continue + result = _class_decl(node, tree, info) + if isinstance(result, str): + skipped[node.name] = result + else: + decls[node.name] = result + return decls, skipped + + +def _class_decl(node: ast.ClassDef, tree: ast.Module, info: _ModuleInfo) -> ir.ClassDecl | str: + """The `ClassDecl` of a class whose layout is fixed in CPython too, else why it is not one.""" + name, at = node.name, f"line {node.lineno}: class `{node.name}`" + other = [d for d in node.decorator_list if not (isinstance(d, ast.Name) and d.id == "compiled")] + if other: + return f"{at}: decorator `@{_src(other[0])}` is not supported (only `@compiled`)" + if node.decorator_list and info.bound("compiled"): + return f"{at}: `compiled` is bound in this module, so `@compiled` is not the builtin" + if node.keywords: + return f"{at}: has a metaclass or class keyword `{node.keywords[0].arg}`" + bases = [b for b in node.bases if not (isinstance(b, ast.Name) and b.id == "object")] + if bases or (node.bases and info.bound("object")): + shown = _src(bases[0]) if bases else "object" + return f"{at}: has a base class `{shown}` (only `object` is allowed)" + if len(info.bindings.get(name, [])) > 1: + lines = ", ".join(str(line) for line, _ in info.bindings[name]) + return f"{at}: `{name}` is bound more than once (lines {lines})" + for n in ast.walk(tree): + if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Del) and n.id == name: + return f"{at}: `{name}` is deleted (line {n.lineno}); bound more than once" + if isinstance(n, ast.NamedExpr) and isinstance(n.target, ast.Name) and n.target.id == name: + return f"{at}: `{name}` is rebound by `:=` (line {n.lineno}); bound more than once" + if isinstance(n, ast.ClassDef) and any(isinstance(b, ast.Name) and b.id == name + for b in n.bases): + return f"{at}: is subclassed by `{n.name}` (line {n.lineno})" + for target in _assigned_attributes(n): + if isinstance(target.value, ast.Name) and target.value.id == name: + return (f"{at}: its class attribute `{target.attr}` is assigned at line " + f"{target.lineno}") + if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) \ + and n.func.id in ("setattr", "delattr") and n.args \ + and isinstance(n.args[0], ast.Name) and n.args[0].id == name: + return f"{at}: its class attribute is assigned by `{_src(n)}` at line {n.lineno}" + + slots: ast.expr | None = None + annotated: list[str] = [] + init: ast.FunctionDef | None = None + for st in node.body: + if isinstance(st, ast.Expr) and isinstance(st.value, ast.Constant) \ + and isinstance(st.value.value, str): + continue + if isinstance(st, ast.Pass): + continue + if isinstance(st, ast.Assign) and len(st.targets) == 1 \ + and isinstance(st.targets[0], ast.Name) and st.targets[0].id == "__slots__": + slots = st.value + elif isinstance(st, ast.AnnAssign) and isinstance(st.target, ast.Name) and st.value is None: + annotated.append(st.target.id) + elif isinstance(st, ast.FunctionDef): + if st.name in SLOT_ACCESS_DUNDERS: + return f"{at}: defines `{st.name}`, which runs user code on slot access" + if st.name == "__init__": + init = st + else: + return (f"{at}: class body statement `{_src(st).splitlines()[0]}` (line {st.lineno}) " + "is not supported (only `__slots__`, bare field annotations and methods)") + if slots is None: + return f"{at}: has no `__slots__`" + if not (isinstance(slots, (ast.Tuple, ast.List)) and all( + isinstance(e, ast.Constant) and isinstance(e.value, str) for e in slots.elts)): + return f"{at}: `__slots__` is not a literal tuple or list of strings" + fields = tuple(e.value for e in slots.elts) # type: ignore[attr-defined] + twice = next((f for f in fields if fields.count(f) > 1), None) + if twice is not None: + return f"{at}: `__slots__` lists `{twice}` twice" + if set(fields) != set(annotated): + return (f"{at}: `__slots__` {fields} does not equal its annotated fields " + f"{tuple(dict.fromkeys(annotated))}") + return ir.ClassDecl(name, fields, init is not None and _trivial_init(init, fields)) + + +def _assigned_attributes(n: ast.AST) -> list[ast.Attribute]: + targets: list[ast.expr] = [] + if isinstance(n, ast.Assign): + targets = list(n.targets) + elif isinstance(n, (ast.AugAssign, ast.AnnAssign)): + targets = [n.target] + elif isinstance(n, ast.Delete): + targets = list(n.targets) + elif isinstance(n, (ast.For, ast.AsyncFor)): + targets = [n.target] + found: list[ast.Attribute] = [] + for t in targets: + found += [x for x in ast.walk(t) if isinstance(x, ast.Attribute)] + return found + + +def _trivial_init(init: ast.FunctionDef, fields: tuple[str, ...]) -> bool: + """`__init__(self, p0, p1, ...)` whose body is exactly `self. = ` once per slot, with + parameter i stored into slot i (`ir.ClassDecl.trivial_init`). Statement order is free.""" + a = init.args + if init.decorator_list or a.posonlyargs or a.kwonlyargs or a.vararg or a.kwarg or a.defaults \ + or init.returns is not None and _src(init.returns) != "None" or not a.args: + return False + self_name, params = a.args[0].arg, [x.arg for x in a.args[1:]] + body = list(init.body) + if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant) \ + and isinstance(body[0].value.value, str): + body = body[1:] + if len(params) != len(fields) or len(set(params)) != len(params) or len(body) != len(fields): + return False + stored: dict[str, str] = {} + for st in body: + if not (isinstance(st, ast.Assign) and len(st.targets) == 1 + and isinstance(st.targets[0], ast.Attribute) + and isinstance(st.targets[0].value, ast.Name) + and st.targets[0].value.id == self_name and isinstance(st.value, ast.Name) + and st.value.id in params and self_name not in params): + return False + field_name = st.targets[0].attr + if field_name not in fields or field_name in stored: + return False + stored[field_name] = st.value.id + return all(stored.get(f) == p for f, p in zip(fields, params)) + + +def _class_of(kind: str, classes: dict[str, ir.ClassDecl]) -> tuple[str, bool] | None: + """`C`, `C | None`, `None | C`, `Optional[C]`, `"C | None"` (an annotation spelled by + `_annotation_kind`) -> (C, optional) for a compiled class C, else None.""" + kind = kind.strip("'\"") + optional = False + for prefix in ("Optional[", "typing.Optional["): + if kind.startswith(prefix) and kind.endswith("]"): + kind, optional = kind[len(prefix):-1], True + parts = [p.strip("'\"") for p in kind.split("|")] + named = [p for p in parts if p != "None"] + if len(named) != 1 or named[0] not in classes: + return None + return named[0], optional or len(named) != len(parts) + + def _targets( tree: ast.Module, source: str, info: _ModuleInfo, ) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]]: - first = next((line.strip() for line in source.splitlines() if line.strip()), "") - whole_module = first == MARKER + whole_module = _whole_module(source) found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]] = [] for node in tree.body: if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): @@ -294,7 +472,7 @@ def _targets( return found -def _signature(fn: ast.FunctionDef) -> _Signature: +def _signature(fn: ast.FunctionDef, classes: dict[str, ir.ClassDecl] | None = None) -> _Signature: args = fn.args line = fn.lineno if args.posonlyargs: @@ -314,6 +492,8 @@ def _signature(fn: ast.FunctionDef) -> _Signature: kind = _annotation_kind(a.annotation) if kind in ARRAY_KIND: params.append(ir.ArrayParam(a.arg, ARRAY_KIND[kind], False)) # `stored` set later + elif (c := _class_of(kind, classes or {})) is not None: + params.append(ir.Param(a.arg, Type.OBJ, c[0], c[1])) else: params.append(ir.Param(a.arg, SCALAR_KIND.get(kind, Type.OBJ))) if fn.returns is None: @@ -475,6 +655,7 @@ def __init__(self, sig: _Signature, env: dict[str, ir.Function], info: _ModuleIn self.deopts = False # a node that can deopt self.stored: set[str] = set() self.current: ast.stmt | None = None # the statement being lowered (for messages) + self._new_locals: dict[str, str] | None = None # --- entry --- @@ -718,11 +899,123 @@ def assign(self, target: ast.expr, value_node: ast.expr, node: ast.stmt) -> list self.store_effect(array) return self.take() + [ir.StoreIndex(array, index, value)] if isinstance(target, ast.Attribute): + found = self.field_set(target, value_node) + if found is not None: + return found raise Skip(line, f"assignment to attribute `{_src(target)}` is not supported") if isinstance(target, (ast.Tuple, ast.List)): raise Skip(line, "tuple unpacking is not supported") raise _unsupported(target) + # --- fixed-layout classes --- + + def new_class(self, node: ast.expr) -> str | None: + """The class C when `node` is `C(args)` that lowers to `ir.New`.""" + if not (isinstance(node, ast.Call) and isinstance(node.func, ast.Name)): + return None + decl = self.info.classes.get(node.func.id) + if decl is None or not decl.trivial_init or self.is_local(decl.name) \ + or node.keywords or any(isinstance(a, ast.Starred) for a in node.args) \ + or len(node.args) != len(decl.fields): + return None + return decl.name + + def new_locals(self) -> dict[str, str]: + """Locals every one of whose bindings is a `C(...)` lowered to `New`: always exactly a C.""" + if self._new_locals is None: + good: dict[str, set[str]] = {} + bad: set[str] = set() + for stmt in rebinding._own_statements(self.fn): + if stmt is self.fn: + continue + value = None + if isinstance(stmt, ast.Assign) and len(stmt.targets) == 1 \ + and isinstance(stmt.targets[0], ast.Name): + name, value = stmt.targets[0].id, stmt.value + elif isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name): + name, value = stmt.target.id, stmt.value + if value is None: + continue + else: + for t in _binding_targets(stmt): + bad |= {n.id for n in ast.walk(t) if isinstance(n, ast.Name)} + continue + c = self.new_class(value) + if c is None: + bad.add(name) + else: + good.setdefault(name, set()).add(c) + self._new_locals = {n: next(iter(cs)) for n, cs in good.items() + if n not in bad and len(cs) == 1 and n not in self.params} + return self._new_locals + + def cls_of_name(self, name: str) -> tuple[str | None, bool]: + """(compiled class the name is declared to hold, whether its exact type is proved).""" + classes = self.info.classes + if name in self.params: + p = self.params[name] + if isinstance(p, ir.Param) and p.cls is not None: + rebound = any(b.how != "annotation" for b in self.bindings.get(name, [])) + return p.cls, not p.optional and not rebound + return None, False + if name in self.new_locals(): + return self.new_locals()[name], True + if name in self.bindings: + kinds = list(self.annotations.get(name, [])) + if name not in self.annotations: + kinds += self.inferred.get(name, []) + found = {_class_of(k, classes) for _, k in kinds} + if len(found) == 1 and None not in found: + return next(iter(found))[0], False # type: ignore[index] + return None, False + + def field_target(self, node: ast.Attribute) -> tuple[ir.Local, str, bool] | None: + """(object, class, exact type proved) for `name.field` of a compiled class's field.""" + if not (isinstance(node.value, ast.Name) and self.is_local(node.value.id)): + return None + cls, exact = self.cls_of_name(node.value.id) + if cls is None or node.attr not in self.info.classes[cls].fields: + return None + return ir.Local(Type.OBJ, node.value.id), cls, exact + + def field_get(self, node: ast.Attribute) -> ir.Expr | None: + found = self.field_target(node) + if found is None: + return None + obj, cls, exact = found + if not exact: + if self.impure_mode: + raise Skip(node.lineno, f"`{_src(node)}`: the exact type of `{obj.name}` is not " + f"proved to be `{cls}`, and an impure function cannot " + "deopt on a failed check") + obj = ir.CheckExact(Type.OBJ, obj, cls) + self.deopts = True + return ir.FieldGet(Type.OBJ, obj, cls, node.attr) + + def field_set(self, target: ast.Attribute, value_node: ast.expr) -> list[ir.Stmt] | None: + found = self.field_target(target) + if found is None: + return None + obj, cls, exact = found + if not exact: + raise Skip(target.lineno, f"`{_src(target)} = ...`: the exact type of `{obj.name}` is " + f"not proved to be `{cls}`, and a store cannot be guarded by " + "a deopting check") + value = self.boxed(self.value(value_node), value_node) + self.impure = True + return self.take() + [ir.FieldSet(obj, cls, target.attr, value)] + + def arg_proved(self, e: ir.Expr, p: ir.Param) -> bool: + """The argument `e` is certainly accepted by the callee's entry guard for `p.cls`.""" + if isinstance(e, ir.Const) and e.value is None: + return p.optional + if isinstance(e, (ir.New, ir.CheckExact)): + return e.cls == p.cls + if isinstance(e, ir.Local) and e.type == Type.OBJ: + cls, exact = self.cls_of_name(e.name) + return exact and cls == p.cls + return False + def array_word(self, name: str) -> str: return "local array" if name in self.local_arrays else "array parameter" @@ -860,6 +1153,8 @@ def range_bound(self, node: ast.expr) -> ir.Expr: def return_(self, node: ast.Return) -> list[ir.Stmt]: returns = self.sig.returns if node.value is None or (isinstance(node.value, ast.Constant) and node.value.value is None): + if returns == Type.OBJ: + return [ir.Return(ir.Const(Type.OBJ, None))] if returns != Type.NONE: raise Skip(node.lineno, f"returns None from a function declared to return " f"{TYPE_NAME[returns]}") @@ -940,6 +1235,8 @@ def expr(self, node: ast.expr, cond: int = 0) -> ir.Expr: return ir.Const(Type.I64, v) if isinstance(v, float): return ir.Const(Type.F64, v) + if v is None: + return ir.Const(Type.OBJ, None) raise Skip(line, f"{type(v).__name__} constant `{_src(node)}` is not supported") if isinstance(node, ast.Name): return self.name(node) @@ -959,6 +1256,9 @@ def expr(self, node: ast.expr, cond: int = 0) -> ir.Expr: if isinstance(node, ast.Attribute): if isinstance(node.value, ast.Name) and node.value.id in self.arrays: raise self.array_misuse(node.value.id, node) + field_get = self.field_get(node) + if field_get is not None: + return field_get obj = self.boxed(self.sub(node.value), node.value) self.impure = True return ir.GetAttr(Type.OBJ, obj, node.attr) @@ -1101,6 +1401,12 @@ def compare(self, node: ast.Compare, cond: int) -> ir.Expr: def link(self, op: ast.cmpop, left: ir.Expr, right: ir.Expr, obj_ok: bool, node: ast.Compare) -> ir.Expr: + if isinstance(op, (ast.Is, ast.IsNot)): + if left.type != Type.OBJ or right.type != Type.OBJ: + raise Skip(node.lineno, f"identity comparison `{_src(node)}` of a native " + f"{TYPE_NAME[left.type if left.type != Type.OBJ else right.type]}" + " is not supported (only of objects)") + return ir.Is(Type.BOOL, left, right, isinstance(op, ast.IsNot)) kind = COMPARE.get(type(op)) if kind is None: raise Skip(node.lineno, f"comparison `{_src(node)}` is not supported " @@ -1161,6 +1467,11 @@ def call(self, node: ast.Call) -> ir.Expr: return arg # float(x) of an exact float is x itself self.impure = True return ir.ObjToFloat(Type.F64, _box(arg)) + new_cls = self.new_class(node) + if new_cls is not None: + self.hoist_ok = False + return ir.New(Type.OBJ, new_cls, + tuple(self.boxed(self.value(a), a) for a in node.args)) if not self.is_local(name) and name in self.env: direct = self.direct_call(name, node) if direct is not None: @@ -1200,13 +1511,19 @@ def direct_call(self, name: str, node: ast.Call) -> ir.Expr | None: for p, e in zip(callee.params, args): if p.type != Type.OBJ and e.type != p.type: direct = False # the callee's guard would deopt; the global keeps CPython's result + # A class parameter's guard (`type(x) is C`) fails for an argument not proved to be a C: + # that is a deopt of the callee at its entry — redoable only by a pure caller. + unproved = any(isinstance(p, ir.Param) and p.cls is not None and not self.arg_proved(e, p) + for p, e in zip(callee.params, args)) + if unproved and self.impure_mode: + direct = False if not direct: self.impure = True return ir.CallObject(Type.OBJ, ir.Global(Type.OBJ, name), tuple(self.boxed(e, a) for e, a in zip(args, node.args))) if not callee.pure: self.impure = True - if callee.may_deopt and not redo: + if (callee.may_deopt and not redo) or unproved: self.deopts = True typed = tuple(self.boxed(e, a) if p.type == Type.OBJ else e for p, e, a in zip(callee.params, args, node.args)) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index 7d0b891fa..dbc8dbe5a 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -242,7 +242,13 @@ class Call(Expr): redone); otherwise the front end calls it as an object (`CallObject(Global(name), ...)`). A call cycle (direct or mutual recursion) is legal: every compiled function counts its frame at entry (runtime `tp_enter_call`), so a recursion that CPython stops with RecursionError is stopped - with the same error, and the C stack cannot overflow.""" + with the same error, and the C stack cannot overflow. + A callee with a `Param.cls` parameter can deopt at entry whenever an argument is not proved to be + exactly that class, so such a call counts as a node that can deopt: the caller is pure (and + `may_deopt`), or the call is `redo`. A pure callee that uses a compiled class can also return the + deopt signal with `may_deopt` False, when the class changed after init (N-11); an impure caller + then redoes that callee interpreted, for OBJ and I64 results as well — such a callee is pure and + its I64 result is interval-proved, so the redone value is the same and fits.""" function: str args: tuple[Expr, ...] @@ -347,6 +353,67 @@ class Tuple(Expr): elements: tuple[Expr, ...] +@dataclass(frozen=True) +class Is(Expr): + """`a is b` / `a is not b` (negate) on OBJ operands: identity, no user code. Type BOOL. With + `Const(None, OBJ)` this is how `x is None` is written.""" + + left: Expr + right: Expr + negate: bool = False + + +@dataclass(frozen=True) +class IsExact(Expr): + """`type(obj) is C` for a compiled fixed-layout class C (SPEC N-11): no user code. Type BOOL. + A true IsExact dominating a FieldGet/FieldSet on the same local is what the verifier accepts as + the exact-type proof for that access. `C` is the module global as CPython reads it: when the + global no longer is the class captured at init, the back end compares with the current binding + (impure) or deopts (pure).""" + + obj: Expr + cls: str + + +@dataclass(frozen=True) +class CheckExact(Expr): + """`obj` itself (type OBJ), after checking `type(obj) is C`; anything else → deopt. Because it + can deopt it is legal only in a pure function — the same rule as Unbox. It is the proof the + verifier accepts for a field access the front end cannot guard with a dominating IsExact.""" + + obj: Expr + cls: str + + +@dataclass(frozen=True) +class FieldGet(Expr): + """`obj.field` of a compiled fixed-layout class C whose exact type is proved (a dominating true + IsExact on the same local, a CheckExact, a New, or a parameter guarded `type(x) is C`). Reads the + slot through the member descriptor captured at module init: a new reference, or CPython's own + AttributeError for an unset slot. Runs no user code, so it is not an effect. Type OBJ. + The class may change after init (a property replacing the field, `obj.__class__ = Other`): the + back end re-checks `type(obj) is C` and C's version tag at the access, and otherwise does what + CPython does — `getattr(obj, field)` in an impure function, a deopt in a pure one (where running + user code could be repeated by a later deopt's redo). FieldSet likewise (setattr).""" + + obj: Expr + cls: str + field: str + + +@dataclass(frozen=True) +class New(Expr): + """`C(args)` for a compiled class whose `__init__` only assigns each slot once from its parameters + (ClassDecl.trivial_init): allocate with C's tp_alloc and fill the slots in __init__'s order — + the observable result of running that __init__. A fresh object is not an effect. Type OBJ; args + are OBJ (scalars through Box). `C` is read as a module global before the args, as CPython's + LOAD_GLOBAL is: if it is no longer the captured class, or the class changed (e.g. __init__), the + back end calls what the global held (impure) or deopts (pure).""" + + cls: str + args: tuple[Expr, ...] + + @dataclass(frozen=True) class Len(Expr): """`len(array)` of an array parameter. Type I64.""" @@ -391,6 +458,17 @@ class StoreIndex(Stmt): proven: bool = False # as in Index +@dataclass(frozen=True) +class FieldSet(Stmt): + """`obj.field = value` on a proved-exact compiled class (as FieldGet). An effect — unless `obj` is + a New made in this same function and not yet escaped (not assumed in M2b: always an effect).""" + + obj: Expr + cls: str + field: str + value: Expr + + @dataclass(frozen=True) class ExprStmt(Stmt): value: Expr @@ -445,6 +523,10 @@ class Continue(Stmt): class Param: name: str type: Type + # For an OBJ parameter annotated with a compiled class C: the entry guard checks + # `type(x) is C` (or `x is None` too when `optional`), deopting before any effect otherwise. + cls: str | None = None + optional: bool = False @dataclass(frozen=True) @@ -515,9 +597,22 @@ class Function: entry_globals: tuple[Param, ...] = () +@dataclass(frozen=True) +class ClassDecl: + """A compiled fixed-layout class (SPEC N-11): `__slots__` written by the user equal to its + annotated fields, base object, no metaclass, no decorator but @compiled. The module init checks + every field's class attribute is a member descriptor and captures it; if not, the class is + not compiled (functions using it deopt at entry or are refused by the verifier).""" + + name: str + fields: tuple[str, ...] + trivial_init: bool # __init__(self, *fields-in-order) doing exactly `self.f = f` once each + + @dataclass(frozen=True) class Module: name: str functions: tuple[Function, ...] # Functions the front end left interpreted, with the reason (for `demo` and the coverage table). skipped: dict[str, str] = field(default_factory=dict) + classes: tuple[ClassDecl, ...] = () diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index 77bd67469..51c18a750 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -97,3 +97,83 @@ CPython's own Python-frame check uses. It then calls `Py_EnterRecursiveCall(" in C-stack guard: a raised recursion limit can never let compiled C overflow the stack. Counters are per module (the header is `static`): frames of a *different* compiled module that sit between two frames of this one are not counted. + +## Fixed-layout classes (ir.ClassDecl / FieldGet / FieldSet / New / IsExact / CheckExact; SPEC N-11) +Guard state, one per ClassDecl in the module state (zeroed = not compiled): + + typedef struct { + unsigned int tag; /* tp_version_tag at which the slots were proved; never 0 while compiled */ + unsigned int bad_tag; /* a tag at which tp_class_refresh failed (0: none); not retried */ + uint64_t gepoch; /* tp_globals_epoch at which module_dict[name] was last seen to be the class */ + PyObject *init; /* strong: the class's own __init__ at capture, or NULL */ + Py_ssize_t slow; /* failed per-access checks (introspection) */ + Py_ssize_t refreshed; /* new tags adopted by tp_class_refresh (introspection) */ + } tp_class_rt; + static uint64_t tp_globals_epoch; /* moves when a watched module dict changes a class-name key */ + +At module init (none runs user code): + + /* `name` in cls's OWN dict (no MRO): 1 found (*out new ref), 0 absent, -1 error */ + int tp_own_attr(PyTypeObject *cls, const char *name, PyObject **out); + /* `field` in cls's own dict must be a member descriptor made for cls, of a writable + Py_T_OBJECT_EX slot of that name inside the instance: *out = its PyMemberDef, *descr = a new + reference to the descriptor. 0 ok, 1 not a plain slot (class not compiled), -1 error */ + int tp_slot_capture(PyTypeObject *cls, const char *field, PyMemberDef **out, PyObject **descr); + /* heap type made by `type`, base object, no __dict__, fixed size, object's tp_new/tp_alloc, + generic getattro/setattro (no __getattribute__/__getattr__/__setattr__/__delattr__), not abstract */ + int tp_class_shape_ok(PyTypeObject *cls); + /* the whole ClassDecl check: shape, every field (slots[i], descrs[i]), rt->init, and a version + tag (PyUnstable_Type_AssignVersionTag; none → not compiled). 1 compiled, 0 not, -1 error; + on 0/-1 nothing is kept */ + int tp_class_capture(PyObject *cls, const char *const *fields, Py_ssize_t n, + PyMemberDef **slots, PyObject **descrs, tp_class_rt *rt); + void tp_class_rt_clear(tp_class_rt *rt, PyObject **descrs, Py_ssize_t n); /* drop refs, zero tags */ + +At every access (the class may have changed since init; cgen.py "Classes changed after init"): + + /* FieldGet/FieldSet (obj) or New (obj == NULL): 1 = type(obj) is cls and cls's tp_version_tag is + rt->tag (or tp_class_refresh re-proved it), 0 = take the slow path / deopt, -1 = error. + Fast case: one type compare, one tag compare. */ + int tp_class_current(PyObject *obj, PyObject *cls, tp_class_rt *rt, const char *const *fields, + Py_ssize_t n, PyObject *const *descrs); + /* the tag moved: get a tag (exhausted → 0), shape still ok, every field still the very + descriptor object and __init__ the very object captured → adopt the tag (1); else remember + it in bad_tag (0). -1 error. Runs no user code. */ + int tp_class_refresh(PyTypeObject *cls, tp_class_rt *rt, const char *const *fields, + Py_ssize_t n, PyObject *const *descrs); + /* LOAD_GLOBAL of the class name as an identity test: 1 module_dict[name] is cls, 0 not (or not a + module global), -1 error. watched: a 1 is cached in rt->gepoch until tp_globals_epoch moves */ + int tp_class_global_ok(PyObject *module_dict, PyObject *name, PyObject *cls, tp_class_rt *rt, + int watched); + unsigned int tp_class_tag(PyObject *cls); /* cls->tp_version_tag; 0 for NULL */ + +Module-dict watcher (one id per module instance; ids 2..7 of 8 per interpreter are free for all +extensions together): + + /* body of the module's PyDict_WatchCallback: move tp_globals_epoch when key is an exact str + equal to one of `names` (NULL-terminated), any other key type or str subclass, or NULL + (cleared/cloned/deallocated). Never raises; returns 0 */ + int tp_globals_watch_event(PyDict_WatchEvent event, PyObject *key, const char *const *names); + /* PyDict_AddWatcher + PyDict_Watch: *id1 = id + 1, or 0 when no id is free / Watch failed + (error cleared: the caller uses the conservative path, a dict lookup per New) */ + void tp_globals_watch(PyObject *module_dict, PyDict_WatchCallback cb, int *id1); + /* PyDict_Unwatch (if module_dict) + PyDict_ClearWatcher; idempotent; keeps a pending exception */ + void tp_globals_unwatch(PyObject *module_dict, int *id1); + +Slot access and the slow paths: + + PyObject *tp_field_get(PyObject *obj, PyMemberDef *m); /* PyMember_GetOne: new ref or AttributeError */ + int tp_field_set(PyObject *obj, PyMemberDef *m, PyObject *v); /* PyMember_SetOne: 0 / -1 */ + int tp_setattr(PyObject *obj, PyObject *name, PyObject *v); /* STORE_ATTR (PyObject_SetAttr): 0 / -1 */ + PyObject *tp_new_fixed(PyTypeObject *cls, PyMemberDef *const *slots, PyObject *const *values, Py_ssize_t n); + /* cls->tp_alloc(cls, 0) then set each slot (new references); NULL + MemoryError */ + int tp_is_exact(PyObject *obj, PyTypeObject *cls); /* Py_IS_TYPE */ +FieldGet's slow path is `tp_getattr`, New's is `tp_global` + `tp_call` (the binding loaded before +the arguments), IsExact's compares with `tp_global`; in a pure function every slow path is a deopt. + +Version tags (CPython 3.14 Objects/typeobject.c): PyType_Modified sets tp_version_tag to 0 for the +type and its subclasses; assign_version_tag gives a heap type `NEXT_VERSION_TAG(interp)++`, a +per-interpreter counter that only grows and, once it wraps to 0, assigns nothing again; a class +gets at most MAX_VERSIONS_PER_CLASS (1000) tags. So a tag never repeats within an interpreter and a +class whose tags are exhausted keeps 0 (never current). Py_TPFLAGS_VALID_VERSION_TAG is "Unused. +Legacy flag" in 3.14 (never set) and is not consulted. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index 2f64563d8..e0b2f3193 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -1008,6 +1008,385 @@ static inline void tp_release(PyObject **slot) Py_CLEAR(*slot); } +/* ------------------------------------------------------------------------------------------ + * Fixed-layout classes (SPEC N-11, ir.ClassDecl; API.md "Fixed-layout classes"). A compiled class + * is a plain heap type whose instances keep every field in an object slot at a fixed offset; the + * slots are reached through the PyMemberDef of the member descriptor captured at module init. + * Nothing here runs user code: PyMember_GetOne/SetOne touch the slot only. + * + * A class can change after init (a property replaces a field, __init__ is replaced, the module + * global is rebound, an instance's __class__ is assigned). Compiled code therefore re-proves its + * preconditions at EVERY FieldGet / FieldSet / New: + * - the class is unchanged: cls->tp_version_tag equals the tag captured (or re-proved) for it. + * CPython clears the tag to 0 on every change of the type (PyType_Modified / + * type_modified_unlocked) and a later lookup assigns a NEW one from a per-interpreter counter + * that only grows and stops at 0 when exhausted (typeobject.c assign_version_tag), so an old + * tag never comes back and a tag of 0 never matches (the captured tag is never 0). + * (Py_TPFLAGS_VALID_VERSION_TAG is "Unused. Legacy flag" in 3.14's object.h and is never set: + * it cannot be part of the test.) + * - FieldGet / FieldSet: Py_IS_TYPE(obj, cls) (covers `obj.__class__ = Other`). + * - New: the module global still is the class (tp_class_global_ok). + * When a check fails the generated code takes a slow path with CPython's own semantics (or, in a + * pure function, deopts); see cgen.py "Fixed-layout classes". + * ------------------------------------------------------------------------------------------ */ + +/* Per-class guard state, one per ClassDecl in the module state (zeroed = not compiled). */ +typedef struct { + unsigned int tag; /* tp_version_tag at which the slots were proved; never 0 while compiled */ + unsigned int bad_tag; /* a tag at which tp_class_refresh failed (0: none); not retried */ + uint64_t gepoch; /* tp_globals_epoch at which module_dict[name] was last seen to be cls */ + PyObject *init; /* strong: the class's own __init__ at capture, or NULL if it had none */ + Py_ssize_t slow; /* per-access checks that failed (introspection only) */ + Py_ssize_t refreshed; /* new tags adopted by tp_class_refresh (introspection only) */ +} tp_class_rt; + +/* Moves whenever a watched module dict changes a key that is (or may be) a compiled class name, is + * cleared, cloned or deallocated (tp_globals_watch_event). Static: one counter per extension, shared + * by every instance of the module in the process; an instance re-proves its own globals after any + * move. The module declares no Py_mod_multiple_interpreters / Py_mod_gil slot, so CPython loads it + * only into interpreters sharing the main GIL and keeps the GIL on in a free-threaded build: plain + * reads and writes are serialised. */ +static uint64_t tp_globals_epoch = 1; + +/* `name` in cls's OWN dict (not through the MRO). 1 found (*out a new reference), 0 absent, -1 error. */ +static inline int tp_own_attr(PyTypeObject *cls, const char *name, PyObject **out) +{ + PyObject *dict = PyType_GetDict(cls); + int found; + + *out = NULL; + if (dict == NULL) { + return -1; + } + found = PyDict_GetItemStringRef(dict, name, out); + Py_DECREF(dict); + return found; +} + +/* Look up `field` in cls's OWN dict (not through the MRO: a subclass or an instance attribute must + * not be consulted) and capture its PyMemberDef. It must be a member descriptor made for exactly + * this class (d_type == cls), of an object slot (Py_T_OBJECT_EX), writable, of that name, whose + * offset lies inside the instance. 0 = captured (*out set, *descr a new reference to the descriptor), + * 1 = not a plain slot (the class is not compiled), -1 = error. The PyMemberDef lives in + * cls->tp_members: the caller must keep `cls` alive for as long as the pointer is used (the + * generated module holds a strong reference). */ +static inline int tp_slot_capture(PyTypeObject *cls, const char *field, PyMemberDef **out, + PyObject **descr_out) +{ + PyObject *descr = NULL; + int found, rc = 1; + PyMemberDescrObject *md; + PyMemberDef *m; + + *descr_out = NULL; + found = tp_own_attr(cls, field, &descr); + if (found < 0) { + return -1; + } + if (found == 0) { + return 1; + } + if (Py_IS_TYPE(descr, &PyMemberDescr_Type)) { + md = (PyMemberDescrObject *)descr; + m = md->d_member; + if (md->d_common.d_type == cls && m != NULL && m->name != NULL + && strcmp(m->name, field) == 0 && m->type == Py_T_OBJECT_EX + && (m->flags & Py_READONLY) == 0 && m->offset >= (Py_ssize_t)sizeof(PyObject) + && m->offset + (Py_ssize_t)sizeof(PyObject *) <= cls->tp_basicsize) { + *out = m; + *descr_out = descr; /* the reference moves to the caller */ + return 0; + } + } + Py_DECREF(descr); + return rc; +} + +/* The type-level conditions of ClassDecl: a heap type made by `type` (no metaclass), base `object`, + * no instance __dict__, no variable size, object's own tp_new and tp_alloc (so tp_new_fixed is what + * `C(...)` does for a trivial __init__), and the generic attribute protocol (no __getattribute__, + * __getattr__, __setattr__ or __delattr__: those would run instead of, or after, the slot access). */ +static inline int tp_class_shape_ok(PyTypeObject *cls) +{ + return (cls->tp_flags & Py_TPFLAGS_HEAPTYPE) && Py_TYPE(cls) == &PyType_Type + && cls->tp_base == &PyBaseObject_Type && cls->tp_dictoffset == 0 + && cls->tp_itemsize == 0 && cls->tp_new == PyBaseObject_Type.tp_new + && cls->tp_alloc == PyType_GenericAlloc && !(cls->tp_flags & Py_TPFLAGS_IS_ABSTRACT) + && cls->tp_getattro == PyObject_GenericGetAttr + && cls->tp_setattro == PyObject_GenericSetAttr; +} + +static inline void tp_class_rt_clear(tp_class_rt *rt, PyObject **descrs, Py_ssize_t n) +{ + Py_ssize_t i; + for (i = 0; i < n; i++) { + Py_CLEAR(descrs[i]); + } + Py_CLEAR(rt->init); + rt->tag = 0; + rt->bad_tag = 0; + rt->gepoch = 0; +} + +/* At module init: the class object must be what ClassDecl promises (tp_class_shape_ok) and every + * field a plain slot. slots[i] receives the PyMemberDef of fields[i], descrs[i] a strong reference + * to its member descriptor, rt->init a strong reference to the class's own __init__ (or NULL), and + * rt->tag the class's version tag (PyUnstable_Type_AssignVersionTag assigns one if it has none; a + * class that cannot get a tag is not compiled). 1 = compiled, 0 = not (nothing kept), -1 = error + * (nothing kept). */ +static inline int tp_class_capture(PyObject *cls_obj, const char *const *fields, Py_ssize_t n, + PyMemberDef **slots, PyObject **descrs, tp_class_rt *rt) +{ + PyTypeObject *cls; + Py_ssize_t i; + int rc; + + if (!PyType_Check(cls_obj)) { + return 0; + } + cls = (PyTypeObject *)cls_obj; + if (!tp_class_shape_ok(cls)) { + return 0; + } + for (i = 0; i < n; i++) { + rc = tp_slot_capture(cls, fields[i], &slots[i], &descrs[i]); + if (rc != 0) { + tp_class_rt_clear(rt, descrs, i); + return rc < 0 ? -1 : 0; + } + } + if (tp_own_attr(cls, "__init__", &rt->init) < 0) { + tp_class_rt_clear(rt, descrs, n); + return -1; + } + if (!PyUnstable_Type_AssignVersionTag(cls) || cls->tp_version_tag == 0) { + tp_class_rt_clear(rt, descrs, n); + return 0; + } + rt->tag = cls->tp_version_tag; + rt->bad_tag = 0; + rt->gepoch = 0; + rt->slow = 0; + rt->refreshed = 0; + return 1; +} + +/* The class's tag moved (or is 0). Re-prove what the captured slots rely on, without running user + * code: get a tag (none left: 0), the shape still holds, every field's own-dict entry is the very + * member descriptor object captured at init, and __init__ is the very object captured at init. Then + * adopt the new tag (1). Otherwise remember the tag as bad (0), so the next access with that tag + * does not search again. -1 = error (a failed dict lookup). */ +static inline int tp_class_refresh(PyTypeObject *cls, tp_class_rt *rt, const char *const *fields, + Py_ssize_t n, PyObject *const *descrs) +{ + unsigned int tag; + Py_ssize_t i; + PyObject *d; + int found; + + if (!PyUnstable_Type_AssignVersionTag(cls) || cls->tp_version_tag == 0) { + return 0; /* tags exhausted for this class */ + } + tag = cls->tp_version_tag; + if (tag == rt->tag) { + return 1; + } + if (tag == rt->bad_tag) { + return 0; + } + if (!tp_class_shape_ok(cls)) { + goto bad; + } + for (i = 0; i < n; i++) { + found = tp_own_attr(cls, fields[i], &d); + if (found < 0) { + return -1; + } + Py_XDECREF(d); /* identity only; descrs[i] keeps it alive */ + if (d != descrs[i]) { + goto bad; + } + } + found = tp_own_attr(cls, "__init__", &d); + if (found < 0) { + return -1; + } + Py_XDECREF(d); + if (d != rt->init) { + goto bad; + } + rt->tag = tag; + rt->refreshed++; + return 1; +bad: + rt->bad_tag = tag; + return 0; +} + +/* cls's current tp_version_tag, 0 for NULL (introspection: __typedpython_class_info__). */ +static inline unsigned int tp_class_tag(PyObject *cls) +{ + return cls != NULL ? ((PyTypeObject *)cls)->tp_version_tag : 0u; +} + +/* The per-access check of FieldGet / FieldSet (obj != NULL) and New (obj == NULL): 1 = the captured + * slots are what `obj.f` / `C(...)` reach now (obj is exactly cls, and cls is unchanged since the + * slots were proved, or re-proved by tp_class_refresh), 0 = take the slow path, -1 = error. The + * fast case is one type compare and one tag compare. */ +static inline int tp_class_current(PyObject *obj, PyObject *cls_obj, tp_class_rt *rt, + const char *const *fields, Py_ssize_t n, PyObject *const *descrs) +{ + PyTypeObject *cls = (PyTypeObject *)cls_obj; + if (cls == NULL || (obj != NULL && !Py_IS_TYPE(obj, cls))) { + return 0; + } + if (cls->tp_version_tag == rt->tag) { /* rt->tag != 0 while cls != NULL */ + return 1; + } + return tp_class_refresh(cls, rt, fields, n, descrs); +} + +/* LOAD_GLOBAL of a class name, as an identity test: 1 = module_dict[name] is cls, 0 = it is not (or + * the name is not a module global: the slow path then asks tp_global), -1 = error. With a dict + * watcher on module_dict (`watched`), a 1 is remembered in rt->gepoch until tp_globals_epoch moves, + * so the steady state is one compare; without one (no watcher id was free) every call looks up. */ +static inline int tp_class_global_ok(PyObject *module_dict, PyObject *name, PyObject *cls, + tp_class_rt *rt, int watched) +{ + PyObject *v; + if (cls == NULL) { + return 0; + } + if (watched && rt->gepoch == tp_globals_epoch) { + return 1; + } + v = PyDict_GetItemWithError(module_dict, name); /* borrowed */ + if (v == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + if (v != cls) { + return 0; + } + if (watched) { + rt->gepoch = tp_globals_epoch; + } + return 1; +} + +/* The body of a module's PyDict_WatchCallback: move tp_globals_epoch when `key` is one of the + * NULL-terminated class `names` (an exact str compared by content), may equal one (any other key + * type, a str subclass), or is NULL (cleared / cloned / deallocated). Never raises; returns 0. */ +static inline int tp_globals_watch_event(PyDict_WatchEvent event, PyObject *key, + const char *const *names) +{ + Py_ssize_t i; + (void)event; + if (key != NULL && PyUnicode_CheckExact(key)) { +#if PY_VERSION_HEX >= 0x030D0000 + for (i = 0; names[i] != NULL; i++) { + if (PyUnicode_EqualToUTF8(key, names[i])) { + tp_globals_epoch++; + return 0; + } + } + return 0; +#else + (void)i; + (void)names; +#endif + } + tp_globals_epoch++; + return 0; +} + +/* Watch module_dict with `cb` (a function calling tp_globals_watch_event). *id1 = watcher id + 1, or + * 0 when no dict-watcher id is free (8 per interpreter, 2 reserved by CPython, shared with every + * other extension) or PyDict_Watch failed: the caller then takes the conservative path (a dict + * lookup per New). Never leaves an error set. */ +static inline void tp_globals_watch(PyObject *module_dict, PyDict_WatchCallback cb, int *id1) +{ + int id; + *id1 = 0; + id = PyDict_AddWatcher(cb); + if (id < 0) { + PyErr_Clear(); + return; + } + if (PyDict_Watch(id, module_dict) < 0) { + PyErr_Clear(); + if (PyDict_ClearWatcher(id) < 0) { + PyErr_Clear(); + } + return; + } + *id1 = id + 1; +} + +/* Undo tp_globals_watch (module clear / free): unwatch module_dict (if still there) and release the + * watcher id. Idempotent; keeps any exception that was already set. */ +static inline void tp_globals_unwatch(PyObject *module_dict, int *id1) +{ + PyObject *exc; + if (*id1 <= 0) { + return; + } + exc = PyErr_GetRaisedException(); + if (module_dict != NULL && PyDict_Unwatch(*id1 - 1, module_dict) < 0) { + PyErr_Clear(); + } + if (PyDict_ClearWatcher(*id1 - 1) < 0) { + PyErr_Clear(); + } + *id1 = 0; + PyErr_SetRaisedException(exc); +} + +/* obj.field of a proved-exact instance: a new reference, or CPython's AttributeError for an unset + * slot ("'C' object has no attribute 'f'"). */ +static inline PyObject *tp_field_get(PyObject *obj, PyMemberDef *m) +{ + return PyMember_GetOne((const char *)obj, m); +} + +/* obj.field = v: the slot takes its own reference to v and releases the old one. 0 or -1. */ +static inline int tp_field_set(PyObject *obj, PyMemberDef *m, PyObject *v) +{ + return PyMember_SetOne((char *)obj, m, v); +} + +/* The slow path of FieldSet: STORE_ATTR (PyObject_SetAttr). 0 or -1. */ +static inline int tp_setattr(PyObject *obj, PyObject *name, PyObject *v) +{ + return PyObject_SetAttr(obj, name, v); +} + +/* The observable result of a trivial __init__: allocate with cls's tp_alloc and set slots[i] to + * values[i] (new references; the caller keeps its own). NULL with the error set on failure; the + * half-built object is released (its unset slots are NULL, which dealloc handles). */ +static inline PyObject *tp_new_fixed(PyTypeObject *cls, PyMemberDef *const *slots, + PyObject *const *values, Py_ssize_t n) +{ + Py_ssize_t i; + PyObject *obj = cls->tp_alloc(cls, 0); + + if (obj == NULL) { + return NULL; + } + for (i = 0; i < n; i++) { + if (PyMember_SetOne((char *)obj, slots[i], values[i]) < 0) { + Py_DECREF(obj); + return NULL; + } + } + return obj; +} + +/* type(obj) is cls */ +static inline int tp_is_exact(PyObject *obj, PyTypeObject *cls) +{ + return Py_IS_TYPE(obj, cls); +} + /* ------------------------------------------------------------------------------------------ * Call depth (issue #57): compiled frames have no Python frame, so a recursion made of them would * neither raise RecursionError nor stop before the C stack ends. tp_enter_call / tp_leave_call diff --git a/python-multiplatform-ksp/src/main/python/typedpython/verify.py b/python-multiplatform-ksp/src/main/python/typedpython/verify.py index e9b164799..2257cd3a2 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/verify.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/verify.py @@ -42,6 +42,52 @@ on a node `proven` does not apply to, or it is not a bool) verify/redo `Call.redo` where it is not legal (see below) verify/entry-global-open a function with entry_globals is not closed + verify/class a class named by FieldGet/FieldSet/New/IsExact/CheckExact/Param.cls is not + a (valid) ClassDecl of the module, a field is not one of its fields, a + New names a class without trivial_init or has the wrong arity, or a + ClassDecl itself is malformed (bad name/fields, duplicates) + verify/field-unproved a FieldGet/FieldSet whose object is not proved to be exactly the class + +Fixed-layout classes (SPEC N-11). The verifier never trusts the front end for them either: + + * Every class named anywhere must be a valid `ClassDecl` of `Module.classes`; every field named + must be one of that class's `fields`; `New` needs `trivial_init` and exactly len(fields) OBJ + arguments; `Param.cls` only on an OBJ parameter (`optional` only together with `cls`). + * Exact-type proof for FieldGet/FieldSet (`verify/field-unproved` otherwise). The object must be + one of exactly these forms, for the same class C as the access: + (a) a `Local` that is a parameter with cls=C and optional=False, until the local is assigned; + (b) a `CheckExact(x, C)` node used directly as the object; + (c) a `New(C, ...)` node used directly as the object; + (d) a `Local` proved exact-C by control flow, with no assignment of it in between: inside the + `then` of `If(IsExact(Local, C), ...)`, or its `orelse` when the condition is false + (`Not(IsExact)`), and inside the right operand of an `And` whose left operand is + such a test (an `Or`'s right operand: when the left is false); or + `Is(Local x, Const(None, OBJ), negate=True)` (true, or `negate=False` false) when x is + an `optional` parameter of class C that the function never assigns anywhere; or + after `if : return/break/continue` (the facts of the branches that fall + through are intersected); or a local assigned from (b), (c) or another proved local + (`x = New(C, ..)`, `x = CheckExact(y, C)`, `x = y`) from there until it is assigned again; + (e) a `Local` that is not a parameter, entry global or loop var, and whose every Assign in the + function is a `New(C, ...)` of that same class: whatever is read from it (definite + assignment is proved separately) is a C created by New. Flow-insensitive, so one other kind + of assignment anywhere in the function ends the form. + Facts are killed by any assignment of the local (and, in a loop, by an assignment anywhere in + the loop body for the whole loop, the loop var included). Nothing else is accepted: not + isinstance, not a fact about another local, not a fact through a call, GetAttr or FieldGet. + * A `Call` of a compiled callee whose parameter has cls=C (the callee's entry guard) with an + argument not proved exact-C by the forms above can fail that guard in the middle of the caller: + it counts toward `may_deopt` (so it is legal only in a pure function) unless it is a legal redo. + * Purity/deopt: CheckExact can deopt (like Unbox to i64): only in a pure function, and it counts + toward `may_deopt` even when redundant. FieldSet is an effect (and makes the function not + *closed*: replacing a slot may drop the last reference to the old value, whose `__del__` is user + code). FieldGet, Is, IsExact, New are not effects. + * Typing: Is/IsExact give BOOL over OBJ operands; FieldGet/New/CheckExact give OBJ; `Const(None, + OBJ)` is the only OBJ constant (anything else `verify/type`) and counts as a reference creator + (a new reference to the None singleton) wherever an OBJ may flow: an operand of Is, a returned + value, an argument. + * Ownership: New, FieldGet and CheckExact are reference creators. CheckExact is read as the IR says, + "`obj` itself ... after checking": the result is a new reference to the operand (the back end + increfs it); the operand's own reference is untouched. Proven i64 operations. The verifier never trusts `BinOp.proven` / `UnaryOp.proven`: for every I64 expression it computes an interval [lo, hi] (exact Python ints) and accepts `proven=True` only on an @@ -130,6 +176,7 @@ Local, MathCall, MathFunc, Module, NewArray, ObjToFloat, Or, Param, Return, StoreIndex, ToFloat, Truth, Tuple, Type, Unbox, UnaryOp, UnaryOpKind, While, ) +from .ir import CheckExact, ClassDecl, FieldGet, FieldSet, Is, IsExact, New DA = "verify/definite-assignment" MISSING_RETURN = "verify/missing-return" @@ -149,6 +196,8 @@ PROVEN = "verify/proven-overflow" REDO = "verify/redo" ENTRY_OPEN = "verify/entry-global-open" +CLASS = "verify/class" +FIELD = "verify/field-unproved" @dataclass(frozen=True) @@ -163,13 +212,15 @@ class Diagnostic: # the one the back end implements). _EXPRS = frozenset({Const, Local, BinOp, UnaryOp, Compare, And, Or, ToFloat, Box, Unbox, MathCall, Call, Global, GetAttr, CallObject, Truth, CompareObj, ObjToFloat, Len, Index, - NewArray, CopyArray, Tuple}) -_STMTS = frozenset({Assign, StoreIndex, ExprStmt, If, While, ForRange, Return, Break, Continue}) + NewArray, CopyArray, Tuple, Is, IsExact, CheckExact, FieldGet, New}) +_STMTS = frozenset({Assign, StoreIndex, ExprStmt, If, While, ForRange, Return, Break, Continue, + FieldSet}) # Nodes whose OBJ result is a new reference (ir.py "ownership": Box, Global, GetAttr, CallObject, # OBJ BinOp, Tuple), plus reads of an OBJ local (the local owns its reference; the back end increfs or # borrows under that rule) and Calls of a compiled function returning OBJ (which returns a new # reference by the same rule). -_OBJ_PRODUCERS = frozenset({Box, Global, GetAttr, CallObject, BinOp, Local, Call, Tuple}) +_OBJ_PRODUCERS = frozenset({Box, Global, GetAttr, CallObject, BinOp, Local, Call, Tuple, New, + FieldGet, CheckExact, Const}) # an OBJ Const is None (e_Const) _VALUE_TYPES = (Type.I64, Type.F64, Type.BOOL, Type.OBJ) # what a local may hold _RETURN_TYPES = _VALUE_TYPES + (Type.NONE,) _ELEMENT = {Type.F64_ARRAY: Type.F64, Type.I64_ARRAY: Type.I64} @@ -205,9 +256,15 @@ class _Checker: """Checks one function against the module's signatures. Collects diagnostics; rebuilds the body with proven bounds.""" - def __init__(self, function: Function, signatures: dict[str, Function]): + def __init__(self, function: Function, signatures: dict[str, Function], + classes: dict[str, ClassDecl] | None = None): self.fn = function self.sigs = signatures + self.classes = classes or {} + self.facts: dict[str, str] = {} # local -> class it is proved exactly (see docstring) + self.optional: dict[str, str] = {} # optional-class parameter -> class + self.reassigned: frozenset[str] = frozenset() + self.new_only: dict[str, str] = {} # local whose every assignment is New(C, ..) -> C self.diags: list[Diagnostic] = [] self.can_deopt = False self.effects: list[str] = [] # reasons the body is impure @@ -246,6 +303,7 @@ def declarations(self) -> frozenset[str]: f"ArrayParam") else: self.scalars[p.name] = p.type + self.param_class(p) else: self.error(UNKNOWN, f"parameter {p!r} is not an ir Param/ArrayParam") continue @@ -267,6 +325,9 @@ def declarations(self) -> frozenset[str]: self.error(TYPE, f"entry global '{p.name}' has type {p.type}; only i64, f64 and " f"bool globals can be read at entry") continue + if p.cls is not None or p.optional is not False: + self.error(TYPE, f"entry global '{p.name}' carries a class annotation") + continue if p.name in params or p.name in self.entry: self.error(STRUCTURE, f"entry global '{p.name}' collides with a parameter or " f"another entry global") @@ -301,6 +362,102 @@ def declarations(self) -> frozenset[str]: self.error(TYPE, f"`{flag}` is not a bool") return frozenset(params) + def param_class(self, p: Param) -> None: + """`Param.cls`/`optional`: only on an OBJ parameter, naming a ClassDecl of the module. A + non-optional one is an exact-C fact at entry (the entry guard); an optional one is C or None.""" + if type(p.optional) is not bool: + self.error(TYPE, f"parameter '{p.name}' has a non-bool `optional`") + return + if p.cls is None: + if p.optional: + self.error(TYPE, f"parameter '{p.name}' is optional but names no class") + return + if p.type is not Type.OBJ: + self.error(TYPE, f"parameter '{p.name}' names class {p.cls!r} but is " + f"{p.type.value}, not obj") + elif not self.known_class(p.cls, f"parameter '{p.name}'"): + return + elif p.optional: + self.optional[p.name] = p.cls + else: + self.facts[p.name] = p.cls + + def known_class(self, cls: object, what: str) -> bool: + if not isinstance(cls, str) or cls not in self.classes: + self.error(CLASS, f"{what} names class {cls!r}, which is not a compiled class of this " + f"module") + return False + return True + + def known_field(self, cls: str, field: object, what: str) -> bool: + if not self.known_class(cls, what): + return False + if not isinstance(field, str) or field not in self.classes[cls].fields: + self.error(CLASS, f"{what}: '{field}' is not a field of class '{cls}' " + f"(fields: {', '.join(self.classes[cls].fields)})") + return False + return True + + # --- exact-type facts --------------------------------------------------------------------- + + def new_only_locals(self, params: frozenset[str]) -> dict[str, str]: + """Form (e): locals (not parameters, entry globals, loop vars or arrays) every Assign of + which is a `New` of one and the same declared class.""" + values: dict[str, list[object]] = {} + _assigned_values(self.fn.body, values) + loop_vars = _loop_vars(self.fn.body) + out: dict[str, str] = {} + for name, vs in values.items(): + if name in params or name in loop_vars or name in self.larrays: + continue + classes = {v.cls for v in vs if type(v) is New and isinstance(v.cls, str)} + if (len(classes) == 1 and all(type(v) is New for v in vs) + and next(iter(classes)) in self.classes): + out[name] = next(iter(classes)) + return out + + def exact_of(self, e: object) -> str | None: + """The class `e` is proved to be exactly (forms (a)-(d) of the module docstring).""" + cls = type(e) + if cls is Local: + return self.facts.get(e.name) or self.new_only.get(e.name) + if cls is CheckExact or cls is New: + return e.cls if isinstance(e.cls, str) and e.cls in self.classes else None + return None + + def _none_test(self, c: Is) -> str | None: + """`x is None` / `x is not None` on an optional-C parameter never assigned: the name.""" + for a, b in ((c.left, c.right), (c.right, c.left)): + if (type(a) is Local and type(b) is Const and b.type is Type.OBJ and b.value is None + and a.name in self.optional and a.name not in self.reassigned): + return a.name + return None + + def facts_if(self, c: object, truth: bool) -> dict[str, str]: + """The exact-type facts that hold when condition `c` evaluated to `truth`.""" + cls = type(c) + if cls is IsExact: + if truth and type(c.obj) is Local and c.cls in self.classes: + return {c.obj.name: c.cls} + return {} + if cls is Is and type(c.negate) is bool: + name = self._none_test(c) + if name is not None and truth is c.negate: # `is not None` true, or `is None` false + return {name: self.optional[name]} + return {} + if cls is UnaryOp and c.op is UnaryOpKind.NOT: + return self.facts_if(c.operand, not truth) + if (cls is And and truth) or (cls is Or and not truth): + return {**self.facts_if(c.left, truth), **self.facts_if(c.right, truth)} + return {} + + def proved(self, obj: ir.Expr, cls: str, what: str) -> None: + if self.exact_of(obj) != cls: + self.error(FIELD, f"{what}: the object is not proved to be exactly '{cls}' (needs an " + f"exact-{cls} parameter, a CheckExact, a New, or a dominating " + f"IsExact / `is not None` of an optional parameter on the same local, " + f"with no assignment in between)") + # --- expressions -------------------------------------------------------------------------- def slot(self, e: ir.Expr, want: Type, what: str) -> None: @@ -348,7 +505,9 @@ def e_Const(self, e: Const, state: _State) -> ir.Expr: or (t is Type.F64 and type(v) is float) or (t is Type.BOOL and type(v) is bool) or (t is Type.NONE and v is None)) - if not ok and t is not Type.OBJ: + if t is Type.OBJ: + ok = v is None # the only OBJ constant: a new reference to the None singleton + if not ok: self.error(TYPE, f"constant {v!r} does not match type {t.value}") return e @@ -488,7 +647,12 @@ def e_Compare(self, e: Compare, state: _State) -> ir.Expr: def _bool_pair(self, e: And | Or, state: _State, cond: bool = False) -> ir.Expr: name = type(e).__name__ - left, right = self.expr(e.left, state, cond), self.expr(e.right, state, cond) + left = self.expr(e.left, state, cond) + saved = self.facts + # the right operand runs only when the left was true (And) / false (Or) + self.facts = {**saved, **self.facts_if(left, type(e) is And)} + right = self.expr(e.right, state, cond) + self.facts = saved self.slot(left, Type.BOOL, f"left operand of {name}") self.slot(right, Type.BOOL, f"right operand of {name}") if e.type is not Type.BOOL: @@ -586,6 +750,10 @@ def e_Call(self, e: Call, state: _State) -> ir.Expr: f"bool and none results are exactly reproducible") else: redo = True + if not redo: + for k, (a, p) in enumerate(zip(args, callee.params)): + if type(p) is Param and p.cls is not None and self.exact_of(a) != p.cls: + self.can_deopt = True # the callee's entry guard `type(x) is C` may fail if callee.may_deopt is True and not redo: self.can_deopt = True if not redo and not self.covers(callee): @@ -723,6 +891,51 @@ def in_bounds(self, array: str, idx: ir.Expr) -> bool: return loop.bounds_array == array return False + # --- fixed-layout classes ----------------------------------------------------------------- + + def e_Is(self, e: Is, state: _State) -> ir.Expr: + left = self.obj_operand(e.left, state, "left operand of is") + right = self.obj_operand(e.right, state, "right operand of is") + if type(e.negate) is not bool: + self.error(TYPE, f"Is `negate` {e.negate!r} is not a bool") + self.result(e, Type.BOOL) + return dataclasses.replace(e, left=left, right=right) + + def e_IsExact(self, e: IsExact, state: _State) -> ir.Expr: + obj = self.obj_operand(e.obj, state, "operand of IsExact") + self.known_class(e.cls, "IsExact") + self.result(e, Type.BOOL) + return dataclasses.replace(e, obj=obj) + + def e_CheckExact(self, e: CheckExact, state: _State) -> ir.Expr: + obj = self.obj_operand(e.obj, state, "operand of CheckExact") + self.known_class(e.cls, "CheckExact") + self.result(e, Type.OBJ) + self.can_deopt = True # a wrong type deopts, even if the check is redundant + return dataclasses.replace(e, obj=obj) + + def e_FieldGet(self, e: FieldGet, state: _State) -> ir.Expr: + obj = self.obj_operand(e.obj, state, f"object of .{e.field}") + self.result(e, Type.OBJ) + if self.known_field(e.cls, e.field, f"FieldGet .{e.field}"): + self.proved(obj, e.cls, f"FieldGet {e.cls}.{e.field}") + return dataclasses.replace(e, obj=obj) # a slot read runs no user code: not an effect + + def e_New(self, e: New, state: _State) -> ir.Expr: + args = self.args(e.args, state, f"New {e.cls}") + for k, a in enumerate(args): + self.slot(a, Type.OBJ, f"argument {k + 1} of New {e.cls}") + self.result(e, Type.OBJ) + if self.known_class(e.cls, "New"): + decl = self.classes[e.cls] + if decl.trivial_init is not True: + self.error(CLASS, f"New of '{e.cls}', whose __init__ is not trivial (it must be " + f"called as an object)") + if len(args) != len(decl.fields): + self.error(CLASS, f"New of '{e.cls}' has {len(args)} argument(s) for " + f"{len(decl.fields)} field(s)") + return dataclasses.replace(e, args=args) + # --- tuples and local arrays -------------------------------------------------------------- def e_Tuple(self, e: Tuple, state: _State) -> ir.Expr: @@ -841,6 +1054,11 @@ def s_Assign(self, s: Assign, state: _State): self.error(TYPE, f"None assigned to '{target}'") if target in self.entry: self.error(STRUCTURE, f"entry global '{target}' is assigned in the body") + proved = None if alloc else self.exact_of(value) + if proved is not None: + self.facts[target] = proved + else: + self.facts.pop(target, None) for loop in self.loops: if loop.var == target: self.error(STRUCTURE, f"loop variable '{target}' is assigned in its loop body") @@ -864,20 +1082,53 @@ def cond(self, c: object, state: _State, what: str) -> ir.Expr: self.slot(c2, Type.BOOL, f"condition of {what}") return c2 + def s_FieldSet(self, s: FieldSet, state: _State): + obj = self.obj_operand(s.obj, state, f"object of .{s.field} =") + value = self.expr(s.value, state) + self.slot(value, Type.OBJ, f"value stored into .{s.field}") + if self.known_field(s.cls, s.field, f"FieldSet .{s.field}"): + self.proved(obj, s.cls, f"FieldSet {s.cls}.{s.field}") + self.effects.append(f"FieldSet .{s.field} (a store)") + self.opens.append(f"FieldSet .{s.field}") # dropping the old value may run its __del__ + return dataclasses.replace(s, obj=obj, value=value), state + def s_If(self, s: If, state: _State): cond = self.cond(s.cond, state, "if") + base = self.facts + self.facts = {**base, **self.facts_if(cond, True)} then, st = self.block(s.then, state) + then_facts = self.facts + self.facts = {**base, **self.facts_if(cond, False)} orelse, se = self.block(s.orelse, state) + else_facts = self.facts + if st is None and se is None: + self.facts = base + elif st is None: + self.facts = else_facts + elif se is None: + self.facts = then_facts + else: + self.facts = {k: v for k, v in then_facts.items() if else_facts.get(k) == v} return dataclasses.replace(s, cond=cond, then=then, orelse=orelse), _meet(st, se) + def loop_facts(self, body: object, extra: tuple[str, ...] = ()) -> dict[str, str]: + """The facts that hold at the head of every iteration: those of the entry that nothing in + the loop body (or its loop var) assigns.""" + killed = _assigned_names(body) | set(extra) + return {k: v for k, v in self.facts.items() if k not in killed} + def s_While(self, s: While, state: _State): + head = self.loop_facts(s.body) + self.facts = head cond = self.cond(s.cond, state, "while") loop = _Loop(None, None) + self.facts = {**head, **self.facts_if(cond, True)} self.loops.append(loop) # Assignments only add names, so the loop-head state is the entry state (the meet of the # entry, end-of-body and continue states equals the entry state). body, _ = self.block(s.body, state) self.loops.pop() + self.facts = dict(head) # leaving by the condition or a break: no more than the head's facts forever = type(s.cond) is Const and s.cond.type is Type.BOOL and s.cond.value is True after: _State = None if forever else state for br in loop.breaks: @@ -924,8 +1175,11 @@ def s_ForRange(self, s: ForRange, state: _State): lo = min(hi, elo + 1) loop.interval = (max(lo, _I64_MIN), min(hi, _I64_MAX)) self.loops.append(loop) + head = self.loop_facts(s.body, (var,)) + self.facts = dict(head) body, _ = self.block(s.body, None if state is None else state | {var}) self.loops.pop() + self.facts = dict(head) # Zero iterations leave `var` (and everything the body assigns) as it was: the state after # the loop is the entry state, met with every break state (each a superset of it). after: _State = state @@ -960,6 +1214,8 @@ def _jump(self, s: Break | Continue, state: _State): def run(self) -> Function: fn = self.fn params = self.declarations() + self.reassigned = frozenset(_assigned_names(fn.body)) + self.new_only = self.new_only_locals(params) if self.larrays: counts: dict[str, int] = {} _assigned(fn.body, counts) @@ -1005,6 +1261,53 @@ def _assigned(body: object, counts: dict[str, int] | None = None) -> dict[str, i return counts +def _assigned_values(body: object, out: dict[str, list[object]]) -> None: + """Every Assign value in `body`, by target (tolerates a malformed body).""" + if isinstance(body, tuple): + for st in body: + cls = type(st) + if cls is Assign: + out.setdefault(st.target, []).append(st.value) + elif cls is If: + _assigned_values(st.then, out) + _assigned_values(st.orelse, out) + elif cls in (While, ForRange): + _assigned_values(st.body, out) + + +def _loop_vars(body: object) -> set[str]: + out: set[str] = set() + if isinstance(body, tuple): + for st in body: + cls = type(st) + if cls is If: + out |= _loop_vars(st.then) | _loop_vars(st.orelse) + elif cls is While: + out |= _loop_vars(st.body) + elif cls is ForRange: + out.add(st.var) + out |= _loop_vars(st.body) + return out + + +def _assigned_names(body: object) -> set[str]: + """Every local assigned anywhere in `body` by an Assign or as a ForRange var.""" + out: set[str] = set() + if isinstance(body, tuple): + for st in body: + cls = type(st) + if cls is Assign: + out.add(st.target) + elif cls is If: + out |= _assigned_names(st.then) | _assigned_names(st.orelse) + elif cls is While: + out |= _assigned_names(st.body) + elif cls is ForRange: + out.add(st.var) + out |= _assigned_names(st.body) + return out + + def _exact(op: BinOpKind, a: tuple[int, int], b: tuple[int, int]) -> tuple[int, int]: """The exact (unbounded) result interval of ADD/SUB/MUL on operand intervals.""" if op is BinOpKind.ADD: @@ -1041,12 +1344,13 @@ def reject(fn_name: str, line: int, rule: str, message: str) -> None: reject(name, g.source_line, STRUCTURE, f"function '{name}' is defined " f"{len(gs)} times") sigs = {name: gs[0] for name, gs in named.items() if len(gs) == 1} + classes, skipped_classes = _check_classes(module, set(named), diags) checked: dict[str, Function] = {} callees: dict[str, set[str]] = {} open_why: dict[str, str] = {} # a function that runs user code -> why for name, g in sigs.items(): - c = _Checker(g, sigs) + c = _Checker(g, sigs, classes) rebuilt = c.run() for d in c.diags: reject(d.function, d.source_line, d.rule, d.message) @@ -1092,6 +1396,44 @@ def reject(fn_name: str, line: int, rule: str, message: str) -> None: kept = tuple(checked[g.name] for g in functions if type(g) is Function and g.name in checked and g.name not in reasons) skipped = dict(module.skipped) - for name, reason in reasons.items(): + for name, reason in {**skipped_classes, **reasons}.items(): skipped.setdefault(name, reason) - return Module(module.name, kept, skipped), diags + return Module(module.name, kept, skipped, tuple(classes.values())), diags + + +def _check_classes(module: Module, function_names: set[str], diags: list[Diagnostic] + ) -> tuple[dict[str, ClassDecl], dict[str, str]]: + """The valid ClassDecls of the module by name, and the reason for each invalid one.""" + declared = module.classes if isinstance(module.classes, tuple) else () + names: dict[str, int] = {} + for c in declared: + if type(c) is ClassDecl and isinstance(c.name, str): + names[c.name] = names.get(c.name, 0) + 1 + valid: dict[str, ClassDecl] = {} + bad: dict[str, str] = {} + for c in declared: + problem = None + if type(c) is not ClassDecl: + problem = f"{type(c).__name__} is not an ir ClassDecl" + elif not (isinstance(c.name, str) and c.name.isidentifier()): + problem = f"class name {c.name!r} is not an identifier" + elif names[c.name] > 1: + problem = f"class '{c.name}' is declared {names[c.name]} times" + elif c.name in function_names: + problem = f"class '{c.name}' has the name of a function of the module" + elif not (isinstance(c.fields, tuple) and all(isinstance(f, str) and f.isidentifier() + for f in c.fields)): + problem = f"fields of class '{c.name}' are not a tuple of identifiers" + elif len(set(c.fields)) != len(c.fields): + problem = f"class '{c.name}' lists a field twice" + elif type(c.trivial_init) is not bool: + problem = f"`trivial_init` of class '{c.name}' is not a bool" + name = getattr(c, "name", repr(c)) + if problem is None: + valid[c.name] = c + else: + d = Diagnostic(str(name), CLASS, problem, 0) + if d not in diags: + diags.append(d) + bad.setdefault(str(name), f"{CLASS}: {problem}") + return valid, bad diff --git a/python-multiplatform-ksp/src/test/python/test_cgen_class_changes.py b/python-multiplatform-ksp/src/test/python/test_cgen_class_changes.py new file mode 100644 index 000000000..23063d209 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_cgen_class_changes.py @@ -0,0 +1,911 @@ +"""TypedPython C back end: a compiled fixed-layout class (SPEC N-11) changed AFTER module init. + +After init, compiled code must still compute what the interpreter computes when user code + * replaces a field's class attribute (`Node.left = property(...)`), with or without a setter; + * rebinds the module global (`Node = Twin`, `Node = Wide`, `del Node`); + * assigns `obj.__class__` to a layout-compatible class; + * changes `Node.__init__`, or adds an unrelated class attribute (`Node.counter = 1`); + * exhausts the class's version tags (tp_version_tag stays 0); +and it must do so mid-body too, when an impure compiled function runs a Python callback +(CallObject) between entry and the field access / New — without a deopt after that effect. + +Every scenario runs in a CHILD process (a crash fails one test, not the run) on a fresh compiled +module instance, and the same scenario runs on the plain interpreted source (loaded as an ordinary +module from the same file): the two normalised outcome lists must be equal. Probes read +`__typedpython_deopts__` and `__typedpython_class_info__()` of the compiled instance only. + +Builds go to `.tmp/cgen-tests/` of the worktree (test_cgen.OUT_ROOT). +""" +from __future__ import annotations + +import gc +import importlib.util +import json +import os +import subprocess +import sys +import sysconfig +import textwrap +from pathlib import Path + +import pytest + +from test_cgen import (PACKAGE, OUT_ROOT, I64, BOOL, OBJ, NONE, F64, c, L, fn, runtime_dir, + need_runtime) +from typedpython import ir +from typedpython.ir import ( + ArrayParam, Assign, BinOp, BinOpKind, Call, CallObject, ClassDecl, Compare, CompareKind, Const, + ExprStmt, FieldGet, FieldSet, If, Is, IsExact, Module, New, Param, Return, StoreIndex, Type, +) + +TEST_DIR = Path(__file__).resolve().parent + +SOURCE = textwrap.dedent('''\ + """typedpython class-change test module — the interpreted reference.""" + + class Node: + __slots__ = ("left", "right") + + def __init__(self, left, right): + self.left = left + self.right = right + + class Twin: # the same layout as Node (so __class__ can be assigned) + __slots__ = ("left", "right") + + def __init__(self, left, right): + self.left = left + self.right = right + + class Shadow: # the same layout; `left` is a property + __slots__ = ("left", "right") + + Shadow.left = property(lambda self: "shadow") + + class Wide: # a different layout (an instance __dict__) + def __init__(self, left, right): + self.left = left + self.right = right + self.extra = "wide" + + SCALE = 2.0 + + def make(depth): + if depth == 0: + return None + return Node(make(depth - 1), make(depth - 1)) + + def check(node): + if node is None: + return 0 + return 1 + check(node.left) + check(node.right) + + def pair(a, b): + return Node(a, b) + + def get_left(n): + return n.left + + def setl(n, v): + n.left = v + + def mid_get(cb, n): + cb() + return n.left + + def mid_set(cb, n, v): + cb() + n.left = v + + def mid_new(cb, a, b): + cb() + return Node(a, b) + + def arg_new(cb, b): + return Node(cb(), b) + + def via_pair(cb, a, b): + cb() + return pair(a, b) + + def is_node(cb, x): + cb() + return type(x) is Node + + def fill(a, n): + x = n.left + a[0] = SCALE + ''') + +NODE = ClassDecl("Node", ("left", "right"), True) +CLASSES = (NODE,) + + +def _o(name): + return L(name, OBJ) + + +def f_make(): + depth = L("depth", I64) + rec = lambda: Call(OBJ, "make", (BinOp(I64, BinOpKind.SUB, depth, c(1)),)) + return fn("make", [Param("depth", I64)], OBJ, {"a": OBJ, "b": OBJ}, + [If(Compare(BOOL, CompareKind.EQ, depth, c(0)), (Return(Const(OBJ, None)),)), + Assign("a", rec()), Assign("b", rec()), + Return(New(OBJ, "Node", (_o("a"), _o("b"))))], + pure=True, may_deopt=True) + + +def f_check(): + node = _o("node") + return fn("check", [Param("node", OBJ, cls="Node", optional=True)], I64, {"l": OBJ, "r": OBJ}, + [If(Is(BOOL, node, Const(OBJ, None)), (Return(c(0)),)), + Assign("l", FieldGet(OBJ, node, "Node", "left")), + Assign("r", FieldGet(OBJ, node, "Node", "right")), + Return(BinOp(I64, BinOpKind.ADD, + BinOp(I64, BinOpKind.ADD, c(1), Call(I64, "check", (_o("l"),))), + Call(I64, "check", (_o("r"),))))], + pure=True, may_deopt=True) + + +def f_pair(): + return fn("pair", [Param("a", OBJ), Param("b", OBJ)], OBJ, {}, + [Return(New(OBJ, "Node", (_o("a"), _o("b"))))], pure=True, may_deopt=False) + + +def f_get_left(): + return fn("get_left", [Param("n", OBJ, cls="Node")], OBJ, {}, + [Return(FieldGet(OBJ, _o("n"), "Node", "left"))], pure=True, may_deopt=False) + + +def f_setl(): + return fn("setl", [Param("n", OBJ, cls="Node"), Param("v", OBJ)], NONE, {}, + [FieldSet(_o("n"), "Node", "left", _o("v")), Return()], pure=False, may_deopt=False) + + +def _cb(): + return ExprStmt(CallObject(OBJ, _o("cb"), ())) + + +def f_mid_get(): + return fn("mid_get", [Param("cb", OBJ), Param("n", OBJ, cls="Node")], OBJ, {}, + [_cb(), Return(FieldGet(OBJ, _o("n"), "Node", "left"))], pure=False, may_deopt=False) + + +def f_mid_set(): + return fn("mid_set", [Param("cb", OBJ), Param("n", OBJ, cls="Node"), Param("v", OBJ)], NONE, {}, + [_cb(), FieldSet(_o("n"), "Node", "left", _o("v")), Return()], + pure=False, may_deopt=False) + + +def f_mid_new(): + return fn("mid_new", [Param("cb", OBJ), Param("a", OBJ), Param("b", OBJ)], OBJ, {}, + [_cb(), Return(New(OBJ, "Node", (_o("a"), _o("b"))))], pure=False, may_deopt=False) + + +def f_arg_new(): + return fn("arg_new", [Param("cb", OBJ), Param("b", OBJ)], OBJ, {}, + [Return(New(OBJ, "Node", (CallObject(OBJ, _o("cb"), ()), _o("b"))))], + pure=False, may_deopt=False) + + +def f_via_pair(): + return fn("via_pair", [Param("cb", OBJ), Param("a", OBJ), Param("b", OBJ)], OBJ, {}, + [_cb(), Return(Call(OBJ, "pair", (_o("a"), _o("b"))))], pure=False, may_deopt=False) + + +def f_is_node(): + return fn("is_node", [Param("cb", OBJ), Param("x", OBJ)], BOOL, {}, + [_cb(), Return(IsExact(BOOL, _o("x"), "Node"))], pure=False, may_deopt=False) + + +def f_fill(): + # closed (no user code), impure (an array store), with an entry global + return fn("fill", [ArrayParam("a", Type.F64_ARRAY, True), Param("n", OBJ, cls="Node")], NONE, + {"x": OBJ}, + [Assign("x", FieldGet(OBJ, _o("n"), "Node", "left")), + StoreIndex("a", c(0), L("SCALE", F64)), Return()], + pure=False, may_deopt=False, entry_globals=(Param("SCALE", F64),)) + + +def functions(): + return (f_make(), f_check(), f_pair(), f_get_left(), f_setl(), f_mid_get(), f_mid_set(), + f_mid_new(), f_arg_new(), f_via_pair(), f_is_node(), f_fill()) + + +def source_file(name: str) -> Path: + import shutil + d = OUT_ROOT / name + if d.exists(): + shutil.rmtree(d) + d.mkdir(parents=True) + p = d / f"{name}.py" + p.write_text(SOURCE) + return p + + +def generate(name, funcs=None): + from typedpython import cgen + funcs = functions() if funcs is None else funcs + return cgen.generate(Module(name=name, functions=tuple(funcs), classes=CLASSES), + source_file(name)) + + +def build(name, flags=None): + from typedpython import cbuild + need_runtime() + src = generate(name) + kwargs = {} if flags is None else {"flags": flags} + return cbuild.build(src, name, OUT_ROOT / name, runtime_dir=runtime_dir(), **kwargs) + + +# --- the child process ------------------------------------------------------------------------------- +# Everything below `child_main` runs in a separate interpreter (subprocess) so a crash or a corrupted +# class cannot leak into other tests. + +def NOOP(): + return None + + +def norm(x): + if x is None or isinstance(x, (bool, int, float, str)): + return x + if isinstance(x, (list, tuple)): + return [norm(i) for i in x] + if isinstance(x, BaseException): + return ["exc", type(x).__name__, str(x)] + d = {"type": type(x).__name__} + for f in ("left", "right", "extra"): + try: + d[f] = norm(getattr(x, f)) + except Exception as e: # noqa: BLE001 — the outcome is the result + d[f] = ["exc", type(e).__name__, str(e)] + return d + + +def sc_unchanged(M, call, P): + P("before") + t = M.make(3) + n = M.Node(1, 2) + out = [norm(t), call("check", t), call("get_left", n), call("setl", n, "x"), norm(n), + call("pair", 1, 2), call("mid_get", NOOP, n), call("mid_set", NOOP, n, "y"), + call("mid_new", NOOP, 3, 4), call("arg_new", lambda: "a", 5), call("via_pair", NOOP, 6, 7), + call("is_node", NOOP, n), call("is_node", NOOP, M.Twin(1, 2))] + a = [0.0] + out += [call("fill", a, n), a] + P("after") + return out + + +def sc_prop_get(M, call, P): + n, n2 = M.Node(1, 2), M.Node(3, 4) + P("before") + M.Node.left = property(lambda s: 42) + out = [call("get_left", n), call("mid_get", NOOP, n2), call("check", n), call("make", 1), + call("pair", 1, 2), call("mid_new", NOOP, 1, 2), call("via_pair", NOOP, 1, 2), + call("setl", n, 3), call("arg_new", NOOP, 2), call("get_left", n2)] + P("after") + return out + + +def sc_prop_set(M, call, P): + log = [] + n = M.Node(1, 2) + P("before") + M.Node.left = property(lambda s: ("got", tuple(log)), lambda s, v: log.append(v)) + out = [call("setl", n, "x"), call("mid_set", NOOP, n, "y"), call("get_left", n), list(log), + call("pair", "p", "q"), list(log), call("mid_new", NOOP, "r", "s"), list(log)] + P("after") + return out + + +def _rebind(M, cls_name, call, P): + old = M.Node(1, 2) + warm = [call("make", 1), call("pair", 0, 0), call("mid_new", NOOP, 0, 0), + call("via_pair", NOOP, 0, 0), call("arg_new", lambda: 0, 0)] # each New has looked once + P("before") + M.Node = getattr(M, cls_name) + out = warm + [call("make", 2), call("pair", 1, 2), call("mid_new", NOOP, 1, 2), + call("via_pair", NOOP, 1, 2), call("arg_new", lambda: "a", 2), + call("is_node", NOOP, M.Twin(1, 2)), call("is_node", NOOP, old), + call("check", old), call("get_left", old)] + P("after") + return out + + +def sc_rebind_twin(M, call, P): + return _rebind(M, "Twin", call, P) + + +def sc_rebind_wide(M, call, P): + return _rebind(M, "Wide", call, P) + + +def sc_rebind_back(M, call, P): + orig = M.Node + out = [call("make", 1)] + M.Node = M.Twin + out += [call("make", 1), call("pair", 1, 2)] + M.Node = orig + P("restored") + out += [call("make", 3), call("pair", 1, 2), call("mid_new", NOOP, 1, 2)] + P("after") + return out + + +def sc_del_global(M, call, P): + seen = [] + out = [call("make", 1), call("pair", 0, 0), call("arg_new", lambda: 0, 0)] + P("before") + del M.Node + out += [call("make", 1), call("pair", 1, 2), call("mid_new", NOOP, 1, 2), + call("arg_new", lambda: seen.append("cb") or "a", 2), list(seen), + call("via_pair", NOOP, 1, 2), call("is_node", NOOP, 5)] + P("after") + return out + + +def sc_mid_prop(M, call, P): + n = M.Node(1, 2) + out = [call("mid_get", NOOP, n)] # warm: the fast path + P("before") + + def cb(): + M.Node.left = property(lambda s: 42) + out += [call("mid_get", cb, n)] + P("after") + return out + + +def sc_mid_prop_set(M, call, P): + log = [] + n = M.Node(1, 2) + out = [call("mid_set", NOOP, n, 0)] + P("before") + + def cb(): + M.Node.left = property(lambda s: "g", lambda s, v: log.append(v)) + out += [call("mid_set", cb, n, "v"), list(log)] + P("after") + return out + + +def sc_mid_rebind(M, call, P): + out = [call("mid_new", NOOP, 0, 0)] + P("before") + out += [call("mid_new", lambda: setattr(M, "Node", M.Twin), 1, 2)] + P("after") + return out + + +def sc_arg_rebind(M, call, P): + out = [call("arg_new", lambda: 0, 0)] + P("before") + + def cb(): + M.Node = M.Twin + return "a" + out += [call("arg_new", cb, 2), call("arg_new", lambda: "b", 3)] # the 2nd builds a Twin + P("after") + return out + + +def sc_mid_class(M, call, P): + n, n2 = M.Node(1, 2), M.Node(3, 4) + out = [call("mid_get", NOOP, n)] + P("before") + + def to_shadow(): + n.__class__ = M.Shadow + + def to_twin(): + n2.__class__ = M.Twin + out += [call("mid_get", to_shadow, n), call("mid_set", to_twin, n2, 9), norm(n2)] + P("after") + out += [call("mid_set", NOOP, n, 7)] # guard: n is a Shadow now → interpreted + return out + + +def sc_mid_is_node(M, call, P): + out = [call("is_node", NOOP, M.Node(1, 2))] + P("before") + out += [call("is_node", lambda: setattr(M, "Node", M.Twin), M.Twin(1, 2))] + P("after") + return out + + +def sc_mid_init(M, call, P): + def swapped(self, l, r): + M.Node.left.__set__(self, r) + M.Node.right.__set__(self, l) + init = swapped + + def cb(): + M.Node.__init__ = init + out = [call("mid_new", NOOP, 0, 0), call("via_pair", NOOP, 0, 0)] + P("before") + out += [call("mid_new", cb, 1, 2)] + P("mid") + out += [call("via_pair", NOOP, 3, 4), call("pair", 5, 6), call("make", 1)] + P("after") + return out + + +def sc_counter(M, call, P): + t0 = M.make(2) + P("before") + M.Node.counter = 1 + n = M.Node(1, 2) + out = [call("make", 3), call("check", t0), call("get_left", n), call("setl", n, "x"), norm(n), + call("pair", 1, 2), call("mid_get", NOOP, n), call("mid_new", NOOP, 1, 2)] + P("after") + return out + + +def sc_exhaust(M, call, P): + n = M.Node(1, 2) + for i in range(1100): # MAX_VERSIONS_PER_CLASS is 1000 in CPython 3.14 + M.Node.x = i + M.Node.x # a lookup assigns a fresh tag (until exhausted) + P("before") + out = [call("make", 2), call("check", M.Node(None, None)), call("get_left", n), + call("setl", n, "s"), call("mid_get", NOOP, n), call("mid_set", NOOP, n, "t"), + call("pair", 1, 2), call("via_pair", NOOP, 1, 2), call("mid_new", NOOP, 1, 2)] + P("after") + return out + + +def sc_entry_global(M, call, P): + n = M.Node(1, 2) + a = [0.0] + out = [call("fill", a, n), list(a)] + P("before") + + def getter(s): + M.SCALE = 5.0 + return "p" + M.Node.left = property(getter) + b = [0.0] + out += [call("fill", b, n), list(b), M.SCALE] + P("after") + return out + + +SCENARIOS = {k[3:]: v for k, v in dict(globals()).items() if k.startswith("sc_")} + + +def _exhaust_dict_watchers(): + """Take every free dict-watcher id of this interpreter (ctypes); return them.""" + import ctypes + CB = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p) + add = ctypes.pythonapi.PyDict_AddWatcher + add.argtypes, add.restype = [CB], ctypes.c_int + keep, ids = [], [] + while True: + cb = CB(lambda *a: 0) + keep.append(cb) + try: + ids.append(add(cb)) + except RuntimeError: + break + if len(ids) > 16: + raise AssertionError(f"PyDict_AddWatcher never ran out: {ids}") + _exhaust_dict_watchers.keep = keep + return ids + + +def _clear_dict_watchers(ids): + import ctypes + clear = ctypes.pythonapi.PyDict_ClearWatcher + clear.argtypes, clear.restype = [ctypes.c_int], ctypes.c_int + for i in ids: + clear(i) + + +_counter = [0] + + +def _load_reference(src): + _counter[0] += 1 + spec = importlib.util.spec_from_file_location(f"tp_ref_{_counter[0]}", src) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def _run(M, scenario, compiled): + probes = [] + + def P(label): + if not compiled: + return + info = getattr(M, "__typedpython_class_info__", None) + probes.append({"label": label, "deopts": M.__typedpython_deopts__, + "info": info() if info is not None else None}) + + def call(name, *args): + try: + return norm(getattr(M, name)(*args)) + except Exception as e: # noqa: BLE001 + return ["exc", type(e).__name__, str(e)] + try: + out = SCENARIOS[scenario](M, call, P) + except Exception as e: # noqa: BLE001 — a scenario must not raise itself + out = ["scenario-raised", type(e).__name__, str(e)] + return out, probes + + +def child_main(so, src, name, scenarios, exhaust=False): + from typedpython import cbuild + report = {"scenarios": {}} + if exhaust: + report["taken_watcher_ids"] = _exhaust_dict_watchers() + for sc in scenarios: + gc.collect() # a dropped instance releases its watcher id + A = cbuild.load(name, so) + R = _load_reference(src) + got, probes = _run(A, sc, True) + want, _ = _run(R, sc, False) + report["scenarios"][sc] = {"compiled": got, "reference": want, "probes": probes} + del A, R + print("REPORT " + json.dumps(report)) + + +def child_instances(so, name, n, keep_alive): + """Load `n` instances; report what each says about its dict watcher.""" + from typedpython import cbuild + kept, out = [], [] + for _ in range(n): + if not keep_alive: + gc.collect() + A = cbuild.load(name, so) + info = getattr(A, "__typedpython_class_info__", None) + watched = info()["Node"]["watched"] if info else None + # correctness on every instance, watched or not: a rebinding seen by New + A.make(1) + A.Node = A.Twin + out.append([watched, type(A.make(1)).__name__, type(A.pair(1, 2)).__name__]) + if keep_alive: + kept.append(A) + del A + print("REPORT " + json.dumps(out)) + + +def child_refcounts(so, name, rounds): + from typedpython import cbuild + shared = object() + A1 = cbuild.load(name, so) # unchanged: fast paths + A2 = cbuild.load(name, so) # a property with a setter: slow / deopt paths + A3 = cbuild.load(name, so) # the global rebound: New slow / deopt paths + A4 = cbuild.load(name, so) # tags exhausted: every check fails + store = [] + A2.Node.left = property(lambda s: shared, lambda s, v: (store.append(v), store.pop())) + n1, n2, n4 = A1.Node(shared, shared), A2.Node.__new__(A2.Node), A4.Node(shared, shared) + A2.Node.right.__set__(n2, shared) + A3.make(1) + A3.Node = A3.Twin + for i in range(1100): + A4.Node.x = i + A4.Node.x + cb = lambda: shared # noqa: E731 + + def round_(): + for M, n in ((A1, n1), (A4, n4)): + t = M.make(3) + M.check(t) + M.get_left(n) + M.setl(n, shared) + M.pair(shared, shared) + M.mid_get(cb, n) + M.mid_set(cb, n, shared) + M.mid_new(cb, shared, shared) + M.via_pair(cb, shared, shared) + M.arg_new(cb, shared) + M.is_node(cb, n) + M.fill([0.0], n) + A2.get_left(n2) + A2.mid_get(cb, n2) + A2.setl(n2, shared) + A2.mid_set(cb, n2, shared) + A2.fill([0.0], n2) + try: + A2.check(n2) + except AttributeError: + pass + try: + A2.pair(shared, shared) + except AttributeError: + pass + A3.make(2) + A3.pair(shared, shared) + A3.mid_new(cb, shared, shared) + A3.via_pair(cb, shared, shared) + A3.arg_new(cb, shared) + A3.is_node(cb, n1) + + for _ in range(30): + round_() + objs = [shared, n1, n2, n4, A1.Node, A2.Node, A3.Node, A3.Twin, A4.Node, cb] + base = [sys.getrefcount(o) for o in objs] + d0 = [M.__typedpython_deopts__ for M in (A1, A2, A3, A4)] + for _ in range(rounds): + round_() + after = [sys.getrefcount(o) for o in objs] + d1 = [M.__typedpython_deopts__ for M in (A1, A2, A3, A4)] + print("REPORT " + json.dumps({"base": base, "after": after, "deopts": [d0, d1]})) + + +def _child_cmd(call: str): + return [sys.executable, "-c", + f"import sys; sys.path[:0] = [{str(TEST_DIR)!r}, {str(PACKAGE.parent)!r}]; " + f"import test_cgen_class_changes as T; T.{call}"] + + +def _report(run): + for line in run.stdout.splitlines(): + if line.startswith("REPORT "): + return json.loads(line[len("REPORT "):]) + raise AssertionError(f"child failed (rc={run.returncode}):\n{run.stdout[-4000:]}\n" + f"{run.stderr[-6000:]}") + + +def run_scenarios(so, name, scenarios, exhaust=False, launcher=None, env=None): + src = OUT_ROOT / name / f"{name}.py" + cmd = _child_cmd(f"child_main({str(so)!r}, {str(src)!r}, {name!r}, {list(scenarios)!r}, " + f"exhaust={exhaust!r})") + if launcher is not None: + cmd[0] = str(launcher) + run = subprocess.run(cmd, capture_output=True, text=True, env=env) + rep = _report(run) + assert run.returncode == 0, run.stderr[-4000:] + return rep, run + + +# --- fixtures ----------------------------------------------------------------------------------------- + +NAME = "tp_cls_chg" + + +@pytest.fixture(scope="module") +def so(): + return build(NAME) + + +def same(rep, sc): + r = rep["scenarios"][sc] + assert r["compiled"] == r["reference"], json.dumps(r, indent=1) + return r + + +def probe(r, label): + for p in r["probes"]: + if p["label"] == label: + return p + raise AssertionError(f"no probe {label}: {r['probes']}") + + +# --- the IR used here is what the verifier accepts -------------------------------------------------- + +def test_the_hand_built_ir_passes_the_verifier(): + from typedpython import verify + proved, diags = verify.verify(Module(name="m", functions=functions(), classes=CLASSES)) + errors = [d for d in diags if getattr(d, "severity", "error") == "error"] + assert not proved.skipped, (proved.skipped, diags) + assert {f.name for f in proved.functions} == {f.name for f in functions()}, errors + + +# --- behaviour: watched module dict (the normal path) ------------------------------------------------ + +ALL_SCENARIOS = ("unchanged", "prop_get", "prop_set", "rebind_twin", "rebind_wide", "rebind_back", + "del_global", "mid_prop", "mid_prop_set", "mid_rebind", "arg_rebind", "mid_class", + "mid_is_node", "mid_init", "counter", "exhaust", "entry_global") + + +@pytest.fixture(scope="module") +def report(so): + rep, _ = run_scenarios(so, NAME, ALL_SCENARIOS) + return rep + + +@pytest.mark.parametrize("sc", ALL_SCENARIOS) +def test_compiled_equals_interpreted_after_the_class_changes(report, sc): + r = same(report, sc) + for p in r["probes"]: + assert p["info"] is not None and p["info"]["Node"]["watched"] is True, p # watcher path + + +def test_unchanged_class_stays_on_the_fast_path(report): + r = same(report, "unchanged") + b, a = probe(r, "before"), probe(r, "after") + assert a["deopts"] == b["deopts"] + assert a["info"]["Node"]["slow"] == b["info"]["Node"]["slow"] == 0 + assert a["info"]["Node"]["current"] is True + + +@pytest.mark.parametrize("sc", ("mid_prop", "mid_prop_set", "mid_rebind", "arg_rebind", "mid_class", + "mid_is_node")) +def test_impure_function_takes_the_slow_path_mid_body_without_a_deopt(report, sc): + r = same(report, sc) + b, a = probe(r, "before"), probe(r, "after") + assert a["deopts"] == b["deopts"], (b, a) # no deopt after the effect + if sc != "arg_rebind": # arg_rebind: New was loaded before cb + assert a["info"]["Node"]["slow"] > b["info"]["Node"]["slow"], (b, a) + + +def test_property_reads_give_the_interpreted_value(report): + r = same(report, "prop_get") + assert r["compiled"][0] == 42 and r["compiled"][1] == 42 + + +def test_property_setter_runs_on_field_set(report): + r = same(report, "prop_set") + assert r["compiled"][3] == ["x", "y"] + + +def test_rebound_global_builds_the_new_class(report): + for sc, cls in (("rebind_twin", "Twin"), ("rebind_wide", "Wide")): + r = same(report, sc) + made = r["compiled"][5:10] + assert all(isinstance(x, dict) and x["type"] == cls for x in made[1:4]), made + + +def test_rebinding_back_returns_to_the_fast_path(report): + r = same(report, "rebind_back") + assert probe(r, "after")["deopts"] == probe(r, "restored")["deopts"] + + +def test_mid_body_init_change_and_class_redo_of_a_pure_callee(report): + r = same(report, "mid_init") + assert r["compiled"][2] == {"type": "Node", "left": 2, "right": 1, "extra": r["compiled"][2]["extra"]} + # mid_new (impure) took the slow path, no deopt; via_pair's pure callee `pair` was redone + assert probe(r, "mid")["deopts"] == probe(r, "before")["deopts"] + + +def test_unrelated_class_attribute_refreshes_the_tag_back_to_the_fast_path(report): + r = same(report, "counter") + b, a = probe(r, "before"), probe(r, "after") + assert a["deopts"] == b["deopts"], (b, a) + assert a["info"]["Node"]["slow"] == 0, a + assert a["info"]["Node"]["refreshed"] >= 1 and a["info"]["Node"]["current"] is True, a + assert a["info"]["Node"]["captured_tag"] != b["info"]["Node"]["captured_tag"], (b, a) + + +def test_exhausted_version_tags_take_the_slow_path(report): + r = same(report, "exhaust") + b, a = probe(r, "before"), probe(r, "after") + assert b["info"]["Node"]["tag"] == 0 and a["info"]["Node"]["tag"] == 0, (b, a) + assert a["info"]["Node"]["current"] is False + assert a["info"]["Node"]["slow"] > b["info"]["Node"]["slow"] + assert a["deopts"] > b["deopts"] # the pure functions deopted + + +def test_closed_function_with_an_entry_global_sees_the_rebinding(report): + r = same(report, "entry_global") + assert r["compiled"][1] == [2.0] and r["compiled"][3] == [5.0] + + +# --- behaviour: no dict watcher available ------------------------------------------------------------ + +FALLBACK = ("rebind_twin", "rebind_wide", "rebind_back", "del_global", "mid_rebind", "arg_rebind", + "mid_is_node", "unchanged") + + +def test_without_a_dict_watcher_rebinding_is_still_seen(so): + rep, run = run_scenarios(so, NAME, FALLBACK, exhaust=True) + assert 1 <= len(rep["taken_watcher_ids"]) <= 6, rep["taken_watcher_ids"] + for sc in FALLBACK: + r = same(rep, sc) + assert r["probes"], sc + for p in r["probes"]: + assert p["info"]["Node"]["watched"] is False, p # the conservative path ran + r = rep["scenarios"]["mid_rebind"] + assert probe(r, "after")["deopts"] == probe(r, "before")["deopts"] + + +def test_each_instance_gets_and_releases_its_own_watcher(so): + run = subprocess.run(_child_cmd(f"child_instances({str(so)!r}, {NAME!r}, 12, False)"), + capture_output=True, text=True) + rows = _report(run) + assert all(w is True for w, _, _ in rows), rows # released on module free + assert all(m == "Twin" and p == "Twin" for _, m, p in rows), rows + run = subprocess.run(_child_cmd(f"child_instances({str(so)!r}, {NAME!r}, 9, True)"), + capture_output=True, text=True) + rows = _report(run) + assert rows[0][0] is True and rows[-1][0] is False, rows # ids run out: fallback + assert all(m == "Twin" and p == "Twin" for _, m, p in rows), rows + + +# --- reference counts ------------------------------------------------------------------------------ + +def test_reference_counts_are_flat_over_fast_and_slow_paths(so): + run = subprocess.run(_child_cmd(f"child_refcounts({str(so)!r}, {NAME!r}, 1500)"), + capture_output=True, text=True) + rep = _report(run) + assert rep["base"] == rep["after"], rep + d0, d1 = rep["deopts"] + assert d1[0] == d0[0] and d1[1] > d0[1] and d1[2] > d0[2] and d1[3] > d0[3], rep + + +# --- generation contract ------------------------------------------------------------------------------- + +def test_gen_uses_only_runtime_helpers(): + text = generate("tp_cls_chg_gen") + for h in ("tp_class_capture", "tp_class_current", "tp_class_global_ok", "tp_getattr", + "tp_setattr", "tp_globals_watch_event", "tp_globals_unwatch"): + assert h in text, h + assert "PyMember_" not in text and "tp_version_tag" not in text + assert "PyDict_Watch(" not in text and "PyDict_AddWatcher" not in text + + +# --- mutations --------------------------------------------------------------------------------------- + +def _mutant(monkeypatch, name, attr, value): + from typedpython import cgen, cbuild + monkeypatch.setattr(cgen, attr, value) + src = generate(name) + monkeypatch.undo() + return cbuild.build(src, name, OUT_ROOT / name, runtime_dir=runtime_dir()) + + +def test_mutation_check_always_current_fails_the_property_test(monkeypatch): + need_runtime() + so = _mutant(monkeypatch, "tp_cls_chg_mut1", "_class_current_expr", lambda *a, **k: "1") + rep, _ = run_scenarios(so, "tp_cls_chg_mut1", ("prop_get",)) + r = rep["scenarios"]["prop_get"] + assert r["compiled"] != r["reference"], "the mutant (no per-access check) passed prop_get" + + +def test_mutation_watcher_never_fires_fails_the_rebinding_test(monkeypatch): + need_runtime() + so = _mutant(monkeypatch, "tp_cls_chg_mut2", "_watch_event_stmt", + lambda: "(void)event; (void)key; return 0;") + rep, _ = run_scenarios(so, "tp_cls_chg_mut2", ("rebind_twin",)) + r = rep["scenarios"]["rebind_twin"] + assert r["compiled"] != r["reference"], "the mutant (silent watcher) passed rebind_twin" + + +# --- sanitizers ----------------------------------------------------------------------------------------- + +def test_class_change_scenarios_under_address_and_undefined_sanitizers(): + need_runtime() + cc = sysconfig.get_config_var("CC") or "cc" + if sys.platform != "darwin": + pytest.skip("sanitizer preload recipe written for macOS only") + probe_rt = subprocess.run(cc.split() + ["-print-file-name=libclang_rt.asan_osx_dynamic.dylib"], + capture_output=True, text=True) + if probe_rt.returncode != 0 or not Path(probe_rt.stdout.strip()).is_file(): + pytest.skip(f"host compiler {cc!r} has no ASan runtime ({probe_rt.stdout.strip()!r})") + from typedpython import cbuild + name = "tp_cls_chg_asan" + flags = list(cbuild.DEFAULT_FLAGS) + ["-O1", "-g", "-fno-omit-frame-pointer", + "-fsanitize=address,undefined", + "-fno-sanitize-recover=undefined"] + so = build(name, flags=flags) + d = so.parent + launcher_c, launcher = d / "asan_python.c", d / "asan_python" + launcher_c.write_text("#include \nint main(int argc, char **argv) " + "{ return Py_BytesMain(argc, argv); }\n") + link = subprocess.run(cc.split() + ["-fsanitize=address,undefined", "-g", + f"-I{sysconfig.get_paths()['include']}", str(launcher_c), + f"-L{sysconfig.get_config_var('LIBDIR')}", + f"-lpython{sysconfig.get_config_var('LDVERSION')}", + "-o", str(launcher)], capture_output=True, text=True) + if link.returncode != 0: + pytest.skip(f"cannot link a sanitized launcher: {link.stderr[-2000:]}") + env = dict(os.environ, PYTHONHOME=sys.base_prefix, + PYTHONPATH=os.pathsep.join(p for p in sys.path if p and "site-packages" in p), + ASAN_OPTIONS="detect_leaks=0:abort_on_error=0:halt_on_error=1", + UBSAN_OPTIONS="halt_on_error=1:print_stacktrace=1") + env.pop("DYLD_INSERT_LIBRARIES", None) + run = subprocess.run([str(launcher)] + _child_cmd( + f"child_main({str(so)!r}, {str(so.parent / (name + '.py'))!r}, {name!r}, " + f"{list(ALL_SCENARIOS)!r})")[1:], capture_output=True, text=True, env=env) + out = run.stdout + run.stderr + if "violates platform policy" in out or "Interceptors are not working" in out: + pytest.skip(f"ASan runtime could not be loaded: {out[-2000:]}") + assert run.returncode == 0, out[-6000:] + rep = _report(run) + for sc in ALL_SCENARIOS: + same(rep, sc) + linked = subprocess.run(["otool", "-L", str(so)], capture_output=True, text=True).stdout + assert "libclang_rt.asan" in linked, linked + assert "runtime error" not in out and "AddressSanitizer" not in out, out[-6000:] diff --git a/python-multiplatform-ksp/src/test/python/test_cgen_classes.py b/python-multiplatform-ksp/src/test/python/test_cgen_classes.py new file mode 100644 index 000000000..c7200cb11 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_cgen_classes.py @@ -0,0 +1,599 @@ +"""TypedPython C back end: fixed-layout classes (SPEC N-11; ir.ClassDecl / FieldGet / FieldSet / New / +IsExact / CheckExact / Is / Param.cls). + +IR is built by hand; every end-to-end test compares the compiled function with the interpreted one the +extension keeps in `__typedpython_interpreted__` (INTENT §1.4). Builds go to `.tmp/cgen-tests/` of the +worktree, inside the caller's heavy.sh lock (see test_cgen.py). + +Ordering of the red/green record: this file was written before the back end supported these nodes; +before the implementation every end-to-end test failed with `CGenError: no lowering for ...`. +""" +from __future__ import annotations + +import subprocess +import sys +import sysconfig +import textwrap +import os +from pathlib import Path + +import pytest + +import test_cgen as tc +from test_cgen import (PACKAGE, OUT_ROOT, I64, BOOL, OBJ, NONE, c, L, fn, runtime_dir, need_runtime) +from typedpython import ir +from typedpython.ir import ( + Assign, BinOp, BinOpKind, Call, CheckExact, ClassDecl, Compare, CompareKind, Const, FieldGet, + FieldSet, If, Is, IsExact, Module, New, Param, Return, +) + +SOURCE = textwrap.dedent('''\ + """typedpython class test module — the interpreted reference.""" + + class Node: + __slots__ = ("left", "right") + + def __init__(self, left, right): + self.left = left + self.right = right + + class Sub(Node): # a subclass whose `left` is NOT the slot + __slots__ = () + + @property + def left(self): + return None + + def mk_sub(l, r): + s = Sub.__new__(Sub) + Node.left.__set__(s, l) + Node.right.__set__(s, r) + return s + + class Fat: # no __slots__: instances have a __dict__ + def __init__(self, left, right): + self.left = left + self.right = right + + class Prop: # a property shadows a field after creation + __slots__ = ("left", "right") + + Prop.left = property(lambda self: "shadowed") + + class Other: + __slots__ = ("left", "right") + + class Stolen: # a member descriptor of ANOTHER class under our name + __slots__ = ("left", "right") + + def __init__(self, left, right): + pass + + Stolen.left = Other.left + + def make(depth): + if depth == 0: + return None + return Node(make(depth - 1), make(depth - 1)) + + def check(node): + if node is None: + return 0 + return 1 + check(node.left) + check(node.right) + + def setl(node, v): + node.left = v + + def left_checked(x): + return x.left + + def is_node(x): + return type(x) is Node + + def is_not_none(x): + return x is not None + + def same(a, b): + return a is b + + def ret_none(): + return None + + def mkfat(depth): + if depth == 0: + return None + return Fat(mkfat(depth - 1), mkfat(depth - 1)) + + def check_fat(node): + if node is None: + return 0 + return 1 + check_fat(node.left) + check_fat(node.right) + + def left_of_prop(p): + return p.left + + def left_of_stolen(p): + return p.left + ''') + +NODE = ClassDecl("Node", ("left", "right"), True) +FAT = ClassDecl("Fat", ("left", "right"), True) +PROP = ClassDecl("Prop", ("left", "right"), True) +STOLEN = ClassDecl("Stolen", ("left", "right"), True) +CLASSES = (NODE, FAT, PROP, STOLEN) + + +def f_make(name="make", cls="Node"): + depth, a, b = L("depth", I64), L("a", OBJ), L("b", OBJ) + rec = lambda: Call(OBJ, name, (BinOp(I64, BinOpKind.SUB, depth, c(1)),)) + return fn(name, [Param("depth", I64)], OBJ, {"a": OBJ, "b": OBJ}, + [If(Compare(BOOL, CompareKind.EQ, depth, c(0)), (Return(Const(OBJ, None)),)), + Assign("a", rec()), Assign("b", rec()), + Return(New(OBJ, cls, (a, b)))], + pure=True, may_deopt=True) + + +def f_check(name="check", cls="Node"): + node, l, r = L("node", OBJ), L("l", OBJ), L("r", OBJ) + ca = Call(I64, name, (l,)) + cb = Call(I64, name, (r,)) + return fn(name, [Param("node", OBJ, cls=cls, optional=True)], I64, {"l": OBJ, "r": OBJ}, + [If(Is(BOOL, node, Const(OBJ, None)), (Return(c(0)),)), + Assign("l", FieldGet(OBJ, node, cls, "left")), + Assign("r", FieldGet(OBJ, node, cls, "right")), + Return(BinOp(I64, BinOpKind.ADD, BinOp(I64, BinOpKind.ADD, c(1), ca), cb))], + pure=True, may_deopt=True) + + +def f_setl(): + node, v = L("node", OBJ), L("v", OBJ) + return fn("setl", [Param("node", OBJ, cls="Node"), Param("v", OBJ)], NONE, {}, + [FieldSet(node, "Node", "left", v), Return()], + pure=False, may_deopt=False) + + +def f_left_checked(): + x = L("x", OBJ) + return fn("left_checked", [Param("x", OBJ)], OBJ, {}, + [Return(FieldGet(OBJ, CheckExact(OBJ, x, "Node"), "Node", "left"))], + pure=True, may_deopt=True) + + +def f_is_node(): + return fn("is_node", [Param("x", OBJ)], BOOL, {}, [Return(IsExact(BOOL, L("x", OBJ), "Node"))], + pure=True, may_deopt=False) + + +def f_is_not_none(): + return fn("is_not_none", [Param("x", OBJ)], BOOL, {}, + [Return(Is(BOOL, L("x", OBJ), Const(OBJ, None), negate=True))], + pure=True, may_deopt=False) + + +def f_same(): + return fn("same", [Param("a", OBJ), Param("b", OBJ)], BOOL, {}, + [Return(Is(BOOL, L("a", OBJ), L("b", OBJ)))], pure=True, may_deopt=False) + + +def f_ret_none(): + return fn("ret_none", [], OBJ, {}, [Return(Const(OBJ, None))], pure=True, may_deopt=False) + + +def f_left_of(name, cls): + return fn(name, [Param("p", OBJ, cls=cls)], OBJ, {}, + [Return(FieldGet(OBJ, L("p", OBJ), cls, "left"))], pure=True, may_deopt=True) + + +def functions(): + return (f_make(), f_check(), f_setl(), f_left_checked(), f_is_node(), f_is_not_none(), f_same(), + f_ret_none(), f_make("mkfat", "Fat"), f_check("check_fat", "Fat"), + f_left_of("left_of_prop", "Prop"), f_left_of("left_of_stolen", "Stolen")) + + +def source_file(name: str) -> Path: + import shutil + d = OUT_ROOT / name + if d.exists(): + shutil.rmtree(d) + d.mkdir(parents=True) + p = d / f"{name}.py" + p.write_text(SOURCE) + return p + + +def generate(name, funcs=None, classes=CLASSES): + from typedpython import cgen + funcs = functions() if funcs is None else funcs + return cgen.generate(Module(name=name, functions=tuple(funcs), classes=tuple(classes)), + source_file(name)) + + +def build(name, flags=None, funcs=None, classes=CLASSES): + from typedpython import cbuild + need_runtime() + src = generate(name, funcs, classes) + kwargs = {} if flags is None else {"flags": flags} + so = cbuild.build(src, name, OUT_ROOT / name, runtime_dir=runtime_dir(), **kwargs) + return cbuild.load(name, so), so + + +@pytest.fixture(scope="module") +def m(): + return build("tp_cgen_cls")[0] + + +def interp(m, name): + return m.__typedpython_interpreted__[name] + + +def tree_size(n): + return 0 if n is None else 1 + tree_size(n.left) + tree_size(n.right) + + +# --- behaviour --------------------------------------------------------------------------------------- + +def test_make_and_check_match_the_interpreted_results(m): + for depth in (0, 1, 2, 5, 10): + before = m.__typedpython_deopts__ + tree = m.make(depth) + assert type(tree) is m.Node or tree is None + assert tree_size(tree) == 2 ** depth - 1 + assert m.check(tree) == interp(m, "check")(tree) == 2 ** depth - 1 + assert m.__typedpython_deopts__ == before # compiled all the way + + +def test_compiled_objects_are_ordinary_instances(m): + t = m.make(2) + assert type(t) is m.Node and not hasattr(t, "__dict__") + assert t.left.left is None and t.left.right is None + assert interp(m, "check")(t) == 3 + + +def test_unset_slot_raises_cpythons_attribute_error(m): + bare = m.Node.__new__(m.Node) + with pytest.raises(AttributeError) as want: + interp(m, "check")(bare) + with pytest.raises(AttributeError) as got: + m.check(bare) + assert str(got.value) == str(want.value) + assert str(got.value).endswith("Node' object has no attribute 'left'") + half = m.Node.__new__(m.Node) + m.Node.left.__set__(half, None) + with pytest.raises(AttributeError) as got2: + m.check(half) + assert str(got2.value).endswith("Node' object has no attribute 'right'") + + +def test_subclass_instance_takes_the_interpreted_path(m): + sub = m.mk_sub(m.Node(None, None), None) + assert interp(m, "check")(sub) == 1 # the property makes `left` None + before = m.__typedpython_deopts__ + assert m.check(sub) == 1 # raw slot would give 2 + assert m.__typedpython_deopts__ == before + 1 + + +def test_nested_subclass_instance_deopts_the_whole_call(m): + sub = m.mk_sub(m.Node(None, None), None) + tree = m.Node(sub, m.Node(None, None)) + want = interp(m, "check")(tree) + before = m.__typedpython_deopts__ + assert m.check(tree) == want == 3 + # the outermost call is redone interpreted; the interpreted body then calls the (compiled) global + # `check` on each child, so the subclass child deopts again: at least one, never zero + assert m.__typedpython_deopts__ >= before + 1 + + +def test_other_argument_types_take_the_interpreted_path(m): + def outcome(f, x): + try: + return ("ok", f(x)) + except Exception as e: # noqa: BLE001 — the type and text are the result + return (type(e), str(e)) + + for bad in (5, "x", object(), m.Fat(None, None), m.Fat(m.Fat(None, None), None)): + want = outcome(interp(m, "check"), bad) + before = m.__typedpython_deopts__ + assert outcome(m.check, bad) == want + assert m.__typedpython_deopts__ >= before + 1 + + +def test_field_set_is_visible_to_python(m): + n = m.Node(1, 2) + assert m.setl(n, "x") is None + assert n.left == "x" and n.right == 2 + assert m.check(m.Node(None, None)) == 1 + keep = object() + m.setl(n, keep) + assert n.left is keep + + +def test_field_set_on_a_subclass_goes_interpreted(m): + sub = m.mk_sub(None, None) + before = m.__typedpython_deopts__ + with pytest.raises(AttributeError) as want: + interp(m, "setl")(sub, 1) + with pytest.raises(AttributeError) as got: + m.setl(sub, 1) + assert str(got.value) == str(want.value) + assert m.__typedpython_deopts__ == before + 1 + + +def test_check_exact_deopts_on_mismatch_and_reads_otherwise(m): + n = m.Node("l", "r") + before = m.__typedpython_deopts__ + assert m.left_checked(n) == "l" + assert m.__typedpython_deopts__ == before + sub = m.mk_sub("l", "r") + assert m.left_checked(sub) is interp(m, "left_checked")(sub) is None + assert m.__typedpython_deopts__ == before + 1 + with pytest.raises(AttributeError): + m.left_checked(5) + assert m.__typedpython_deopts__ == before + 2 + + +def test_is_exact_is_and_none_constants(m): + assert m.is_node(m.Node(1, 2)) is True + assert m.is_node(m.mk_sub(1, 2)) is False + assert m.is_node(None) is False and m.is_node(5) is False + x = object() + assert m.same(x, x) is True and m.same(x, object()) is False + assert m.same(None, None) is True + assert m.is_not_none(None) is False and m.is_not_none(0) is True + assert m.ret_none() is None + + +# --- classes that are not compiled --------------------------------------------------------------------- + +def test_class_with_a_dict_is_not_compiled_and_nothing_crashes(m): + before = m.__typedpython_deopts__ + tree = m.mkfat(4) # New of a class without __slots__ + assert type(tree) is m.Fat and tree.left is not None + assert m.check_fat(tree) == 15 == interp(m, "check_fat")(tree) + assert m.__typedpython_deopts__ > before # the calls took the interpreted path + assert m.check_fat(None) == 0 + + +def test_property_shadowing_a_field_is_not_compiled(m): + p = m.Prop() + before = m.__typedpython_deopts__ + assert m.left_of_prop(p) == "shadowed" == interp(m, "left_of_prop")(p) + assert m.__typedpython_deopts__ == before + 1 + + +def test_descriptor_of_another_class_is_not_compiled(m): + s = m.Stolen.__new__(m.Stolen) + before = m.__typedpython_deopts__ + with pytest.raises(TypeError) as want: + interp(m, "left_of_stolen")(s) + with pytest.raises(TypeError) as got: + m.left_of_stolen(s) + assert str(got.value) == str(want.value) + assert m.__typedpython_deopts__ == before + 1 + + +def test_missing_class_is_not_compiled(tmp_path): + mod, _ = build("tp_cgen_cls_missing", funcs=(f_make(),), classes=(ClassDecl("Nope", ("a",), True), + NODE)) + before = mod.__typedpython_deopts__ + assert type(mod.make(2)) is mod.Node # Node is fine; the missing one is unused + assert mod.__typedpython_deopts__ == before + + +# --- generation contract ------------------------------------------------------------------------------- + +def test_gen_rejects_check_exact_in_an_impure_function(): + bad2 = fn("bad2", [Param("x", OBJ)], OBJ, {}, + [Return(CheckExact(OBJ, L("x", OBJ), "Node"))], pure=False, may_deopt=False) + from typedpython import cgen + with pytest.raises(cgen.CGenError, match="CheckExact"): + generate("tp_gen_cls_bad", funcs=(bad2,)) + + +def test_gen_rejects_field_set_in_a_pure_function(): + bad = fn("bad", [Param("x", OBJ, cls="Node")], NONE, {}, + [FieldSet(L("x", OBJ), "Node", "left", Const(OBJ, None)), Return()], + pure=True, may_deopt=False) + from typedpython import cgen + with pytest.raises(cgen.CGenError, match="pure"): + generate("tp_gen_cls_bad2", funcs=(bad,)) + + +def test_gen_rejects_an_impure_caller_of_a_class_guarded_callee(): + setter = f_setl() + caller = fn("caller", [Param("x", OBJ)], NONE, {}, + [ir.ExprStmt(Call(NONE, "setl", (L("x", OBJ), L("x", OBJ)))), Return()], + pure=False, may_deopt=False) + from typedpython import cgen + with pytest.raises(cgen.CGenError, match="deopt"): + generate("tp_gen_cls_bad3", funcs=(setter, caller)) + + +def test_gen_rejects_unknown_class_field_or_arity(): + from typedpython import cgen + unknown = fn("u", [Param("x", OBJ)], OBJ, {}, [Return(New(OBJ, "Nope", ()))], + pure=True, may_deopt=False) + badfield = fn("f", [Param("x", OBJ, cls="Node")], OBJ, {}, + [Return(FieldGet(OBJ, L("x", OBJ), "Node", "middle"))], pure=True, may_deopt=True) + arity = fn("a", [], OBJ, {}, [Return(New(OBJ, "Node", (Const(OBJ, None),)))], + pure=True, may_deopt=False) + for i, f in enumerate((unknown, badfield, arity)): + with pytest.raises(cgen.CGenError): + generate(f"tp_gen_cls_bad4_{i}", funcs=(f,)) + + +def test_gen_uses_only_the_runtime_helpers_for_slots(): + text = generate("tp_gen_cls_only") + for h in ("tp_field_get", "tp_field_set", "tp_new_fixed", "tp_is_exact", "tp_class_capture"): + assert h in text, h + assert "PyMember_" not in text and "tp_alloc" not in text and "->tp_dictoffset" not in text + + +# --- refcounts --------------------------------------------------------------------------------------------- + +def test_reference_counts_are_unchanged_over_many_calls(m): + shared = object() + node = m.Node(shared, shared) + bare = m.Node.__new__(m.Node) + + def churn(rounds): + for _ in range(rounds): + m.make(4) + m.left_checked(node) + m.setl(node, shared) + m.is_node(node) + m.same(node, node) + m.check(m.Node(None, None)) + m.check(m.mk_sub(None, None)) # deopt path + try: + m.check(bare) + except AttributeError: + pass + try: + m.setl(5, 5) + except AttributeError: + pass + + churn(20) # warm up caches, then measure + counts = lambda: (sys.getrefcount(shared), sys.getrefcount(node), sys.getrefcount(m.Node), + sys.getrefcount(m.Sub), sys.getrefcount(bare)) + base = counts() + churn(3000) + assert counts() == base, (base, counts()) + + +# --- the mutation: without the exact-type guard the subclass test must fail --------------------------- + +MUTANT_SCRIPT = textwrap.dedent('''\ + import sys + sys.path.insert(0, {pkg!r}) + from typedpython import cbuild + m = cbuild.load({name!r}, {so!r}) + sub = m.mk_sub(m.Node(None, None), None) + want = m.__typedpython_interpreted__["check"](sub) + got = m.check(sub) + print("RESULT", want, got, m.__typedpython_deopts__) + bad = 0 if (want == got and m.__typedpython_deopts__ == 1) else 3 + # the DIRECT native call path: a subclass / a non-Node reached as a child of a Node + def outcome(f, x): + try: + return ("ok", f(x)) + except Exception as e: + return (type(e).__name__, str(e)) + for child in (sub, 5, m.Fat(None, None)): + tree = m.Node(child, None) + w = outcome(m.__typedpython_interpreted__["check"], tree) + before = m.__typedpython_deopts__ + g = outcome(m.check, tree) + print("NESTED", type(child).__name__, w, g, m.__typedpython_deopts__ - before) + if w != g or m.__typedpython_deopts__ <= before: + bad = 3 + sys.exit(bad) + ''') + + +def run_child(name, so): + script = MUTANT_SCRIPT.format(pkg=str(PACKAGE.parent), name=name, so=str(so)) + return subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + + +def test_mutation_unmutated_build_passes_the_subclass_script(): + need_runtime() + _, so = build("tp_cgen_cls_ok") + run = run_child("tp_cgen_cls_ok", so) + assert run.returncode == 0, run.stdout + run.stderr + + +def test_mutation_without_the_param_guard_the_subclass_test_fails(monkeypatch): + need_runtime() + from typedpython import cgen, cbuild + monkeypatch.setattr(cgen, "_param_cls_fail", lambda *a, **k: "0") # the guard never fires + # Since the per-access class check (test_cgen_class_changes.py) every FieldGet also tests + # `type(obj) is C` itself, so the entry guard alone is no longer the only defence: the mutant + # removes both (with the per-access check kept, this mutant computes the right results). + monkeypatch.setattr(cgen, "_class_current_expr", lambda *a, **k: "1") + src = generate("tp_cgen_cls_mut") + monkeypatch.undo() + so = cbuild.build(src, "tp_cgen_cls_mut", OUT_ROOT / "tp_cgen_cls_mut", runtime_dir=runtime_dir()) + run = run_child("tp_cgen_cls_mut", so) + assert run.returncode != 0, ("the mutant (no exact-type guard on Param.cls) passed the subclass " + "test:\n" + run.stdout + run.stderr) + + +# --- sanitizers --------------------------------------------------------------------------------------------- + +ASAN_SCRIPT = textwrap.dedent('''\ + import sys + sys.path.insert(0, {pkg!r}) + from typedpython import cbuild + m = cbuild.load({name!r}, {so!r}) + I = m.__typedpython_interpreted__ + for d in (0, 1, 6): + t = m.make(d) + assert m.check(t) == I["check"](t) == 2 ** d - 1 + for _ in range(300): + t = m.make(5) + n = m.Node(t, t) + assert m.check(n) == 2 * 31 + 1 + m.setl(n, None) + m.left_checked(n) + sub = m.mk_sub(m.Node(None, None), None) + assert m.check(sub) == 1 and m.check(m.Node(sub, None)) == I["check"](m.Node(sub, None)) + for bad in (5, "x", object(), m.Fat(None, None)): + try: + m.check(bad) + except AttributeError: + pass + try: + m.setl(bad, 1) + except AttributeError: + pass + bare = m.Node.__new__(m.Node) + try: + m.check(bare) + except AttributeError: + pass + assert m.check_fat(m.mkfat(3)) == 7 + assert m.left_of_prop(m.Prop()) == "shadowed" + print("ASAN-RUN-OK", m.__typedpython_deopts__) + ''') + + +def test_class_functions_under_address_and_undefined_sanitizers(): + need_runtime() + cc = sysconfig.get_config_var("CC") or "cc" + if sys.platform != "darwin": + pytest.skip("sanitizer preload recipe written for macOS only") + probe = subprocess.run(cc.split() + ["-print-file-name=libclang_rt.asan_osx_dynamic.dylib"], + capture_output=True, text=True) + if probe.returncode != 0 or not Path(probe.stdout.strip()).is_file(): + pytest.skip(f"host compiler {cc!r} has no ASan runtime ({probe.stdout.strip()!r})") + from typedpython import cbuild + name = "tp_cgen_cls_asan" + flags = list(cbuild.DEFAULT_FLAGS) + ["-O1", "-g", "-fno-omit-frame-pointer", + "-fsanitize=address,undefined", + "-fno-sanitize-recover=undefined"] + so = cbuild.build(generate(name), name, OUT_ROOT / name, runtime_dir=runtime_dir(), flags=flags) + d = so.parent + launcher_c, launcher = d / "asan_python.c", d / "asan_python" + launcher_c.write_text("#include \nint main(int argc, char **argv) " + "{ return Py_BytesMain(argc, argv); }\n") + link = subprocess.run(cc.split() + ["-fsanitize=address,undefined", "-g", + f"-I{sysconfig.get_paths()['include']}", str(launcher_c), + f"-L{sysconfig.get_config_var('LIBDIR')}", + f"-lpython{sysconfig.get_config_var('LDVERSION')}", + "-o", str(launcher)], capture_output=True, text=True) + if link.returncode != 0: + pytest.skip(f"cannot link a sanitized launcher: {link.stderr[-2000:]}") + env = dict(os.environ, PYTHONHOME=sys.base_prefix, + ASAN_OPTIONS="detect_leaks=0:abort_on_error=0:halt_on_error=1", + UBSAN_OPTIONS="halt_on_error=1:print_stacktrace=1") + env.pop("DYLD_INSERT_LIBRARIES", None) + script = ASAN_SCRIPT.format(pkg=str(PACKAGE.parent), name=name, so=str(so)) + run = subprocess.run([str(launcher), "-c", script], capture_output=True, text=True, env=env) + out = run.stdout + run.stderr + if "violates platform policy" in out or "Interceptors are not working" in out: + pytest.skip(f"ASan runtime could not be loaded: {out[-2000:]}") + assert run.returncode == 0 and "ASAN-RUN-OK" in run.stdout, out[-6000:] + linked = subprocess.run(["otool", "-L", str(so)], capture_output=True, text=True).stdout + assert "libclang_rt.asan" in linked, linked + assert "runtime error" not in out and "AddressSanitizer" not in out, out[-6000:] diff --git a/python-multiplatform-ksp/src/test/python/test_frontend_classes.py b/python-multiplatform-ksp/src/test/python/test_frontend_classes.py new file mode 100644 index 000000000..4def4ed49 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_frontend_classes.py @@ -0,0 +1,608 @@ +"""TypedPython front end: fixed-layout classes, None and identity (SPEC N-11, ir.py contract). + +A class is lowered to an `ir.ClassDecl` only when its layout is fixed in CPython too: the user wrote +`__slots__` equal to the annotated fields, no base but object, no metaclass, no decorator but +`@compiled`, and the module never subclasses it, assigns its class attributes or rebinds its name. +Every rule has an accepting and a rejecting test. A rejected class has no ClassDecl, an entry in +`Module.skipped` (keyed by the class name) with the reason, and functions using it lower as before +(`GetAttr` / `CallObject`). + +Contract choice (ir.ClassDecl says "`__init__(self, *fields-in-order)`"): `trivial_init` requires the +i-th `__init__` parameter to be stored into the i-th slot of `__slots__`; the statements may come in +any order and the parameter names are free. `ClassDecl.fields` is the `__slots__` order and +`New(cls, args)` takes the call's arguments in that order. +""" +import dataclasses +import textwrap +from pathlib import Path + +import pytest + +from typedpython import frontend +from typedpython.ir import ( + And, Assign, Box, Call, CallObject, CheckExact, ClassDecl, Const, ExprStmt, FieldGet, FieldSet, + GetAttr, Global, If, Is, Local, New, Or, Param, Return, Type, +) + +I64, F64, BOOL, OBJ, NONE = Type.I64, Type.F64, Type.BOOL, Type.OBJ, Type.NONE +BINARY_TREES = Path(__file__).resolve().parents[4] / "benchmarks/typedpython/py/binary_trees.py" +SLOTS_LINE = ' __slots__ = ("left", "right")\n' + +MARK = "# typedpython: compiled\n" + +POINT = """ +class P: + __slots__ = ("x", "y") + x: float + y: float + + def __init__(self, x: float, y: float) -> None: + self.x = x + self.y = y +""" + + +def lower(tmp_path, source, name="mod.py"): + path = tmp_path / name + path.write_text(MARK + textwrap.dedent(source).lstrip("\n")) + return frontend.lower(path) + + +def function(module, name): + found = [f for f in module.functions if f.name == name] + assert found, f"{name} was not lowered: {module.skipped.get(name)!r}" + return found[0] + + +def cls(module, name): + found = [c for c in module.classes if c.name == name] + assert found, f"{name} has no ClassDecl: {module.skipped.get(name)!r}" + return found[0] + + +def nodes(root, kind): + found, stack = [], [root] + while stack: + x = stack.pop() + if isinstance(x, (tuple, list)): + stack.extend(x) + elif dataclasses.is_dataclass(x) and not isinstance(x, type): + if isinstance(x, kind): + found.append(x) + stack.extend(getattr(x, f.name) for f in dataclasses.fields(x)) + return found + + +def rejected(tmp_path, source, name, reason): + m = lower(tmp_path, source) + assert all(c.name != name for c in m.classes), f"{name} should not be a ClassDecl" + assert reason in m.skipped[name], m.skipped[name] + return m + + +# --- ClassDecl: accepted --------------------------------------------------------------------------- + +def test_slots_class_in_marked_module_is_a_classdecl(tmp_path): + m = lower(tmp_path, POINT) + assert cls(m, "P") == ClassDecl("P", ("x", "y"), True) + assert "P" not in m.skipped + + +def test_decorated_class_in_unmarked_module(tmp_path): + path = tmp_path / "d.py" + path.write_text(textwrap.dedent(""" + @compiled + class P: + __slots__ = ["a"] + a: int + + def __init__(self, a: int) -> None: + self.a = a + + class Q: + __slots__ = ("b",) + b: int + """)) + m = frontend.lower(path) + assert [c.name for c in m.classes] == ["P"] # Q is neither decorated nor in a marked module + assert m.classes[0].fields == ("a",) + + +def test_fields_keep_the_slots_order_not_the_annotation_order(tmp_path): + m = lower(tmp_path, """ + class P: + __slots__ = ("y", "x") + x: int + y: int + def __init__(self, a: int, b: int) -> None: + self.y = a + self.x = b + """) + assert cls(m, "P") == ClassDecl("P", ("y", "x"), True) + + +def test_explicit_object_base_and_docstring_are_accepted(tmp_path): + m = lower(tmp_path, """ + class P(object): + "doc" + __slots__ = ("x",) + x: int + """) + assert cls(m, "P").fields == ("x",) + assert cls(m, "P").trivial_init is False # no __init__: object's, not an assign-each-slot one + + +def test_methods_are_allowed_and_stay_interpreted(tmp_path): + m = lower(tmp_path, POINT + """ + def norm(self) -> float: + return self.x +""") + assert cls(m, "P").fields == ("x", "y") + + +# --- ClassDecl: rejected --------------------------------------------------------------------------- + +def test_missing_slots_is_rejected(tmp_path): + rejected(tmp_path, """ + class P: + x: int + def __init__(self, x: int) -> None: + self.x = x + """, "P", "`__slots__`") + + +def test_slots_differing_from_annotations_is_rejected(tmp_path): + rejected(tmp_path, """ + class P: + __slots__ = ("x", "z") + x: int + y: int + """, "P", "annotated fields") + + +def test_slots_not_a_literal_is_rejected(tmp_path): + rejected(tmp_path, """ + NAMES = ("x",) + class P: + __slots__ = NAMES + x: int + """, "P", "literal") + + +def test_duplicate_slot_is_rejected(tmp_path): + rejected(tmp_path, """ + class P: + __slots__ = ("x", "x") + x: int + """, "P", "twice") + + +def test_base_class_is_rejected(tmp_path): + m = rejected(tmp_path, """ + class B: + pass + class P(B): + __slots__ = ("x",) + x: int + """, "P", "base") + assert all(c.name != "B" for c in m.classes) # B has no slots either + + +def test_metaclass_keyword_is_rejected(tmp_path): + rejected(tmp_path, """ + class P(metaclass=type): + __slots__ = ("x",) + x: int + """, "P", "metaclass") + + +def test_decorator_other_than_compiled_is_rejected(tmp_path): + rejected(tmp_path, """ + import dataclasses + @dataclasses.dataclass + class P: + __slots__ = ("x",) + x: int + """, "P", "decorator") + + +def test_subclass_in_the_module_rejects_the_base(tmp_path): + m = rejected(tmp_path, """ + class P: + __slots__ = ("x",) + x: int + class Q(P): + __slots__ = () + """, "P", "subclass") + assert m.classes == () + + +def test_subclass_nested_in_a_function_still_counts(tmp_path): + rejected(tmp_path, """ + class P: + __slots__ = ("x",) + x: int + def make(): + class Q(P): + pass + return Q + """, "P", "subclass") + + +def test_class_attribute_assignment_is_rejected(tmp_path): + rejected(tmp_path, """ + class P: + __slots__ = ("x",) + x: int + P.count = 0 + """, "P", "class attribute") + + +def test_class_attribute_assignment_in_a_function_is_rejected(tmp_path): + rejected(tmp_path, """ + class P: + __slots__ = ("x",) + x: int + def f() -> None: + setattr(P, "count", 1) + """, "P", "class attribute") + + +def test_class_level_value_is_rejected(tmp_path): + rejected(tmp_path, """ + class P: + __slots__ = ("x",) + x: int + limit = 3 + """, "P", "class body") + + +def test_rebound_class_name_is_rejected(tmp_path): + rejected(tmp_path, """ + class P: + __slots__ = ("x",) + x: int + P = 3 + """, "P", "bound more than once") + + +def test_global_rebinding_in_a_function_is_rejected(tmp_path): + rejected(tmp_path, """ + class P: + __slots__ = ("x",) + x: int + def f() -> None: + global P + P = None + """, "P", "bound more than once") + + +@pytest.mark.parametrize("dunder", ["__getattribute__", "__setattr__", "__getattr__", "__new__"]) +def test_slot_access_overrides_are_rejected(tmp_path, dunder): + # these run user code on a slot read/write/construct, which the direct slot access would skip + rejected(tmp_path, f""" + class P: + __slots__ = ("x",) + x: int + def {dunder}(self, *a): + return 0 + """, "P", dunder) + + +# --- trivial_init ------------------------------------------------------------------------------------ + +def test_init_with_free_parameter_names_and_statement_order_is_trivial(tmp_path): + m = lower(tmp_path, """ + class P: + __slots__ = ("x", "y") + x: int + y: int + def __init__(self, a: int, b: int) -> None: + self.y = b + self.x = a + """) + assert cls(m, "P").trivial_init is True + + +@pytest.mark.parametrize("body, why", [ + ("self.x = x", "assigns 1 of 2"), # a slot left unset + ("self.x = x\n self.y = y\n self.x = x", "more than once"), + ("self.x = x\n self.y = x + 1", "exactly"), # not a plain parameter + ("self.x = x\n self.y = y\n print(1)", "exactly"), # extra statement + ("self.x = y\n self.y = x", "order"), # params not in slot order +]) +def test_non_trivial_init_keeps_the_class_but_not_new(tmp_path, body, why): # `why` documents the case + m = lower(tmp_path, f""" + class P: + __slots__ = ("x", "y") + x: int + y: int + def __init__(self, x: int, y: int) -> None: + {body} + @compiled + def make(a: int) -> P: + return P(a, a) + """) + assert cls(m, "P") == ClassDecl("P", ("x", "y"), False) + f = function(m, "make") + assert nodes(f, New) == [] + assert f.body == (Return(CallObject(OBJ, Global(OBJ, "P"), ( + Box(OBJ, Local(I64, "a")), Box(OBJ, Local(I64, "a"))))),) + assert not f.pure + + +def test_init_with_defaults_or_extra_params_is_not_trivial(tmp_path): + m = lower(tmp_path, """ + class P: + __slots__ = ("x",) + x: int + def __init__(self, x: int = 0) -> None: + self.x = x + """) + assert cls(m, "P").trivial_init is False + + +# --- None and identity ------------------------------------------------------------------------------- + +def test_none_literal_is_an_obj_const(tmp_path): + m = lower(tmp_path, POINT + """ +@compiled +def nothing() -> P | None: + return None +""") + f = function(m, "nothing") + assert f.body == (Return(Const(OBJ, None)),) + assert f.pure and not f.may_deopt + + +def test_is_none_and_is_not_none_and_identity(tmp_path): + m = lower(tmp_path, POINT + """ +@compiled +def f(a: P | None, b: P | None) -> int: + if a is None: + return 0 + if a is not b: + return 1 + return 2 +""") + f = function(m, "f") + none = Const(OBJ, None) + assert f.body[0] == If(Is(BOOL, Local(OBJ, "a"), none), (Return(Const(I64, 0)),)) + assert f.body[1] == If(Is(BOOL, Local(OBJ, "a"), Local(OBJ, "b"), True), (Return(Const(I64, 1)),)) + assert f.pure and not f.may_deopt + + +def test_is_in_and_or_conditions(tmp_path): + m = lower(tmp_path, POINT + """ +@compiled +def f(a: P | None, b: P | None) -> int: + if a is None or b is None: + return 0 + return 1 +""") + f = function(m, "f") + none = Const(OBJ, None) + assert f.body[0] == If(Or(BOOL, Is(BOOL, Local(OBJ, "a"), none), Is(BOOL, Local(OBJ, "b"), none)), + (Return(Const(I64, 0)),)) + + +def test_identity_of_a_scalar_is_not_lowered(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(a: int) -> bool: + return a is None + """) + assert "f" not in [fn.name for fn in m.functions] + assert "identity" in m.skipped["f"] + + +# --- parameters and locals ----------------------------------------------------------------------------- + +def test_class_parameters_are_obj_with_cls_and_optional(tmp_path): + m = lower(tmp_path, "from typing import Optional\n" + POINT + """ +@compiled +def f(a: P, b: P | None, c: "P | None", d: Optional[P], e: None | P) -> int: + return 1 +""") + f = function(m, "f") + assert f.params == ( + Param("a", OBJ, "P", False), Param("b", OBJ, "P", True), Param("c", OBJ, "P", True), + Param("d", OBJ, "P", True), Param("e", OBJ, "P", True)) + + +def test_annotation_naming_a_rejected_class_is_a_plain_object(tmp_path): + m = lower(tmp_path, """ + class P: + x: int + @compiled + def f(a: P) -> int: + return 1 + """) + assert function(m, "f").params == (Param("a", OBJ),) + + +# --- field reads ----------------------------------------------------------------------------------------- + +def test_field_read_of_a_non_optional_parameter_is_proved(tmp_path): + m = lower(tmp_path, POINT + """ +@compiled +def f(p: P) -> object: + return p.x +""") + f = function(m, "f") + assert f.body == (Return(FieldGet(OBJ, Local(OBJ, "p"), "P", "x")),) + assert f.pure and not f.may_deopt # a slot read is not an effect, nothing deopts + assert nodes(f, CheckExact) == [] and nodes(f, GetAttr) == [] + + +def test_field_read_of_an_optional_parameter_is_checked_in_a_pure_function(tmp_path): + m = lower(tmp_path, POINT + """ +@compiled +def f(p: P | None) -> object: + return p.x +""") + f = function(m, "f") + assert f.body == (Return(FieldGet(OBJ, CheckExact(OBJ, Local(OBJ, "p"), "P"), "P", "x")),) + assert f.pure and f.may_deopt + + +def test_field_read_after_a_rebinding_is_not_trusted(tmp_path): + m = lower(tmp_path, POINT + """ +@compiled +def f(p: P, q: P | None) -> object: + p = q + return p.x +""") + f = function(m, "f") + assert len(nodes(f, CheckExact)) == 1 and f.may_deopt + + +def test_unproved_field_read_in_an_impure_function_is_skipped(tmp_path): + m = lower(tmp_path, POINT + """ +@compiled +def f(p: P | None) -> object: + print(1) + return p.x +""") + assert "f" not in [fn.name for fn in m.functions] + assert "not proved" in m.skipped["f"] + + +def test_proved_field_read_in_an_impure_function_is_lowered(tmp_path): + m = lower(tmp_path, POINT + """ +@compiled +def f(p: P) -> object: + print(1) + return p.x +""") + f = function(m, "f") + assert nodes(f, FieldGet) == [FieldGet(OBJ, Local(OBJ, "p"), "P", "x")] + assert not f.pure and not f.may_deopt + + +def test_attribute_that_is_not_a_field_stays_getattr(tmp_path): + m = lower(tmp_path, POINT + """ + def norm(self) -> float: + return self.x + +""" + """ +@compiled +def f(p: P) -> object: + return p.norm +""") + f = function(m, "f") + assert nodes(f, GetAttr) and nodes(f, FieldGet) == [] + assert not f.pure + + +def test_fresh_new_local_is_proved_exact(tmp_path): + m = lower(tmp_path, POINT + """ +@compiled +def f(a: float) -> object: + p = P(a, a) + return p.y +""") + f = function(m, "f") + assert nodes(f, FieldGet) == [FieldGet(OBJ, Local(OBJ, "p"), "P", "y")] + assert nodes(f, CheckExact) == [] and f.pure and not f.may_deopt + + +# --- field writes ------------------------------------------------------------------------------------------ + +def test_field_write_is_an_effect(tmp_path): + m = lower(tmp_path, POINT + """ +@compiled +def f(p: P, v: float) -> None: + p.x = v +""") + f = function(m, "f") + assert f.body == (FieldSet(Local(OBJ, "p"), "P", "x", Box(OBJ, Local(F64, "v"))),) + assert not f.pure and not f.may_deopt + + +def test_field_write_on_an_unproved_object_is_skipped(tmp_path): + m = lower(tmp_path, POINT + """ +@compiled +def f(p: P | None, v: float) -> None: + p.x = v +""") + assert "f" not in [fn.name for fn in m.functions] + assert "not proved" in m.skipped["f"] + + +def test_write_to_an_attribute_of_an_unknown_object_is_still_skipped(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(p: object) -> None: + p.x = 1 + """) + assert "attribute" in m.skipped["f"] + + +# --- construction ------------------------------------------------------------------------------------------- + +def test_construction_with_a_trivial_init_is_new_in_field_order(tmp_path): + m = lower(tmp_path, POINT + """ +@compiled +def f(a: float, b: float) -> object: + return P(a, b) +""") + f = function(m, "f") + assert f.body == (Return(New(OBJ, "P", (Box(OBJ, Local(F64, "a")), Box(OBJ, Local(F64, "b"))))),) + assert f.pure and not f.may_deopt # allocating is not an effect + + +def test_construction_with_wrong_arity_or_keywords_is_a_plain_call(tmp_path): + m = lower(tmp_path, POINT + """ +@compiled +def f(a: float) -> object: + return P(a) + +@compiled +def g(a: float) -> object: + return P(x=a, y=a) +""") + for name in ("f", "g"): + assert nodes(function(m, name), New) == [] + assert len(nodes(function(m, name), CallObject)) == 1 + + +# --- the binary-trees benchmark ------------------------------------------------------------------------------ + +def test_binary_trees_with_slots_lowers_make_and_check(tmp_path): + source = BINARY_TREES.read_text() + assert "class Node:\n" + SLOTS_LINE in source # the benchmark itself declares the slots + path = tmp_path / "binary_trees.py" + path.write_text(MARK + source) + m = frontend.lower(path) + + assert cls(m, "Node") == ClassDecl("Node", ("left", "right"), True) + make, check = function(m, "make"), function(m, "check") + assert make.pure and check.pure + assert make.params == (Param("depth", I64),) + assert check.params == (Param("node", OBJ, "Node", False),) + assert len(nodes(make, New)) == 2 + assert sorted(nodes(check, FieldGet), key=lambda g: g.field) == [ + FieldGet(OBJ, Local(OBJ, "node"), "Node", "left"), + FieldGet(OBJ, Local(OBJ, "node"), "Node", "right")] + assert nodes(check, Is) + assert nodes(check, CallObject) == [] and nodes(check, GetAttr) == [] + # report: make() and check() both may deopt (i64 arithmetic, and `check(left)` passes an + # `Node | None` local to a parameter guarded `type(x) is Node`) + print(f"make.may_deopt={make.may_deopt} check.may_deopt={check.may_deopt}") + assert make.may_deopt and check.may_deopt + + +def test_binary_trees_without_slots_stays_object_code(tmp_path): + path = tmp_path / "binary_trees.py" + source = BINARY_TREES.read_text() + assert SLOTS_LINE in source + path.write_text(MARK + source.replace(SLOTS_LINE, "", 1)) # the run.py no-slots baseline + m = frontend.lower(path) + assert m.classes == () + assert "`__slots__`" in m.skipped["Node"] + # as before: object code. `check` reads attributes (impure), so its int arithmetic would be + # CPython ints and it cannot return a native int: it stays interpreted; `make` calls `Node`. + assert "check" not in [f.name for f in m.functions] + make = function(m, "make") + assert nodes(make, New) == [] and nodes(make, CallObject) and not make.pure diff --git a/python-multiplatform-ksp/src/test/python/test_verify.py b/python-multiplatform-ksp/src/test/python/test_verify.py index 38063296c..e2df11290 100644 --- a/python-multiplatform-ksp/src/test/python/test_verify.py +++ b/python-multiplatform-ksp/src/test/python/test_verify.py @@ -646,7 +646,7 @@ def test_scalar_into_an_obj_slot_without_box_is_rejected(): def test_obj_from_a_node_that_does_not_create_a_reference_is_rejected(): - assert_rejected(OWNERSHIP, fn(returns=OBJ, body=[Return(Const(OBJ, None))])) + # Const(OBJ, None) creates a reference since N-11 (`return None`); see test_verify_classes.py. assert_rejected(OWNERSHIP, fn(params=[Param("o", OBJ)], returns=OBJ, body=[Return(UnaryOp(OBJ, UnaryOpKind.NEG, lo("o")))])) assert_rejected(OWNERSHIP, fn(params=[ArrayParam("a", F64A, stored=False)], returns=OBJ, diff --git a/python-multiplatform-ksp/src/test/python/test_verify_classes.py b/python-multiplatform-ksp/src/test/python/test_verify_classes.py new file mode 100644 index 000000000..78132918c --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_verify_classes.py @@ -0,0 +1,728 @@ +"""The IR safety verifier for fixed-layout classes (SPEC N-9 safety + N-11; IR contract 158a9b4b). + +Every test builds IR by hand: the verifier must prove, never trust the front end, that + + * every class named by FieldGet/FieldSet/New/IsExact/CheckExact/Param.cls is a ClassDecl of the + module, every field named belongs to it, and New is used only for trivial_init classes with one + OBJ argument per field; + * a FieldGet/FieldSet's object is proved exactly the class (`verify/field-unproved` otherwise); + * CheckExact (can deopt) is only in pure functions, FieldSet is an effect, FieldGet/Is/IsExact/New + are not; `pure` and `may_deopt` are recomputed and compared; + * the node types are right and the OBJ-creating nodes follow the ownership rule. + +The last section mutates correct class-using functions at random (seeded) and checks that every +property-breaking mutation is caught. +""" +from __future__ import annotations + +import dataclasses +import random + +import pytest + +from typedpython import ir +from typedpython.ir import ( + And, ArrayParam, Assign, Break, CheckExact, ClassDecl, Const, ExprStmt, FieldGet, FieldSet, + ForRange, Function, GetAttr, Global, If, Is, IsExact, Len, Local, Module, New, Or, Param, + Return, StoreIndex, Type, UnaryOp, UnaryOpKind, While, CallObject, Box, Call, BinOp, BinOpKind, +) + +I64, F64, BOOL, NONE, OBJ = Type.I64, Type.F64, Type.BOOL, Type.NONE, Type.OBJ +F64A = Type.F64_ARRAY + +TYPE = "verify/type" +OWNERSHIP = "verify/ownership" +DEOPT = "verify/deopt-impure" +FLAG_DEOPT = "verify/flag-may-deopt" +FLAG_PURE = "verify/flag-pure" +STRUCTURE = "verify/structure" +CLASS = "verify/class" +FIELD = "verify/field-unproved" +ENTRY_OPEN = "verify/entry-global-open" +UNVERIFIED = "verify/call-unverified" + +POINT = ClassDecl("Point", ("x", "y"), True) +OTHER = ClassDecl("Other", ("x", "z"), True) +SLOWINIT = ClassDecl("SlowInit", ("a",), False) +CLASSES = (POINT, OTHER, SLOWINIT) + + +def _verify(): + # Imported lazily so that, before the class rules exist, each test fails on its own assertion. + from typedpython import verify + return verify + + +# --- builders ------------------------------------------------------------------------------------ + +def lo(n): return Local(OBJ, n) +def li(n): return Local(I64, n) +def g_(n): return Global(OBJ, n) +def fget(obj, field, cls="Point"): return FieldGet(OBJ, obj, cls, field) +def fset(obj, field, value, cls="Point"): return FieldSet(obj, cls, field, value) +def exact(obj, cls="Point"): return IsExact(BOOL, obj, cls) +def check(obj, cls="Point"): return CheckExact(OBJ, obj, cls) +def new(*args, cls="Point"): return New(OBJ, cls, tuple(args)) +def none(): return Const(OBJ, None) +def is_none(x, negate=False): return Is(BOOL, x, none(), negate) +def not_(x): return UnaryOp(BOOL, UnaryOpKind.NOT, x) +def pt(name="p", optional=False, cls="Point"): return Param(name, OBJ, cls, optional) +def obj(name="q"): return Param(name, OBJ) +def band(a, b): return And(BOOL, a, b) +def bor(a, b): return Or(BOOL, a, b) +IMPURE = ExprStmt(CallObject(OBJ, Global(OBJ, "print"), (), ())) + + +def fn(name="f", body=(), params=(), locals_=None, returns=OBJ, pure=True, may_deopt=False, + line=10, entry_globals=()): + return Function(name=name, params=tuple(params), returns=returns, locals=dict(locals_ or {}), + body=tuple(body), pure=pure, may_deopt=may_deopt, source_line=line, + entry_globals=tuple(entry_globals)) + + +def run(*functions, classes=CLASSES): + return _verify().verify(Module("m", tuple(functions), {}, tuple(classes))) + + +def kept(out): + return [g.name for g in out.functions] + + +def rules_for(diags, name): + return {d.rule for d in diags if d.function == name} + + +def accepted(*functions, classes=CLASSES): + out, diags = run(*functions, classes=classes) + assert kept(out) == [g.name for g in functions], [dataclasses.astuple(d) for d in diags] + assert diags == [] + return out + + +def rejected(rule, *functions, name="f", classes=CLASSES): + out, diags = run(*functions, classes=classes) + assert name not in kept(out) + assert name in out.skipped + assert rule in rules_for(diags, name), [dataclasses.astuple(d) for d in diags] + return out, diags + + +def getter(body, params=(pt(),), locals_=None, **kw): + """A pure function returning an OBJ read from fields (the common shape of these tests).""" + return fn(body=body, params=params, locals_=locals_, **kw) + + +# --- the module keeps its classes ---------------------------------------------------------------- + +def test_the_verified_module_keeps_its_valid_classes(): + out = accepted(getter([Return(fget(lo("p"), "x"))])) + assert out.classes == CLASSES + + +# --- 1. class and field names -------------------------------------------------------------------- + +def test_a_function_without_classes_is_unaffected(): + accepted(fn(params=[obj()], body=[Return(lo("q"))]), classes=()) + + +@pytest.mark.parametrize("stmt", [ + lambda: Return(fget(lo("p"), "x", cls="Nowhere")), + lambda: Return(new(g_("a"), g_("b"), cls="Nowhere")), + lambda: If(exact(lo("p"), "Nowhere"), (Return(g_("a")),)), + lambda: Return(check(lo("p"), "Nowhere")), + lambda: fset(lo("p"), "x", g_("v"), cls="Nowhere"), +]) +def test_every_node_must_name_a_class_of_the_module(stmt): + rejected(CLASS, fn(params=[pt()], pure=False, body=[stmt(), Return(g_("r"))])) + + +def test_a_param_may_only_name_a_class_of_the_module(): + rejected(CLASS, fn(params=[pt(cls="Nowhere")], body=[Return(lo("p"))])) + rejected(CLASS, fn(params=[pt(cls="Nowhere", optional=True)], body=[Return(lo("p"))])) + + +def test_a_field_must_belong_to_the_named_class(): + rejected(CLASS, getter([Return(fget(lo("p"), "z"))])) # z is Other's field + rejected(CLASS, fn(params=[pt()], pure=False, body=[fset(lo("p"), "nope", g_("v")), Return(lo("p"))])) + accepted(getter([Return(fget(lo("p"), "y"))])) + + +def test_new_needs_a_trivial_init_class_and_one_object_per_field(): + accepted(getter([Return(new(g_("a"), g_("b")))], params=())) + rejected(CLASS, getter([Return(new(g_("a")))], params=())) + rejected(CLASS, getter([Return(new(g_("a"), g_("b"), g_("c")))], params=())) + rejected(CLASS, getter([Return(new(cls="Point"))], params=())) + rejected(CLASS, getter([Return(new(g_("a"), cls="SlowInit"))], params=())) # not trivial_init + accepted(getter([Return(new(g_("a"), g_("b"), cls="Other"))], params=())) + + +def test_new_arguments_must_be_objects(): + rejected(OWNERSHIP, getter([Return(new(Const(I64, 1), g_("b")))], params=())) + accepted(getter([Return(new(Box(OBJ, Const(I64, 1)), g_("b")))], params=())) + + +def test_param_cls_only_on_obj_params_and_optional_only_with_cls(): + rejected(TYPE, fn(params=[Param("n", I64, "Point")], body=[Return(g_("r"))])) + rejected(TYPE, fn(params=[Param("n", OBJ, None, True)], body=[Return(lo("n"))])) + rejected(TYPE, fn(params=[Param("n", OBJ, "Point", 1)], body=[Return(lo("n"))])) + rejected(TYPE, fn(params=[], entry_globals=[Param("N", I64, "Point")], returns=NONE, + body=[Return()])) + + +@pytest.mark.parametrize("bad, why", [ + (ClassDecl("Dup", ("a",), True), "dup"), + (ClassDecl("Twice", ("a", "a"), True), "field twice"), + (ClassDecl("not an id", ("a",), True), "name"), + (ClassDecl("Fld", ("a b",), True), "field name"), + (ClassDecl("Ti", ("a",), 1), "trivial_init"), +]) +def test_a_malformed_class_decl_is_reported_and_unusable(bad, why): + classes = (bad, bad) if why == "dup" else (bad,) + f = getter([Return(fget(lo("p"), "a", cls=bad.name))], params=(pt(cls=bad.name),)) + out, diags = run(f, classes=classes) + assert any(d.rule == CLASS and d.function == bad.name for d in diags), diags + assert bad.name not in [c.name for c in out.classes] + assert "f" not in kept(out) # the function that used it is not compiled + assert CLASS in rules_for(diags, "f") + + +def test_a_class_with_the_name_of_a_function_is_rejected(): + out, diags = run(getter([Return(lo("p"))], params=(pt(),), name="Point")) + assert any(d.rule == CLASS and d.function == "Point" for d in diags) + assert out.classes == (OTHER, SLOWINIT) + + +# --- 2. the exact-type proof --------------------------------------------------------------------- + +def test_exact_param_is_proved(): + accepted(getter([Return(fget(lo("p"), "x"))])) + + +def test_a_param_exact_proof_is_for_its_own_class(): + rejected(FIELD, getter([Return(fget(lo("p"), "x", cls="Other"))])) + + +def test_a_plain_object_param_is_not_proved(): + rejected(FIELD, getter([Return(fget(lo("q"), "x"))], params=(obj(),))) + + +def test_an_optional_param_alone_is_not_proved(): + rejected(FIELD, getter([Return(fget(lo("p"), "x"))], params=(pt(optional=True),))) + + +def test_check_exact_is_the_proof_for_its_own_class_only(): + accepted(getter([Return(fget(check(lo("q")), "x"))], params=(obj(),), may_deopt=True)) + rejected(FIELD, getter([Return(fget(check(lo("q"), "Other"), "x"))], params=(obj(),), + may_deopt=True)) + + +def test_new_is_the_proof_for_its_own_class_only(): + accepted(getter([Return(fget(new(g_("a"), g_("b")), "x"))], params=())) + rejected(FIELD, getter([Return(fget(new(g_("a"), g_("b"), cls="Other"), "x"))], params=())) + + +def test_a_local_assigned_from_a_new_or_check_exact_is_proved_until_reassigned(): + accepted(getter([Assign("t", new(g_("a"), g_("b"))), Return(fget(lo("t"), "x"))], + params=(), locals_={"t": OBJ})) + accepted(getter([Assign("t", check(lo("q"))), Return(fget(lo("t"), "x"))], params=(obj(),), + locals_={"t": OBJ}, may_deopt=True)) + rejected(FIELD, getter([Assign("t", new(g_("a"), g_("b"))), Assign("t", g_("other")), + Return(fget(lo("t"), "x"))], params=(), locals_={"t": OBJ})) + # a copy of a proved local is proved; a copy of an unproved one is not + accepted(getter([Assign("t", lo("p")), Return(fget(lo("t"), "x"))], locals_={"t": OBJ})) + rejected(FIELD, getter([Assign("t", lo("q")), Return(fget(lo("t"), "x"))], params=(obj(),), + locals_={"t": OBJ})) + + +def test_a_local_assigned_only_new_of_one_class_is_proved_everywhere(): + two = [If(exact(lo("q")), (Assign("t", new(g_("a"), g_("b"))),), + (Assign("t", new(g_("c"), g_("d"))),)), Return(fget(lo("t"), "x"))] + accepted(getter(two, params=(obj(),), locals_={"t": OBJ})) + # in a loop, flow-insensitively: the read comes before the second assignment in the text + loop = [Assign("t", new(g_("a"), g_("b"))), Assign("u", g_("o")), + While(exact(g_("zz")), (Assign("u", fget(lo("t"), "x")), Assign("t", new(g_("c"), g_("d"))), + Break())), Return(lo("u"))] + accepted(getter(loop, params=(), locals_={"t": OBJ, "u": OBJ}, pure=False)) + + +def test_one_other_assignment_to_the_local_ends_the_new_only_form(): + other = [Assign("t", new(g_("a"), g_("b"))), If(exact(g_("zz")), (Assign("t", g_("o")),)), + Return(fget(lo("t"), "x"))] + rejected(FIELD, getter(other, params=(), locals_={"t": OBJ}, pure=False)) + mixed = [Assign("t", new(g_("a"), g_("b"))), If(exact(g_("zz")), + (Assign("t", new(g_("a"), g_("b"), cls="Other")),)), + Return(fget(lo("t"), "x"))] + rejected(FIELD, getter(mixed, params=(), locals_={"t": OBJ}, pure=False)) + # a parameter is never new-only, even if every body assignment is a New + rejected(FIELD, getter([If(exact(g_("zz")), (Assign("q", new(g_("a"), g_("b"))),)), + Return(fget(lo("q"), "x"))], params=(obj(),), pure=False)) + # the flow-sensitive forms still apply to a local with other assignments: the last assignment + # before the read is a New, so form (d) proves it (the front end need not rely on this) + accepted(getter([Assign("t", g_("o")), Assign("t", new(g_("a"), g_("b"))), + Return(fget(lo("t"), "x"))], params=(), locals_={"t": OBJ})) + + +def test_a_reassigned_param_loses_its_proof(): + rejected(FIELD, getter([Assign("p", g_("other")), Return(fget(lo("p"), "x"))])) + accepted(getter([Assign("p", new(g_("a"), g_("b"))), Return(fget(lo("p"), "x"))])) + + +def test_a_dominating_is_exact_is_the_proof(): + body = [If(exact(lo("q")), (Return(fget(lo("q"), "x")),)), Return(g_("r"))] + accepted(getter(body, params=(obj(),))) + + +def test_is_exact_proves_only_its_branch_local_and_class(): + p = (obj(),) + rejected(FIELD, getter([If(exact(lo("q")), (), (Return(fget(lo("q"), "x")),)), + Return(g_("r"))], params=p)) # else branch + rejected(FIELD, getter([If(exact(lo("q")), ()), Return(fget(lo("q"), "x"))], params=p)) + rejected(FIELD, getter([If(exact(lo("q"), "Other"), (Return(fget(lo("q"), "x")),)), + Return(g_("r"))], params=p)) # wrong class + rejected(FIELD, getter([If(exact(lo("q")), (Return(fget(lo("w"), "x")),)), Return(g_("r"))], + params=(obj(), obj("w")))) # another local + rejected(FIELD, getter([If(exact(g_("k")), (Return(fget(g_("k"), "x")),)), Return(g_("r"))], + params=p)) # not a local at all + + +def test_an_assignment_between_the_guard_and_the_access_kills_the_proof(): + body = [If(exact(lo("q")), (Assign("q", g_("o")), Return(fget(lo("q"), "x")))), Return(g_("r"))] + rejected(FIELD, getter(body, params=(obj(),))) + body = [If(exact(lo("q")), (Assign("q", new(g_("a"), g_("b"))), Return(fget(lo("q"), "x")))), + Return(g_("r"))] + accepted(getter(body, params=(obj(),))) # ... unless it is itself proved + + +def test_the_condition_forms_that_prove(): + p = (obj(),) + # And: the left test holds in the right operand and in the then branch + accepted(getter([If(band(exact(lo("q")), Is(BOOL, fget(lo("q"), "x"), g_("k"))), + (Return(fget(lo("q"), "y")),)), Return(g_("r"))], params=p)) + # `not exact` false in the else + accepted(getter([If(not_(exact(lo("q"))), (), (Return(fget(lo("q"), "x")),)), + Return(g_("r"))], params=p)) + # Or: the right operand runs when the left was false + accepted(getter([If(bor(not_(exact(lo("q"))), Is(BOOL, fget(lo("q"), "x"), g_("k"))), + (Return(g_("r")),)), Return(g_("r"))], params=p)) + # ... but not the other way round + rejected(FIELD, getter([If(bor(exact(lo("q")), Is(BOOL, fget(lo("q"), "x"), g_("k"))), + (Return(g_("r")),)), Return(g_("r"))], params=p)) + rejected(FIELD, getter([If(band(Is(BOOL, fget(lo("q"), "x"), g_("k")), exact(lo("q"))), + (Return(g_("r")),)), Return(g_("r"))], params=p)) + + +def test_the_proof_survives_a_branch_that_leaves(): + p = (obj(),) + accepted(getter([If(not_(exact(lo("q"))), (Return(g_("r")),)), Return(fget(lo("q"), "x"))], + params=p)) + # a fall-through branch does not + rejected(FIELD, getter([If(not_(exact(lo("q"))), (Assign("t", g_("r")),)), + Return(fget(lo("q"), "x"))], params=p, locals_={"t": OBJ})) + + +def test_a_fact_both_branches_establish_survives_the_join_and_one_branch_does_not(): + def join(else_value): + return getter([If(exact(lo("q")), (Assign("t", lo("q")),), (Assign("t", else_value),)), + Return(fget(lo("t"), "x"))], params=(obj(),), locals_={"t": OBJ}) + accepted(join(new(g_("a"), g_("b")))) + rejected(FIELD, join(g_("o"))) + + +def test_optional_param_is_not_none_proves_the_class(): + p = (pt(optional=True),) + accepted(getter([If(is_none(lo("p"), negate=True), (Return(fget(lo("p"), "x")),)), + Return(g_("r"))], params=p)) + accepted(getter([If(is_none(lo("p")), (Return(g_("r")),)), Return(fget(lo("p"), "x"))], + params=p)) + accepted(getter([If(is_none(lo("p")), (Return(g_("r")),), (Return(fget(lo("p"), "x")),)), + Return(g_("r"))], params=p)) + # the `None` may stand on either side + accepted(getter([If(Is(BOOL, none(), lo("p"), True), (Return(fget(lo("p"), "x")),)), + Return(g_("r"))], params=p)) + + +def test_the_none_test_proves_nothing_where_the_param_is_not_a_never_assigned_optional_class(): + rejected(FIELD, getter([If(is_none(lo("q"), negate=True), (Return(fget(lo("q"), "x")),)), + Return(g_("r"))], params=(obj(),))) + rejected(FIELD, getter([If(is_none(lo("p")), (Return(fget(lo("p"), "x")),)), Return(g_("r"))], + params=(pt(optional=True),))) # wrong sense + rejected(FIELD, getter([If(is_none(lo("p"), negate=True), (Return(fget(lo("p"), "x")),)), + Assign("p", g_("o")), Return(g_("r"))], params=(pt(optional=True),))) + # reassigned anywhere in the function -> no fact, even where the assignment is later + rejected(FIELD, getter([If(is_none(lo("p"), negate=True), (Return(fget(lo("p"), "x")),)), + Assign("p", g_("o")), Return(g_("r"))], params=(pt(optional=True),))) + # the identity of another object is not a fact about this one + rejected(FIELD, getter([If(Is(BOOL, lo("p"), g_("k"), True), (Return(fget(lo("p"), "x")),)), + Return(g_("r"))], params=(pt(optional=True),))) + + +def test_loops_kill_what_the_body_assigns(): + p = (obj(),) + guard = lambda *body: getter([If(exact(lo("q")), (While(Const(BOOL, True), tuple(body)),)), + Return(g_("r"))], params=p, pure=False) + accepted(guard(fset(lo("q"), "x", g_("v")), Break())) + # q is assigned later in the body: at the head of the next iteration the guard no longer holds + rejected(FIELD, guard(fset(lo("q"), "x", g_("v")), Assign("q", g_("o")))) + rejected(FIELD, getter([If(exact(lo("q")), (ForRange("k", Const(I64, 0), Const(I64, 3), + Const(I64, 1), ( + fset(lo("q"), "x", g_("v")), Assign("q", g_("o")))),)), Return(g_("r"))], + params=p, locals_={"k": I64}, pure=False)) + + +def test_a_proof_before_a_loop_that_assigns_does_not_outlive_it(): + p = (obj(),) + loop = ForRange("k", Const(I64, 0), Const(I64, 3), Const(I64, 1), (Assign("q", g_("o")),)) + rejected(FIELD, getter([If(exact(lo("q")), (loop, Return(fget(lo("q"), "x")))), + Return(g_("r"))], params=p, locals_={"k": I64}, pure=False)) + quiet = ForRange("k", Const(I64, 0), Const(I64, 3), Const(I64, 1), (IMPURE,)) + accepted(getter([If(exact(lo("q")), (quiet, Return(fget(lo("q"), "x")))), Return(g_("r"))], + params=p, locals_={"k": I64}, pure=False)) + + +def test_a_proof_does_not_cross_a_call_or_another_nodes_result(): + # a FieldGet result is just an object + rejected(FIELD, getter([Return(fget(fget(lo("p"), "x"), "x"))])) + rejected(FIELD, getter([Return(fget(g_("o"), "x"))])) + rejected(FIELD, getter([Return(fget(CallObject(OBJ, g_("mk"), ()), "x"))], pure=False)) + + +def test_field_set_needs_the_same_proof(): + accepted(fn(params=[pt()], pure=False, body=[fset(lo("p"), "x", g_("v")), Return(lo("p"))])) + rejected(FIELD, fn(params=[obj("p")], pure=False, body=[fset(lo("p"), "x", g_("v")), + Return(lo("p"))])) + rejected(FIELD, fn(params=[pt(optional=True)], pure=False, + body=[fset(lo("p"), "x", g_("v")), Return(lo("p"))])) + accepted(fn(params=[obj("p")], pure=False, + body=[If(exact(lo("p")), (fset(lo("p"), "x", g_("v")),)), Return(lo("p"))])) + + +def test_field_set_value_must_be_an_object(): + rejected(OWNERSHIP, fn(params=[pt()], pure=False, + body=[fset(lo("p"), "x", Const(I64, 1)), Return(lo("p"))])) + accepted(fn(params=[pt()], pure=False, body=[fset(lo("p"), "x", Box(OBJ, Const(I64, 1))), + Return(lo("p"))])) + + +# --- 3. purity and deopt ------------------------------------------------------------------------- + +def test_check_exact_in_a_pure_function_is_a_deopt_point(): + accepted(getter([Return(check(lo("q")))], params=(obj(),), may_deopt=True)) + rejected(FLAG_DEOPT, getter([Return(check(lo("q")))], params=(obj(),), may_deopt=False)) + + +def test_check_exact_in_an_impure_function_is_rejected(): + rejected(DEOPT, fn(params=[obj()], pure=False, may_deopt=False, + body=[IMPURE, Return(check(lo("q")))])) + rejected(DEOPT, fn(params=[obj()], pure=False, may_deopt=True, + body=[IMPURE, Return(check(lo("q")))])) + + +def test_a_redundant_check_exact_still_counts_as_a_deopt_point(): + rejected(DEOPT, fn(params=[pt()], pure=False, body=[IMPURE, Return(check(lo("p")))])) + + +def test_a_caller_of_a_check_exact_function_inherits_its_deopt(): + callee = fn("g", params=[obj()], may_deopt=True, body=[Return(check(lo("q")))]) + impure = fn("f", params=[obj()], pure=False, body=[IMPURE, Return(Call(OBJ, "g", (lo("q"),)))]) + rejected(DEOPT, impure, callee) + accepted(fn("f", params=[obj()], may_deopt=True, body=[Return(Call(OBJ, "g", (lo("q"),)))]), + callee) + + +def test_a_call_whose_callee_guards_a_class_param_may_deopt_unless_the_argument_is_proved(): + callee = fn("g", params=[pt()], body=[Return(fget(lo("p"), "x"))]) + unproved = fn("f", params=[obj()], may_deopt=True, body=[Return(Call(OBJ, "g", (lo("q"),)))]) + accepted(unproved, callee) + rejected(FLAG_DEOPT, dataclasses.replace(unproved, may_deopt=False), callee) + rejected(DEOPT, fn("f", params=[obj()], pure=False, body=[IMPURE, + Return(Call(OBJ, "g", (lo("q"),)))]), + callee) + # proved arguments cannot fail the callee's guard: no deopt + for arg, params in ((lo("p"), [pt()]), (new(g_("a"), g_("b")), [])): + accepted(fn("f", params=params, may_deopt=False, body=[Return(Call(OBJ, "g", (arg,)))]), + callee) + accepted(fn("f", params=[obj()], may_deopt=True, + body=[Return(Call(OBJ, "g", (check(lo("q")),)))]), callee) # the CheckExact deopts + accepted(fn("f", params=[pt()], pure=False, body=[IMPURE, Return(Call(OBJ, "g", (lo("p"),)))]), + callee) + # the wrong class proves nothing for this callee + rejected(FLAG_DEOPT, fn("f", params=[pt("p", cls="Other")], may_deopt=False, + body=[Return(Call(OBJ, "g", (lo("p"),)))]), callee) + # an optional class param is satisfied by a proved exact argument too, not by an unproved one + opt = fn("g", params=[pt(optional=True)], body=[Return(g_("a"))]) + accepted(fn("f", params=[pt()], may_deopt=False, body=[Return(Call(OBJ, "g", (lo("p"),)))]), opt) + rejected(FLAG_DEOPT, fn("f", params=[obj()], may_deopt=False, + body=[Return(Call(OBJ, "g", (lo("q"),)))]), opt) + + +def test_field_set_is_an_effect(): + rejected(FLAG_PURE, fn(params=[pt()], pure=True, body=[fset(lo("p"), "x", g_("v")), + Return(lo("p"))])) + callee = fn("g", params=[pt()], pure=False, body=[fset(lo("p"), "x", g_("v")), Return(lo("p"))]) + rejected(FLAG_PURE, fn("f", params=[pt()], pure=True, body=[Return(Call(OBJ, "g", (lo("p"),)))]), + callee) + + +def test_field_set_makes_a_function_not_closed(): + rejected(ENTRY_OPEN, fn(params=[pt()], pure=False, returns=F64, entry_globals=[Param("G", F64)], + body=[fset(lo("p"), "x", g_("v")), Return(Local(F64, "G"))])) + + +def test_reads_identity_and_allocation_are_not_effects(): + accepted(getter([Return(fget(lo("p"), "x"))])) + accepted(getter([If(is_none(lo("q")), (Return(g_("a")),)), Return(g_("b"))], + params=(obj(),))) + accepted(getter([If(exact(lo("q")), (Return(g_("a")),)), Return(g_("b"))], params=(obj(),))) + accepted(getter([Return(new(g_("a"), g_("b")))], params=())) + accepted(fn(params=[pt()], returns=F64, entry_globals=[Param("G", F64)], + body=[Assign("t", fget(lo("p"), "x")), Return(Local(F64, "G"))], + locals_={"t": OBJ})) # a field read keeps a function closed + + +# --- 4. typing ----------------------------------------------------------------------------------- + +def test_is_and_is_exact_give_bool_over_objects(): + accepted(getter([If(Is(BOOL, lo("q"), g_("k")), (Return(g_("a")),)), Return(g_("b"))], + params=(obj(),))) + rejected(TYPE, getter([If(Is(OBJ, lo("q"), g_("k")), (Return(g_("a")),)), Return(g_("b"))], + params=(obj(),))) + rejected(OWNERSHIP, getter([If(Is(BOOL, Const(I64, 1), g_("k")), (Return(g_("a")),)), + Return(g_("b"))], params=(obj(),))) + rejected(TYPE, getter([If(Is(BOOL, lo("q"), g_("k"), 1), (Return(g_("a")),)), + Return(g_("b"))], params=(obj(),))) + rejected(TYPE, getter([If(IsExact(OBJ, lo("q"), "Point"), (Return(g_("a")),)), + Return(g_("b"))], params=(obj(),))) + rejected(OWNERSHIP, getter([If(IsExact(BOOL, Const(F64, 1.0), "Point"), (Return(g_("a")),)), + Return(g_("b"))], params=(obj(),))) + + +def test_field_get_new_and_check_exact_give_obj(): + for node in (fget(lo("p"), "x"), check(lo("p")), new(g_("a"), g_("b"))): + for wrong in (I64, F64, BOOL): + rejected(TYPE, getter([Return(dataclasses.replace(node, type=wrong))], + may_deopt=isinstance(node, CheckExact), returns=wrong)) + + +def test_the_operand_of_a_field_node_must_be_an_object(): + rejected(OWNERSHIP, getter([Return(fget(Const(I64, 1), "x"))], params=())) + rejected(OWNERSHIP, getter([Return(check(Const(I64, 1)))], params=(), may_deopt=True)) + rejected(OWNERSHIP, getter([If(exact(Const(I64, 1)), (Return(g_("a")),)), Return(g_("b"))], + params=())) + + +def test_none_is_an_obj_constant_and_nothing_else_is(): + rejected(TYPE, getter([If(Is(BOOL, lo("q"), Const(OBJ, 0)), (Return(g_("a")),)), + Return(g_("b"))], params=(obj(),))) + rejected(TYPE, getter([If(Is(BOOL, lo("q"), Const(OBJ, "x")), (Return(g_("a")),)), + Return(g_("b"))], params=(obj(),))) + # a None constant is a new reference to the singleton wherever an object may flow + accepted(getter([Return(none())], params=())) + accepted(getter([Assign("t", none()), Return(lo("t"))], params=(), locals_={"t": OBJ})) + accepted(getter([Return(new(none(), g_("b")))], params=())) + rejected(TYPE, getter([Return(Const(I64, None))], params=(), returns=I64)) + rejected(TYPE, getter([Return(Const(OBJ, 0))], params=())) + + +def test_a_none_constant_still_fits_no_scalar_slot(): + rejected(OWNERSHIP, getter([Return(none())], params=(), returns=F64)) + + +# --- 5. ownership -------------------------------------------------------------------------------- + +def test_new_field_get_and_check_exact_are_reference_creators(): + accepted(getter([Return(new(g_("a"), g_("b")))], params=())) + accepted(getter([Assign("t", fget(lo("p"), "x")), Return(lo("t"))], locals_={"t": OBJ})) + accepted(getter([Return(check(lo("q")))], params=(obj(),), may_deopt=True)) + + +def test_an_object_cannot_flow_into_a_scalar_slot_from_these_nodes(): + rejected(OWNERSHIP, getter([Return(fget(lo("p"), "x"))], returns=F64)) + rejected(OWNERSHIP, getter([Assign("n", fget(lo("p"), "x")), Return(li("n"))], + locals_={"n": I64}, returns=I64)) + + +# --- "nothing slips through": seeded mutation of class-using functions --------------------------- + +def _base_functions(): + pure = fn("mix", params=[pt("p"), obj("q")], locals_={"t": OBJ, "r": OBJ}, returns=OBJ, pure=True, + may_deopt=True, line=3, body=[ + Assign("t", check(lo("q"))), + Assign("r", new(fget(lo("t"), "x"), fget(lo("p"), "y"))), + If(exact(lo("q")), (Return(fget(lo("q"), "x")),)), + Return(fget(lo("r"), "y")), + ]) + impure = fn("move", params=[pt("p"), pt("o", optional=True), obj("n")], locals_={"t": OBJ}, + returns=NONE, pure=False, may_deopt=False, line=20, body=[ + IMPURE, + If(is_none(lo("o"), negate=True), + (fset(lo("o"), "x", fget(lo("p"), "y")),)), + If(not_(exact(lo("n"))), (Return(),)), + fset(lo("p"), "x", fget(lo("n"), "x")), + Assign("t", new(fget(lo("n"), "y"), g_("k"))), + fset(lo("t"), "y", lo("n")), + Return(), + ]) + return pure, impure + + +def test_the_unmutated_class_functions_are_accepted(): + for g in _base_functions(): + accepted(g) + + +def _paths(node, pred, path=()): + out = [] + if pred(node): + out.append((path, node)) + if dataclasses.is_dataclass(node) and not isinstance(node, type): + for fl in dataclasses.fields(node): + out += _paths(getattr(node, fl.name), pred, path + (fl.name,)) + elif isinstance(node, tuple): + for k, x in enumerate(node): + out += _paths(x, pred, path + (k,)) + return out + + +def _replace(node, path, new_): + if not path: + return new_ + head, rest = path[0], path[1:] + if isinstance(node, tuple): + return node[:head] + (_replace(node[head], rest, new_),) + node[head + 1:] + return dataclasses.replace(node, **{head: _replace(getattr(node, head), rest, new_)}) + + +def _classy(n): return isinstance(n, (FieldGet, FieldSet, New, IsExact, CheckExact)) + + +def _class_mutations(rng, g): + """(description, mutant, acceptable rules): each mutant breaks a class property by construction.""" + kind = rng.randrange(11) + sites = _paths(g.body, _classy) + path, node = rng.choice(sites) + if kind == 0: # a class that is not declared + return "unknown class", dataclasses.replace(g, body=_replace( + g.body, path, dataclasses.replace(node, cls="Nowhere"))), {CLASS} + if kind == 1: # a field that is not in the class + fields = _paths(g.body, lambda n: isinstance(n, (FieldGet, FieldSet))) + fpath, fnode = rng.choice(fields) + return "unknown field", dataclasses.replace(g, body=_replace( + g.body, fpath, dataclasses.replace(fnode, field="nope"))), {CLASS} + if kind == 2: # New with the wrong arity + news = _paths(g.body, lambda n: isinstance(n, New)) + npath, nnode = rng.choice(news) + args = nnode.args[:-1] if rng.random() < 0.5 else nnode.args + (g_("extra"),) + return "new arity", dataclasses.replace(g, body=_replace( + g.body, npath, dataclasses.replace(nnode, args=args))), {CLASS} + if kind == 3: # New of a class whose __init__ is not trivial + news = _paths(g.body, lambda n: isinstance(n, New)) + npath, nnode = rng.choice(news) + return "new nontrivial", dataclasses.replace(g, body=_replace( + g.body, npath, dataclasses.replace(nnode, cls="SlowInit", args=(g_("a"),)))), { + CLASS, FIELD} + if kind == 4: # CheckExact in an impure function / FieldSet in a pure one + if g.pure: + st = fset(lo("p"), "x", g_("v")) + return "fieldset in pure", dataclasses.replace(g, body=(st,) + g.body), {FLAG_PURE} + st = ExprStmt(check(lo("n"))) + return "checkexact in impure", dataclasses.replace(g, body=(IMPURE, st) + g.body[1:]), { + DEOPT} + if kind == 5: # drop the entry guard of the exact param + ps = tuple(dataclasses.replace(p, cls=None) if p.name == "p" else p for p in g.params) + return "drop param guard", dataclasses.replace(g, params=ps), {FIELD} + if kind == 6: # drop the IsExact / is-not-None guard: run the then branch unconditionally + ifs = _paths(g.body, lambda n: isinstance(n, If) and _guards(n.cond)) + ipath, inode = rng.choice(ifs) + # the then body replaces the If inside its block + block_path, idx = ipath[:-1], ipath[-1] + block = _replace_get(g.body, block_path) + new_block = block[:idx] + inode.then + block[idx + 1:] + return "drop guard", dataclasses.replace(g, body=_replace(g.body, block_path, new_block)), { + FIELD} + if kind == 7: # assign over a proved local right before its use + body = g.body + victim = "t" if g.pure else "p" + k = next(j for j, s in enumerate(body) + if _uses(s, victim) and not (isinstance(s, Assign) and s.target == victim)) + st = Assign(victim, g_("unknown")) + return "kill proof", dataclasses.replace(g, body=body[:k] + (st,) + body[k:]), {FIELD} + if kind == 8: # break the pure/may_deopt flags + if g.pure: + return "flip may_deopt", dataclasses.replace(g, may_deopt=False), {FLAG_DEOPT} + return "flip pure", dataclasses.replace(g, pure=True), {FLAG_PURE} + if kind == 9: # retype a class node + path, node = rng.choice(_paths(g.body, lambda n: isinstance(n, ir.Expr) and _classy(n))) + other = rng.choice([t for t in (I64, F64, BOOL, NONE) if t is not node.type + and (t is not OBJ)]) + if isinstance(node, IsExact): + other = rng.choice([OBJ, I64]) + return "retype", dataclasses.replace(g, body=_replace( + g.body, path, dataclasses.replace(node, type=other))), {TYPE, OWNERSHIP} + # kind 10: an operand that is a scalar + objs = _paths(g.body, lambda n: isinstance(n, (FieldGet, FieldSet, IsExact, CheckExact))) + opath, onode = rng.choice(objs) + return "scalar operand", dataclasses.replace(g, body=_replace( + g.body, opath, dataclasses.replace(onode, obj=Const(I64, 1)))), {OWNERSHIP, TYPE} + + +def _guards(cond): + return (isinstance(cond, IsExact) or isinstance(cond, Is) + or (isinstance(cond, UnaryOp) and isinstance(cond.operand, IsExact))) + + +def _replace_get(node, path): + for step in path: + node = node[step] if isinstance(node, tuple) else getattr(node, step) + return node + + +def _uses(stmt, name): + return bool(_paths(stmt, lambda n: isinstance(n, Local) and n.name == name)) + + +@pytest.mark.parametrize("seed", range(40)) +def test_every_property_breaking_class_mutation_is_rejected(seed): + rng = random.Random(seed) + base = _base_functions() + seen = set() + for _ in range(60): + g = rng.choice(base) + try: + desc, mutant, rules = _class_mutations(rng, g) + except (IndexError, StopIteration): + continue # the mutation has no site in this function + out, diags = run(mutant) + seen.add(desc) + assert mutant.name not in kept(out), f"{desc} was accepted" + assert rules & rules_for(diags, mutant.name), (desc, rules, [d.rule for d in diags]) + + +def test_all_the_mutation_kinds_are_exercised_across_the_seeds(): + kinds = set() + for seed in range(40): + rng = random.Random(seed) + for _ in range(60): + g = rng.choice(_base_functions()) + try: + kinds.add(_class_mutations(rng, g)[0]) + except (IndexError, StopIteration): + pass + assert kinds >= {"unknown class", "unknown field", "new arity", "fieldset in pure", + "checkexact in impure", "drop param guard", "drop guard", "kill proof", + "flip may_deopt", "flip pure", "retype", "scalar operand", "new nontrivial"} + + +def test_dropping_each_guard_of_the_base_functions_is_caught_one_by_one(): + pure, impure = _base_functions() + # the three guards: p's entry guard (both), q's IsExact (pure), o's is-not-None and n's early exit + for g, guard_param in ((pure, "p"), (impure, "p"), (impure, "o")): + ps = tuple(dataclasses.replace(p, cls=None, optional=False) if p.name == guard_param else p + for p in g.params) + rejected(FIELD, dataclasses.replace(g, params=ps), name=g.name) + drop_if = dataclasses.replace(pure, body=pure.body[:2] + pure.body[2].then + pure.body[3:]) + rejected(FIELD, drop_if, name="mix") + no_none_guard = dataclasses.replace( + impure, body=impure.body[:1] + impure.body[1].then + impure.body[2:]) + rejected(FIELD, no_none_guard, name="move") + no_exit = dataclasses.replace(impure, body=impure.body[:2] + impure.body[3:]) + rejected(FIELD, no_exit, name="move")