From 801670ab72165ef6cf8e194d3c77cb633bec201e Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 07:04:39 +0900 Subject: [PATCH 01/36] Feat: Add the TypedPython soundness gate (Any leaks become errors) Step 1 of docs/design/typedpython.md: python -m typedpython check runs Pyrefly with its Any-leak kinds raised to errors, reports user-code expressions of type Any outside cast() (any-flow), and reports the forbidden list as warnings in checked mode and errors in compiled mode. Lives in python-multiplatform-ksp. INTENT gains TypedPython as purpose 4 (maintainer statements quoted); SPEC gains N-6 (partial: not yet run by the build). Refs #16 --- docs/INTENT.md | 19 ++- docs/SPEC.md | 11 ++ docs/design/typedpython.md | 17 ++ python-multiplatform-ksp/.gitignore | 5 + python-multiplatform-ksp/pyproject.toml | 19 +++ .../src/main/python/typedpython/__init__.py | 4 + .../src/main/python/typedpython/__main__.py | 43 +++++ .../src/main/python/typedpython/diagnostic.py | 17 ++ .../src/main/python/typedpython/forbidden.py | 110 +++++++++++++ .../src/main/python/typedpython/gate.py | 114 ++++++++++++++ .../src/main/python/typedpython/pyrefly.py | 147 ++++++++++++++++++ .../src/test/python/conftest.py | 27 ++++ .../test/python/fixtures/deps/untyped_dep.py | 7 + .../src/test/python/test_any_flow.py | 43 +++++ .../src/test/python/test_cli.py | 48 ++++++ .../src/test/python/test_forbidden.py | 54 +++++++ .../src/test/python/test_measurement.py | 50 ++++++ .../src/test/python/test_pyrefly_kinds.py | 69 ++++++++ 18 files changed, 803 insertions(+), 1 deletion(-) create mode 100644 python-multiplatform-ksp/.gitignore create mode 100644 python-multiplatform-ksp/pyproject.toml create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/__init__.py create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/__main__.py create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/diagnostic.py create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/forbidden.py create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/gate.py create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py create mode 100644 python-multiplatform-ksp/src/test/python/conftest.py create mode 100644 python-multiplatform-ksp/src/test/python/fixtures/deps/untyped_dep.py create mode 100644 python-multiplatform-ksp/src/test/python/test_any_flow.py create mode 100644 python-multiplatform-ksp/src/test/python/test_cli.py create mode 100644 python-multiplatform-ksp/src/test/python/test_forbidden.py create mode 100644 python-multiplatform-ksp/src/test/python/test_measurement.py create mode 100644 python-multiplatform-ksp/src/test/python/test_pyrefly_kinds.py diff --git a/docs/INTENT.md b/docs/INTENT.md index 97f365c16..b103c1a70 100644 --- a/docs/INTENT.md +++ b/docs/INTENT.md @@ -26,6 +26,23 @@ CPython interpreter inside a Kotlin Multiplatform application so that: written (`Greeter('Kotlin').greet(2)` in `sample/`). 3. **The same code runs on every target** a Kotlin Multiplatform app ships to: desktop JVM, Android, iOS, Android native, and the web (wasm, experimental). +4. **The project's own Python is statically typed by default** (TypedPython). The maintainer's goal: + + > "파이썬을 컴파일 언어 및 타입 안정화, 정적 타입 검사, 컴파일 타임 타입검사를 지원할 수 있도록" + > — *Support Python as a compiled language with type stability, static type checking and + > compile-time type checking*, so that Python stays usable as AI-friendly languages such as Rust + > and TypeScript gain ground. + + > "TypedPython을 별도 레포가 아니라 PythonMultiplatform의 기본 동작이 될거야" + > — *TypedPython is not a separate repository; it will be PythonMultiplatform's default behaviour.* + + The maintainer chose the scope "검사만 기본, 컴파일은 opt-in" (2026-10-02): every user Python source + is **checked** by default; **compiling** it to native code is opt-in. The compiled code still runs on + the embedded CPython, as an extension module (`docs/design/typedpython.md`). + + > The maintainer also asked to *explore* GPU kernels written in Python without a separate language + > ("GPU 관련 최적화까지도 별도 언어 없이 지원할 방안을 모색해보자"). That is an exploration, not yet + > a commitment; it is recorded in the design note only. The CPython version moved on from 3.13 (the build pins 3.14.x today and offers opt-in free-threaded 3.14t builds on desktop only); the intent — embed **CPython itself**, not a re-implementation — is unchanged. @@ -88,7 +105,7 @@ no per-platform Python → Kotlin binding code to maintain. | A replacement for the `pythonx` packages | §2.3 — they are real packages in their own repository | | Runtime-reflection or name-lookup based binding | §2.1 — impossible under Kotlin/Native and native image | | A hand-written wrapper per Kotlin function or class | Generic adaptation plus generated tables; per-declaration wrappers were tried in 2024 and rotted (`docs/design/pythonx-adapter-design.md`) | -| A Python re-implementation, transpiler, or JVM-hosted Python | It embeds CPython itself, so C extensions keep working | +| A Python re-implementation, transpiler, or JVM-hosted Python | It embeds CPython itself, so C extensions keep working. TypedPython (§1.4) does not change this: checked code is ordinary Python, and compiled code is a CPython extension module, not a translation to another runtime | | A build tool, dependency resolver or packager | That is pypackpack (the work) and toolchain (the Gradle vocabulary) | | A sub-interpreter parallelism model | Rejected: extensions that do not declare multi-interpreter support fail to import; parallelism comes from free-threaded CPython | | A vendored interpreter | The build downloads and verifies CPython per platform | diff --git a/docs/SPEC.md b/docs/SPEC.md index 539a84a03..414045d7d 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -239,6 +239,17 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. - **N-3** Binding Kotlin/Native klib declarations at run time (B-2). `Status: planned`. - **N-4** Linux and Windows desktop runs in CI (P-2). `Status: planned`. - **N-5** Native-image upcall verification as an automated test (U-6). `Status: planned`. +- **N-6** TypedPython soundness gate (INTENT §1.4; `docs/design/typedpython.md` §4.2): checking a set of + user Python files reports every Pyrefly type error, the Any-leak kinds (`implicit-any-parameter`, + `unannotated-return`, `unknown-variable-type`, `no-any-return-implicit`, `no-any-return-explicit`, + `explicit-any`) and any user-code expression of type `Any` outside `cast()` (`any-flow`) as errors; + the forbidden list (`eval`/`exec`, computed-name `getattr`/`setattr`/`delattr`, patching an imported + module or a module-level class) as warnings in `checked` mode and errors in `compiled` mode. Nothing + inside dependencies is reported. `python -m typedpython check` exits 0 / 1 / 2 for clean / errors / + tool failure. `Status: partial` — the gate is asserted by + `python-multiplatform-ksp/src/test/python/` (run with that module's `.venv`); it is **not yet run by the + build** (design §6 step 1b), and rebinding a variable to a different type and mixed-element containers + are not detected (Pyrefly 1.3.2 reports neither). --- diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index fb126c89e..42b3ea035 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -207,6 +207,23 @@ TypedPython 은 그래서 자체 텐서 런타임을 만들지 않고 **torch op 각 단계는 AGENTS.md §5·§14 의 TDD 와 측정 테스트 규정을 따른다. 특히 3 은 경계 비용 감소를, 4 는 upstream 대비 수치 오차를 기준선으로 박아 둔다. +### 6.1 단계 1 의 현재 상태 (2026-10-03, #16) + +- **위치:** `python-multiplatform-ksp/src/main/python/typedpython/`, 테스트는 + `python-multiplatform-ksp/src/test/python/`. 사용자 지시("ksp랑 통합")에 따라 KSP 모듈 안에 둔다. + KSP 프로세서는 Kotlin 심볼만 보고 파이썬 파일 변경으로는 다시 돌지 않으므로, **실행**은 1b 에서 + Gradle 태스크가 맡는다 — KSP 모듈은 소스와 배포(jar 리소스)의 자리다. +- **구현된 규칙:** SPEC N-6. Pyrefly 의 `Any` 유출 kind 6개를 에러로 올리고, Pyrefly 의 + per-expression 타입 리포트(`--report-pysa`)로 사용자 코드 안의 `Any` 식을 찾는다(`any-flow`). + `cast()` 인자, 버려지는 식 문장의 값, 함수 시그니처 자체는 제외한다. +- **미구현:** 타입이 다른 값으로의 재할당, 원소 타입이 섞인 컨테이너. Pyrefly 1.3.2 는 둘 다 + 보고하지 않고, 타입 리포트에 대입 대상의 타입이 없다. +- **측정:** 1800줄 합성 모듈에서 plain `pyrefly check` 160–235 ms, 게이트 1.3–1.9 s — 두 번 잰 비율이 + 모두 약 8배다. 단독 측정이 아니다(두 번째는 load average 18.6). 절대값보다 비율을 기준으로 본다. 차이의 + 대부분은 타입 리포트가 typeshed 사본까지 디스크에 쓰는 비용으로 보인다(추정, 미분해). 1b 에서 + 빌드마다 돌리기 전에 줄일 대상이다. +- **고정:** Pyrefly `==1.3.2`. 타입 리포트는 Pyrefly 가 버전을 매기는 포맷이지 공개 API 가 아니다. + ## 7. 위험 - **Pyrefly 의 외부 API 안정성.** 라이브러리 임베드는 확인 전까지 미룬다 (§4.1). diff --git a/python-multiplatform-ksp/.gitignore b/python-multiplatform-ksp/.gitignore new file mode 100644 index 000000000..4dc423b60 --- /dev/null +++ b/python-multiplatform-ksp/.gitignore @@ -0,0 +1,5 @@ +.venv/ +__pycache__/ +.pytest_cache/ +*.egg-info/ +.pytest_tmp/ diff --git a/python-multiplatform-ksp/pyproject.toml b/python-multiplatform-ksp/pyproject.toml new file mode 100644 index 000000000..73c9238fa --- /dev/null +++ b/python-multiplatform-ksp/pyproject.toml @@ -0,0 +1,19 @@ +# TypedPython soundness gate (docs/design/typedpython.md §4.2). +# The Kotlin processor in this module is built by Gradle; this file only describes the Python +# package that lives next to it in src/main/python. +[project] +name = "typedpython" +version = "0.1.0" +requires-python = ">=3.13" +# Pinned exactly: the gate reads Pyrefly's per-expression type report (--report-pysa), whose +# format is versioned by Pyrefly, not by a public API. +dependencies = ["pyrefly==1.3.2"] + +[project.optional-dependencies] +test = ["pytest"] + +[tool.pytest.ini_options] +testpaths = ["src/test/python"] +pythonpath = ["src/main/python"] +addopts = "--basetemp=.pytest_tmp" +markers = ["measurement: records cost; asserts only sanity bounds"] diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__init__.py b/python-multiplatform-ksp/src/main/python/typedpython/__init__.py new file mode 100644 index 000000000..507809dab --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/__init__.py @@ -0,0 +1,4 @@ +"""TypedPython — static checking for the Python code of a PythonMultiplatform project. + +See docs/design/typedpython.md. This package is the soundness gate (§4.2). +""" diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py new file mode 100644 index 000000000..ab05b7dad --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py @@ -0,0 +1,43 @@ +"""`python -m typedpython check [--mode checked|compiled] [--format text|json] PATHS...` + +Exit status: 0 no errors (warnings allowed), 1 errors, 2 the gate itself could not run. +""" +import argparse +import json +import sys +from pathlib import Path + +from typedpython import gate +from typedpython.pyrefly import PyreflyError + +EXIT_CLEAN, EXIT_ERRORS, EXIT_FAILURE = 0, 1, 2 + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(prog="typedpython") + commands = parser.add_subparsers(dest="command", required=True) + check = commands.add_parser("check", help="type-check user Python code") + check.add_argument("paths", nargs="+", type=Path) + check.add_argument("--mode", choices=["checked", "compiled"], default="checked") + check.add_argument("--format", choices=["text", "json"], default="text") + check.add_argument("--search-path", action="append", type=Path, default=[], + help="directory to resolve imports from (repeatable)") + args = parser.parse_args(argv) + + try: + found = gate.check(args.paths, mode=args.mode, search_paths=args.search_path) + except PyreflyError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_FAILURE + + if args.format == "json": + print(json.dumps([d.to_json() for d in found], indent=2)) + else: + for d in found: + print(f"{d.severity.upper()} {d.path}:{d.line}:{d.column}: {d.message} [{d.rule}]") + + return EXIT_ERRORS if any(d.severity == "error" for d in found) else EXIT_CLEAN + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/diagnostic.py b/python-multiplatform-ksp/src/main/python/typedpython/diagnostic.py new file mode 100644 index 000000000..2ec322edc --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/diagnostic.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass, asdict +from typing import Literal + +Severity = Literal["error", "warning"] + + +@dataclass(frozen=True, order=True) +class Diagnostic: + path: str + line: int + column: int + rule: str + severity: Severity + message: str + + def to_json(self) -> dict[str, object]: + return asdict(self) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/forbidden.py b/python-multiplatform-ksp/src/main/python/typedpython/forbidden.py new file mode 100644 index 000000000..906cc2b9c --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/forbidden.py @@ -0,0 +1,110 @@ +"""The forbidden list (docs/design/typedpython.md §3.1), found by walking the AST. + +These are the dynamic features that make a static type unreliable at compile time. The gate +reports them as warnings for `checked` code and as errors for `compiled` code; this module only +finds them. + +Not covered here: rebinding a variable to a different type and containers that mix element +types. Both need inferred types for assignment targets, which Pyrefly 1.3.2 does not report. +""" +import ast +from dataclasses import dataclass + +DYNAMIC_ATTRIBUTE_CALLS = {"getattr", "setattr", "delattr"} +# Index of the attribute-name argument in each of the calls above. +NAME_ARGUMENT = 1 + + +@dataclass(frozen=True) +class Finding: + rule: str + line: int + column: int + message: str + + +def scan(source: str, path: str) -> list[Finding]: + tree = ast.parse(source, filename=path) + patchable = _module_level_modules_and_classes(tree) + findings: list[Finding] = [] + + for node in ast.walk(tree): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + findings += _check_call(node, node.func.id, patchable) + elif isinstance(node, (ast.Assign, ast.AugAssign, ast.AnnAssign, ast.Delete)): + for target in _targets(node): + if _is_patch_target(target, patchable): + assert isinstance(target, ast.Attribute) and isinstance(target.value, ast.Name) + findings.append(_finding( + "forbidden/monkeypatch", target, + f"`{target.value.id}.{target.attr}` changes a module or class from outside it", + )) + + return sorted(findings, key=lambda f: (f.line, f.column, f.rule)) + + +def _check_call(call: ast.Call, name: str, patchable: set[str]) -> list[Finding]: + if name in ("eval", "exec"): + return [_finding(f"forbidden/{name}", call, f"`{name}` runs code the checker cannot see")] + if name not in DYNAMIC_ATTRIBUTE_CALLS or len(call.args) <= NAME_ARGUMENT: + return [] + + attribute = call.args[NAME_ARGUMENT] + if not (isinstance(attribute, ast.Constant) and isinstance(attribute.value, str)): + return [_finding( + "forbidden/dynamic-attr", call, + f"`{name}` with a computed attribute name hides the attribute from the checker", + )] + + # A literal name is static, but setattr/delattr on a module or class is still a patch. + receiver = call.args[0] + if name != "getattr" and isinstance(receiver, ast.Name) and receiver.id in patchable: + return [_finding( + "forbidden/monkeypatch", call, + f"`{name}` on `{receiver.id}` changes a module or class from outside it", + )] + return [] + + +def _module_level_modules_and_classes(tree: ast.Module) -> set[str]: + """Names bound at module scope to an imported module/name or to a class definition.""" + names: set[str] = set() + for node in tree.body: + if isinstance(node, ast.Import): + names |= {(alias.asname or alias.name).split(".")[0] for alias in node.names} + elif isinstance(node, ast.ImportFrom): + names |= {alias.asname or alias.name for alias in node.names if alias.name != "*"} + elif isinstance(node, ast.ClassDef): + names.add(node.name) + return names + + +def _targets(node: ast.stmt) -> list[ast.expr]: + if isinstance(node, ast.Assign): + return [t for target in node.targets for t in _flatten(target)] + if isinstance(node, ast.Delete): + return [t for target in node.targets for t in _flatten(target)] + if isinstance(node, (ast.AugAssign, ast.AnnAssign)): + return [node.target] + return [] + + +def _flatten(target: ast.expr) -> list[ast.expr]: + if isinstance(target, (ast.Tuple, ast.List)): + return [t for element in target.elts for t in _flatten(element)] + if isinstance(target, ast.Starred): + return _flatten(target.value) + return [target] + + +def _is_patch_target(target: ast.expr, patchable: set[str]) -> bool: + return ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id in patchable + ) + + +def _finding(rule: str, node: ast.expr, message: str) -> Finding: + # ast columns are 0-based; Pyrefly's (and every editor's) are 1-based. + return Finding(rule, node.lineno, node.col_offset + 1, message) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/gate.py b/python-multiplatform-ksp/src/main/python/typedpython/gate.py new file mode 100644 index 000000000..f13be0080 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/gate.py @@ -0,0 +1,114 @@ +"""The soundness gate (docs/design/typedpython.md §4.2). + +A type checker may fall back to `Any` when it is unsure; a compiler cannot. The gate closes that +gap for the files it is given: + + Pyrefly errors, Any-leak kinds included -> error + an expression of type Any in user code -> error (`any-flow`), unless inside cast() + the §3.1 forbidden list -> warning (`checked`) / error (`compiled`) + +Only the given files are user code. Their dependencies are type-checked by Pyrefly only as far +as their stubs or sources say; nothing inside them is reported. +""" +import ast +from collections.abc import Sequence +from pathlib import Path +from typing import Literal + +from typedpython import forbidden, pyrefly +from typedpython.diagnostic import Diagnostic, Severity + +Mode = Literal["checked", "compiled"] + +Position = tuple[int, int] + + +def check( + paths: Sequence[Path], + mode: Mode = "checked", + search_paths: Sequence[Path] = (), +) -> list[Diagnostic]: + report = pyrefly.run(paths, search_paths) + diagnostics = [ + Diagnostic(e.path, e.line, e.column, f"pyrefly/{e.kind}", "error", e.message) + for e in report.errors + ] + reported = {(d.path, d.line, d.column) for d in diagnostics} + + forbidden_severity: Severity = "error" if mode == "compiled" else "warning" + for path in (str(Path(p).resolve()) for p in paths): + source = Path(path).read_text() + try: + tree = ast.parse(source, filename=path) + except SyntaxError: + continue # Pyrefly has already reported the parse error + + cast_spans = _cast_argument_spans(tree) + discarded = _discarded_values(tree) + for expr in report.any_expressions: + if expr.path != path or (expr.path, expr.line, expr.column) in reported: + continue + if _is_signature(expr.type): + continue # a callable whose signature mentions Any; its result is judged on its own + if ((expr.line, expr.column), (expr.stop_line, expr.stop_column)) in discarded: + continue # an expression statement's value goes nowhere + if any(_inside((expr.line, expr.column), span) for span in cast_spans): + continue + diagnostics.append(Diagnostic( + path, expr.line, expr.column, "any-flow", "error", + f"this expression has type `{expr.type}`; give it a type, or cast() it " + "where it leaves untyped code", + )) + reported.add((expr.path, expr.line, expr.column)) + + for finding in forbidden.scan(source, path): + diagnostics.append(Diagnostic( + path, finding.line, finding.column, finding.rule, forbidden_severity, + finding.message, + )) + + return sorted(diagnostics) + + +def _cast_argument_spans(tree: ast.Module) -> list[tuple[Position, Position]]: + """Source spans of the value argument of every `cast(T, value)` / `typing.cast(T, value)`.""" + spans = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call) or len(node.args) != 2: + continue + func = node.func + is_cast = (isinstance(func, ast.Name) and func.id == "cast") or ( + isinstance(func, ast.Attribute) and func.attr == "cast" + and isinstance(func.value, ast.Name) and func.value.id == "typing" + ) + value = node.args[1] + if is_cast and value.end_lineno is not None and value.end_col_offset is not None: + spans.append(( + (value.lineno, value.col_offset + 1), + (value.end_lineno, value.end_col_offset + 1), + )) + return spans + + +def _discarded_values(tree: ast.Module) -> set[tuple[Position, Position]]: + """Spans of values computed and thrown away by an expression statement (`eval(s)` alone).""" + spans = set() + for node in ast.walk(tree): + if isinstance(node, ast.Expr): + value = node.value + if value.end_lineno is not None and value.end_col_offset is not None: + spans.add(( + (value.lineno, value.col_offset + 1), + (value.end_lineno, value.end_col_offset + 1), + )) + return spans + + +def _is_signature(spelled: str) -> bool: + # Pyrefly spells callables as `(x: int) -> str` and overload sets as `Overload[...]`. + return "->" in spelled or spelled.startswith("Overload[") + + +def _inside(position: Position, span: tuple[Position, Position]) -> bool: + start, stop = span + return start <= position <= stop diff --git a/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py b/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py new file mode 100644 index 000000000..f6ef9b82d --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py @@ -0,0 +1,147 @@ +"""Running Pyrefly and reading what it reports (docs/design/typedpython.md §4.1). + +Pyrefly is used as a CLI, not embedded. Two outputs are read: + +- its JSON error list, with the Any-leak error kinds raised to `error` by a generated config; +- its per-expression type report (`--report-pysa`), which says which expressions in user code + have type `Any`. That report is a Pyrefly-versioned format, which is why pyproject.toml pins + Pyrefly exactly. +""" +import json +import os +import re +import subprocess +import sys +import tempfile +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + +# Pyrefly error kinds that mean "an Any got in". Pyrefly leaves several of them off by default. +ANY_LEAK_KINDS = ( + "implicit-any-parameter", + "unannotated-return", + "unknown-variable-type", + "no-any-return-implicit", + "no-any-return-explicit", + "explicit-any", +) + +# How Pyrefly spells an Any in the type report: `typing.Any` for an explicit one and `Unknown` +# for an implicit one, alone or inside a larger type such as `list[typing.Any]`. +ANY_IN_TYPE = re.compile(r"(? list[str]: + """The Pyrefly command line: `TYPEDPYTHON_PYREFLY`, or the copy installed next to this Python.""" + override = os.environ.get("TYPEDPYTHON_PYREFLY") + return [override] if override else [sys.executable, "-m", "pyrefly"] + + +def run(paths: Sequence[Path], search_paths: Sequence[Path] = ()) -> Report: + files = [str(Path(p).resolve()) for p in paths] + with tempfile.TemporaryDirectory(prefix="typedpython-") as work: + config = Path(work) / "pyrefly.toml" + config.write_text(_config(search_paths)) + report_dir = Path(work) / "pysa" + argv = [ + *command(), "check", *files, + "--config", str(config), + "--output-format", "json", + "--summary=none", + "--report-pysa", str(report_dir), + "--report-pysa-format", "json", + ] + try: + result = subprocess.run(argv, capture_output=True, text=True, check=False) + except OSError as e: + raise PyreflyError(f"cannot run {argv[0]}: {e}") from e + + # Exit code 1 only means "type errors found"; anything else is a failure of the tool. + if result.returncode not in (0, 1): + raise PyreflyError(f"pyrefly exited {result.returncode}: {result.stderr.strip()}") + try: + raw_errors = json.loads(result.stdout)["errors"] + except (json.JSONDecodeError, KeyError) as e: + raise PyreflyError(f"unreadable pyrefly output: {result.stdout[:200]!r}") from e + + errors = [ + Error(_resolve(e["path"]), e["line"], e["column"], e["name"], e["description"]) + for e in raw_errors + ] + return Report(errors, _any_expressions(report_dir, set(files))) + + +def _config(search_paths: Sequence[Path]) -> str: + lines = [] + if search_paths: + quoted = ", ".join(json.dumps(str(Path(p).resolve())) for p in search_paths) + lines.append(f"search-path = [{quoted}]") + lines.append("[errors]") + lines += [f'{kind} = "error"' for kind in ANY_LEAK_KINDS] + return "\n".join(lines) + "\n" + + +def _any_expressions(report_dir: Path, files: set[str]) -> list[AnyExpression]: + try: + index = json.loads((report_dir / "pyrefly.pysa.json").read_text()) + except (OSError, json.JSONDecodeError) as e: + raise PyreflyError(f"pyrefly wrote no readable type report: {e}") from e + + found: list[AnyExpression] = [] + for module in index["modules"].values(): + source = module.get("source_path", {}).get("FileSystem") + if source is None or _resolve(source) not in files: + continue # typeshed and dependencies: not user code + table_file = report_dir / "type_of_expressions" / module["info_filename"] + if not table_file.exists(): + continue + table = json.loads(table_file.read_text()) + for function in table["functions"].values(): + types = function["type_table"] + for location, type_index in function["locations"].items(): + spelled = types[type_index]["string"] + if ANY_IN_TYPE.search(spelled): + found.append(_expression(_resolve(source), location, spelled)) + return found + + +def _expression(path: str, location: str, spelled: str) -> AnyExpression: + # "6:9-6:15" -> start line/column, stop line/column (1-based, as in Pyrefly's errors). + start, stop = location.split("-") + line, column = map(int, start.split(":")) + stop_line, stop_column = map(int, stop.split(":")) + return AnyExpression(path, line, column, stop_line, stop_column, spelled) + + +def _resolve(path: str) -> str: + return str(Path(path).resolve()) diff --git a/python-multiplatform-ksp/src/test/python/conftest.py b/python-multiplatform-ksp/src/test/python/conftest.py new file mode 100644 index 000000000..65b1e5f0a --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/conftest.py @@ -0,0 +1,27 @@ +from pathlib import Path +import textwrap + +import pytest + +FIXTURES = Path(__file__).parent / "fixtures" + + +@pytest.fixture +def write(tmp_path): + """Write a dedented source file under tmp_path and return its path.""" + + def _write(name: str, source: str) -> Path: + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(source)) + return path + + return _write + + +def rules(diagnostics): + return sorted({d.rule for d in diagnostics}) + + +def errors(diagnostics): + return [d for d in diagnostics if d.severity == "error"] diff --git a/python-multiplatform-ksp/src/test/python/fixtures/deps/untyped_dep.py b/python-multiplatform-ksp/src/test/python/fixtures/deps/untyped_dep.py new file mode 100644 index 000000000..af73da304 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/fixtures/deps/untyped_dep.py @@ -0,0 +1,7 @@ +# Stands in for a third-party package without stubs. It is deliberately unannotated: the gate +# must not report anything inside it, only where its untyped results flow into user code. +import json + + +def load(text): + return json.loads(text) diff --git a/python-multiplatform-ksp/src/test/python/test_any_flow.py b/python-multiplatform-ksp/src/test/python/test_any_flow.py new file mode 100644 index 000000000..fae430c44 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_any_flow.py @@ -0,0 +1,43 @@ +"""Any reaching user code from an untyped dependency (design §2.3, §4.2).""" +from conftest import FIXTURES, errors +from typedpython import gate + +DEPS = FIXTURES / "deps" + + +def test_untyped_result_flowing_into_user_code_is_an_error(write): + path = write("user.py", """ + from untyped_dep import load + + def total(text: str) -> None: + value = load(text) + print(value) + """) + found = errors(gate.check([path], search_paths=[DEPS])) + flow = [d for d in found if d.rule == "any-flow"] + assert flow, found + assert all(d.path == str(path) for d in flow) + + +def test_cast_at_the_boundary_resolves_it(write): + path = write("user.py", """ + from typing import cast + from untyped_dep import load + + def total(text: str) -> int: + value = cast(int, load(text)) + return value + 1 + """) + assert gate.check([path], search_paths=[DEPS]) == [] + + +def test_the_dependency_itself_is_not_checked(write): + path = write("user.py", """ + from typing import cast + from untyped_dep import load + + def total(text: str) -> int: + return cast(int, load(text)) + """) + found = gate.check([path], search_paths=[DEPS]) + assert not [d for d in found if d.path.endswith("untyped_dep.py")], found diff --git a/python-multiplatform-ksp/src/test/python/test_cli.py b/python-multiplatform-ksp/src/test/python/test_cli.py new file mode 100644 index 000000000..6b5d1f383 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_cli.py @@ -0,0 +1,48 @@ +"""`python -m typedpython check` exit codes: 0 clean, 1 errors, 2 tool failure.""" +import json +import os +import subprocess +import sys +from pathlib import Path + +SRC = Path(__file__).parents[2] / "main" / "python" + + +def run(*args, env_extra=None): + env = dict(os.environ, PYTHONPATH=str(SRC), **(env_extra or {})) + return subprocess.run( + [sys.executable, "-m", "typedpython", "check", *map(str, args)], + capture_output=True, text=True, env=env, + ) + + +def test_clean_exits_0(write): + path = write("user.py", "def f(x: int) -> int:\n return x\n") + result = run(path) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_error_exits_1(write): + path = write("user.py", "def f(x):\n return x\n") + result = run(path) + assert result.returncode == 1, result.stdout + result.stderr + assert "implicit-any-parameter" in result.stdout + + +def test_warnings_alone_exit_0(write): + path = write("user.py", "def f(s: str) -> None:\n eval(s)\n") + assert run(path).returncode == 0 + assert run("--mode", "compiled", path).returncode == 1 + + +def test_json_output(write): + path = write("user.py", "def f(x):\n return x\n") + result = run("--format", "json", path) + data = json.loads(result.stdout) + assert {"path", "line", "column", "rule", "severity", "message"} <= set(data[0]) + + +def test_missing_pyrefly_exits_2(write): + path = write("user.py", "def f(x: int) -> int:\n return x\n") + result = run(path, env_extra={"TYPEDPYTHON_PYREFLY": "/nonexistent/pyrefly"}) + assert result.returncode == 2, result.stdout + result.stderr diff --git a/python-multiplatform-ksp/src/test/python/test_forbidden.py b/python-multiplatform-ksp/src/test/python/test_forbidden.py new file mode 100644 index 000000000..b672a22f8 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_forbidden.py @@ -0,0 +1,54 @@ +"""The §3.1 forbidden list: warning under `checked`, error under `compiled`.""" +import pytest + +from conftest import rules +from typedpython import forbidden, gate + +FLAGGED = { + "forbidden/eval": "def f(s: str) -> None:\n eval(s)\n", + "forbidden/exec": "def f(s: str) -> None:\n exec(s)\n", + "forbidden/dynamic-attr": "def f(o: object, n: str) -> None:\n getattr(o, n)\n", + "forbidden/monkeypatch": "import json\n\ndef f() -> None:\n json.dumps = print\n", +} + + +@pytest.mark.parametrize("rule", sorted(FLAGGED)) +def test_rule_is_found(rule): + assert rule in {f.rule for f in forbidden.scan(FLAGGED[rule], "user.py")} + + +@pytest.mark.parametrize("call", ["setattr(o, n, 1)", "delattr(o, n)"]) +def test_other_dynamic_attribute_calls(call): + source = f"def f(o: object, n: str) -> None:\n {call}\n" + assert {f.rule for f in forbidden.scan(source, "user.py")} == {"forbidden/dynamic-attr"} + + +def test_monkeypatching_a_module_level_class(): + source = ( + "class Greeter:\n pass\n\n" + "def patch() -> None:\n Greeter.hello = 1\n" + ) + assert {f.rule for f in forbidden.scan(source, "user.py")} == {"forbidden/monkeypatch"} + + +@pytest.mark.parametrize("source", [ + "def f(o: object) -> None:\n getattr(o, 'name')\n", + "class C:\n def __init__(self) -> None:\n self.x = 1\n", + "def f() -> None:\n class Local:\n pass\n obj = Local()\n obj.y = 2\n", +]) +def test_static_forms_are_not_flagged(source): + assert forbidden.scan(source, "user.py") == [] + + +def test_findings_carry_a_location(): + [finding] = forbidden.scan("x = 1\neval('x')\n", "user.py") + assert (finding.line, finding.column) == (2, 1) + + +def test_severity_follows_the_mode(write): + path = write("user.py", "def f(s: str) -> None:\n eval(s)\n") + checked = gate.check([path], mode="checked") + compiled = gate.check([path], mode="compiled") + assert [d.severity for d in checked if d.rule == "forbidden/eval"] == ["warning"] + assert [d.severity for d in compiled if d.rule == "forbidden/eval"] == ["error"] + assert rules(checked) == rules(compiled) diff --git a/python-multiplatform-ksp/src/test/python/test_measurement.py b/python-multiplatform-ksp/src/test/python/test_measurement.py new file mode 100644 index 000000000..45e5a15b2 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_measurement.py @@ -0,0 +1,50 @@ +"""What the gate costs on top of a plain `pyrefly check` (design §7, AGENTS.md §14). + +The extra work is the per-expression type report (`--report-pysa`) and the AST pass. The numbers +are printed so a run records them; only a loose sanity bound is asserted. +""" +import subprocess +import sys +import time + +import pytest + +from typedpython import gate + +FUNCTIONS = 300 + + +def synthetic_module() -> str: + lines = [] + for i in range(FUNCTIONS): + lines += [ + f"def f{i}(a: int, b: int) -> int:", + " total = 0", + " for k in range(a):", + " total += k * b", + " return total", + "", + ] + return "\n".join(lines) + + +@pytest.mark.measurement +def test_gate_cost_against_plain_pyrefly(write): + path = write("big.py", synthetic_module()) + lines = synthetic_module().count("\n") + 1 + + start = time.perf_counter() + subprocess.run( + [sys.executable, "-m", "pyrefly", "check", str(path), "--summary=none"], + capture_output=True, check=False, + ) + plain = time.perf_counter() - start + + start = time.perf_counter() + found = gate.check([path]) + gated = time.perf_counter() - start + + print(f"\n[measure] {lines} lines: plain pyrefly {plain * 1000:.0f} ms, " + f"gate {gated * 1000:.0f} ms, ratio {gated / plain:.2f}x") + assert found == [] + assert gated < 60 diff --git a/python-multiplatform-ksp/src/test/python/test_pyrefly_kinds.py b/python-multiplatform-ksp/src/test/python/test_pyrefly_kinds.py new file mode 100644 index 000000000..f7e864dfb --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_pyrefly_kinds.py @@ -0,0 +1,69 @@ +"""Each Pyrefly Any-leak kind the gate raises to error (design §4.2).""" +import pytest + +from conftest import errors, rules +from typedpython import gate + +CASES = { + "implicit-any-parameter": """ + def f(x, y: int) -> int: + return y + """, + "unannotated-return": """ + def f(y: int): + return y + """, + "unknown-variable-type": """ + import json + def f() -> None: + d = json.loads("1") + z = d["k"] + """, + "no-any-return-implicit": """ + import json + def f() -> int: + d = json.loads("1") + return d["k"] + """, + "no-any-return-explicit": """ + from typing import Any + def f(v: Any) -> int: + return v + """, + "explicit-any": """ + from typing import Any + def f(v: Any) -> None: + pass + """, +} + + +@pytest.mark.parametrize("kind", sorted(CASES)) +def test_kind_is_an_error(write, kind): + path = write("user.py", CASES[kind]) + found = gate.check([path]) + assert f"pyrefly/{kind}" in rules(errors(found)), found + + +def test_ordinary_type_error_is_an_error(write): + path = write("user.py", """ + def f(y: int) -> int: + return y + "a" + """) + assert errors(gate.check([path])), "a plain type error must still fail the gate" + + +def test_fully_typed_code_is_clean(write): + path = write("user.py", """ + def area(w: int, h: int) -> int: + scale = 2 + return w * h * scale + + class Box: + def __init__(self, w: int) -> None: + self.w = w + + def double(self) -> int: + return self.w * 2 + """) + assert gate.check([path]) == [] From 62d3332b85973dea9433afe4747f0076b8c3995f Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 07:12:12 +0900 Subject: [PATCH 02/36] Build: Package the TypedPython gate as a typedpython wheel toolchain installs the gate with uv at a pinned version (1b, #20), so the package needs a build backend and a console script. Wheel built and smoke- tested: installed from --find-links, typedpython check exits 0 on clean code and 1 on an unannotated function. Refs #16 --- python-multiplatform-ksp/pyproject.toml | 10 ++++++++++ .../src/main/python/typedpython/__main__.py | 7 ++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/python-multiplatform-ksp/pyproject.toml b/python-multiplatform-ksp/pyproject.toml index 73c9238fa..62f6993b2 100644 --- a/python-multiplatform-ksp/pyproject.toml +++ b/python-multiplatform-ksp/pyproject.toml @@ -9,6 +9,16 @@ requires-python = ">=3.13" # format is versioned by Pyrefly, not by a public API. dependencies = ["pyrefly==1.3.2"] +[project.scripts] +typedpython = "typedpython.__main__:cli" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/main/python/typedpython"] + [project.optional-dependencies] test = ["pytest"] diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py index ab05b7dad..78fbba395 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py @@ -39,5 +39,10 @@ def main(argv: list[str]) -> int: return EXIT_ERRORS if any(d.severity == "error" for d in found) else EXIT_CLEAN -if __name__ == "__main__": +def cli() -> None: + """Console-script entry point (`typedpython check ...`).""" sys.exit(main(sys.argv[1:])) + + +if __name__ == "__main__": + cli() From 38b51189eecae3fac2ca7241437d97b534eb54ee Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 07:20:31 +0900 Subject: [PATCH 03/36] Feat: Report type-changing rebinding and mixed-element containers Pyrefly 1.3.2 reports neither and records no types for assignment targets, but it does record the type of every name load. The gate now writes a probed copy in which each candidate assignment is followed on the same line by a load of its targets (line numbers unchanged), runs Pyrefly on it, and judges the types at the probe positions. The second run happens only when a file assigns a name twice or has a container display. Refs #22 --- docs/SPEC.md | 9 +- docs/design/typedpython.md | 10 +- .../src/main/python/typedpython/forbidden.py | 4 +- .../src/main/python/typedpython/gate.py | 44 ++- .../src/main/python/typedpython/pyrefly.py | 23 +- .../src/main/python/typedpython/rebinding.py | 258 ++++++++++++++++++ .../src/test/python/test_rebinding.py | 124 +++++++++ 7 files changed, 458 insertions(+), 14 deletions(-) create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/rebinding.py create mode 100644 python-multiplatform-ksp/src/test/python/test_rebinding.py diff --git a/docs/SPEC.md b/docs/SPEC.md index 414045d7d..c2a5fa92b 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -244,12 +244,15 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. `unannotated-return`, `unknown-variable-type`, `no-any-return-implicit`, `no-any-return-explicit`, `explicit-any`) and any user-code expression of type `Any` outside `cast()` (`any-flow`) as errors; the forbidden list (`eval`/`exec`, computed-name `getattr`/`setattr`/`delattr`, patching an imported - module or a module-level class) as warnings in `checked` mode and errors in `compiled` mode. Nothing + module or a module-level class, rebinding an unannotated name to a different type, and a list/set/dict + display whose inferred element type is a union) as warnings in `checked` mode and errors in + `compiled` mode. Nothing inside dependencies is reported. `python -m typedpython check` exits 0 / 1 / 2 for clean / errors / tool failure. `Status: partial` — the gate is asserted by `python-multiplatform-ksp/src/test/python/` (run with that module's `.venv`); it is **not yet run by the - build** (design §6 step 1b), and rebinding a variable to a different type and mixed-element containers - are not detected (Pyrefly 1.3.2 reports neither). + build** (design §6 step 1b). Rebinding and mixed containers are found by a second Pyrefly pass over a + probed copy (design §6.1), which runs only when a file has a name assigned twice or a container + display; `for` targets and comprehension variables are not probed. --- diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 42b3ea035..108f57523 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -216,10 +216,14 @@ upstream 대비 수치 오차를 기준선으로 박아 둔다. - **구현된 규칙:** SPEC N-6. Pyrefly 의 `Any` 유출 kind 6개를 에러로 올리고, Pyrefly 의 per-expression 타입 리포트(`--report-pysa`)로 사용자 코드 안의 `Any` 식을 찾는다(`any-flow`). `cast()` 인자, 버려지는 식 문장의 값, 함수 시그니처 자체는 제외한다. -- **미구현:** 타입이 다른 값으로의 재할당, 원소 타입이 섞인 컨테이너. Pyrefly 1.3.2 는 둘 다 - 보고하지 않고, 타입 리포트에 대입 대상의 타입이 없다. +- **재할당·혼합 컨테이너 (#22):** Pyrefly 1.3.2 는 둘 다 보고하지 않고, 타입 리포트에 대입 대상의 + 타입이 없다. 대신 이름을 *읽는* 위치의 타입은 기록한다. 그래서 대입문 바로 뒤에 같은 줄로 + `; x` 를 붙인 사본을 만들어(줄 번호 불변) 한 번 더 돌리고, 그 위치의 타입으로 판정한다. 두 번 + 이상 대입되는 이름이나 컨테이너 디스플레이가 있는 파일에서만 돈다. `for` 대상과 컴프리헨션 + 변수는 아직 탐침하지 않는다. - **측정:** 1800줄 합성 모듈에서 plain `pyrefly check` 160–235 ms, 게이트 1.3–1.9 s — 두 번 잰 비율이 - 모두 약 8배다. 단독 측정이 아니다(두 번째는 load average 18.6). 절대값보다 비율을 기준으로 본다. 차이의 + 모두 약 8배다. 탐침 패스가 붙은 뒤로는 약 15배(232 ms → 3.6 s, load 41)다 — 합성 모듈의 + `total += ...` 가 탐침 대상이라 두 번째 실행이 돈다. 단독 측정이 아니다(두 번째는 load average 18.6). 절대값보다 비율을 기준으로 본다. 차이의 대부분은 타입 리포트가 typeshed 사본까지 디스크에 쓰는 비용으로 보인다(추정, 미분해). 1b 에서 빌드마다 돌리기 전에 줄일 대상이다. - **고정:** Pyrefly `==1.3.2`. 타입 리포트는 Pyrefly 가 버전을 매기는 포맷이지 공개 API 가 아니다. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/forbidden.py b/python-multiplatform-ksp/src/main/python/typedpython/forbidden.py index 906cc2b9c..cccc1651f 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/forbidden.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/forbidden.py @@ -4,8 +4,8 @@ reports them as warnings for `checked` code and as errors for `compiled` code; this module only finds them. -Not covered here: rebinding a variable to a different type and containers that mix element -types. Both need inferred types for assignment targets, which Pyrefly 1.3.2 does not report. +Rebinding a name to a different type and containers that mix element types need inferred types +for assignment targets; they live in typedpython.rebinding. """ import ast from dataclasses import dataclass diff --git a/python-multiplatform-ksp/src/main/python/typedpython/gate.py b/python-multiplatform-ksp/src/main/python/typedpython/gate.py index f13be0080..af839b2a7 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/gate.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/gate.py @@ -11,11 +11,12 @@ as their stubs or sources say; nothing inside them is reported. """ import ast +import tempfile from collections.abc import Sequence from pathlib import Path from typing import Literal -from typedpython import forbidden, pyrefly +from typedpython import forbidden, pyrefly, rebinding from typedpython.diagnostic import Diagnostic, Severity Mode = Literal["checked", "compiled"] @@ -67,9 +68,50 @@ def check( finding.message, )) + for d in _rebinding_findings(paths, search_paths): + diagnostics.append(Diagnostic( + d.path, d.line, d.column, d.rule, forbidden_severity, d.message, + )) + return sorted(diagnostics) +def _rebinding_findings( + paths: Sequence[Path], search_paths: Sequence[Path], +) -> list[Diagnostic]: + """Second Pyrefly pass over probed copies; skipped when no file has anything to probe.""" + plans: dict[str, tuple[str, list[rebinding.Probe]]] = {} + for path in (str(Path(p).resolve()) for p in paths): + try: + tree = ast.parse(Path(path).read_text(), filename=path) + except SyntaxError: + continue + wanted = rebinding.candidates(rebinding.assignments(tree)) + if wanted: + plans[path] = rebinding.probe(Path(path).read_text(), wanted) + if not plans: + return [] + + found: list[Diagnostic] = [] + with tempfile.TemporaryDirectory(prefix="typedpython-probe-") as work: + copies: dict[str, str] = {} + for i, (original, (probed, _)) in enumerate(plans.items()): + # One directory per file keeps module names intact; the original's directory goes + # on the search path so sibling imports still resolve. + copy = Path(work) / str(i) / Path(original).name + copy.parent.mkdir() + copy.write_text(probed) + copies[str(copy.resolve())] = original + dirs = sorted({str(Path(o).parent) for o in plans}) + report = pyrefly.run([Path(c) for c in copies], [*search_paths, *map(Path, dirs)]) + + for copy, original in copies.items(): + types = report.expression_types.get(copy, {}) + for f in rebinding.judge(plans[original][1], types): + found.append(Diagnostic(original, f.line, f.column, f.rule, "warning", f.message)) + return found + + def _cast_argument_spans(tree: ast.Module) -> list[tuple[Position, Position]]: """Source spans of the value argument of every `cast(T, value)` / `typing.cast(T, value)`.""" spans = [] diff --git a/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py b/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py index f6ef9b82d..a9b944361 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py @@ -59,6 +59,8 @@ class AnyExpression: class Report: errors: list[Error] any_expressions: list[AnyExpression] + # file -> (line, column) -> type, for every expression the type report records. + expression_types: dict[str, dict[tuple[int, int], str]] def command() -> list[str]: @@ -98,7 +100,8 @@ def run(paths: Sequence[Path], search_paths: Sequence[Path] = ()) -> Report: Error(_resolve(e["path"]), e["line"], e["column"], e["name"], e["description"]) for e in raw_errors ] - return Report(errors, _any_expressions(report_dir, set(files))) + types = _expression_types(report_dir, set(files)) + return Report(errors, _any_expressions(types), _types_by_start(types)) def _config(search_paths: Sequence[Path]) -> str: @@ -111,7 +114,8 @@ def _config(search_paths: Sequence[Path]) -> str: return "\n".join(lines) + "\n" -def _any_expressions(report_dir: Path, files: set[str]) -> list[AnyExpression]: +def _expression_types(report_dir: Path, files: set[str]) -> list[AnyExpression]: + """Every typed expression the report records in the given files (typeshed and deps skipped).""" try: index = json.loads((report_dir / "pyrefly.pysa.json").read_text()) except (OSError, json.JSONDecodeError) as e: @@ -129,12 +133,21 @@ def _any_expressions(report_dir: Path, files: set[str]) -> list[AnyExpression]: for function in table["functions"].values(): types = function["type_table"] for location, type_index in function["locations"].items(): - spelled = types[type_index]["string"] - if ANY_IN_TYPE.search(spelled): - found.append(_expression(_resolve(source), location, spelled)) + found.append(_expression(_resolve(source), location, types[type_index]["string"])) return found +def _any_expressions(expressions: list[AnyExpression]) -> list[AnyExpression]: + return [e for e in expressions if ANY_IN_TYPE.search(e.type)] + + +def _types_by_start(expressions: list[AnyExpression]) -> dict[str, dict[tuple[int, int], str]]: + table: dict[str, dict[tuple[int, int], str]] = {} + for e in expressions: + table.setdefault(e.path, {})[(e.line, e.column)] = e.type + return table + + def _expression(path: str, location: str, spelled: str) -> AnyExpression: # "6:9-6:15" -> start line/column, stop line/column (1-based, as in Pyrefly's errors). start, stop = location.split("-") diff --git a/python-multiplatform-ksp/src/main/python/typedpython/rebinding.py b/python-multiplatform-ksp/src/main/python/typedpython/rebinding.py new file mode 100644 index 000000000..8f26cef8e --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/rebinding.py @@ -0,0 +1,258 @@ +"""Rebinding a name to a different type, and mixed-element containers (design §3.1). + +Pyrefly 1.3.2 reports neither, and its per-expression type report has no entries for assignment +targets. It does record every *load* of a name, with the name's type at that point. So the gate +writes a probed copy of the file in which each assignment of interest is followed, on the same +line, by a bare load of each target (`x = 1` becomes `x = 1; x`), runs Pyrefly on the copy, and +reads the target types back at the probe positions. Line numbers do not change, and every column +before the insertion point stays where it was. +""" +import ast +import re +from collections.abc import Iterable +from dataclasses import dataclass, field + +# Container types whose element type should be one type, not a union. Tuples are excluded: a +# fixed-length tuple is typed per slot, so `tuple[int, str]` is not "mixed". +MIXABLE = re.compile(r"^(list|set|frozenset|dict)\[(.*)\]$", re.DOTALL) +LITERAL = re.compile(r"^Literal\[(.*)\]$", re.DOTALL) + + +@dataclass(frozen=True) +class Assignment: + """One assignment to a plain name, in the original source.""" + + scope: int + name: str + line: int + column: int + end_line: int + end_byte: int # ast end_col_offset of the whole statement (UTF-8 bytes) + augmented: bool + display: bool # the value is a list/set/dict display with more than one element + + +@dataclass(frozen=True) +class Probe: + name: str + line: int + column: int + assignment: Assignment | None = field(default=None, compare=False) + + +@dataclass(frozen=True) +class Finding: + rule: str + line: int + column: int + message: str + + +def assignments(tree: ast.Module) -> list[Assignment]: + """Assignments to unannotated names, scope by scope (module, class bodies, functions).""" + found: list[Assignment] = [] + for scope_id, scope in enumerate(_scopes(tree)): + declared = _declared_names(scope) + for stmt in _own_statements(scope): + found += [a for a in _assignments_in(stmt, scope_id) if a.name not in declared] + return found + + +def candidates(found: Iterable[Assignment]) -> list[Assignment]: + """The assignments worth probing: names assigned more than once, and container displays.""" + found = list(found) + counts: dict[tuple[int, str], int] = {} + for a in found: + counts[(a.scope, a.name)] = counts.get((a.scope, a.name), 0) + 1 + return [a for a in found if counts[(a.scope, a.name)] > 1 or a.display] + + +def probe(source: str, wanted: Iterable[Assignment] | None = None) -> tuple[str, list[Probe]]: + """Insert a load of each target after its statement; `wanted=None` probes every assignment.""" + if wanted is None: + wanted = assignments(ast.parse(source)) + lines = source.splitlines(keepends=True) + # Insert right-to-left within a line so earlier offsets stay valid. + by_position: dict[tuple[int, int], list[Assignment]] = {} + for a in wanted: + by_position.setdefault((a.end_line, a.end_byte), []).append(a) + + probes: list[Probe] = [] + for (line_no, end_byte) in sorted(by_position, reverse=True): + encoded = lines[line_no - 1].encode() + head, tail = encoded[:end_byte].decode(), encoded[end_byte:].decode() + inserted = "" + added: list[Probe] = [] + for a in by_position[(line_no, end_byte)]: + inserted += "; " + added.append(Probe(a.name, line_no, len(head) + len(inserted) + 1, a)) + inserted += a.name + lines[line_no - 1] = head + inserted + tail + # Probes already placed further right on this line moved by what was just inserted. + probes = [ + Probe(q.name, q.line, q.column + len(inserted), q.assignment) + if q.line == line_no and q.column > len(head) else q + for q in probes + ] + probes += added + + return "".join(lines), probes + + +def judge(probes: list[Probe], types: dict[tuple[int, int], str]) -> list[Finding]: + """Turn probed types into findings.""" + findings: list[Finding] = [] + history: dict[tuple[int, str], tuple[str, Assignment]] = {} + + for p in sorted(probes, key=lambda p: (p.line, p.column)): + a = p.assignment + spelled = types.get((p.line, p.column)) + if a is None or spelled is None or _is_unknown(spelled): + continue + + if a.display and (mixed := _mixed_element(spelled)): + findings.append(Finding( + "forbidden/mixed-container", a.line, a.column, + f"`{a.name}` is `{spelled}`: its elements mix `{mixed}`; annotate the union " + "on purpose or keep one element type", + )) + + key = (a.scope, a.name) + current = _base(spelled) + if key in history and history[key][0] != current: + before = history[key][0] + findings.append(Finding( + "forbidden/rebinding", a.line, a.column, + f"`{a.name}` was `{before}` and is rebound to `{current}`; annotate it " + f"(`{a.name}: {before} | {current}`) or use another name", + )) + history[key] = (current, a) + + return findings + + +# --- AST helpers ------------------------------------------------------------------------------- + +Scope = ast.Module | ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef + + +def _scopes(tree: ast.Module) -> list[Scope]: + return [tree] + [ + n for n in ast.walk(tree) + if isinstance(n, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)) + ] + + +def _own_statements(scope: Scope) -> Iterable[ast.stmt]: + """Statements of this scope, descending into blocks but not into nested scopes.""" + stack = list(reversed(scope.body)) + while stack: + stmt = stack.pop() + yield stmt + if isinstance(stmt, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for name in ("body", "orelse", "finalbody"): + stack.extend(reversed(getattr(stmt, name, []))) + for handler in getattr(stmt, "handlers", []): + stack.extend(reversed(handler.body)) + for case in getattr(stmt, "cases", []): + stack.extend(reversed(case.body)) + + +def _declared_names(scope: Scope) -> set[str]: + """Names whose type is declared (annotated, parameters) or owned elsewhere (global).""" + names: set[str] = set() + if isinstance(scope, (ast.FunctionDef, ast.AsyncFunctionDef)): + args = scope.args + for arg in [*args.posonlyargs, *args.args, *args.kwonlyargs]: + names.add(arg.arg) + names |= {a.arg for a in (args.vararg, args.kwarg) if a is not None} + for stmt in _own_statements(scope): + if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name): + names.add(stmt.target.id) + elif isinstance(stmt, (ast.Global, ast.Nonlocal)): + names |= set(stmt.names) + return names + + +def _assignments_in(stmt: ast.stmt, scope_id: int) -> list[Assignment]: + if isinstance(stmt, ast.Assign): + targets = [t for target in stmt.targets for t in _names(target)] + display = len(stmt.targets) == 1 and isinstance(stmt.targets[0], ast.Name) \ + and _is_display(stmt.value) + augmented = False + elif isinstance(stmt, ast.AugAssign) and isinstance(stmt.target, ast.Name): + targets, display, augmented = [stmt.target], False, True + else: + return [] + assert stmt.end_lineno is not None and stmt.end_col_offset is not None + return [ + Assignment(scope_id, t.id, t.lineno, t.col_offset + 1, + stmt.end_lineno, stmt.end_col_offset, augmented, display) + for t in targets + ] + + +def _names(target: ast.expr) -> list[ast.Name]: + if isinstance(target, ast.Name): + return [target] + if isinstance(target, (ast.Tuple, ast.List)): + return [n for element in target.elts for n in _names(element)] + if isinstance(target, ast.Starred): + return _names(target.value) + return [] + + +def _is_display(value: ast.expr) -> bool: + if isinstance(value, (ast.List, ast.Set)): + return len(value.elts) > 1 + if isinstance(value, ast.Dict): + return len(value.keys) > 1 + return False + + +# --- type spelling ----------------------------------------------------------------------------- + +def _is_unknown(spelled: str) -> bool: + return spelled in ("Unknown", "typing.Any", "Any") + + +def _base(spelled: str) -> str: + """`Literal[1]` -> `int`, `Literal['a']` -> `str`, `Literal[True]` -> `bool`.""" + m = LITERAL.match(spelled) + if not m: + return spelled + value = m.group(1).strip() + if value in ("True", "False"): + return "bool" + if value[:1] in ("'", '"'): + return "str" + if value[:1] in ("b",) and value[1:2] in ("'", '"'): + return "bytes" + return "int" + + +def _mixed_element(spelled: str) -> str | None: + m = MIXABLE.match(spelled) + if not m: + return None + for argument in _split_top_level(m.group(2)): + if len(_split_top_level(argument, "|")) > 1: + return argument.strip() + return None + + +def _split_top_level(text: str, separator: str = ",") -> list[str]: + parts, depth, current = [], 0, "" + for ch in text: + if ch in "[(": + depth += 1 + elif ch in "])": + depth -= 1 + if ch == separator and depth == 0: + parts.append(current) + current = "" + else: + current += ch + parts.append(current) + return parts diff --git a/python-multiplatform-ksp/src/test/python/test_rebinding.py b/python-multiplatform-ksp/src/test/python/test_rebinding.py new file mode 100644 index 000000000..d87e4e18d --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_rebinding.py @@ -0,0 +1,124 @@ +"""Rebinding a name to a different type, and mixed-element containers (design §3.1, #22). + +Both are forbidden-list items: warnings under `checked`, errors under `compiled`. They need the +inferred type of an assignment target, which Pyrefly's type report does not give directly; the +gate reads each target back through a probe load (see typedpython.rebinding). +""" +import pytest + +from typedpython import gate, rebinding + +REBIND = "forbidden/rebinding" +MIXED = "forbidden/mixed-container" + + +def found(write, source, mode="checked"): + return gate.check([write("user.py", source)], mode=mode) + + +def by_rule(diagnostics, rule): + return [d for d in diagnostics if d.rule == rule] + + +def test_rebinding_to_another_type_is_reported(write): + result = found(write, """ + def f() -> None: + x = 1 + x = "a" + print(x) + """) + [d] = by_rule(result, REBIND) + assert (d.line, d.severity) == (4, "warning") + assert "int" in d.message and "str" in d.message + + +def test_compiled_mode_makes_it_an_error(write): + result = found(write, """ + def f() -> None: + x = 1 + x = "a" + print(x) + """, mode="compiled") + assert [d.severity for d in by_rule(result, REBIND)] == ["error"] + + +def test_augmented_assignment_that_changes_the_type(write): + result = found(write, """ + def f() -> float: + total = 0 + total += 0.5 + return total + """) + assert by_rule(result, REBIND), result + + +def test_module_level_rebinding(write): + result = found(write, """ + limit = 10 + limit = "ten" + print(limit) + """) + assert by_rule(result, REBIND), result + + +@pytest.mark.parametrize("source", [ + # same type again + "def f() -> None:\n x = 1\n x = 2\n print(x)\n", + # a declared union governs; Pyrefly checks each assignment against it + "def f() -> None:\n x: int | str = 1\n x = 'a'\n print(x)\n", + # an annotated parameter reassigned within its type + "def f(n: int) -> int:\n n = n + 1\n return n\n", +]) +def test_not_rebinding(write, source): + assert by_rule(gate.check([write("user.py", source)]), REBIND) == [] + + +def test_mixed_list_is_reported(write): + result = found(write, """ + def f() -> None: + mixed = [1, "a"] + print(mixed) + """) + [d] = by_rule(result, MIXED) + assert d.line == 3 and "int | str" in d.message + + +def test_mixed_dict_values_are_reported(write): + result = found(write, """ + def f(a: int) -> None: + d = {"k": a, "j": "s"} + print(d) + """) + assert by_rule(result, MIXED), result + + +@pytest.mark.parametrize("source", [ + "def f() -> None:\n nums = [1, 2]\n print(nums)\n", + "def f() -> None:\n pair = (1, 'a')\n print(pair)\n", # tuples are typed per slot + "def f() -> None:\n xs: list[int | str] = [1, 'a']\n print(xs)\n", # declared on purpose +]) +def test_not_mixed(write, source): + assert by_rule(gate.check([write("user.py", source)]), MIXED) == [] + + +def test_probe_keeps_every_line_number(): + source = "def f() -> None:\n x = 1 # one\n x = 'a'\n y, z = 1, 2\n" + probed, probes = rebinding.probe(source) + assert probed.count("\n") == source.count("\n") + assert {p.name for p in probes} >= {"x", "y", "z"} + for p in probes: + line = probed.splitlines()[p.line - 1] + assert line[p.column - 1:].startswith(p.name), (p, line) + + +def test_no_candidates_means_no_second_pyrefly_run(write, monkeypatch): + calls = [] + real = gate.pyrefly.run + + def counting(*args, **kwargs): + calls.append(args) + return real(*args, **kwargs) + + monkeypatch.setattr(gate.pyrefly, "run", counting) + gate.check([write("user.py", "def f(a: int) -> int:\n b = a + 1\n return b\n")]) + assert len(calls) == 1 From 71ee70d7531bfd8aa58d69524293a7ffc688482d Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 07:25:07 +0900 Subject: [PATCH 04/36] Fix: Resolve a package's own imports in the gate The gate's Pyrefly config lives in a temporary directory, which Pyrefly took as the import root, so relative and absolute self-imports of user packages did not resolve and every value from them failed as any-flow. Each checked file's import root (above its outermost package) now goes on the search path. Red before the fix: 3 failed in test_imports.py. Refs #16 --- .../src/main/python/typedpython/pyrefly.py | 18 ++++++++++- .../src/test/python/test_imports.py | 31 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 python-multiplatform-ksp/src/test/python/test_imports.py diff --git a/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py b/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py index f6ef9b82d..781d6e5dd 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py @@ -71,7 +71,7 @@ def run(paths: Sequence[Path], search_paths: Sequence[Path] = ()) -> Report: files = [str(Path(p).resolve()) for p in paths] with tempfile.TemporaryDirectory(prefix="typedpython-") as work: config = Path(work) / "pyrefly.toml" - config.write_text(_config(search_paths)) + config.write_text(_config([*search_paths, *import_roots(files)])) report_dir = Path(work) / "pysa" argv = [ *command(), "check", *files, @@ -101,6 +101,22 @@ def run(paths: Sequence[Path], search_paths: Sequence[Path] = ()) -> Report: return Report(errors, _any_expressions(report_dir, set(files))) +def import_roots(files: Sequence[str]) -> list[Path]: + """The directory each file is imported from: above its outermost package, else its own. + + The config lives in a temporary directory, and Pyrefly would otherwise take that as the import + root, so a package's own imports (relative or absolute) would not resolve. + """ + roots: list[Path] = [] + for file in files: + root = Path(file).parent + while (root / "__init__.py").exists(): + root = root.parent + if root not in roots: + roots.append(root) + return roots + + def _config(search_paths: Sequence[Path]) -> str: lines = [] if search_paths: diff --git a/python-multiplatform-ksp/src/test/python/test_imports.py b/python-multiplatform-ksp/src/test/python/test_imports.py new file mode 100644 index 000000000..4a9a9a876 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_imports.py @@ -0,0 +1,31 @@ +"""User code inside a package must resolve its own imports (defect found 2026-10-03). + +The gate writes its Pyrefly config into a temporary directory. Pyrefly infers the import root +from the config's location, so without help a package's relative and absolute self-imports did +not resolve: every value from them became `Unknown` and failed the gate as `any-flow`. +""" +from conftest import errors +from typedpython import gate + + +def package(write): + write("src/pkg/__init__.py", "") + write("src/pkg/b.py", "def make() -> int:\n return 1\n") + + +def test_relative_import_inside_a_package(write): + package(write) + path = write("src/pkg/a.py", "from .b import make\n\ndef f() -> int:\n x = make()\n return x\n") + assert errors(gate.check([path])) == [] + + +def test_absolute_import_of_the_own_package(write): + package(write) + path = write("src/pkg/c.py", "from pkg.b import make\n\ndef f() -> int:\n return make()\n") + assert errors(gate.check([path])) == [] + + +def test_sibling_module_without_a_package(write): + write("flat/helper.py", "def make() -> int:\n return 1\n") + path = write("flat/main.py", "from helper import make\n\ndef f() -> int:\n return make()\n") + assert errors(gate.check([path])) == [] From f8f6663294de190da1cb27d678a769e713be6e60 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 07:26:54 +0900 Subject: [PATCH 05/36] Fix: Mirror the package tree for the probe pass A probed copy written alone into a temporary directory lost its package, so relative imports in it did not resolve, their values were Unknown, and rebinding went unreported in exactly the code that lives in packages. The probe pass now copies each file's package tree with the same layout. Red before the fix: test_rebinding_inside_a_package_with_relative_imports. Refs #22 --- .../src/main/python/typedpython/gate.py | 48 ++++++++++++++----- .../src/test/python/test_rebinding.py | 16 +++++++ 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/gate.py b/python-multiplatform-ksp/src/main/python/typedpython/gate.py index af839b2a7..303b5c19f 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/gate.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/gate.py @@ -11,6 +11,7 @@ as their stubs or sources say; nothing inside them is reported. """ import ast +import shutil import tempfile from collections.abc import Sequence from pathlib import Path @@ -94,17 +95,14 @@ def _rebinding_findings( found: list[Diagnostic] = [] with tempfile.TemporaryDirectory(prefix="typedpython-probe-") as work: - copies: dict[str, str] = {} - for i, (original, (probed, _)) in enumerate(plans.items()): - # One directory per file keeps module names intact; the original's directory goes - # on the search path so sibling imports still resolve. - copy = Path(work) / str(i) / Path(original).name - copy.parent.mkdir() - copy.write_text(probed) - copies[str(copy.resolve())] = original - dirs = sorted({str(Path(o).parent) for o in plans}) - report = pyrefly.run([Path(c) for c in copies], [*search_paths, *map(Path, dirs)]) - + copies = _mirror(Path(work), {o: probed for o, (probed, _) in plans.items()}) + roots = pyrefly.import_roots(list(plans)) + mirrored = sorted({str(Path(c).parent) for c in copies}) # flat files: their copy's dir + report = pyrefly.run( + [Path(c) for c in copies], + [*[Path(work) / str(i) for i in range(len(roots))], *map(Path, mirrored), + *search_paths, *roots], + ) for copy, original in copies.items(): types = report.expression_types.get(copy, {}) for f in rebinding.judge(plans[original][1], types): @@ -112,6 +110,34 @@ def _rebinding_findings( return found +def _mirror(work: Path, probed: dict[str, str]) -> dict[str, str]: + """Copy each file's package tree under `work`, with the probed text in place of the file. + + The copy keeps the file's path relative to its import root, so relative imports and imports + of sibling modules inside the package resolve to the copied tree. Returns copy -> original. + """ + roots = pyrefly.import_roots(list(probed)) + copies: dict[str, str] = {} + copied: set[Path] = set() + for original, text in probed.items(): + root = pyrefly.import_roots([original])[0] + base = work / str(roots.index(root)) + relative = Path(original).relative_to(root) + if len(relative.parts) > 1: + package = root / relative.parts[0] + if package not in copied: + shutil.copytree( + package, base / relative.parts[0], dirs_exist_ok=True, + ignore=shutil.ignore_patterns("__pycache__", ".*"), + ) + copied.add(package) + target = base / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + copies[str(target.resolve())] = original + return copies + + def _cast_argument_spans(tree: ast.Module) -> list[tuple[Position, Position]]: """Source spans of the value argument of every `cast(T, value)` / `typing.cast(T, value)`.""" spans = [] diff --git a/python-multiplatform-ksp/src/test/python/test_rebinding.py b/python-multiplatform-ksp/src/test/python/test_rebinding.py index d87e4e18d..3d50e272e 100644 --- a/python-multiplatform-ksp/src/test/python/test_rebinding.py +++ b/python-multiplatform-ksp/src/test/python/test_rebinding.py @@ -122,3 +122,19 @@ def counting(*args, **kwargs): monkeypatch.setattr(gate.pyrefly, "run", counting) gate.check([write("user.py", "def f(a: int) -> int:\n b = a + 1\n return b\n")]) assert len(calls) == 1 + + +def test_rebinding_inside_a_package_with_relative_imports(write): + write("pkg/__init__.py", "") + write("pkg/b.py", "def make() -> int:\n return 1\n") + path = write("pkg/a.py", """ + from .b import make + + def f() -> None: + x = make() + x = "s" + print(x) + """) + result = gate.check([path]) + assert by_rule(result, REBIND), result + assert not [d for d in result if d.severity == "error"], result From 077fed8872ec4e22ac88f4905b40757f02f40ec5 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:22:15 +0900 Subject: [PATCH 06/36] Fix: Accept directories in typedpython check A directory argument crashed with IsADirectoryError (found by the toolchain typedpythonCheck work). Directories now expand to their .py files, skipping hidden directories, __pycache__ and build. Red before the fix: test_a_directory_is_checked_file_by_file. Refs #16 --- .../src/main/python/typedpython/__main__.py | 5 +++-- .../src/main/python/typedpython/paths.py | 21 +++++++++++++++++++ .../src/test/python/test_cli.py | 12 +++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/paths.py diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py index 78fbba395..7f3cc1503 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py @@ -8,6 +8,7 @@ from pathlib import Path from typedpython import gate +from typedpython.paths import python_files from typedpython.pyrefly import PyreflyError EXIT_CLEAN, EXIT_ERRORS, EXIT_FAILURE = 0, 1, 2 @@ -17,7 +18,7 @@ def main(argv: list[str]) -> int: parser = argparse.ArgumentParser(prog="typedpython") commands = parser.add_subparsers(dest="command", required=True) check = commands.add_parser("check", help="type-check user Python code") - check.add_argument("paths", nargs="+", type=Path) + check.add_argument("paths", nargs="+", type=Path, help="files, or directories to check recursively") check.add_argument("--mode", choices=["checked", "compiled"], default="checked") check.add_argument("--format", choices=["text", "json"], default="text") check.add_argument("--search-path", action="append", type=Path, default=[], @@ -25,7 +26,7 @@ def main(argv: list[str]) -> int: args = parser.parse_args(argv) try: - found = gate.check(args.paths, mode=args.mode, search_paths=args.search_path) + found = gate.check(python_files(args.paths), mode=args.mode, search_paths=args.search_path) except PyreflyError as e: print(f"typedpython: {e}", file=sys.stderr) return EXIT_FAILURE diff --git a/python-multiplatform-ksp/src/main/python/typedpython/paths.py b/python-multiplatform-ksp/src/main/python/typedpython/paths.py new file mode 100644 index 000000000..34f342238 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/paths.py @@ -0,0 +1,21 @@ +"""Turning the paths a caller gives into the Python files to check.""" +from collections.abc import Sequence +from pathlib import Path + +SKIPPED_DIRS = {"__pycache__", "build"} + + +def python_files(paths: Sequence[Path]) -> list[Path]: + """Files as given; directories expanded to their `.py` files, skipping hidden and cache dirs.""" + found: list[Path] = [] + for path in paths: + path = Path(path) + if not path.is_dir(): + found.append(path) + continue + for file in sorted(path.rglob("*.py")): + parts = file.relative_to(path).parts[:-1] + if any(p.startswith(".") or p in SKIPPED_DIRS for p in parts): + continue + found.append(file) + return found diff --git a/python-multiplatform-ksp/src/test/python/test_cli.py b/python-multiplatform-ksp/src/test/python/test_cli.py index 6b5d1f383..4a7e003a8 100644 --- a/python-multiplatform-ksp/src/test/python/test_cli.py +++ b/python-multiplatform-ksp/src/test/python/test_cli.py @@ -46,3 +46,15 @@ def test_missing_pyrefly_exits_2(write): path = write("user.py", "def f(x: int) -> int:\n return x\n") result = run(path, env_extra={"TYPEDPYTHON_PYREFLY": "/nonexistent/pyrefly"}) assert result.returncode == 2, result.stdout + result.stderr + + +def test_a_directory_is_checked_file_by_file(write, tmp_path): + write("proj/pkg/__init__.py", "") + write("proj/pkg/ok.py", "def f(x: int) -> int:\n return x\n") + write("proj/pkg/bad.py", "def g(x):\n return x\n") + write("proj/.venv/lib/junk.py", "def h(x):\n return x\n") # hidden: skipped + write("proj/pkg/__pycache__/stale.py", "def k(x):\n return x\n") # cache: skipped + result = run(tmp_path / "proj") + assert result.returncode == 1, result.stdout + result.stderr + assert "bad.py" in result.stdout + assert "junk.py" not in result.stdout and "stale.py" not in result.stdout From a85f7ebfe359244ed1e578cad7d54423cd23c25f Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:26:42 +0900 Subject: [PATCH 07/36] Feat: Define the TypedPython typed IR (stage 2 contract) The IR between the front end (AST + Pyrefly types) and the C back end, with each node specified by its CPython behaviour: checked i64 arithmetic that deopts on overflow, Python floor division and modulo, exact int/float comparison, list-semantics indexing, math errors, and native arrays for list[float]/list[int] parameters with entry guards and write-back. Also brings the TypedPython CI workflow (without Cython). Refs #41 --- .github/workflows/typedpython.yml | 40 ++ .../src/main/python/typedpython/ir.py | 376 ++++++++++++++++++ 2 files changed, 416 insertions(+) create mode 100644 .github/workflows/typedpython.yml create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/ir.py diff --git a/.github/workflows/typedpython.yml b/.github/workflows/typedpython.yml new file mode 100644 index 000000000..84dba3f04 --- /dev/null +++ b/.github/workflows/typedpython.yml @@ -0,0 +1,40 @@ +name: TypedPython CI + +# The TypedPython gate and compiler are Python (python-multiplatform-ksp/src/{main,test}/python). +# Gradle does not run them; this workflow does. +on: + push: + branches: [ "develop", "main" ] + paths: [ "python-multiplatform-ksp/**", ".github/workflows/typedpython.yml" ] + pull_request: + branches: [ "develop", "main" ] + paths: [ "python-multiplatform-ksp/**", ".github/workflows/typedpython.yml" ] + +jobs: + test: + name: TypedPython tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + defaults: + run: + working-directory: python-multiplatform-ksp + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + + - name: Create the environment + # pyrefly is pinned exactly: the gate reads its versioned type-report format. + run: | + uv venv .venv --python 3.13 + uv pip install --python .venv/bin/python "pyrefly==1.3.2" pytest + + - name: Behaviour tests + run: .venv/bin/python -m pytest -q -m "not measurement" + + - name: Measurements (recorded, sanity bounds only) + # A shared runner is not an idle machine (AGENTS.md §8): read ratios, not absolute times. + run: .venv/bin/python -m pytest -q -s -m measurement diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py new file mode 100644 index 000000000..7d8ac2b23 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -0,0 +1,376 @@ +"""TypedPython typed IR (design §4.3 stage 2, #41). + +The IR sits between the front end (AST + Pyrefly types → IR, `frontend.py`) and the back end +(IR → C, `cgen.py`). Both sides are written against this file; it is the contract. + +The governing rule is INTENT §1.4: **compiled code computes what CPython computes.** The IR +encodes that rule, so a back end that implements each node as specified below cannot change a +result: + +- Every node says what it does in CPython terms. Where a C operation differs from CPython (integer + overflow, floor division and modulo of negatives, true division of large ints, int/float + comparison, list indexing), the node is defined by the CPython behaviour and the back end must + implement that behaviour, not the C one. +- When the C path cannot produce CPython's result, the IR says **deopt**: the compiled call gives + up and the whole call is redone by the interpreted function. A deopt is allowed only where + redoing is unobservable: before any effect (`Function.guards`, evaluated at entry), or anywhere + inside a `pure` function. The front end guarantees that an impure function contains no node + that can deopt (`Function.may_deopt` is False for impure functions); the back end may assert it. +- Exceptions are CPython's exceptions, with CPython's messages (`ZeroDivisionError("division by + zero")`, `IndexError("list index out of range")`, `ValueError("math domain error")`, ...). + The traceback differs (there is no Python frame for compiled code); that is the one accepted + difference, as for every C extension. + +Module shape (back end). The extension module executes the module's original source in its own +namespace first, so every name, class and global is the interpreted one and lives in one +namespace. Then each compiled function `f` replaces the global `f` with a C function, and the +original stays reachable as `__typedpython_interpreted__[f]` for deopt. Because there is one +namespace, interpreted code that calls `f` gets the compiled `f`, and a deopted call sees the same +globals and classes as compiled code. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + + +# --- types --------------------------------------------------------------------------------------- + +class Type(Enum): + """The C representation of a value. Python semantics are attached per type below.""" + + I64 = "i64" # a Python int known to fit in [-2**63, 2**63-1]; arithmetic is checked + F64 = "f64" # a Python float (IEEE double, exactly as CPython) + BOOL = "bool" # a Python bool + NONE = "none" # the value None (only as a return type) + OBJ = "obj" # any Python object, held as a strong reference (PyObject*) + F64_ARRAY = "f64[]" # a list[float] parameter lowered to a native array (see ArrayParam) + I64_ARRAY = "i64[]" # a list[int] parameter lowered to a native array (see ArrayParam) + + +SCALARS = (Type.I64, Type.F64, Type.BOOL) +ARRAYS = (Type.F64_ARRAY, Type.I64_ARRAY) + + +# --- expressions --------------------------------------------------------------------------------- + +@dataclass(frozen=True) +class Expr: + type: Type + + +@dataclass(frozen=True) +class Const(Expr): + """A literal. `value` is a Python int/float/bool/None matching `type`.""" + + value: int | float | bool | None + + +@dataclass(frozen=True) +class Local(Expr): + """Reading a local variable or parameter (always definitely assigned: the front end checks).""" + + name: str + + +class BinOpKind(Enum): + ADD = "+" + SUB = "-" + MUL = "*" + TRUEDIV = "/" + FLOORDIV = "//" + MOD = "%" + + +@dataclass(frozen=True) +class BinOp(Expr): + """Arithmetic with CPython's result. Both operands have the same scalar type (the front end + inserts `ToFloat` for mixed int/float, as CPython converts the int). + + I64 (result I64, except TRUEDIV → F64): + ADD/SUB/MUL — checked; overflow → deopt. + FLOORDIV/MOD — Python floor semantics (result sign follows the divisor for MOD); divisor 0 + → ZeroDivisionError("integer division or modulo by zero") for FLOORDIV and + ZeroDivisionError("integer modulo by zero") for MOD (CPython 3.13/3.14 messages; the + runtime's differential tests pin them); -2**63 // -1 overflows → deopt. + TRUEDIV — CPython rounds int/int correctly; when both |operands| <= 2**53 the C double + division is exact-rounded too, otherwise → deopt. Divisor 0 → ZeroDivisionError("division + by zero"). + F64: + ADD/SUB/MUL — IEEE, as CPython. + TRUEDIV — divisor 0.0 (either sign) → ZeroDivisionError("float division by zero"). + FLOORDIV/MOD — CPython's float floor division and modulo (sign of divisor, `-0.0` cases), + divisor 0.0 → ZeroDivisionError("float floor division by zero") / ("float modulo by + zero"). + OBJ: CPython's generic operation on objects (PyNumber_*), never deopts. + """ + + op: BinOpKind + left: Expr + right: Expr + + +class UnaryOpKind(Enum): + NEG = "-" + POS = "+" + NOT = "not" + + +@dataclass(frozen=True) +class UnaryOp(Expr): + """NEG of I64 is checked (-(-2**63) → deopt). NOT takes a BOOL and gives a BOOL.""" + + op: UnaryOpKind + operand: Expr + + +class CompareKind(Enum): + EQ = "==" + NE = "!=" + LT = "<" + LE = "<=" + GT = ">" + GE = ">=" + + +@dataclass(frozen=True) +class Compare(Expr): + """Result BOOL. Chains (`a < b < c`) are lowered by the front end into `And` with each middle + operand evaluated once (via a temporary). Mixed I64/F64 compares **exactly**, as CPython does — + not by converting the int to double (2**53+1 != float(2**53)). NaN compares as in IEEE.""" + + op: CompareKind + left: Expr + right: Expr + + +@dataclass(frozen=True) +class And(Expr): + """`a and b` on BOOL operands only (result BOOL), short-circuit. The front end does not lower + `and`/`or` on non-bool operands (they return an operand, not a bool).""" + + left: Expr + right: Expr + + +@dataclass(frozen=True) +class Or(Expr): + left: Expr + right: Expr + + +@dataclass(frozen=True) +class ToFloat(Expr): + """`float(x)` of an I64 (round-to-nearest-even, as CPython's int→float), or the implicit + int→float conversion of mixed arithmetic. Type F64.""" + + operand: Expr + + +@dataclass(frozen=True) +class Box(Expr): + """A scalar or array element converted to a Python object (type OBJ): I64 → int, F64 → float, + BOOL → True/False. Used when a typed value flows into OBJ code.""" + + operand: Expr + + +@dataclass(frozen=True) +class Unbox(Expr): + """An OBJ known by the front end to be exactly int/float/bool converted to a scalar. For I64, + a value outside the i64 range → deopt (so only legal in a pure function or a guard).""" + + operand: Expr + + +class MathFunc(Enum): + SQRT = "sqrt" + EXP = "exp" + LOG = "log" + SIN = "sin" + COS = "cos" + TAN = "tan" + ATAN2 = "atan2" + FABS = "fabs" + HYPOT = "hypot" + + +@dataclass(frozen=True) +class MathCall(Expr): + """`math.(x...)` on F64, type F64, with the stdlib `math` module's errors: + a domain error → ValueError("math domain error"), a range overflow → + OverflowError("math range error") (CPython's math_1/math_2 rules on errno/inf/nan).""" + + func: MathFunc + args: tuple[Expr, ...] + + +@dataclass(frozen=True) +class Call(Expr): + """A direct call of another compiled function in the same module (C to C). If the callee may + deopt, the caller must be pure (the deopt propagates and the outermost compiled call is + redone); the front end otherwise emits `PyCall`.""" + + function: str + args: tuple[Expr, ...] + + +@dataclass(frozen=True) +class PyCall(Expr): + """Calling a Python-level callable found by global name in the module namespace (type OBJ), + with OBJ arguments. Makes the function impure.""" + + function: str + args: tuple[Expr, ...] + + +@dataclass(frozen=True) +class Len(Expr): + """`len(array)` of an array parameter. Type I64.""" + + array: str + + +@dataclass(frozen=True) +class Index(Expr): + """`array[i]` of an array parameter, with list semantics: a negative index counts from the + end; out of range → IndexError("list index out of range"). Element type F64 or I64.""" + + array: str + index: Expr + + +# --- statements ---------------------------------------------------------------------------------- + +@dataclass(frozen=True) +class Stmt: + pass + + +@dataclass(frozen=True) +class Assign(Stmt): + target: str + value: Expr + + +@dataclass(frozen=True) +class StoreIndex(Stmt): + """`array[i] = value` with list semantics (negative index; out of range → + IndexError("list assignment index out of range")). Marks the element dirty for write-back.""" + + array: str + index: Expr + value: Expr + + +@dataclass(frozen=True) +class ExprStmt(Stmt): + value: Expr + + +@dataclass(frozen=True) +class If(Stmt): + cond: Expr # BOOL + then: tuple[Stmt, ...] + orelse: tuple[Stmt, ...] = () + + +@dataclass(frozen=True) +class While(Stmt): + cond: Expr # BOOL + body: tuple[Stmt, ...] + + +@dataclass(frozen=True) +class ForRange(Stmt): + """`for var in range(start, stop, step)`: `step` is a non-zero I64 Const; `start`/`stop` are + evaluated once before the loop, as range() does. `var` is I64 and is not assigned in the body + (the front end checks), so it never overflows: it stays between start and stop. After the loop + `var` keeps its last value, and if the loop ran zero times it keeps its previous binding — the + front end rejects reading it after the loop unless it was definitely assigned before.""" + + var: str + start: Expr + stop: Expr + step: Const + body: tuple[Stmt, ...] + + +@dataclass(frozen=True) +class Return(Stmt): + value: Expr | None = None + + +@dataclass(frozen=True) +class Break(Stmt): + pass + + +@dataclass(frozen=True) +class Continue(Stmt): + pass + + +# --- functions and modules ----------------------------------------------------------------------- + +@dataclass(frozen=True) +class Param: + name: str + type: Type + + +@dataclass(frozen=True) +class ArrayParam: + """How a `list[float]`/`list[int]` parameter becomes a native array, keeping list semantics: + + At entry (guards, before any effect): the argument is exactly a `list`, every element is exactly + `float` (F64_ARRAY) or exactly `int` within i64 (I64_ARRAY), and no two array parameters are + the same list object (aliasing would make separate copies diverge). Then the elements are copied + into a C array. The function body only indexes, stores into and takes `len` of it (no calls that + could observe the list, no appends: the front end checks), so the list and the array cannot + diverge observably. On **every** exit — return, exception, or deopt — dirty elements are + written back as new float/int objects before control leaves the function, so CPython's + partial-update-then-raise behaviour is preserved, and a deopt redo starts from the same list + state the interpreted function would see... which is only correct because a function with array + stores is impure and therefore can never deopt after entry (`Function.may_deopt` is False). + """ + + name: str + type: Type # F64_ARRAY or I64_ARRAY + stored: bool # whether the body stores into it (write-back needed) + + +@dataclass(frozen=True) +class Function: + """One compiled function. `params` lists every positional parameter in order (scalar Param, + ArrayParam, or Param of type OBJ for anything else). Keyword-only, *args, **kwargs, defaults + and decorators other than `compiled` are not compiled in this stage (the function stays + interpreted; `frontend` records why). + + `pure`: no effect outside its own locals — no stores into anything but locals, no PyCall, no + attribute reads (a property can have effects), only `Call`s to pure functions. Only a pure + function may contain deopting nodes after entry. + + Entry guards are generated by the back end from the params: exact runtime type per scalar/array + param (`type(x) is float`, not isinstance — CPython would compute with the int or the subclass + it was given), i64 range for I64, the ArrayParam checks. A failing guard → deopt (always safe: + nothing has happened yet). Wrong arity or keyword use → deopt too, so CPython raises its own + TypeError. + """ + + name: str + params: tuple[Param | ArrayParam, ...] + returns: Type + locals: dict[str, Type] + body: tuple[Stmt, ...] + pure: bool + may_deopt: bool + source_line: int + + +@dataclass(frozen=True) +class Module: + name: str + functions: tuple[Function, ...] + # Functions the front end left interpreted, with the reason (for `demo` and the coverage table). + skipped: dict[str, str] = field(default_factory=dict) From e9fe49e7a1885152bb58754bb5bf72e0f418bff1 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:28:39 +0900 Subject: [PATCH 08/36] Spec: TypedPython compilation, IR-level safety proof, incremental builds SPEC N-7 (compile to C), N-8 (memory safety proved on the IR; a function that cannot be proved stays interpreted with a diagnostic) and N-9 (module-level incremental compilation keyed by source, dependency interfaces, compiler version, platform and flags), all planned. The IR gains 'proven' on array accesses, set only by the verifier. Refs #41 --- docs/SPEC.md | 28 +++++++++++++++++++ .../src/main/python/typedpython/ir.py | 25 +++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/docs/SPEC.md b/docs/SPEC.md index c2a5fa92b..0ac409a8c 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -253,6 +253,34 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. build** (design §6 step 1b). Rebinding and mixed containers are found by a second Pyrefly pass over a probed copy (design §6.1), which runs only when a file has a name assigned twice or a container display; `for` targets and comprehension variables are not probed. +- **N-7** TypedPython compilation to C (INTENT §1.4; design §4.3; #41). `@compiled` (a builtin name, no + import — #42) and a first-line `# typedpython: compiled` mark functions; the compiler lowers them + through the typed IR (`typedpython/ir.py`) to C against the CPython C API and builds an extension with + the platform C compiler. Compiled functions return what CPython returns (IR contract: checked i64 + with deopt to the interpreted function, Python floor division/modulo, exact int/float comparison, + CPython's exception types and messages; tracebacks are the one accepted difference). `Status: planned`. +- **N-8** TypedPython memory safety is proved on the IR before C is generated (#41; maintainer decision + 2026-10-03). Each property, and what happens when it cannot be proved: + + | Property proved on the IR | If not proved | + |---|---| + | No local read before assignment on every path | function not compiled; diagnostic names the local | + | Every node's operand types match; no implicit conversion | not compiled; diagnostic | + | No deopting node in an impure function | not compiled; diagnostic | + | Array access in bounds, or a runtime bounds check kept | the check stays (never an unchecked access) | + | Array parameters only indexed, stored, `len`-ed; no aliasing (entry guard) | not compiled; diagnostic | + | OBJ references follow one ownership rule; no IR form for free, borrowed alias or NULL read | not compiled; diagnostic | + | Integer overflow checked and promoted (deopt) | — (every I64 operation is checked by construction) | + + Generated C reaches memory and `PyObject`s only through the runtime helpers. ASan/UBSan CI, + differential tests against the interpreter and fuzzing back the proof. `Status: planned`. +- **N-9** TypedPython incremental compilation (#41; maintainer decision 2026-10-03). The unit is a module. + Its cache key is the source hash, the interface hashes (typed signatures) of the modules it depends + on, the compiler version, the target platform and the build flags; a body-only change leaves the + interface hash unchanged, so dependent modules are not recompiled. C objects are cached per module. + The Gradle task declares exact inputs and outputs so up-to-date checks and the build cache apply. + Tests: changing one function body recompiles that module only; an unchanged rebuild spends ~0 s in + the compile step. `Status: planned`. --- diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index 7d8ac2b23..dd1df5180 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -21,6 +21,26 @@ The traceback differs (there is no Python frame for compiled code); that is the one accepted difference, as for every C extension. +Safety (`verify.py`, maintainer decision 2026-10-03: "IR 단계에서 안전성을 증명하도록 해"). Before any +C is generated, the verifier proves on the IR that the C will be memory-safe and keep the rules +above; **a function it cannot prove is not compiled** — it stays interpreted and the reason is +reported as a diagnostic. What it proves: + +- definite assignment: no local is read before it is assigned on every path; +- types: every node's operands have the types the node requires; no implicit conversion; +- deopt placement: no node that can deopt in an impure function (`may_deopt` consistent); +- arrays: every `Index`/`StoreIndex` is either proved in bounds (and marked `proven`) or keeps its + runtime check; array params are used only by `Index`/`StoreIndex`/`Len`; +- ownership: OBJ values are created only by nodes that return a new reference (Box, PyCall, OBJ + BinOp) and the back end releases them by one rule (every OBJ local owns its reference; assigning + releases the old one; every exit releases all); the IR has no node that frees, aliases a + borrowed reference, or reads a possibly-NULL object, so use-after-free and NULL dereference have + no IR form. + +The generated C touches memory and `PyObject`s only through the runtime helpers (`tp_runtime.h`), +which are tested separately; AddressSanitizer/UBSan runs, differential tests against the +interpreter, and fuzzing are the safety net outside the proof. + Module shape (back end). The extension module executes the module's original source in its own namespace first, so every name, class and global is the interpreted one and lives in one namespace. Then each compiled function `f` replaces the global `f` with a C function, and the @@ -238,6 +258,10 @@ class Index(Expr): array: str index: Expr + # Set only by `verify` when it has proved 0 <= index < len for the non-negative case (e.g. a + # `ForRange` index over range(len(array))); then the back end may omit the bounds check. + # Never set by the front end. Unproved accesses keep the full check. + proven: bool = False # --- statements ---------------------------------------------------------------------------------- @@ -261,6 +285,7 @@ class StoreIndex(Stmt): array: str index: Expr value: Expr + proven: bool = False # as in Index @dataclass(frozen=True) From c2c80fc9121be2725249a130825401371ebda675 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:30:37 +0900 Subject: [PATCH 09/36] Spec: Fix the C runtime API the generated code may call Refs #41 --- .../main/python/typedpython/runtime/API.md | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md new file mode 100644 index 000000000..30bcdba1a --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -0,0 +1,62 @@ +# tp_runtime.h — the C runtime API generated code may call + +Generated C (cgen.py) reaches memory and `PyObject`s **only** through these helpers (SPEC N-8). +Header-only, `static inline`, C99 + CPython C API, no other dependencies. Every helper implements +the CPython behaviour named in `ir.py`. Return convention unless stated: `0` ok, `1` deopt (no +Python error set, nothing to undo), `-1` Python error set (CPython's exception type and message). + +## Integers (int64_t) + int tp_add_i64(int64_t a, int64_t b, int64_t *out); /* 0 / 1 on overflow */ + int tp_sub_i64(int64_t a, int64_t b, int64_t *out); + int tp_mul_i64(int64_t a, int64_t b, int64_t *out); + int tp_neg_i64(int64_t a, int64_t *out); /* 1 for INT64_MIN */ + int tp_floordiv_i64(int64_t a, int64_t b, int64_t *out); /* -1 ZeroDivisionError; 1 for MIN // -1 */ + int tp_mod_i64(int64_t a, int64_t b, int64_t *out); /* -1 ZeroDivisionError; sign of b */ + int tp_truediv_i64(int64_t a, int64_t b, double *out); /* -1 ZeroDivisionError; 1 if |a| or |b| > 2**53 */ + double tp_i64_to_f64(int64_t a); /* as float(int): round half to even */ + +## Floats (double) + int tp_truediv_f64(double a, double b, double *out); /* 0 / -1 ZeroDivisionError("float division by zero") */ + int tp_floordiv_f64(double a, double b, double *out); /* CPython float_floor_div semantics */ + int tp_mod_f64(double a, double b, double *out); /* CPython float_rem semantics */ + +## Comparisons + int tp_cmp_i64_f64(int64_t a, double b, int op); /* exact, op is Py_LT..Py_GE; returns 0/1 */ + int tp_cmp_f64_i64(double a, int64_t b, int op); + +## math (stdlib math module semantics: math_1 / math_2 errno and inf/nan rules) + int tp_math_sqrt(double x, double *out); /* -1 ValueError("math domain error") */ + int tp_math_exp(double x, double *out); /* -1 OverflowError("math range error") */ + int tp_math_log(double x, double *out); + int tp_math_sin(double x, double *out); + int tp_math_cos(double x, double *out); + int tp_math_tan(double x, double *out); + int tp_math_fabs(double x, double *out); + int tp_math_atan2(double y, double x, double *out); + int tp_math_hypot(double x, double y, double *out); + +## Boxing (new references; NULL with MemoryError set on failure) + PyObject *tp_box_i64(int64_t v); + PyObject *tp_box_f64(double v); + PyObject *tp_box_bool(int v); + /* exact-type unboxing for guards: 0 ok, 1 wrong exact type or out of i64 range (deopt) */ + int tp_unbox_i64(PyObject *o, int64_t *out); /* type(o) is int and fits */ + int tp_unbox_f64(PyObject *o, double *out); /* type(o) is float */ + int tp_unbox_bool(PyObject *o, int *out); /* o is Py_True / Py_False */ + +## Native arrays for list[float] / list[int] parameters (ir.ArrayParam) + typedef struct { PyObject *list; Py_ssize_t len; double *data; unsigned char *dirty; } tp_f64_array; + typedef struct { PyObject *list; Py_ssize_t len; int64_t *data; unsigned char *dirty; } tp_i64_array; + + int tp_f64_array_enter(PyObject *obj, tp_f64_array *a); /* 1 unless exactly list of exactly float; -1 MemoryError */ + int tp_i64_array_enter(PyObject *obj, tp_i64_array *a); /* 1 unless exactly list of exactly int in i64 range */ + /* resolve a Python index (negative counts from the end) to a slot; -1 IndexError with + "list index out of range" (load) or "list assignment index out of range" (store) */ + int tp_f64_array_slot(const tp_f64_array *a, int64_t i, int store, Py_ssize_t *slot); + int tp_i64_array_slot(const tp_i64_array *a, int64_t i, int store, Py_ssize_t *slot); + /* write dirty elements back as new float/int objects and free the buffers. Always frees, even + when it fails (then -1 with MemoryError). Safe to call on a zeroed struct. */ + int tp_f64_array_exit(tp_f64_array *a); + int tp_i64_array_exit(tp_i64_array *a); + /* aliasing guard: 1 if any two of the n objects are the same object */ + int tp_any_same(PyObject *const *objs, Py_ssize_t n); From 6492bed320cac46b0386bf1f1173972c4fc49fae Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:34:12 +0900 Subject: [PATCH 10/36] Docs: Record the IR-to-C backend, IR safety proof, incremental builds and the compiled builtin Refs #41 --- docs/design/typedpython.md | 60 ++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 108f57523..d7720ea8d 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -55,7 +55,7 @@ TypedPython 의 목표는 **새 언어를 만들지 않고, 파이썬 문법 그 |---|---|---|---| | 0 | 서드파티 패키지 (의존성) | 없음 — 검사하지 않는다 | 바이트코드 (pypackpack 이 받은 그대로) | | 1 `checked` | **사용자 프로젝트의 파이썬 코드 전부 — 기본값** | 빌드 타임 타입 오류, `Any` 유출 0 (§4.2) | 바이트코드 | -| 1 `compiled` | `@compiled` 또는 모듈 선언 | `checked` + §3.1 금지 목록 | C 확장 모듈 | +| 1 `compiled` | 내장 데코레이터 `@compiled`(import 없음, #42) 또는 첫 줄 `# typedpython: compiled` | `checked` + §3.1 금지 목록 | C 확장 모듈 | | 2 `kernel` | `@kernel` | 객체·힙 할당 없음, 고정 dtype | SPIR-V / MSL / WGSL / PTX | - **사용자 코드는 표시하지 않아도 검사된다.** 타입 오류가 있으면 Kotlin 컴파일 에러처럼 빌드가 @@ -119,22 +119,49 @@ API 안정성을 확인한 뒤로 미룬다. **이 게이트만으로도 독립적인 가치가 있다.** 컴파일러가 없어도 "사용자 코드에는 `Any` 가 없다" 는 보장이 생기고, 그것이 AI 친화적 피드백 루프의 핵심이다. 이후 모든 단계가 이 게이트를 전제로 한다. -### 4.3 백엔드 — 단계별 +### 4.3 백엔드 — 자체 typed IR → C (결정 2026-10-03, #41) -| 단계 | 경로 | 이유 | -|---|---|---| -| 1 | 추론 결과 → Cython pure-Python 모드 애노테이션 → C | 성숙한 백엔드. pypackpack 의 빈 Cython 슬롯(`native`/`mixed` 빌드 레벨)에 그대로 들어간다 | -| 2 | 자체 typed IR → C / LLVM | Cython 의 의미론 제약에서 벗어남. 등급 2 와 IR 을 공유할 여지 | +**Cython 은 쓰지 않는다** (사용자: "Cython 의존 별로야. 2단계로 바로 진행하도록 해"). Cython 경로 +(PR #39)는 닫았고, 거기서 검증된 의미 보존 규칙만 IR 계약으로 옮겼다. -Cython 은 착지용 디딤돌이지 종착지가 아니다. + 소스 ─ Pyrefly 추론 ─▶ frontend ─▶ typed IR ─▶ verify ─▶ cgen ─▶ C ─▶ 플랫폼 C 컴파일러 ─▶ 확장 모듈 + (ir.py) (증명) (CPython C API + tp_runtime.h) + +- **IR 이 계약이다** (`typedpython/ir.py`). 각 노드는 CPython 동작으로 정의된다: 검사되는 i64 산술과 + 오버플로 시 deopt(인터프리터 재실행), 파이썬식 floor 나눗셈·나머지, 정확한 int/float 비교, 리스트식 + 인덱싱, `math` 모듈의 예외. C 런타임 API 는 `typedpython/runtime/API.md` 에 고정했다. +- **왜 C 인가, LLVM 이 아니라.** 모든 타깃에 C 컴파일러가 이미 있다(clang, NDK, Xcode) — 새 의존이 없다. + CPython C API 가 C 이고, 플랫폼 디버거가 그대로 쓰인다. 타입이 확정된 스칼라·배열 코드에서 중요한 + 최적화(인라이닝, 벡터화)는 C 컴파일러가 이미 한다. IR 이 안정 계층이라 LLVM 백엔드는 나중에도 붙일 수 있다. +- **왜 Rust 가 아닌가.** 파이썬의 별칭과 공유 변경은 Rust 소유권에 맞지 않아 생성 코드가 + `Rc`/`unsafe` 로 채워지고, 앱 빌드마다 Rust 툴체인 비용이 붙는다(사용자 확정). +- **모듈 모양.** 확장 모듈은 원래 소스를 자기 네임스페이스에서 먼저 실행하고, 컴파일된 함수만 C 함수로 + 바꾼다. 원래 함수는 `__typedpython_interpreted__` 에 남아 deopt 때 같은 전역·같은 클래스로 실행된다. +- **순서.** 스칼라 int/float/bool 과 컴파일된 함수끼리의 호출 → `list[float]`/`list[int]` 의 네이티브 + 배열 → 고정 레이아웃 클래스. + +### 4.3.1 안전성은 IR 에서 증명한다 (SPEC N-8) + +사용자 결정: "IR 단계에서 안전성을 증명하도록 해". C 를 만들기 전에 `verify.py` 가 IR 에서 증명한다 — +정의 전 사용 없음, 노드 타입 일치, 비순수 함수에 deopt 노드 없음, 배열 접근은 경계 증명 또는 런타임 검사 +유지, 배열 파라미터는 인덱싱·저장·len 만, OBJ 참조는 하나의 소유 규칙. **증명하지 못한 함수는 컴파일하지 +않고** 인터프리터에 남기며 이유를 진단으로 낸다. 생성 C 는 `tp_runtime.h` 헬퍼로만 메모리와 `PyObject` +를 다룬다. 증명 밖의 안전망: ASan/UBSan, 인터프리터와의 차등 테스트, IR 수준 퍼징. + +### 4.3.2 증분 컴파일 (SPEC N-9) + +사용자 결정: "증분 컴파일도 지원하도록 해. 빌드 속도 너무 오래 안걸리게". 단위는 모듈. 키는 소스 해시 + +의존 모듈의 **인터페이스** 해시(타입 시그니처) + 컴파일러 버전 + 플랫폼 + 빌드 플래그. 본문만 바뀌면 +인터페이스 해시가 그대로라 의존 모듈은 다시 컴파일하지 않는다. C 오브젝트는 모듈별로 캐시한다. +Gradle 태스크는 입력·출력을 정확히 선언해 up-to-date 판정과 빌드 캐시가 함께 작동한다. 검토하고 기각한 경로: +- **Cython** — 사용자 결정으로 폐기(위). - **mypyc** — 타입 체커가 mypy 에 묶여 있어 Pyrefly 와 판정이 갈라진다. -- **Nuitka** — 동작 동일성을 우선해 타입 특수화를 하지 않는다. 등급 0 모듈의 전체 컴파일 폴백으로는 유효. -- **LPython** — ASR 설계는 좋지만 업스트림 활동이 LFortran 쪽으로 옮겨간 것으로 보인다. -- **Rust / Kotlin 으로 번역** — 생성기가 이미 아는 사실을 borrow checker 나 다른 런타임에 다시 - 증명해야 하고, 파이썬 의미론(임의 정밀도 정수, 슬라이싱, 예외)을 그쪽에 재구현해야 한다. +- **Nuitka** — 동작 동일성을 우선해 타입 특수화를 하지 않는다. +- **LPython** — 업스트림 활동이 LFortran 쪽으로 옮겨간 것으로 보인다. +- **Rust 생성** — 위. ### 4.4 Kotlin 경계의 직접 호출 @@ -199,7 +226,7 @@ TypedPython 은 그래서 자체 텐서 런타임을 만들지 않고 **torch op |---|---|---|---| | 1 | 건전성 게이트 | Pyrefly 출력을 받아 `Any` 유출을 에러로, 금지 목록 위반을 경고/에러로 내는 도구 | 없음 | | 1b | 빌드 기본 단계로 연결 | PythonMultiplatform Gradle 플러그인이 사용자 파이썬 소스에 게이트를 항상 돌리고, 에러면 빌드 실패 | 1 | -| 2 | Cython 백엔드 | `@compiled` 모듈 → C 확장, 데스크톱 타깃에서 끝까지 | pypackpack Cython 슬롯 인터페이스 결정 | +| 2 | typed IR → C 백엔드 (#41) | 프론트엔드·검증기·C 생성·런타임, 데스크톱에서 끝까지 | 없음 (pypackpack 슬롯은 3 의 의존) | | 3 | Kotlin 직접 호출 | 스텁 타입 기반 C ABI 호출 생성 | 스텁 배선 완료 | | 4 | 커널 하나 | 융합 softmax 를 Vulkan · Metal 양쪽에서 upstream 과 수치 일치 | torchnative 착지 승인 | | 5 | 커널 컴파일러 일반화 | SIMT 서브셋 전체, 이후 타일 | 4 | @@ -231,7 +258,8 @@ upstream 대비 수치 오차를 기준선으로 박아 둔다. ## 7. 위험 - **Pyrefly 의 외부 API 안정성.** 라이브러리 임베드는 확인 전까지 미룬다 (§4.1). -- **Cython pure 모드의 표현 한계.** 게이트가 통과시킨 코드를 Cython 이 거부하는 경우가 나올 수 있다. +- **IR 이 표현하지 못하는 코드.** 게이트를 통과한 코드라도 IR 밖의 구문이 있으면 그 함수는 컴파일되지 + 않고 인터프리터에 남는다(`Module.skipped`). 그 목록이 IR 확장의 요구사항이 된다. 그 목록이 2단계 IR 의 요구사항이 된다. - **등급 2 컴파일러의 규모.** 가장 큰 작업이다. 커널 하나를 끝까지 하는 것(§6 의 4)을 첫 검증 단위로 삼아 범위를 제한한다. @@ -245,9 +273,9 @@ upstream 대비 수치 오차를 기준선으로 박아 둔다. ## 8. 미결 사항 (사용자 결정) -1. `compiled` 의 표기 — 모듈 선언 방식, 데코레이터 이름. -2. 고정폭 정수의 오버플로 의미. -3. pypackpack Cython 백엔드 슬롯의 인터페이스를 누가 정하는가. +1. ~~`compiled` 의 표기~~ — **결정됨 (2026-10-03):** import 없는 내장 이름 `@compiled`(사용자: "특정 패키지 아래로 넣지 마"). 런타임이 `builtins` 에 넣고(#42), 일반 CPython 에서는 개발용 심, 타입 검사기는 내장 확장 스텁으로 안다. 모듈 전체 표식은 도구 프라그마 관례(`# pyright: strict` 처럼)를 따라 `# typedpython: compiled` 로 둔다. +2. ~~고정폭 정수의 오버플로 의미~~ — **결정됨:** `int` 는 i64 로 컴파일하고 오버플로 시 인터프리터 재실행으로 큰 정수로 승격(의미 보존). +3. ~~pypackpack Cython 슬롯 인터페이스 소유~~ — **결정됨:** pypackpack 이 정한다. TypedPython 은 생성한 C 소스, 빌드 플래그, 모듈 목록, 모듈별 `.py` 폴백을 넘긴다(Cython 폐기 후 갱신). 4. GPU 착지를 torchnative 커스텀 op 로 할 것인가 (다른 레포 변경). 5. ~~TypedPython 을 별도 레포로 분리할 것인가~~ — **결정됨 (2026-10-02):** PythonMultiplatform 의 기본 동작. 검사는 기본, 컴파일은 opt-in. From 6110e47543bba993a304dcd38b24cefe5dfedd0f Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 08:39:38 +0900 Subject: [PATCH 11/36] Spec: Let compiled code call Kotlin through opaque Python objects New IR nodes Global, GetAttr, CallObject (vectorcall with keywords), Truth, CompareObj and ObjToFloat, with runtime helpers. A function that calls the binder becomes impure, so its unbounded ints stay CPython int objects, while floats, bounded indices and the surrounding loops stay native. PyCall is replaced by CallObject(Global(name)). Refs #41 --- .../src/main/python/typedpython/ir.py | 75 +++++++++++++++++-- .../main/python/typedpython/runtime/API.md | 11 +++ 2 files changed, 78 insertions(+), 8 deletions(-) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index dd1df5180..2caa21177 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -31,8 +31,8 @@ - deopt placement: no node that can deopt in an impure function (`may_deopt` consistent); - arrays: every `Index`/`StoreIndex` is either proved in bounds (and marked `proven`) or keeps its runtime check; array params are used only by `Index`/`StoreIndex`/`Len`; -- ownership: OBJ values are created only by nodes that return a new reference (Box, PyCall, OBJ - BinOp) and the back end releases them by one rule (every OBJ local owns its reference; assigning +- ownership: OBJ values are created only by nodes that return a new reference (Box, Global, + GetAttr, CallObject, OBJ BinOp) and the back end releases them by one rule (every OBJ local owns its reference; assigning releases the old one; every exit releases all); the IR has no node that frees, aliases a borrowed reference, or reads a possibly-NULL object, so use-after-free and NULL dereference have no IR form. @@ -236,12 +236,65 @@ class Call(Expr): @dataclass(frozen=True) -class PyCall(Expr): - """Calling a Python-level callable found by global name in the module namespace (type OBJ), - with OBJ arguments. Makes the function impure.""" +class Global(Expr): + """Reading a module-level name (then builtins), looked up **at the time of the read** in the + module namespace, exactly as CPython's LOAD_GLOBAL: rebinding the global later is seen. + Unbound → NameError("name 'x' is not defined"). Type OBJ. This is how compiled code reaches + functions and classes imported from Kotlin packages through the binder (they are ordinary + module globals). A read alone has no effect, so it does not make a function impure.""" - function: str + name: str + + +@dataclass(frozen=True) +class GetAttr(Expr): + """`obj.name` on an OBJ (PyObject_GetAttr — descriptors, properties and binder proxies' attribute + hooks run exactly as in CPython). Type OBJ. Makes the function impure: a property may have + effects, so the call can never be redone.""" + + obj: Expr + name: str + + +@dataclass(frozen=True) +class CallObject(Expr): + """Calling an OBJ callable — a binder Kotlin function or proxy method, any Python callable — + with positional `args` and keyword arguments `kwnames` (the last len(kwnames) args are the + keyword values, as in vectorcall). Arguments are OBJ (scalars arrive through Box). Type OBJ. + Makes the function impure. This is the general path that keeps Kotlin interop available to + compiled code: whatever the binder accepts from interpreted code, it accepts from here, and + whatever it returns (a proxy, a str, a list) flows on as an opaque OBJ.""" + + callee: Expr args: tuple[Expr, ...] + kwnames: tuple[str, ...] = () + + +@dataclass(frozen=True) +class Truth(Expr): + """`bool(obj)` as a condition (PyObject_IsTrue; may raise as CPython does). Type BOOL.""" + + operand: Expr + + +@dataclass(frozen=True) +class CompareObj(Expr): + """A comparison with an OBJ operand **in a condition** (If/While cond, And/Or operand): + PyObject_RichCompareBool semantics — the rich comparison runs, then its truth value. Type BOOL. + The front end must not use it where the comparison's own result object is kept (`x = a < b`), + because that object need not be a bool.""" + + op: CompareKind + left: Expr + right: Expr + + +@dataclass(frozen=True) +class ObjToFloat(Expr): + """`float(obj)` (PyNumber_Float: __float__/__index__ run as in CPython), result as F64. Raises as + CPython raises; never deopts, so it is allowed in impure functions. Type F64.""" + + operand: Expr @dataclass(frozen=True) @@ -372,8 +425,14 @@ class Function: and decorators other than `compiled` are not compiled in this stage (the function stays interpreted; `frontend` records why). - `pure`: no effect outside its own locals — no stores into anything but locals, no PyCall, no - attribute reads (a property can have effects), only `Call`s to pure functions. Only a pure + `pure`: no effect outside its own locals — no stores into anything but locals, no CallObject, + no GetAttr (a property can have effects), no Truth/CompareObj/ObjToFloat/OBJ BinOp (they run + user methods), only `Call`s to pure functions. + + Kotlin interop (maintainer: compiled code must still talk to Kotlin freely): a function that + calls into the binder is impure, so its int arithmetic cannot use the deopting I64 path — + unbounded ints there are OBJ (CPython's own int objects, PyNumber_*), while floats (F64 never + deopts), bounded loop indices, and the loops around the calls are still native. Only a pure function may contain deopting nodes after entry. Entry guards are generated by the back end from the params: exact runtime type per scalar/array diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index 30bcdba1a..24be459a4 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -60,3 +60,14 @@ Python error set, nothing to undo), `-1` Python error set (CPython's exception t int tp_i64_array_exit(tp_i64_array *a); /* aliasing guard: 1 if any two of the n objects are the same object */ int tp_any_same(PyObject *const *objs, Py_ssize_t n); + +## Opaque objects (Kotlin interop and any Python object; ir.Global/GetAttr/CallObject/Truth/CompareObj/ObjToFloat) +All return new references or NULL with CPython's exception set; none deopts. + PyObject *tp_global(PyObject *module_dict, PyObject *name); /* globals, then builtins; NameError("name 'x' is not defined") */ + PyObject *tp_getattr(PyObject *obj, PyObject *name); /* PyObject_GetAttr */ + PyObject *tp_call(PyObject *callee, PyObject *const *args, size_t nargs, PyObject *kwnames); /* PyObject_Vectorcall */ + int tp_truth(PyObject *obj); /* 0/1, -1 error (PyObject_IsTrue) */ + int tp_compare_bool(PyObject *a, PyObject *b, int op); /* 0/1, -1 error (PyObject_RichCompareBool) */ + int tp_obj_to_f64(PyObject *obj, double *out); /* 0 / -1 (PyNumber_Float, then exact double) */ + PyObject *tp_binop_obj(PyObject *a, PyObject *b, int op); /* PyNumber_Add/... for ir.BinOpKind on OBJ */ + void tp_release(PyObject **slot); /* Py_CLEAR: the only way generated code drops a reference */ From 4f99132ed809885fe513013d40651517d69c4c6c Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:02:47 +0900 Subject: [PATCH 12/36] Perf: Run Pyrefly once per gate check, probes included When a file needs probing, every file of the check is mirrored and Pyrefly runs once on the copies; errors and any-flow positions are mapped back to the original columns (_Remap), and an error inside an inserted probe surfaces as internal/probe-error instead of being dropped. Red: test_pyrefly_runs_once_when_a_file_has_a_candidate (2 runs). Mutation: removing the column shift fails the remap test. Refs #21 --- docs/design/typedpython.md | 7 +- .../src/main/python/typedpython/gate.py | 168 +++++++++++++----- .../src/test/python/test_rebinding.py | 44 +++++ 3 files changed, 171 insertions(+), 48 deletions(-) diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 108f57523..d80ea9f50 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -222,8 +222,11 @@ upstream 대비 수치 오차를 기준선으로 박아 둔다. 이상 대입되는 이름이나 컨테이너 디스플레이가 있는 파일에서만 돈다. `for` 대상과 컴프리헨션 변수는 아직 탐침하지 않는다. - **측정:** 1800줄 합성 모듈에서 plain `pyrefly check` 160–235 ms, 게이트 1.3–1.9 s — 두 번 잰 비율이 - 모두 약 8배다. 탐침 패스가 붙은 뒤로는 약 15배(232 ms → 3.6 s, load 41)다 — 합성 모듈의 - `total += ...` 가 탐침 대상이라 두 번째 실행이 돈다. 단독 측정이 아니다(두 번째는 load average 18.6). 절대값보다 비율을 기준으로 본다. 차이의 + 모두 약 8배다. 탐침 패스를 별도 실행으로 붙이면 약 15배(232 ms → 3.6 s, load 41)였다 — 합성 모듈의 + `total += ...` 가 탐침 대상이라 두 번째 실행이 돌았기 때문이다. 지금은 탐침 대상이 있으면 + **탐침 사본(과 같은 검사의 나머지 파일)에 대해 Pyrefly 를 한 번만** 돌리고, 에러·`any-flow`· + 타입을 모두 그 한 번에서 읽는다(에러 열은 삽입 길이만큼 왼쪽으로 되돌린다). 같은 합성 모듈에서 + 825 ms → 6.2 s, 약 7.6배(load 30 — 유효한 측정이 아니다. load 4 초과). 단독 측정이 아니다(두 번째는 load average 18.6). 절대값보다 비율을 기준으로 본다. 차이의 대부분은 타입 리포트가 typeshed 사본까지 디스크에 쓰는 비용으로 보인다(추정, 미분해). 1b 에서 빌드마다 돌리기 전에 줄일 대상이다. - **고정:** Pyrefly `==1.3.2`. 타입 리포트는 Pyrefly 가 버전을 매기는 포맷이지 공개 API 가 아니다. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/gate.py b/python-multiplatform-ksp/src/main/python/typedpython/gate.py index 303b5c19f..1f34e2303 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/gate.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/gate.py @@ -14,6 +14,7 @@ import shutil import tempfile from collections.abc import Sequence +from dataclasses import dataclass from pathlib import Path from typing import Literal @@ -25,30 +26,107 @@ Position = tuple[int, int] +@dataclass(frozen=True) +class _Run: + """One user file as Pyrefly sees it: the text it was given and where that text differs.""" + + original: str + text: str # the source Pyrefly reads: the original, or the probed copy of it + probes: list[rebinding.Probe] + remap: "_Remap" + + +class _Remap: + """Maps positions in probed text back to the original source. + + A probe inserts `; name` on the line of its assignment, so a column to the right of an + insertion is `len("; name")` further right than in the original; line numbers never change. + """ + + def __init__(self, probes: Sequence[rebinding.Probe]) -> None: + self._inserted: dict[int, list[tuple[int, int]]] = {} + for p in probes: + self._inserted.setdefault(p.line, []).append((p.column - 2, len(p.name) + 2)) + + def to_original(self, line: int, column: int) -> Position | None: + """The original position, or None when it lies inside an inserted probe load.""" + shift = 0 + for start, length in self._inserted.get(line, ()): + if column >= start + length: + shift += length + elif column >= start: + return None + return line, column - shift + + def check( paths: Sequence[Path], mode: Mode = "checked", search_paths: Sequence[Path] = (), ) -> list[Diagnostic]: - report = pyrefly.run(paths, search_paths) - diagnostics = [ - Diagnostic(e.path, e.line, e.column, f"pyrefly/{e.kind}", "error", e.message) - for e in report.errors - ] - reported = {(d.path, d.line, d.column) for d in diagnostics} - + runs = _plan([str(Path(p).resolve()) for p in paths]) forbidden_severity: Severity = "error" if mode == "compiled" else "warning" - for path in (str(Path(p).resolve()) for p in paths): - source = Path(path).read_text() + diagnostics: list[Diagnostic] = [] + + # One Pyrefly run for everything. When any file has something to probe, it runs on a mirrored + # package tree holding the probed copies (and the other files unchanged); otherwise on the + # files themselves. + with tempfile.TemporaryDirectory(prefix="typedpython-probe-") as work: + probing = any(r.probes for r in runs.values()) + if probing: + copies = _mirror(Path(work), {o: r.text for o, r in runs.items()}) + roots = pyrefly.import_roots(list(runs)) + mirror = _Mirror(copies, [((Path(work) / str(i)).resolve(), r) + for i, r in enumerate(roots)]) + report = pyrefly.run( + [Path(c) for c in copies], + [*[Path(work) / str(i) for i in range(len(roots))], + *sorted({Path(c).parent for c in copies}), + *search_paths, *roots], + ) + else: + mirror = _Mirror({o: o for o in runs}, []) + report = pyrefly.run([Path(p) for p in runs], search_paths) + original_of = mirror.copies + run_of = {c: runs[o] for c, o in original_of.items()} + + reported: set[tuple[str, int, int]] = set() + for e in report.errors: + run = run_of.get(e.path) + if run is None: + diagnostics.append(Diagnostic( + mirror.original(e.path), e.line, e.column, f"pyrefly/{e.kind}", "error", e.message, + )) + continue + position = run.remap.to_original(e.line, e.column) + if position is None: + # An error inside an inserted `; name` means the probe itself is wrong. + diagnostics.append(Diagnostic( + run.original, e.line, e.column, "internal/probe-error", "error", + f"pyrefly/{e.kind} inside a probe load (`{e.message}`); this is a gate bug", + )) + continue + diagnostics.append(Diagnostic( + run.original, *position, f"pyrefly/{e.kind}", "error", e.message, + )) + reported.add((run.original, *position)) + + for copy, run in run_of.items(): + path = run.original try: - tree = ast.parse(source, filename=path) + tree = ast.parse(run.text, filename=path) except SyntaxError: continue # Pyrefly has already reported the parse error + # Judged on the probed text: the probe loads are expression statements, so they are + # discarded values and never become any-flow findings. cast_spans = _cast_argument_spans(tree) discarded = _discarded_values(tree) for expr in report.any_expressions: - if expr.path != path or (expr.path, expr.line, expr.column) in reported: + if expr.path != copy: + continue + start = run.remap.to_original(expr.line, expr.column) + if start is None or (path, *start) in reported: continue if _is_signature(expr.type): continue # a callable whose signature mentions Any; its result is judged on its own @@ -57,57 +135,55 @@ def check( if any(_inside((expr.line, expr.column), span) for span in cast_spans): continue diagnostics.append(Diagnostic( - path, expr.line, expr.column, "any-flow", "error", + path, *start, "any-flow", "error", f"this expression has type `{expr.type}`; give it a type, or cast() it " "where it leaves untyped code", )) - reported.add((expr.path, expr.line, expr.column)) + reported.add((path, *start)) - for finding in forbidden.scan(source, path): + for finding in forbidden.scan(Path(path).read_text(), path): diagnostics.append(Diagnostic( path, finding.line, finding.column, finding.rule, forbidden_severity, finding.message, )) - for d in _rebinding_findings(paths, search_paths): - diagnostics.append(Diagnostic( - d.path, d.line, d.column, d.rule, forbidden_severity, d.message, - )) + types = report.expression_types.get(copy, {}) + for f in rebinding.judge(run.probes, types): + diagnostics.append(Diagnostic(path, f.line, f.column, f.rule, forbidden_severity, + f.message)) return sorted(diagnostics) -def _rebinding_findings( - paths: Sequence[Path], search_paths: Sequence[Path], -) -> list[Diagnostic]: - """Second Pyrefly pass over probed copies; skipped when no file has anything to probe.""" - plans: dict[str, tuple[str, list[rebinding.Probe]]] = {} - for path in (str(Path(p).resolve()) for p in paths): +def _plan(paths: Sequence[str]) -> dict[str, _Run]: + """Each file's text for Pyrefly: probed when it has rebinding/mixed-container candidates.""" + runs: dict[str, _Run] = {} + for path in paths: + source = Path(path).read_text() + probed, probes = source, [] try: - tree = ast.parse(Path(path).read_text(), filename=path) + tree = ast.parse(source, filename=path) except SyntaxError: - continue - wanted = rebinding.candidates(rebinding.assignments(tree)) - if wanted: - plans[path] = rebinding.probe(Path(path).read_text(), wanted) - if not plans: - return [] + tree = None + if tree is not None and (wanted := rebinding.candidates(rebinding.assignments(tree))): + probed, probes = rebinding.probe(source, wanted) + runs[path] = _Run(path, probed, probes, _Remap(probes)) + return runs - found: list[Diagnostic] = [] - with tempfile.TemporaryDirectory(prefix="typedpython-probe-") as work: - copies = _mirror(Path(work), {o: probed for o, (probed, _) in plans.items()}) - roots = pyrefly.import_roots(list(plans)) - mirrored = sorted({str(Path(c).parent) for c in copies}) # flat files: their copy's dir - report = pyrefly.run( - [Path(c) for c in copies], - [*[Path(work) / str(i) for i in range(len(roots))], *map(Path, mirrored), - *search_paths, *roots], - ) - for copy, original in copies.items(): - types = report.expression_types.get(copy, {}) - for f in rebinding.judge(plans[original][1], types): - found.append(Diagnostic(original, f.line, f.column, f.rule, "warning", f.message)) - return found + +@dataclass(frozen=True) +class _Mirror: + copies: dict[str, str] # copy -> original + bases: list[tuple[Path, Path]] # (mirrored root, original root) + + def original(self, path: str) -> str: + """The original path of a file in the mirrored tree (a dependency's, for instance).""" + for base, root in self.bases: + try: + return str(root / Path(path).relative_to(base)) + except ValueError: + continue + return path def _mirror(work: Path, probed: dict[str, str]) -> dict[str, str]: diff --git a/python-multiplatform-ksp/src/test/python/test_rebinding.py b/python-multiplatform-ksp/src/test/python/test_rebinding.py index 3d50e272e..731707041 100644 --- a/python-multiplatform-ksp/src/test/python/test_rebinding.py +++ b/python-multiplatform-ksp/src/test/python/test_rebinding.py @@ -138,3 +138,47 @@ def f() -> None: result = gate.check([path]) assert by_rule(result, REBIND), result assert not [d for d in result if d.severity == "error"], result + + +def test_pyrefly_runs_once_when_a_file_has_a_candidate(write, monkeypatch): + """The probed copy is the only Pyrefly run: errors, any-flow and types all come from it.""" + calls = [] + real = gate.pyrefly.run + + def counting(*args, **kwargs): + calls.append(args) + return real(*args, **kwargs) + + monkeypatch.setattr(gate.pyrefly, "run", counting) + result = found(write, """ + def f() -> None: + x = 1 + x = "a" + print(x) + """) + assert by_rule(result, REBIND) + assert len(calls) == 1 + + +def test_error_column_after_a_probe_points_at_the_original_source(write): + """`x = 1; y = undefined_name`: the probe goes after `x = 1`, left of the error.""" + source = "x = 1\nx = 2\nx = 1; y = undefined_name\n" + path = write("user.py", source) + result = gate.check([path]) + [d] = [d for d in result if d.rule == "pyrefly/unknown-name"] + original_column = source.splitlines()[2].index("undefined_name") + 1 + assert (d.line, d.column) == (3, original_column) + + +def test_error_left_of_a_probe_keeps_its_column(write): + source = "x = 1\nx = 2\nprint(undefined_name); x = 3\n" + path = write("user.py", source) + [d] = [d for d in gate.check([path]) if d.rule == "pyrefly/unknown-name"] + assert (d.line, d.column) == (3, source.splitlines()[2].index("undefined_name") + 1) + + +def test_probe_loads_are_not_any_flow_findings(write): + """The probe load `; x` has type Unknown here, but it is a discarded value, not a finding.""" + path = write("user.py", "x = eval('1')\nx = eval('2')\nprint(x)\n") + flows = {(d.line, d.column) for d in gate.check([path]) if d.rule == "any-flow"} + assert flows == {(3, 7)} # the user's own load of x, not the probe loads on lines 1 and 2 From cd9a41252fbf5b1a390b3d25fbd8987ff67fe614 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:03:17 +0900 Subject: [PATCH 13/36] Feat: TypedPython incremental compilation cache (module unit) Cache key = source hash + dependency interface hashes (typed signatures only) + compiler version + platform + flags. A body-only change leaves the interface hash unchanged, so dependents are reused. Entries are written atomically and a corrupt or missing entry is a miss. BuildReport says why each module rebuilt. Red: module missing. Green: 10/10; unchanged rebuild 1.21 ms with 0 compile calls. Mutation (deps keyed by source hash): body-change test fails. Gradle up-to-date wiring of SPEC N-9 is not part of this commit. Refs #41 --- .../main/python/typedpython/incremental.py | 330 ++++++++++++++++++ .../src/test/python/test_incremental.py | 211 +++++++++++ 2 files changed, 541 insertions(+) create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/incremental.py create mode 100644 python-multiplatform-ksp/src/test/python/test_incremental.py diff --git a/python-multiplatform-ksp/src/main/python/typedpython/incremental.py b/python-multiplatform-ksp/src/main/python/typedpython/incremental.py new file mode 100644 index 000000000..820e5d346 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/incremental.py @@ -0,0 +1,330 @@ +"""Incremental compilation (SPEC N-9, #41): a per-module build cache. + +The unit is a module. Its cache key covers the source, the *interface* hashes of the project modules +it imports, the compiler version, the target platform and the build flags. A body-only change in a +dependency leaves that dependency's interface hash unchanged, so dependents stay cached. + +The front end, the C back end and the C compiler are passed in as callables, so this file depends on +none of them (only on `ir`, the contract). + +Key encoding (v1): sha256 of the compact, key-sorted JSON of + {"v": 1, "source": sha256-hex(source bytes), + "deps": [[, ], ...] sorted, + "compiler_version": str, "platform": str, "flags": [str, ...] in the given order} +Interface hash (v1): sha256 of the compact, key-sorted JSON of + {"v": 1, "functions": [[name, [[param name, type value, "array" | "scalar"], ...], return type value], ...] + sorted by function name, "skipped": sorted skipped names} +Only signatures enter it: no bodies, locals, purity flags, line numbers or skip reasons. + +Cache layout: /entries//{meta.json, } and /index.json (the last +key components per module, used only to explain a rebuild). An entry is written to a temporary +directory inside cache_dir and renamed into place, so a crash never leaves a half entry under a key. +An entry whose metadata is unreadable or whose artefact is missing or has a different sha256 is a +miss and is rebuilt. +""" +from __future__ import annotations + +import ast +import hashlib +import json +import os +import shutil +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable + +from typedpython import ir +from typedpython.pyrefly import import_roots + +_V = 1 + + +def _sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _canon(obj) -> bytes: + return json.dumps(obj, sort_keys=True, separators=(",", ":")).encode() + + +def interface_hash(module: ir.Module) -> str: + """sha256 over the compiled functions' signatures and the skipped names — nothing else.""" + functions = [] + for f in sorted(module.functions, key=lambda f: f.name): + params = [[p.name, p.type.value, "array" if isinstance(p, ir.ArrayParam) else "scalar"] + for p in f.params] + functions.append([f.name, params, f.returns.value]) + return _sha(_canon({"v": _V, "functions": functions, "skipped": sorted(module.skipped)})) + + +# --- dependencies -------------------------------------------------------------------------------- + +def _abs(p: Path) -> Path: + return Path(os.path.abspath(p)) + + +def _within(p: Path, root: Path) -> bool: + try: + p.relative_to(root) + return True + except ValueError: + return False + + +def _find(base: Path, parts: list[str]) -> list[Path]: + """Files executed by importing dotted `parts` under `base`: each package's __init__ on the way, + then the module itself (module file or package __init__). Stops at the first missing part.""" + found: list[Path] = [] + cur = base + for part in parts: + pkg = cur / part / "__init__.py" + mod = cur / f"{part}.py" + if pkg.is_file(): + found.append(pkg) + cur = cur / part + elif mod.is_file(): + found.append(mod) + break + else: + break + return found + + +def dependencies(path: Path, project_root: Path) -> list[Path]: + """Project-local modules `path` imports (stdlib and third-party are ignored), in first-seen order.""" + path, project_root = _abs(Path(path)), _abs(Path(project_root)) + try: + tree = ast.parse(path.read_bytes(), filename=str(path)) + except (SyntaxError, ValueError, OSError): + return [] + roots = [_abs(r) for r in import_roots([str(path)])] + if project_root not in roots: + roots.append(project_root) + + out: list[Path] = [] + + def add(found: list[Path]) -> None: + for p in found: + p = _abs(p) + if p != path and _within(p, project_root) and p not in out: + out.append(p) + + def resolve(parts: list[str], bases: list[Path]) -> list[Path]: + best: list[Path] = [] + for b in bases: + got = _find(b, parts) + if len(got) > len(best): + best = got + return best + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + add(resolve(alias.name.split("."), roots)) + elif isinstance(node, ast.ImportFrom): + parts = node.module.split(".") if node.module else [] + if node.level: + base = path.parent + for _ in range(node.level - 1): + base = base.parent + bases = [base] + if not parts: + add([base / "__init__.py"] if (base / "__init__.py").is_file() else []) + else: + bases = roots + got = resolve(parts, bases) if parts else [] + add(got) + # `from pkg import mod`: the imported name may itself be a submodule + for alias in node.names: + if alias.name == "*": + continue + for b in bases: + if parts: + sub_base = b + for part in parts: + sub_base = sub_base / part + else: + sub_base = b + add(_find(sub_base, [alias.name]) if sub_base.is_dir() else []) + return out + + +# --- report -------------------------------------------------------------------------------------- + +@dataclass +class ModuleReport: + path: Path + name: str + status: str # "rebuilt" | "reused" + reasons: list[str] = field(default_factory=list) + artifact: Path | None = None + interface: str = "" + key: str = "" + + +@dataclass +class BuildReport: + modules: list[ModuleReport] = field(default_factory=list) + + @property + def rebuilt(self) -> list[ModuleReport]: + return [m for m in self.modules if m.status == "rebuilt"] + + @property + def reused(self) -> list[ModuleReport]: + return [m for m in self.modules if m.status == "reused"] + + +# --- cache --------------------------------------------------------------------------------------- + +class BuildCache: + def __init__(self, cache_dir: Path): + self.dir = Path(cache_dir) + self.entries = self.dir / "entries" + self.entries.mkdir(parents=True, exist_ok=True) + + # entries + + def _load(self, key: str) -> dict | None: + """The entry's metadata plus its artefact path, or None if it is absent or damaged.""" + d = self.entries / key + try: + meta = json.loads((d / "meta.json").read_text()) + art = d / meta["artifact"] + if meta["key"] != key or _sha(art.read_bytes()) != meta["artifact_sha256"]: + return None + meta["path"] = art + return meta + except (OSError, ValueError, KeyError, TypeError): + return None + + def _store(self, key: str, name: str, interface: str, c_source: str, + compile_c: Callable[[str, str, Path], Path]) -> Path: + tmp = self.dir / f"tmp-{uuid.uuid4().hex}" + tmp.mkdir() + try: + art = Path(compile_c(c_source, name, tmp)) + if art.parent != tmp: # the artefact must live in the entry + shutil.copy2(art, tmp / art.name) + art = tmp / art.name + meta = {"key": key, "name": name, "interface": interface, "artifact": art.name, + "artifact_sha256": _sha(art.read_bytes())} + (tmp / "meta.json").write_text(json.dumps(meta)) + final = self.entries / key + if final.exists(): + shutil.rmtree(final, ignore_errors=True) # a damaged entry under this key + try: + os.rename(tmp, final) + except OSError: + if self._load(key) is None: # lost a race to a valid entry? else fail + raise + shutil.rmtree(tmp, ignore_errors=True) + return final / art.name + except BaseException: + shutil.rmtree(tmp, ignore_errors=True) + raise + + # index (explains rebuilds only; never decides a hit) + + def _read_index(self) -> dict: + try: + data = json.loads((self.dir / "index.json").read_text()) + return data if isinstance(data, dict) else {} + except (OSError, ValueError): + return {} + + def _write_index(self, index: dict) -> None: + tmp = self.dir / f"index-{uuid.uuid4().hex}.tmp" + tmp.write_text(json.dumps(index, sort_keys=True)) + os.replace(tmp, self.dir / "index.json") + + # build + + def build_project(self, sources: list[Path], project_root: Path, *, + lower: Callable[[Path], ir.Module], + generate: Callable[[ir.Module, Path], str], + compile_c: Callable[[str, str, Path], Path], + compiler_version: str, platform: str, flags: list[str]) -> BuildReport: + root = _abs(Path(project_root)) + paths = [_abs(Path(s)) for s in sources] + known = set(paths) + deps = {p: dependencies(p, root) for p in paths} + order = _topological(paths, deps) + rel = lambda p: p.relative_to(root).as_posix() + + index = self._read_index() + interfaces: dict[Path, str] = {} + by_path: dict[Path, ModuleReport] = {} + + for p in order: + source_hash = _sha(p.read_bytes()) + dep_hashes: dict[str, str] = {} + for d in deps[p]: + if d in interfaces: + dep_hashes[rel(d)] = interfaces[d] + else: # not compiled here, or in a cycle: its whole source is its interface + dep_hashes[rel(d)] = "src:" + _sha(d.read_bytes()) + comps = {"source": source_hash, "deps": dep_hashes, "compiler_version": compiler_version, + "platform": platform, "flags": list(flags)} + key = _sha(_canon({"v": _V, "source": source_hash, "deps": sorted(dep_hashes.items()), + "compiler_version": compiler_version, "platform": platform, + "flags": list(flags)})) + name = _module_name(p, root) + hit = self._load(key) + if hit is not None: + interfaces[p] = hit["interface"] + by_path[p] = ModuleReport(p, name, "reused", [], hit["path"], hit["interface"], key) + else: + reasons = _why(index.get(rel(p)), comps, (self.entries / key).exists()) + module = lower(p) + c_source = generate(module, p) + iface = interface_hash(module) + art = self._store(key, name, iface, c_source, compile_c) + interfaces[p] = iface + by_path[p] = ModuleReport(p, name, "rebuilt", reasons, art, iface, key) + index[rel(p)] = comps + + self._write_index(index) + return BuildReport([by_path[p] for p in paths]) + + +def _module_name(p: Path, root: Path) -> str: + parts = list(p.relative_to(root).with_suffix("").parts) + if parts[-1] == "__init__" and len(parts) > 1: + parts.pop() + return ".".join(parts) + + +def _why(prev: dict | None, now: dict, entry_dir_exists: bool) -> list[str]: + if not prev: + return ["new"] + reasons: list[str] = [] + if prev.get("source") != now["source"]: + reasons.append("source") + old_deps, new_deps = prev.get("deps", {}), now["deps"] + for d in sorted(set(old_deps) | set(new_deps)): + if old_deps.get(d) != new_deps.get(d): + reasons.append(f"dependency interface: {d}") + for k in ("compiler_version", "platform", "flags"): + if prev.get(k) != now[k]: + reasons.append(k) + return reasons or ["corrupt" if entry_dir_exists else "missing"] + + +def _topological(paths: list[Path], deps: dict[Path, list[Path]]) -> list[Path]: + """Dependencies first, input order otherwise; a cycle is broken where it is first re-entered.""" + known, order, state = set(paths), [], {} + + def visit(p: Path) -> None: + if state.get(p): + return + state[p] = 1 + for d in deps[p]: + if d in known: + visit(d) + order.append(p) + + for p in paths: + visit(p) + return order diff --git a/python-multiplatform-ksp/src/test/python/test_incremental.py b/python-multiplatform-ksp/src/test/python/test_incremental.py new file mode 100644 index 000000000..ba45de5d2 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_incremental.py @@ -0,0 +1,211 @@ +"""Incremental compilation (SPEC N-9, #41): module cache keyed by source + dependency interface hashes.""" +from __future__ import annotations + +import dataclasses +import re +import shutil +import time +import uuid +from pathlib import Path + +import pytest + +from typedpython import ir +from typedpython.incremental import BuildCache, dependencies, interface_hash + +TMP_ROOT = Path(__file__).resolve().parents[4] / ".tmp" +TYPES = {"int": ir.Type.I64, "float": ir.Type.F64, "bool": ir.Type.BOOL, "None": ir.Type.NONE} +DEF = re.compile(r"^def (\w+)\((.*?)\) -> (\w+):", re.M) + + +@pytest.fixture +def work(): + root = TMP_ROOT / f"incr-{uuid.uuid4().hex[:8]}" + root.mkdir(parents=True) + yield root + shutil.rmtree(root, ignore_errors=True) + + +class Fakes: + """Counting stand-ins for the front end, the C back end and the C compiler.""" + + def __init__(self): + self.lowered, self.generated, self.compiled = [], [], [] + + def lower(self, path: Path) -> ir.Module: + self.lowered.append(path.name) + params_of = lambda text: tuple( + ir.Param(n.strip().split(":")[0], TYPES[n.strip().split(":")[1].strip()]) + for n in text.split(",") if n.strip()) + funcs = tuple( + ir.Function(m[1], params_of(m[2]), TYPES[m[3]], {}, (), True, False, 1) + for m in DEF.finditer(path.read_text())) + return ir.Module(path.stem, funcs) + + def generate(self, module: ir.Module, path: Path) -> str: + self.generated.append(path.name) + return f"/* {module.name} */ {path.read_text()!r}" + + def compile_c(self, c_source: str, name: str, out_dir: Path) -> Path: + self.compiled.append(name) + out = Path(out_dir) / f"{name}.so" + out.write_text("OBJ:" + c_source) + return out + + def counts(self): + return (len(self.lowered), len(self.generated), len(self.compiled)) + + def reset(self): + self.lowered.clear(); self.generated.clear(); self.compiled.clear() + + +def build(cache, fakes, root, files, **over): + args = dict(lower=fakes.lower, generate=fakes.generate, compile_c=fakes.compile_c, + compiler_version="1", platform="linux-x86_64", flags=["-O2"]) + args.update(over) + return cache.build_project([root / f for f in files], root, **args) + + +def project(root: Path): + (root / "a.py").write_text("def f(x: int) -> int:\n return x + 1\n") + (root / "b.py").write_text("from a import f\ndef g(y: int) -> int:\n return f(y)\n") + (root / "c.py").write_text("def h(z: float) -> float:\n return z\n") + return ["a.py", "b.py", "c.py"] + + +def status(report): + return {Path(m.path).name: m.status for m in report.modules} + + +def test_1_first_builds_all_second_builds_nothing(work): + files, fakes, cache = project(work), Fakes(), BuildCache(work / "cache") + r1 = build(cache, fakes, work, files) + assert fakes.counts() == (3, 3, 3) + assert set(status(r1).values()) == {"rebuilt"} + fakes.reset() + t = time.perf_counter() + r2 = build(cache, fakes, work, files) + elapsed = time.perf_counter() - t + assert fakes.counts() == (0, 0, 0) + assert set(status(r2).values()) == {"reused"} + print(f"unchanged rebuild wall time: {elapsed * 1000:.2f} ms") + assert elapsed < 0.5 + assert all(m.artifact.exists() for m in r2.modules) + + +def test_2_body_change_rebuilds_only_that_module(work): + files, fakes, cache = project(work), Fakes(), BuildCache(work / "cache") + build(cache, fakes, work, files) + fakes.reset() + (work / "a.py").write_text("def f(x: int) -> int:\n return x + 2\n") + r = build(cache, fakes, work, files) + assert fakes.compiled == ["a"] + assert status(r) == {"a.py": "rebuilt", "b.py": "reused", "c.py": "reused"} + assert "source" in next(m for m in r.modules if m.path.name == "a.py").reasons + + +def test_3_signature_change_rebuilds_dependents_not_independents(work): + files, fakes, cache = project(work), Fakes(), BuildCache(work / "cache") + build(cache, fakes, work, files) + fakes.reset() + (work / "a.py").write_text("def f(x: float) -> float:\n return x + 1\n") + r = build(cache, fakes, work, files) + assert sorted(fakes.compiled) == ["a", "b"] + assert status(r) == {"a.py": "rebuilt", "b.py": "rebuilt", "c.py": "reused"} + b = next(m for m in r.modules if m.path.name == "b.py") + assert any("dependency interface" in x and "a.py" in x for x in b.reasons), b.reasons + + +@pytest.mark.parametrize("over,component", [ + ({"compiler_version": "2"}, "compiler_version"), + ({"platform": "darwin-arm64"}, "platform"), + ({"flags": ["-O0"]}, "flags"), +]) +def test_4_toolchain_change_rebuilds_everything(work, over, component): + files, fakes, cache = project(work), Fakes(), BuildCache(work / "cache") + build(cache, fakes, work, files) + fakes.reset() + r = build(cache, fakes, work, files, **over) + assert fakes.counts() == (3, 3, 3) + assert set(status(r).values()) == {"rebuilt"} + assert all(component in m.reasons for m in r.modules), [m.reasons for m in r.modules] + + +def test_5_corrupt_entry_is_a_miss_others_reused(work): + files, fakes, cache = project(work), Fakes(), BuildCache(work / "cache") + r1 = build(cache, fakes, work, files) + victim = next(m for m in r1.modules if m.path.name == "c.py").artifact + victim.write_text("OBJ:trunc") # damaged contents + fakes.reset() + r2 = build(cache, fakes, work, files) + assert fakes.compiled == ["c"] + assert status(r2) == {"a.py": "reused", "b.py": "reused", "c.py": "rebuilt"} + assert "corrupt" in next(m for m in r2.modules if m.path.name == "c.py").reasons + # and a deleted artefact, and garbage metadata + fakes.reset() + for m in r2.modules: + if m.path.name == "a.py": + m.artifact.unlink() + if m.path.name == "b.py": + (m.artifact.parent / "meta.json").write_text("{not json") + r3 = build(cache, fakes, work, files) + assert sorted(fakes.compiled) == ["a", "b"] + assert status(r3)["c.py"] == "reused" + + +def test_6_relative_import_in_package_detected(work): + pkg = work / "pkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("") + (pkg / "base.py").write_text("def f(x: int) -> int:\n return x\n") + (pkg / "sub").mkdir() + (pkg / "sub" / "__init__.py").write_text("") + (pkg / "sub" / "leaf.py").write_text("") + (pkg / "user.py").write_text("from .base import f\nfrom .sub import leaf\nimport os, json\n") + (pkg / "deep.py").write_text("from ..pkg import base\nfrom . import base as b2\nimport pkg.sub.leaf\n") + d = dependencies(pkg / "user.py", work) + assert pkg / "base.py" in d and pkg / "sub" / "leaf.py" in d + assert not any(p.name in ("os.py", "json.py") for p in d) + d2 = dependencies(pkg / "deep.py", work) + assert pkg / "base.py" in d2 and pkg / "sub" / "leaf.py" in d2 + assert pkg / "deep.py" not in d2 + + +def test_6b_relative_dependency_drives_rebuild(work): + pkg = work / "pkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("") + (pkg / "base.py").write_text("def f(x: int) -> int:\n return x\n") + (pkg / "user.py").write_text("from .base import f\ndef g(x: int) -> int:\n return f(x)\n") + files, fakes, cache = ["pkg/base.py", "pkg/user.py"], Fakes(), BuildCache(work / "cache") + build(cache, fakes, work, files) + fakes.reset() + (pkg / "base.py").write_text("def f(x: float) -> int:\n return 1\n") + build(cache, fakes, work, files) + assert sorted(fakes.compiled) == ["pkg.base", "pkg.user"] + + +def fn(name="f", params=(ir.Param("x", ir.Type.I64),), ret=ir.Type.I64, body=(), locals_=None, line=1): + return ir.Function(name, tuple(params), ret, locals_ or {}, tuple(body), True, False, line) + + +def test_7_interface_hash(): + base = ir.Module("m", (fn(),)) + h = interface_hash(base) + body = (ir.Return(ir.Const(ir.Type.I64, 1)),) + assert interface_hash(ir.Module("m", (fn(body=body, locals_={"t": ir.Type.F64}, line=99),))) == h + assert interface_hash(ir.Module("m", (dataclasses.replace(fn(), pure=False, may_deopt=True),))) == h + assert interface_hash(ir.Module("m", (fn(params=(ir.Param("x", ir.Type.F64),)),))) != h + assert interface_hash(ir.Module("m", (fn(params=(ir.Param("y", ir.Type.I64),)),))) != h + assert interface_hash(ir.Module("m", (fn(ret=ir.Type.F64),))) != h + assert interface_hash(ir.Module("m", (fn(name="g"),))) != h + arr = lambda t, s: fn(params=(ir.ArrayParam("x", t, s),)) + assert interface_hash(ir.Module("m", (arr(ir.Type.F64_ARRAY, False),))) != \ + interface_hash(ir.Module("m", (arr(ir.Type.I64_ARRAY, False),))) + assert interface_hash(ir.Module("m", (fn(),), {"z": "why"})) != h # skipped names count + assert interface_hash(ir.Module("m", (fn(),), {"z": "why"})) == \ + interface_hash(ir.Module("m", (fn(),), {"z": "other reason"})) + # order of params matters, order of functions does too (it is the module's order) + p2 = (ir.Param("a", ir.Type.I64), ir.Param("b", ir.Type.F64)) + assert interface_hash(ir.Module("m", (fn(params=p2),))) != \ + interface_hash(ir.Module("m", (fn(params=p2[::-1]),))) From a01747327159c342dcdacc2e9f1c7aedea130bf4 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:08:26 +0900 Subject: [PATCH 14/36] Feat: TypedPython IR safety verifier (SPEC N-8) verify(module) proves, per function, definite assignment, exact node typing, deopt placement (no deopting node in an impure function; pure/may_deopt/stored flags recomputed and compared), array use and bounds (proven only for range(k>=0, len(a)) indices; everything else keeps the runtime check), OBJ ownership, resolved calls and no recursion. A function it cannot prove leaves 'functions' for 'skipped' with the rule and reason. Red: 84/84 failing on import. Green: 84/84. Disabling deopt-impure, bounds, definite assignment or condition-only placement fails 10/9/6/2 tests; 200 seeded IR mutants all caught. Refs #41 --- .../src/main/python/typedpython/verify.py | 754 +++++++++++++++ .../src/test/python/test_verify.py | 904 ++++++++++++++++++ 2 files changed, 1658 insertions(+) create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/verify.py create mode 100644 python-multiplatform-ksp/src/test/python/test_verify.py diff --git a/python-multiplatform-ksp/src/main/python/typedpython/verify.py b/python-multiplatform-ksp/src/main/python/typedpython/verify.py new file mode 100644 index 000000000..949c0bb2c --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/verify.py @@ -0,0 +1,754 @@ +"""TypedPython IR safety verifier (SPEC N-8, #41; maintainer decision 2026-10-03). + +`verify(module)` proves, per function and on the IR alone, the properties listed in the "Safety" +section of `ir.py`. A function it cannot prove is removed from `Module.functions` and moved to +`Module.skipped` with the reason, so it stays interpreted; every reason is also returned as a +`Diagnostic`. A proved function comes back with each `Index`/`StoreIndex` whose bounds were proved +marked `proven=True`; every other access has `proven=False` (an input `proven` is never trusted). + +The verifier reads nothing but `ir` — it is independent of the front end — and it trusts no flag +the front end set (`pure`, `may_deopt`, `ArrayParam.stored`, node `type`s): each is recomputed from +the body and compared. + +Rules (the `rule` of a Diagnostic): + + verify/definite-assignment a local may be read before it is assigned on some path + verify/missing-return a function returning a value can reach the end of its body + verify/type an operand, result, declaration, constant, or call signature is wrong + verify/ownership an OBJ is produced by a node that does not create a new reference, or + crosses into a scalar slot without Unbox (or a scalar into an OBJ + slot without Box) + verify/deopt-impure a node that can deopt appears in an impure function + verify/flag-may-deopt `may_deopt` differs from what the body computes + verify/flag-pure `pure` is set but the body has an effect + verify/array-use an array is used other than by Index/StoreIndex/Len, or those nodes + name something that is not an array parameter + verify/array-stored `ArrayParam.stored` differs from the body + verify/unknown-node a node (or enum field) that `ir` does not define + verify/call-unresolved a Call names a function that is not compiled in this module + verify/call-unverified a Call names a function that the verifier rejected + verify/recursion the function is on a cycle of Calls (C stack depth is unbounded; + CPython would raise RecursionError instead) + verify/condition-only a CompareObj outside a condition (If/While cond, or an And/Or + operand that is itself in a condition): its rich-comparison result + need not be a bool, so only its truth value may be used + verify/structure break/continue outside a loop, the ForRange var assigned in its body, + a zero or non-constant step, duplicate names, bad declarations +""" +from __future__ import annotations + +import dataclasses +from dataclasses import dataclass + +from . import ir +from .ir import ( + And, ArrayParam, Assign, BinOp, BinOpKind, Box, Break, Call, CallObject, Compare, CompareKind, + CompareObj, Const, Continue, ExprStmt, ForRange, Function, GetAttr, Global, If, Index, Len, + Local, MathCall, MathFunc, Module, ObjToFloat, Or, Param, Return, StoreIndex, ToFloat, Truth, + Type, Unbox, UnaryOp, UnaryOpKind, While, +) + +DA = "verify/definite-assignment" +MISSING_RETURN = "verify/missing-return" +TYPE = "verify/type" +OWNERSHIP = "verify/ownership" +DEOPT = "verify/deopt-impure" +FLAG_DEOPT = "verify/flag-may-deopt" +FLAG_PURE = "verify/flag-pure" +ARRAY_USE = "verify/array-use" +ARRAY_STORED = "verify/array-stored" +UNKNOWN = "verify/unknown-node" +UNRESOLVED = "verify/call-unresolved" +UNVERIFIED = "verify/call-unverified" +RECURSION = "verify/recursion" +STRUCTURE = "verify/structure" +CONDITION_ONLY = "verify/condition-only" + + +@dataclass(frozen=True) +class Diagnostic: + function: str + rule: str + message: str + source_line: int + + +# Exact classes, not isinstance: a subclass of an ir node is not an ir node (its meaning is not +# the one the back end implements). +_EXPRS = frozenset({Const, Local, BinOp, UnaryOp, Compare, And, Or, ToFloat, Box, Unbox, MathCall, + Call, Global, GetAttr, CallObject, Truth, CompareObj, ObjToFloat, Len, Index}) +_STMTS = frozenset({Assign, StoreIndex, ExprStmt, If, While, ForRange, Return, Break, Continue}) +# Nodes whose OBJ result is a new reference (ir.py "ownership": Box, Global, GetAttr, CallObject, +# OBJ BinOp), plus reads of an OBJ local (the local owns its reference; the back end increfs or +# borrows under that rule) and Calls of a compiled function returning OBJ (which returns a new +# reference by the same rule). +_OBJ_PRODUCERS = frozenset({Box, Global, GetAttr, CallObject, BinOp, Local, Call}) +_VALUE_TYPES = (Type.I64, Type.F64, Type.BOOL, Type.OBJ) # what a local may hold +_RETURN_TYPES = _VALUE_TYPES + (Type.NONE,) +_ELEMENT = {Type.F64_ARRAY: Type.F64, Type.I64_ARRAY: Type.I64} +_MATH_ARITY = {MathFunc.ATAN2: 2, MathFunc.HYPOT: 2} # every other MathFunc: 1 +_I64_DEOPT_OPS = frozenset({BinOpKind.ADD, BinOpKind.SUB, BinOpKind.MUL, BinOpKind.FLOORDIV, + BinOpKind.MOD, BinOpKind.TRUEDIV}) +_I64_MIN, _I64_MAX = -2 ** 63, 2 ** 63 - 1 + +# Definite assignment uses `None` for "unreachable" (bottom: every name counts as assigned). +_State = frozenset[str] | None + + +def _meet(a: _State, b: _State) -> _State: + if a is None: + return b + if b is None: + return a + return a & b + + +class _Loop: + def __init__(self, var: str | None, bounds_array: str | None): + self.var = var # ForRange var (None for While) + self.bounds_array = bounds_array # array whose indices by `var` are in bounds + self.breaks: list[_State] = [] + + +class _Checker: + """Checks one function against the module's signatures. Collects diagnostics; rebuilds the + body with proven bounds.""" + + def __init__(self, function: Function, signatures: dict[str, Function]): + self.fn = function + self.sigs = signatures + self.diags: list[Diagnostic] = [] + self.can_deopt = False + self.effects: list[str] = [] # reasons the body is impure + self.stored: set[str] = set() + self.callees: set[str] = set() + self.loops: list[_Loop] = [] + self.scalars: dict[str, Type] = {} # name -> type of every scalar/OBJ variable + self.arrays: dict[str, Type] = {} # name -> array type of every ArrayParam + + def error(self, rule: str, message: str) -> None: + d = Diagnostic(self.fn.name, rule, message, self.fn.source_line) + if d not in self.diags: + self.diags.append(d) + + # --- declarations ------------------------------------------------------------------------- + + def declarations(self) -> frozenset[str]: + fn = self.fn + params: set[str] = set() + for p in fn.params: + if type(p) is ArrayParam: + if p.type not in ir.ARRAYS: + self.error(TYPE, f"array parameter '{p.name}' has non-array type {p.type}") + elif not isinstance(p.stored, bool): + self.error(TYPE, f"array parameter '{p.name}' has a non-bool `stored`") + else: + self.arrays[p.name] = p.type + elif type(p) is Param: + if p.type not in _VALUE_TYPES: + self.error(TYPE, f"parameter '{p.name}' has type {p.type}; arrays must be " + f"ArrayParam") + else: + self.scalars[p.name] = p.type + else: + self.error(UNKNOWN, f"parameter {p!r} is not an ir Param/ArrayParam") + continue + if p.name in params: + self.error(STRUCTURE, f"parameter '{p.name}' appears twice") + params.add(p.name) + if not isinstance(fn.locals, dict): + self.error(STRUCTURE, "locals is not a dict") + return frozenset(params) + for name, t in fn.locals.items(): + if name in self.arrays: + self.error(ARRAY_USE, f"local '{name}' shadows array parameter '{name}'") + elif t not in _VALUE_TYPES: + self.error(TYPE, f"local '{name}' has type {t}") + elif name in self.scalars and self.scalars[name] is not t: + self.error(TYPE, f"local '{name}' declared {t} but parameter is " + f"{self.scalars[name]}") + else: + self.scalars[name] = t + if fn.returns not in _RETURN_TYPES: + self.error(TYPE, f"return type {fn.returns} is not a value type") + for flag in ("pure", "may_deopt"): + if not isinstance(getattr(fn, flag), bool): + self.error(TYPE, f"`{flag}` is not a bool") + return frozenset(params) + + # --- expressions -------------------------------------------------------------------------- + + def slot(self, e: ir.Expr, want: Type, what: str) -> None: + """`e` flows into a slot of type `want` (already checked as an expression).""" + got = getattr(e, "type", None) + if got is want: + return + if got is Type.OBJ and want in ir.SCALARS: + self.error(OWNERSHIP, f"{what}: OBJ flows into a {want.value} slot without Unbox") + elif want is Type.OBJ and got in ir.SCALARS: + self.error(OWNERSHIP, f"{what}: {got.value} flows into an OBJ slot without Box") + elif got in ir.ARRAYS: + pass # reported as array-use where the array was read + else: + self.error(TYPE, f"{what}: expected {want.value}, got " + f"{got.value if isinstance(got, Type) else got!r}") + + def expr(self, e: object, state: _State, cond: bool = False) -> ir.Expr: + """Check `e`, returning it rebuilt (with proven bounds). `state` is the definitely + assigned set at this point; `cond` says `e` is used only for its truth value (an If/While + condition, or an And/Or operand in one).""" + cls = type(e) + if cls not in _EXPRS: + self.error(UNKNOWN, f"expression node {cls.__name__} is not defined by ir") + return e # type: ignore[return-value] + t = e.type + if not isinstance(t, Type): + self.error(UNKNOWN, f"{cls.__name__} has non-Type type {t!r}") + return e + if t is Type.OBJ and cls not in _OBJ_PRODUCERS: + self.error(OWNERSHIP, f"{cls.__name__} cannot produce an OBJ (only Box, Global, GetAttr, CallObject, OBJ " + f"BinOp, OBJ locals and OBJ-returning Calls create references)") + if cls is CompareObj and not cond: + self.error(CONDITION_ONLY, "CompareObj outside a condition: the rich comparison's " + "result object would be kept, and it need not be a bool") + method = getattr(self, "e_" + cls.__name__) + if cls in (And, Or): + return method(e, state, cond) + return method(e, state) + + def e_Const(self, e: Const, state: _State) -> ir.Expr: + v, t = e.value, e.type + ok = ((t is Type.I64 and type(v) is int and _I64_MIN <= v <= _I64_MAX) + or (t is Type.F64 and type(v) is float) + or (t is Type.BOOL and type(v) is bool) + or (t is Type.NONE and v is None)) + if not ok and t is not Type.OBJ: + self.error(TYPE, f"constant {v!r} does not match type {t.value}") + return e + + def e_Local(self, e: Local, state: _State) -> ir.Expr: + name = e.name + if name in self.arrays: + self.error(ARRAY_USE, f"array parameter '{name}' is used as a value (only Index, " + f"StoreIndex and Len may use it)") + return e + declared = self.scalars.get(name) + if declared is None: + self.error(TYPE, f"local '{name}' is not declared") + return e + if declared is not e.type: + self.error(TYPE, f"local '{name}' is {declared.value} but read as {e.type.value}") + if state is not None and name not in state: + self.error(DA, f"local '{name}' may be read before it is assigned") + return e + + def e_BinOp(self, e: BinOp, state: _State) -> ir.Expr: + if type(e.op) is not BinOpKind: + self.error(UNKNOWN, f"BinOp op {e.op!r} is not a BinOpKind") + return e + left, right = self.expr(e.left, state), self.expr(e.right, state) + lt, rt = getattr(left, "type", None), getattr(right, "type", None) + if lt not in (Type.I64, Type.F64, Type.OBJ): + self.error(TYPE if lt is not Type.OBJ else OWNERSHIP, + f"BinOp {e.op.value} on {getattr(lt, 'value', lt)} operands") + elif lt is not rt: + self.slot(right, lt, f"right operand of {e.op.value}") + else: + want = Type.F64 if (lt is Type.I64 and e.op is BinOpKind.TRUEDIV) else lt + if e.type is not want: + self.error(TYPE, f"{lt.value} {e.op.value} {rt.value} is {want.value}, node says " + f"{e.type.value}") + if lt is Type.I64 and e.op in _I64_DEOPT_OPS: + self.can_deopt = True + if lt is Type.OBJ: + self.effects.append(f"OBJ {e.op.value} (runs user methods)") + return dataclasses.replace(e, left=left, right=right) + + def e_UnaryOp(self, e: UnaryOp, state: _State) -> ir.Expr: + if type(e.op) is not UnaryOpKind: + self.error(UNKNOWN, f"UnaryOp op {e.op!r} is not a UnaryOpKind") + return e + operand = self.expr(e.operand, state) + ot = getattr(operand, "type", None) + if e.op is UnaryOpKind.NOT: + self.slot(operand, Type.BOOL, "operand of not") + if e.type is not Type.BOOL: + self.error(TYPE, f"not gives bool, node says {e.type.value}") + else: + if ot is Type.OBJ: + self.error(OWNERSHIP, f"unary {e.op.value} on OBJ has no IR form") + elif ot not in (Type.I64, Type.F64): + self.error(TYPE, f"unary {e.op.value} on {getattr(ot, 'value', ot)}") + elif e.type is not ot: + self.error(TYPE, f"unary {e.op.value} of {ot.value} is {ot.value}, node says " + f"{e.type.value}") + elif ot is Type.I64 and e.op is UnaryOpKind.NEG: + self.can_deopt = True + return dataclasses.replace(e, operand=operand) + + def e_Compare(self, e: Compare, state: _State) -> ir.Expr: + if type(e.op) is not CompareKind: + self.error(UNKNOWN, f"Compare op {e.op!r} is not a CompareKind") + return e + left, right = self.expr(e.left, state), self.expr(e.right, state) + lt, rt = getattr(left, "type", None), getattr(right, "type", None) + numeric = (Type.I64, Type.F64) + if Type.OBJ in (lt, rt): + self.error(OWNERSHIP, "Compare on an OBJ operand (rich comparison returns an object, " + "not a bool)") + elif not ((lt in numeric and rt in numeric) or (lt is Type.BOOL and rt is Type.BOOL)): + self.error(TYPE, f"Compare of {getattr(lt, 'value', lt)} and " + f"{getattr(rt, 'value', rt)}") + if e.type is not Type.BOOL: + self.error(TYPE, f"Compare gives bool, node says {e.type.value}") + return dataclasses.replace(e, left=left, right=right) + + def _bool_pair(self, e: And | Or, state: _State, cond: bool = False) -> ir.Expr: + name = type(e).__name__ + left, right = self.expr(e.left, state, cond), self.expr(e.right, state, cond) + self.slot(left, Type.BOOL, f"left operand of {name}") + self.slot(right, Type.BOOL, f"right operand of {name}") + if e.type is not Type.BOOL: + self.error(TYPE, f"{name} gives bool, node says {e.type.value}") + return dataclasses.replace(e, left=left, right=right) + + e_And = _bool_pair + e_Or = _bool_pair + + def e_ToFloat(self, e: ToFloat, state: _State) -> ir.Expr: + operand = self.expr(e.operand, state) + self.slot(operand, Type.I64, "operand of ToFloat") + if e.type is not Type.F64: + self.error(TYPE, f"ToFloat gives f64, node says {e.type.value}") + return dataclasses.replace(e, operand=operand) + + def e_Box(self, e: Box, state: _State) -> ir.Expr: + operand = self.expr(e.operand, state) + ot = getattr(operand, "type", None) + if ot is Type.OBJ: + self.error(OWNERSHIP, "Box of an OBJ (it is already an object)") + elif ot not in ir.SCALARS and ot not in ir.ARRAYS: + self.error(TYPE, f"Box of {getattr(ot, 'value', ot)}") + if e.type is not Type.OBJ: + self.error(TYPE, f"Box gives obj, node says {e.type.value}") + return dataclasses.replace(e, operand=operand) + + def e_Unbox(self, e: Unbox, state: _State) -> ir.Expr: + operand = self.expr(e.operand, state) + ot = getattr(operand, "type", None) + if ot in ir.SCALARS: + self.error(OWNERSHIP, f"Unbox of a {ot.value} (only an OBJ can be unboxed)") + elif ot is not Type.OBJ and ot not in ir.ARRAYS: + self.error(TYPE, f"Unbox of {getattr(ot, 'value', ot)}") + if e.type not in ir.SCALARS: + self.error(TYPE, f"Unbox gives a scalar, node says {e.type.value}") + elif e.type is Type.I64: + self.can_deopt = True + return dataclasses.replace(e, operand=operand) + + def e_MathCall(self, e: MathCall, state: _State) -> ir.Expr: + if type(e.func) is not MathFunc: + self.error(UNKNOWN, f"MathCall func {e.func!r} is not a MathFunc") + return e + args = self.args(e.args, state, f"math.{e.func.value}") + arity = _MATH_ARITY.get(e.func, 1) + if len(args) != arity: + self.error(TYPE, f"math.{e.func.value} takes {arity} argument(s), got {len(args)}") + for k, a in enumerate(args): + self.slot(a, Type.F64, f"argument {k + 1} of math.{e.func.value}") + if e.type is not Type.F64: + self.error(TYPE, f"math.{e.func.value} gives f64, node says {e.type.value}") + return dataclasses.replace(e, args=args) + + def args(self, args: object, state: _State, what: str) -> tuple[ir.Expr, ...]: + if not isinstance(args, tuple): + self.error(STRUCTURE, f"arguments of {what} are not a tuple") + return () + return tuple(self.expr(a, state) for a in args) + + def e_Call(self, e: Call, state: _State) -> ir.Expr: + args = self.args(e.args, state, f"call of '{e.function}'") + callee = self.sigs.get(e.function) + if callee is None: + self.error(UNRESOLVED, f"call of '{e.function}', which is not a compiled function of " + f"this module") + return dataclasses.replace(e, args=args) + self.callees.add(e.function) + if len(args) != len(callee.params): + self.error(TYPE, f"'{e.function}' takes {len(callee.params)} argument(s), got " + f"{len(args)}") + for k, (a, p) in enumerate(zip(args, callee.params)): + if type(p) is ArrayParam: + self.error(ARRAY_USE, f"'{e.function}' takes array parameter '{p.name}'; an array " + f"cannot be passed by Call (the entry guard and write-back " + f"belong to the Python-level call)") + else: + self.slot(a, p.type, f"argument {k + 1} of '{e.function}'") + if e.type is not callee.returns: + self.error(TYPE, f"'{e.function}' returns {callee.returns.value}, node says " + f"{e.type.value}") + if callee.may_deopt is True: + self.can_deopt = True + if callee.pure is not True: + self.effects.append(f"calls impure '{e.function}'") + return dataclasses.replace(e, args=args) + + def obj_operand(self, e: object, state: _State, what: str) -> ir.Expr: + x = self.expr(e, state) + self.slot(x, Type.OBJ, what) + return x + + def result(self, e: ir.Expr, want: Type) -> None: + if e.type is not want: + self.error(TYPE, f"{type(e).__name__} gives {want.value}, node says {e.type.value}") + + def e_Global(self, e: Global, state: _State) -> ir.Expr: + if not isinstance(e.name, str): + self.error(TYPE, f"Global name {e.name!r} is not a str") + self.result(e, Type.OBJ) + return e # a read alone has no effect (ir.Global) + + def e_GetAttr(self, e: GetAttr, state: _State) -> ir.Expr: + obj = self.obj_operand(e.obj, state, f"object of .{e.name}") + if not isinstance(e.name, str): + self.error(TYPE, f"GetAttr name {e.name!r} is not a str") + self.result(e, Type.OBJ) + self.effects.append(f"GetAttr .{e.name} (a property may have effects)") + return dataclasses.replace(e, obj=obj) + + def e_CallObject(self, e: CallObject, state: _State) -> ir.Expr: + callee = self.obj_operand(e.callee, state, "callee of CallObject") + args = self.args(e.args, state, "CallObject") + for k, a in enumerate(args): + self.slot(a, Type.OBJ, f"argument {k + 1} of CallObject") + kw = e.kwnames + if not (isinstance(kw, tuple) and all(isinstance(n, str) for n in kw)): + self.error(TYPE, f"CallObject kwnames {kw!r} is not a tuple of str") + elif len(kw) > len(args) or len(set(kw)) != len(kw): + self.error(TYPE, f"CallObject has {len(kw)} keyword name(s) {kw!r} for {len(args)} " + f"argument(s) (they must name distinct trailing arguments)") + self.result(e, Type.OBJ) + self.effects.append("CallObject") + return dataclasses.replace(e, callee=callee, args=args) + + def e_Truth(self, e: Truth, state: _State) -> ir.Expr: + operand = self.obj_operand(e.operand, state, "operand of Truth") + self.result(e, Type.BOOL) + self.effects.append("Truth (runs __bool__/__len__)") + return dataclasses.replace(e, operand=operand) + + def e_CompareObj(self, e: CompareObj, state: _State) -> ir.Expr: + if type(e.op) is not CompareKind: + self.error(UNKNOWN, f"CompareObj op {e.op!r} is not a CompareKind") + return e + left, right = self.expr(e.left, state), self.expr(e.right, state) + types = (getattr(left, "type", None), getattr(right, "type", None)) + if Type.OBJ not in types: + self.error(TYPE, "CompareObj without an OBJ operand (use Compare)") + for t in types: + if t is not Type.OBJ and t not in ir.SCALARS and t not in ir.ARRAYS: + self.error(TYPE, f"CompareObj operand of type {getattr(t, 'value', t)}") + self.result(e, Type.BOOL) + self.effects.append("CompareObj (runs rich comparison)") + return dataclasses.replace(e, left=left, right=right) + + def e_ObjToFloat(self, e: ObjToFloat, state: _State) -> ir.Expr: + operand = self.obj_operand(e.operand, state, "operand of ObjToFloat") + self.result(e, Type.F64) + self.effects.append("ObjToFloat (runs __float__/__index__)") + return dataclasses.replace(e, operand=operand) + + def array(self, name: str, node: str) -> Type | None: + t = self.arrays.get(name) + if t is None: + self.error(ARRAY_USE, f"{node} of '{name}', which is not an array parameter") + return t + + def e_Len(self, e: Len, state: _State) -> ir.Expr: + self.array(e.array, "len") + if e.type is not Type.I64: + self.error(TYPE, f"len gives i64, node says {e.type.value}") + return e + + def index(self, array: str, index: object, state: _State, node: str + ) -> tuple[ir.Expr, Type | None, bool]: + at = self.array(array, node) + idx = self.expr(index, state) + self.slot(idx, Type.I64, f"index of {node} '{array}'") + return idx, (_ELEMENT[at] if at else None), self.in_bounds(array, idx) + + def e_Index(self, e: Index, state: _State) -> ir.Expr: + idx, element, proven = self.index(e.array, e.index, state, "index") + if element is not None and e.type is not element and e.type is not Type.OBJ: + self.error(TYPE, f"element of '{e.array}' is {element.value}, node says " + f"{e.type.value}") + return dataclasses.replace(e, index=idx, proven=proven) + + def in_bounds(self, array: str, idx: ir.Expr) -> bool: + """0 <= idx < len(array): idx is the var of an enclosing ForRange over + range(c >= 0, len(array), step > 0). The var is never assigned in the body (checked), the + stop is evaluated once, and no IR node changes an array's length.""" + if type(idx) is not Local or array not in self.arrays: + return False + for loop in reversed(self.loops): + if loop.var == idx.name: + return loop.bounds_array == array + return False + + # --- statements --------------------------------------------------------------------------- + + def block(self, body: object, state: _State) -> tuple[tuple[ir.Stmt, ...], _State]: + if not isinstance(body, tuple): + self.error(STRUCTURE, "a statement block is not a tuple") + return (), state + out = [] + for s in body: + s2, state = self.stmt(s, state) + out.append(s2) + return tuple(out), state + + def stmt(self, s: object, state: _State) -> tuple[ir.Stmt, _State]: + cls = type(s) + if cls not in _STMTS: + self.error(UNKNOWN, f"statement node {cls.__name__} is not defined by ir") + return s, state # type: ignore[return-value] + return getattr(self, "s_" + cls.__name__)(s, state) + + def s_Assign(self, s: Assign, state: _State): + value = self.expr(s.value, state) + target = s.target + if target in self.arrays: + self.error(ARRAY_USE, f"assignment to array parameter '{target}'") + elif target not in self.scalars: + self.error(TYPE, f"assignment to undeclared local '{target}'") + else: + self.slot(value, self.scalars[target], f"assignment to '{target}'") + if getattr(value, "type", None) is Type.NONE: + self.error(TYPE, f"None assigned to '{target}'") + for loop in self.loops: + if loop.var == target: + self.error(STRUCTURE, f"loop variable '{target}' is assigned in its loop body") + return dataclasses.replace(s, value=value), (None if state is None else state | {target}) + + def s_StoreIndex(self, s: StoreIndex, state: _State): + idx, element, proven = self.index(s.array, s.index, state, "store") + value = self.expr(s.value, state) + if element is not None: + self.slot(value, element, f"value stored into '{s.array}'") + self.stored.add(s.array) + self.effects.append(f"stores into '{s.array}'") + return dataclasses.replace(s, index=idx, value=value, proven=proven), state + + def s_ExprStmt(self, s: ExprStmt, state: _State): + return dataclasses.replace(s, value=self.expr(s.value, state)), state + + def cond(self, c: object, state: _State, what: str) -> ir.Expr: + c2 = self.expr(c, state, cond=True) + self.slot(c2, Type.BOOL, f"condition of {what}") + return c2 + + def s_If(self, s: If, state: _State): + cond = self.cond(s.cond, state, "if") + then, st = self.block(s.then, state) + orelse, se = self.block(s.orelse, state) + return dataclasses.replace(s, cond=cond, then=then, orelse=orelse), _meet(st, se) + + def s_While(self, s: While, state: _State): + cond = self.cond(s.cond, state, "while") + loop = _Loop(None, None) + self.loops.append(loop) + # Assignments only add names, so the loop-head state is the entry state (the meet of the + # entry, end-of-body and continue states equals the entry state). + body, _ = self.block(s.body, state) + self.loops.pop() + forever = type(s.cond) is Const and s.cond.type is Type.BOOL and s.cond.value is True + after: _State = None if forever else state + for br in loop.breaks: + after = _meet(after, br) + return dataclasses.replace(s, cond=cond, body=body), after + + def s_ForRange(self, s: ForRange, state: _State): + start = self.expr(s.start, state) + stop = self.expr(s.stop, state) + self.slot(start, Type.I64, "range start") + self.slot(stop, Type.I64, "range stop") + step = s.step + step_ok = (type(step) is Const and step.type is Type.I64 and type(step.value) is int + and step.value != 0 and _I64_MIN <= step.value <= _I64_MAX) + if not step_ok: + self.error(STRUCTURE, f"range step must be a non-zero i64 Const, got {step!r}") + var = s.var + if var in self.arrays: + self.error(ARRAY_USE, f"loop variable '{var}' is an array parameter") + elif self.scalars.get(var) is not Type.I64: + self.error(TYPE, f"loop variable '{var}' must be a declared i64 local") + for outer in self.loops: + if outer.var == var: + self.error(STRUCTURE, f"loop variable '{var}' is assigned in its loop body " + f"(by an inner loop)") + bounds = None + if (step_ok and step.value > 0 and type(start) is Const and start.type is Type.I64 + and type(start.value) is int and start.value >= 0 and type(stop) is Len + and stop.array in self.arrays): + bounds = stop.array + loop = _Loop(var, bounds) + self.loops.append(loop) + body, _ = self.block(s.body, None if state is None else state | {var}) + self.loops.pop() + # Zero iterations leave `var` (and everything the body assigns) as it was: the state after + # the loop is the entry state, met with every break state (each a superset of it). + after: _State = state + for br in loop.breaks: + after = _meet(after, br) + return dataclasses.replace(s, start=start, stop=stop, body=body), after + + def s_Return(self, s: Return, state: _State): + returns = self.fn.returns + if s.value is None: + if returns is not Type.NONE: + self.error(TYPE, f"return without a value in a function returning " + f"{getattr(returns, 'value', returns)}") + return s, None + value = self.expr(s.value, state) + if isinstance(returns, Type): + self.slot(value, returns, "return value") + return dataclasses.replace(s, value=value), None + + def _jump(self, s: Break | Continue, state: _State): + if not self.loops: + self.error(STRUCTURE, f"{type(s).__name__.lower()} outside a loop") + elif type(s) is Break: + self.loops[-1].breaks.append(state) + return s, None + + s_Break = _jump + s_Continue = _jump + + # --- the function ------------------------------------------------------------------------- + + def run(self) -> Function: + fn = self.fn + params = self.declarations() + body, end = self.block(fn.body, params) + if end is not None and fn.returns is not Type.NONE: + self.error(MISSING_RETURN, f"the end of '{fn.name}' is reachable but it returns " + f"{getattr(fn.returns, 'value', fn.returns)}") + # deopt placement and flags + if self.can_deopt and fn.pure is not True: + self.error(DEOPT, "a node that can deopt (checked i64 arithmetic, i64 negation, Unbox to " + "i64, or a call of a may_deopt function) is in an impure function; a redo " + "would repeat its effects") + if fn.may_deopt is not self.can_deopt: + if fn.pure is True or fn.may_deopt is True: + self.error(FLAG_DEOPT, f"may_deopt is {fn.may_deopt} but the body " + f"{'can' if self.can_deopt else 'cannot'} deopt") + if fn.pure is True and self.effects: + self.error(FLAG_PURE, "pure is set but the body " + "; ".join(sorted(set(self.effects)))) + for p in fn.params: + if type(p) is ArrayParam and p.name in self.arrays and p.stored is not (p.name in + self.stored): + self.error(ARRAY_STORED, f"array '{p.name}' has stored={p.stored} but the body " + f"{'stores' if p.name in self.stored else 'never stores'}" + f" into it") + return dataclasses.replace(fn, body=body) + + +def _cycles(graph: dict[str, set[str]]) -> set[str]: + """Names on a cycle of `graph` (Tarjan's strongly connected components).""" + index: dict[str, int] = {} + low: dict[str, int] = {} + stack: list[str] = [] + on: set[str] = set() + result: set[str] = set() + counter = [0] + + def visit(v: str) -> None: + index[v] = low[v] = counter[0] + counter[0] += 1 + stack.append(v) + on.add(v) + for w in graph.get(v, ()): + if w not in graph: + continue + if w not in index: + visit(w) + low[v] = min(low[v], low[w]) + elif w in on: + low[v] = min(low[v], index[w]) + if low[v] == index[v]: + comp = [] + while True: + w = stack.pop() + on.discard(w) + comp.append(w) + if w == v: + break + if len(comp) > 1 or v in graph.get(v, ()): + result.update(comp) + + for v in graph: + if v not in index: + visit(v) + return result + + +def verify(module: Module) -> tuple[Module, list[Diagnostic]]: + """Prove each function of `module`; return the module of proved functions (with proven bounds + marked) and every diagnostic. Rejected functions move to `skipped` with the reason.""" + diags: list[Diagnostic] = [] + reasons: dict[str, str] = {} + functions = module.functions if isinstance(module.functions, tuple) else () + + def reject(fn_name: str, line: int, rule: str, message: str) -> None: + d = Diagnostic(fn_name, rule, message, line) + if d not in diags: + diags.append(d) + reasons.setdefault(fn_name, f"{rule}: {message}") + + named: dict[str, list[Function]] = {} + for g in functions: + if type(g) is not Function: + reject(getattr(g, "name", repr(g)), getattr(g, "source_line", 0), UNKNOWN, + f"{type(g).__name__} is not an ir Function") + continue + named.setdefault(g.name, []).append(g) + for name, gs in named.items(): + if len(gs) > 1: + for g in gs: + reject(name, g.source_line, STRUCTURE, f"function '{name}' is defined " + f"{len(gs)} times") + sigs = {name: gs[0] for name, gs in named.items() if len(gs) == 1} + + checked: dict[str, Function] = {} + callees: dict[str, set[str]] = {} + for name, g in sigs.items(): + c = _Checker(g, sigs) + rebuilt = c.run() + for d in c.diags: + reject(d.function, d.source_line, d.rule, d.message) + checked[name] = rebuilt + callees[name] = c.callees + + for name in sorted(_cycles(callees)): + reject(name, sigs[name].source_line, RECURSION, + f"'{name}' is on a cycle of calls; compiled recursion has no depth limit") + + # A call into a function that is not compiled cannot be a C call: propagate to a fixpoint. + changed = True + while changed: + changed = False + for name, cs in callees.items(): + if name in reasons: + continue + bad = sorted(c for c in cs if c in reasons) + if bad: + reject(name, sigs[name].source_line, UNVERIFIED, + f"calls '{bad[0]}', which is not compiled ({reasons[bad[0]]})") + changed = True + + kept = tuple(checked[g.name] for g in functions + if type(g) is Function and g.name in checked and g.name not in reasons) + skipped = dict(module.skipped) + for name, reason in reasons.items(): + skipped.setdefault(name, reason) + return Module(module.name, kept, skipped), diags diff --git a/python-multiplatform-ksp/src/test/python/test_verify.py b/python-multiplatform-ksp/src/test/python/test_verify.py new file mode 100644 index 000000000..903b06f77 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_verify.py @@ -0,0 +1,904 @@ +"""The IR safety verifier (SPEC N-8, #41). + +`verify.verify(module)` proves each function's safety properties on the IR; a function it cannot +prove is removed from `functions`, moved to `skipped` with the reason, and reported. Every test +builds IR by hand: the verifier must not depend on the front end. + +Each property has an accepted example and rejected examples with the expected rule id. The last +section mutates correct functions at random (seeded) and checks every property-breaking mutation +is caught, and that property-preserving rewrites are not rejected. +""" +from __future__ import annotations + +import dataclasses +import random +from dataclasses import dataclass + +import pytest + +from typedpython import ir +from typedpython.ir import ( + And, ArrayParam, Assign, BinOp, BinOpKind, Box, Break, Call, CallObject, Compare, CompareKind, + CompareObj, Const, Continue, ExprStmt, ForRange, Function, GetAttr, Global, If, Index, Len, + Local, MathCall, MathFunc, Module, ObjToFloat, Or, Param, Return, StoreIndex, ToFloat, Truth, + Type, Unbox, UnaryOp, UnaryOpKind, While, +) + +I64, F64, BOOL, NONE, OBJ = Type.I64, Type.F64, Type.BOOL, Type.NONE, Type.OBJ +F64A, I64A = Type.F64_ARRAY, Type.I64_ARRAY + +DA = "verify/definite-assignment" +MISSING_RETURN = "verify/missing-return" +TYPE = "verify/type" +OWNERSHIP = "verify/ownership" +DEOPT = "verify/deopt-impure" +FLAG_DEOPT = "verify/flag-may-deopt" +FLAG_PURE = "verify/flag-pure" +ARRAY_USE = "verify/array-use" +ARRAY_STORED = "verify/array-stored" +UNKNOWN = "verify/unknown-node" +UNRESOLVED = "verify/call-unresolved" +UNVERIFIED = "verify/call-unverified" +RECURSION = "verify/recursion" +STRUCTURE = "verify/structure" +CONDITION_ONLY = "verify/condition-only" + + +def _verify(): + # Imported lazily so that, before verify.py exists, every test fails on this line (red + # stage) instead of the whole file failing to collect. + from typedpython import verify + return verify + + +# --- builders ------------------------------------------------------------------------------------ + +def i(v): return Const(I64, v) +def f(v): return Const(F64, v) +def b(v): return Const(BOOL, v) +def li(n): return Local(I64, n) +def lf(n): return Local(F64, n) +def lb(n): return Local(BOOL, n) +def lo(n): return Local(OBJ, n) +def add(l, r, t=I64): return BinOp(t, BinOpKind.ADD, l, r) +def lt(l, r): return Compare(BOOL, CompareKind.LT, l, r) +def g_(n): return Global(OBJ, n) +def call_obj(callee, *args, kw=()): return CallObject(OBJ, callee, tuple(args), tuple(kw)) +def pycall(name, *args): return call_obj(g_(name), *args) + + +def fn(name="f", body=(), params=(), locals_=None, returns=NONE, pure=False, may_deopt=False, + line=10): + return Function(name=name, params=tuple(params), returns=returns, locals=dict(locals_ or {}), + body=tuple(body), pure=pure, may_deopt=may_deopt, source_line=line) + + +def run(*functions, skipped=None): + return _verify().verify(Module("m", tuple(functions), dict(skipped or {}))) + + +def kept(out): + return [g.name for g in out.functions] + + +def rules_for(diags, name): + return {d.rule for d in diags if d.function == name} + + +def assert_accepted(*functions, skipped=None): + out, diags = run(*functions, skipped=skipped) + assert kept(out) == [g.name for g in functions], [dataclasses.astuple(d) for d in diags] + assert diags == [] + return out + + +def assert_rejected(rule, *functions, name="f"): + out, diags = run(*functions) + assert name not in kept(out) + assert name in out.skipped + assert rule in rules_for(diags, name), [dataclasses.astuple(d) for d in diags] + return out, diags + + +# --- output shape -------------------------------------------------------------------------------- + +def test_output_keeps_proved_functions_and_moves_others_to_skipped(): + good = fn("good", [Return()]) + bad = fn("bad", [Return(li("nope"))], returns=I64, line=42) + out, diags = run(good, bad, skipped={"front": "uses **kwargs"}) + assert kept(out) == ["good"] + assert out.name == "m" + assert out.skipped["front"] == "uses **kwargs" + assert "bad" in out.skipped and out.skipped["bad"] + d = [d for d in diags if d.function == "bad"][0] + assert d.source_line == 42 and d.rule and d.message + assert all(d.function == "bad" for d in diags) + + +# --- 1. definite assignment ---------------------------------------------------------------------- + +def test_assigned_on_both_branches_is_accepted(): + assert_accepted(fn(params=[Param("c", BOOL)], locals_={"x": I64}, returns=I64, body=[ + If(lb("c"), (Assign("x", i(1)),), (Assign("x", i(2)),)), + Return(li("x")), + ])) + + +def test_read_before_assign_on_one_branch_is_rejected_and_names_the_local(): + _, diags = assert_rejected(DA, fn(params=[Param("c", BOOL)], locals_={"x": I64}, returns=I64, + body=[If(lb("c"), (Assign("x", i(1)),)), Return(li("x"))])) + assert any("'x'" in d.message for d in diags if d.rule == DA) + + +def test_branch_that_returns_does_not_need_the_assignment(): + assert_accepted(fn(params=[Param("c", BOOL)], locals_={"x": I64}, returns=I64, body=[ + If(lb("c"), (Return(i(0)),), (Assign("x", i(2)),)), + Return(li("x")), + ])) + + +def test_loop_var_read_after_possibly_zero_iteration_loop_is_rejected(): + assert_rejected(DA, fn(params=[Param("n", I64)], locals_={"k": I64}, returns=I64, body=[ + ForRange("k", i(0), li("n"), i(1), (ExprStmt(li("k")),)), + Return(li("k")), + ])) + + +def test_loop_var_assigned_before_the_loop_may_be_read_after_it(): + assert_accepted(fn(params=[Param("n", I64)], locals_={"k": I64}, returns=I64, body=[ + Assign("k", i(-1)), + ForRange("k", i(0), li("n"), i(1), (ExprStmt(li("k")),)), + Return(li("k")), + ])) + + +def test_assignment_inside_a_loop_body_does_not_count_after_the_loop(): + assert_rejected(DA, fn(params=[Param("c", BOOL)], locals_={"x": I64}, returns=I64, body=[ + While(lb("c"), (Assign("x", i(1)), Break())), + Return(li("x")), + ])) + + +def test_read_in_loop_body_before_its_assignment_in_the_body_is_rejected(): + assert_rejected(DA, fn(params=[Param("c", BOOL)], locals_={"x": I64, "y": I64}, body=[ + While(lb("c"), (Assign("y", li("x")), Assign("x", i(1)))), + Return(), + ])) + + +def test_continue_skips_the_rest_of_the_loop_body(): + # `continue` before the assignment: the read after the If in the body is reached only on the + # path that assigned x — accepted. The read after the loop is not (zero iterations). + body_ok = (If(lb("c"), (Continue(),), (Assign("x", i(1)),)), ExprStmt(li("x"))) + assert_accepted(fn(params=[Param("c", BOOL)], locals_={"x": I64}, + body=[While(lb("c"), body_ok), Return()])) + assert_rejected(DA, fn(params=[Param("c", BOOL)], locals_={"x": I64}, returns=I64, + body=[While(lb("c"), body_ok), Return(li("x"))])) + + +def test_while_true_exits_only_through_break_or_return(): + assert_accepted(fn(locals_={"x": I64}, returns=I64, body=[ + While(b(True), (Assign("x", i(3)), Break())), + Return(li("x")), + ])) + assert_accepted(fn(returns=I64, body=[While(b(True), (Return(i(1)),))])) + + +def test_falling_off_the_end_of_a_valued_function_is_rejected(): + assert_rejected(MISSING_RETURN, fn(params=[Param("c", BOOL)], returns=I64, + body=[If(lb("c"), (Return(i(1)),))])) + + +def test_parameters_are_assigned_at_entry(): + assert_accepted(fn(params=[Param("x", F64)], returns=F64, body=[Return(lf("x"))])) + + +def test_break_outside_a_loop_is_rejected(): + assert_rejected(STRUCTURE, fn(body=[Break()])) + assert_rejected(STRUCTURE, fn(body=[Continue()])) + + +def test_assigning_the_loop_var_in_the_body_is_rejected(): + assert_rejected(STRUCTURE, fn(params=[Param("n", I64)], locals_={"k": I64}, body=[ + ForRange("k", i(0), li("n"), i(1), (Assign("k", i(5)),)), + ])) + + +def test_zero_step_is_rejected(): + assert_rejected(STRUCTURE, fn(params=[Param("n", I64)], locals_={"k": I64}, body=[ + ForRange("k", i(0), li("n"), i(0), ()), + ])) + + +# --- 2. typing ----------------------------------------------------------------------------------- + +def test_mixed_arithmetic_through_to_float_is_accepted(): + assert_accepted(fn(params=[Param("x", F64), Param("n", I64)], returns=F64, + body=[Return(add(lf("x"), ToFloat(F64, li("n")), F64))])) + + +def test_f64_plus_i64_without_to_float_is_rejected(): + assert_rejected(TYPE, fn(params=[Param("x", F64), Param("n", I64)], returns=F64, + body=[Return(add(lf("x"), li("n"), F64))])) + + +def test_i64_truediv_result_must_be_f64(): + ok = BinOp(F64, BinOpKind.TRUEDIV, li("a"), li("b")) + bad = BinOp(I64, BinOpKind.TRUEDIV, li("a"), li("b")) + ps = [Param("a", I64), Param("b", I64)] + assert_accepted(fn(params=ps, returns=F64, pure=True, may_deopt=True, body=[Return(ok)])) + assert_rejected(TYPE, fn(params=ps, returns=I64, pure=True, may_deopt=True, + body=[Return(bad)])) + + +def test_bool_arithmetic_is_rejected(): + assert_rejected(TYPE, fn(params=[Param("c", BOOL)], returns=BOOL, + body=[Return(add(lb("c"), lb("c"), BOOL))])) + + +def test_compare_result_must_be_bool_and_mixed_numeric_compare_is_allowed(): + assert_accepted(fn(params=[Param("x", F64), Param("n", I64)], returns=BOOL, + body=[Return(lt(li("n"), lf("x")))])) + assert_rejected(TYPE, fn(params=[Param("x", F64), Param("n", I64)], returns=I64, + body=[Return(Compare(I64, CompareKind.LT, li("n"), lf("x")))])) + + +def test_and_or_need_bool_operands(): + assert_accepted(fn(params=[Param("c", BOOL)], returns=BOOL, + body=[Return(Or(BOOL, And(BOOL, lb("c"), b(True)), lb("c")))])) + assert_rejected(TYPE, fn(params=[Param("n", I64), Param("c", BOOL)], returns=BOOL, + body=[Return(And(BOOL, li("n"), lb("c")))])) + + +def test_if_and_while_conditions_must_be_bool(): + assert_rejected(TYPE, fn(params=[Param("n", I64)], body=[If(li("n"), (Return(),))])) + assert_rejected(TYPE, fn(params=[Param("n", I64)], body=[While(li("n"), (Break(),))])) + + +def test_assign_value_must_match_declared_local_type(): + assert_rejected(TYPE, fn(locals_={"x": I64}, body=[Assign("x", f(1.0))])) + + +def test_assign_to_undeclared_local_is_rejected(): + assert_rejected(TYPE, fn(body=[Assign("x", f(1.0))])) + + +def test_local_read_must_match_its_declaration(): + assert_rejected(TYPE, fn(params=[Param("x", F64)], returns=I64, body=[Return(li("x"))])) + + +def test_return_type_must_match(): + assert_rejected(TYPE, fn(returns=I64, body=[Return(f(1.0))])) + assert_rejected(TYPE, fn(returns=I64, body=[Return()])) + assert_rejected(TYPE, fn(returns=NONE, body=[Return(i(1))])) + + +def test_const_value_must_match_its_type(): + assert_rejected(TYPE, fn(returns=I64, body=[Return(i(1.5))])) + assert_rejected(TYPE, fn(returns=I64, body=[Return(i(True))])) + assert_rejected(TYPE, fn(returns=I64, body=[Return(i(2 ** 63))])) + assert_rejected(TYPE, fn(returns=F64, body=[Return(f(1))])) + + +def test_call_arity_and_argument_types_are_checked_against_the_callee(): + g = fn("g", params=[Param("x", F64)], returns=F64, body=[Return(lf("x"))]) + ok = fn("f", returns=F64, body=[Return(Call(F64, "g", (f(1.0),)))]) + assert_accepted(ok, g) + assert_rejected(TYPE, fn("f", returns=F64, body=[Return(Call(F64, "g", ()))]), g) + assert_rejected(TYPE, fn("f", returns=F64, body=[Return(Call(F64, "g", (i(1),)))]), g) + assert_rejected(TYPE, fn("f", returns=I64, body=[Return(Call(I64, "g", (f(1.0),)))]), g) + + +def test_index_store_len_need_array_params(): + ps = [Param("a", F64)] + assert_rejected(ARRAY_USE, fn(params=ps, returns=F64, body=[Return(Index(F64, "a", i(0)))])) + assert_rejected(ARRAY_USE, fn(params=ps, returns=I64, body=[Return(Len(I64, "a"))])) + assert_rejected(ARRAY_USE, fn(params=ps, body=[StoreIndex("a", i(0), f(1.0)), Return()])) + + +def test_index_type_and_element_type(): + a = [ArrayParam("a", F64A, stored=True)] + assert_rejected(TYPE, fn(params=a, body=[StoreIndex("a", i(0), i(1)), Return()])) + assert_rejected(TYPE, fn(params=a, returns=I64, body=[Return(Index(I64, "a", i(0)))])) + assert_rejected(TYPE, fn(params=[ArrayParam("a", F64A, stored=False)], returns=F64, + body=[Return(Index(F64, "a", f(0.0)))])) + + +def test_math_call_needs_f64_args_and_the_right_arity(): + ps = [Param("x", F64), Param("n", I64)] + assert_accepted(fn(params=ps, returns=F64, body=[ + Return(MathCall(F64, MathFunc.ATAN2, (MathCall(F64, MathFunc.SQRT, (lf("x"),)), lf("x")))) + ])) + assert_rejected(TYPE, fn(params=ps, returns=F64, + body=[Return(MathCall(F64, MathFunc.SQRT, (li("n"),)))])) + assert_rejected(TYPE, fn(params=ps, returns=F64, + body=[Return(MathCall(F64, MathFunc.ATAN2, (lf("x"),)))])) + assert_rejected(TYPE, fn(params=ps, returns=F64, + body=[Return(MathCall(F64, MathFunc.SQRT, (lf("x"), lf("x"))))])) + + +def test_not_needs_bool_and_neg_keeps_type(): + assert_accepted(fn(params=[Param("c", BOOL), Param("x", F64)], returns=BOOL, body=[ + ExprStmt(UnaryOp(F64, UnaryOpKind.NEG, lf("x"))), + Return(UnaryOp(BOOL, UnaryOpKind.NOT, lb("c"))), + ])) + assert_rejected(TYPE, fn(params=[Param("n", I64)], returns=BOOL, + body=[Return(UnaryOp(BOOL, UnaryOpKind.NOT, li("n")))])) + + +# --- 2b. object nodes (Global, GetAttr, CallObject, Truth, CompareObj, ObjToFloat) --------------- + +def test_global_read_alone_keeps_a_function_pure(): + assert_accepted(fn(returns=OBJ, pure=True, body=[Return(g_("Color"))])) + assert_rejected(TYPE, fn(returns=F64, body=[Return(Global(F64, "x"))])) + + +def test_get_attr_is_obj_to_obj_and_impure(): + assert_accepted(fn(params=[Param("o", OBJ)], returns=OBJ, + body=[Return(GetAttr(OBJ, lo("o"), "width"))])) + assert_rejected(FLAG_PURE, fn(params=[Param("o", OBJ)], returns=OBJ, pure=True, + body=[Return(GetAttr(OBJ, lo("o"), "width"))])) + assert_rejected(OWNERSHIP, fn(params=[Param("x", F64)], returns=OBJ, + body=[Return(GetAttr(OBJ, lf("x"), "real"))])) + assert_rejected(TYPE, fn(params=[Param("o", OBJ)], returns=F64, + body=[Return(GetAttr(F64, lo("o"), "width"))])) + + +def test_call_object_takes_obj_callee_and_args_and_is_impure(): + assert_accepted(fn(params=[Param("x", F64), Param("o", OBJ)], returns=OBJ, body=[ + Return(call_obj(GetAttr(OBJ, g_("Modifier"), "padding"), lo("o"), Box(OBJ, lf("x")), + kw=("dp",))), + ])) + assert_rejected(FLAG_PURE, fn(returns=OBJ, pure=True, body=[Return(pycall("make"))])) + assert_rejected(OWNERSHIP, fn(params=[Param("x", F64)], returns=OBJ, + body=[Return(pycall("make", lf("x")))])) + assert_rejected(OWNERSHIP, fn(params=[Param("x", F64)], returns=OBJ, + body=[Return(call_obj(lf("x")))])) + assert_rejected(TYPE, fn(params=[Param("o", OBJ)], returns=OBJ, + body=[Return(call_obj(g_("make"), lo("o"), kw=("a", "b")))])) + assert_rejected(TYPE, fn(params=[Param("o", OBJ)], returns=F64, + body=[Return(CallObject(F64, g_("make"), (lo("o"),), ()))])) + + +def test_truth_is_obj_to_bool_and_impure(): + assert_accepted(fn(params=[Param("o", OBJ)], returns=BOOL, + body=[Return(Truth(BOOL, lo("o")))])) + assert_rejected(FLAG_PURE, fn(params=[Param("o", OBJ)], returns=BOOL, pure=True, + body=[Return(Truth(BOOL, lo("o")))])) + assert_rejected(OWNERSHIP, fn(params=[Param("c", BOOL)], returns=BOOL, + body=[Return(Truth(BOOL, lb("c")))])) + assert_rejected(TYPE, fn(params=[Param("o", OBJ)], returns=I64, + body=[Return(Truth(I64, lo("o")))])) + + +def test_compare_obj_in_a_condition_is_accepted(): + cmp = CompareObj(BOOL, CompareKind.LT, lo("o"), lf("x")) + assert_accepted(fn(params=[Param("o", OBJ), Param("x", F64), Param("c", BOOL)], body=[ + If(cmp, (Return(),)), + While(And(BOOL, lb("c"), Or(BOOL, cmp, b(False))), (Break(),)), + Return(), + ])) + + +def test_compare_obj_outside_a_condition_is_rejected(): + cmp = CompareObj(BOOL, CompareKind.LT, lo("o"), lo("p")) + ps = [Param("o", OBJ), Param("p", OBJ), Param("c", BOOL)] + assert_rejected(CONDITION_ONLY, fn(params=ps, returns=BOOL, body=[Return(cmp)])) + assert_rejected(CONDITION_ONLY, fn(params=ps, locals_={"r": BOOL}, + body=[Assign("r", cmp), Return()])) + # `x = c and (a < b)` yields the rich comparison's own object in CPython, not a bool + assert_rejected(CONDITION_ONLY, fn(params=ps, locals_={"r": BOOL}, + body=[Assign("r", And(BOOL, lb("c"), cmp)), Return()])) + assert_rejected(CONDITION_ONLY, fn(params=ps, + body=[If(UnaryOp(BOOL, UnaryOpKind.NOT, cmp), ()), + Return()])) + + +def test_compare_obj_needs_an_obj_operand_and_is_impure(): + ps = [Param("o", OBJ), Param("x", F64)] + assert_rejected(TYPE, fn(params=ps, body=[ + If(CompareObj(BOOL, CompareKind.LT, lf("x"), lf("x")), ()), Return()])) + assert_rejected(FLAG_PURE, fn(params=ps, pure=True, body=[ + If(CompareObj(BOOL, CompareKind.LT, lo("o"), lf("x")), ()), Return()])) + assert_rejected(TYPE, fn(params=ps, body=[ + If(CompareObj(OBJ, CompareKind.LT, lo("o"), lo("o")), ()), Return()])) + + +def test_obj_to_float_is_obj_to_f64_impure_and_never_deopts(): + assert_accepted(fn(params=[Param("o", OBJ)], returns=F64, + body=[Return(ObjToFloat(F64, lo("o")))])) + assert_rejected(FLAG_PURE, fn(params=[Param("o", OBJ)], returns=F64, pure=True, + body=[Return(ObjToFloat(F64, lo("o")))])) + assert_rejected(OWNERSHIP, fn(params=[Param("n", I64)], returns=F64, + body=[Return(ObjToFloat(F64, li("n")))])) + assert_rejected(TYPE, fn(params=[Param("o", OBJ)], returns=I64, + body=[Return(ObjToFloat(I64, lo("o")))])) + + +def test_obj_binop_is_impure(): + assert_rejected(FLAG_PURE, fn(params=[Param("o", OBJ)], returns=OBJ, pure=True, + body=[Return(BinOp(OBJ, BinOpKind.ADD, lo("o"), lo("o")))])) + + +# --- 3. deopt placement and flags ---------------------------------------------------------------- + +def test_i64_add_in_a_pure_may_deopt_function_is_accepted(): + assert_accepted(fn(params=[Param("n", I64)], returns=I64, pure=True, may_deopt=True, + body=[Return(add(li("n"), i(1)))])) + + +def test_i64_add_in_an_impure_function_is_rejected(): + assert_rejected(DEOPT, fn(params=[Param("n", I64)], returns=I64, pure=False, may_deopt=False, + body=[Return(add(li("n"), i(1)))])) + assert_rejected(DEOPT, fn(params=[Param("n", I64)], returns=I64, pure=False, may_deopt=True, + body=[Return(add(li("n"), i(1)))])) + + +@pytest.mark.parametrize("node", [ + BinOp(I64, BinOpKind.SUB, Local(I64, "n"), Const(I64, 1)), + BinOp(I64, BinOpKind.MUL, Local(I64, "n"), Const(I64, 2)), + BinOp(I64, BinOpKind.FLOORDIV, Local(I64, "n"), Const(I64, 2)), + BinOp(I64, BinOpKind.MOD, Local(I64, "n"), Const(I64, 2)), + BinOp(F64, BinOpKind.TRUEDIV, Local(I64, "n"), Const(I64, 2)), + UnaryOp(I64, UnaryOpKind.NEG, Local(I64, "n")), + Unbox(I64, Box(OBJ, Local(I64, "n"))), +]) +def test_every_deopting_node_is_rejected_in_an_impure_function(node): + assert_rejected(DEOPT, fn(params=[Param("n", I64)], body=[ExprStmt(node), Return()])) + assert_accepted(fn(params=[Param("n", I64)], pure=True, may_deopt=True, + body=[ExprStmt(node), Return()])) + + +def test_non_deopting_nodes_are_fine_in_an_impure_function(): + assert_accepted(fn(params=[Param("x", F64), Param("n", I64), Param("o", OBJ)], returns=F64, + body=[ + ExprStmt(pycall("print", Box(OBJ, lf("x")))), + ExprStmt(UnaryOp(I64, UnaryOpKind.POS, li("n"))), + ExprStmt(Unbox(F64, Box(OBJ, lf("x")))), + ExprStmt(ObjToFloat(F64, GetAttr(OBJ, lo("o"), "x"))), + ExprStmt(BinOp(OBJ, BinOpKind.MUL, lo("o"), lo("o"))), + Return(BinOp(F64, BinOpKind.FLOORDIV, lf("x"), f(2.0))), + ])) + + +def test_may_deopt_flag_must_match_the_body(): + assert_rejected(FLAG_DEOPT, fn(params=[Param("n", I64)], returns=I64, pure=True, + may_deopt=False, body=[Return(add(li("n"), i(1)))])) + assert_rejected(FLAG_DEOPT, fn(params=[Param("x", F64)], returns=F64, pure=True, + may_deopt=True, body=[Return(lf("x"))])) + + +def test_pure_flag_lying_store_index_is_rejected(): + assert_rejected(FLAG_PURE, fn(params=[ArrayParam("a", F64A, stored=True)], pure=True, + body=[StoreIndex("a", i(0), f(1.0)), Return()])) + + +def test_pure_flag_lying_call_object_is_rejected(): + assert_rejected(FLAG_PURE, fn(pure=True, body=[ExprStmt(pycall("print")), Return()])) + + +def test_pure_function_may_only_call_pure_functions(): + g = fn("g", body=[ExprStmt(pycall("print")), Return()]) + assert_rejected(FLAG_PURE, fn("f", pure=True, body=[ExprStmt(Call(NONE, "g", ())), Return()]), + g) + + +def test_impure_caller_of_a_may_deopt_callee_is_rejected(): + g = fn("g", params=[Param("n", I64)], returns=I64, pure=True, may_deopt=True, + body=[Return(add(li("n"), i(1)))]) + caller = fn("f", returns=I64, body=[ExprStmt(pycall("print")), + Return(Call(I64, "g", (i(1),)))]) + out, _ = assert_rejected(DEOPT, caller, g) + assert "g" in kept(out) + + +def test_pure_caller_of_a_may_deopt_callee_propagates_may_deopt(): + g = fn("g", params=[Param("n", I64)], returns=I64, pure=True, may_deopt=True, + body=[Return(add(li("n"), i(1)))]) + assert_accepted(fn("f", returns=I64, pure=True, may_deopt=True, + body=[Return(Call(I64, "g", (i(1),)))]), g) + assert_rejected(FLAG_DEOPT, fn("f", returns=I64, pure=True, may_deopt=False, + body=[Return(Call(I64, "g", (i(1),)))]), g) + + +# --- 4. arrays ----------------------------------------------------------------------------------- + +def _sum(stop, index, step=1, start=0, store=False): + """for k in range(start, stop, step): s = s + a[index] (or a[index] = 0.0)""" + body = (StoreIndex("a", index, f(0.0)),) if store else ( + Assign("s", add(lf("s"), Index(F64, "a", index), F64)),) + return fn(params=[ArrayParam("a", F64A, stored=store), ArrayParam("b", F64A, stored=False)], + locals_={"s": F64, "k": I64}, returns=F64, body=[ + Assign("s", f(0.0)), + ForRange("k", i(start), stop, i(step), body), + Return(lf("s")), + ]) + + +def _accesses(function): + found = [] + + def walk(node): + if isinstance(node, (Index, StoreIndex)): + found.append(node) + if dataclasses.is_dataclass(node): + for fl in dataclasses.fields(node): + walk(getattr(node, fl.name)) + elif isinstance(node, tuple): + for x in node: + walk(x) + + walk(function.body) + return found + + +def _proven(function): + out = assert_accepted(function) + acc = _accesses(out.functions[0]) + assert len(acc) == 1 + return acc[0].proven + + +def test_range_len_index_is_proven(): + assert _proven(_sum(Len(I64, "a"), li("k"))) is True + assert _proven(_sum(Len(I64, "a"), li("k"), store=True)) is True + assert _proven(_sum(Len(I64, "a"), li("k"), start=2, step=3)) is True + + +@pytest.mark.parametrize("stop,index,step,start", [ + (add(Len(I64, "a"), i(1)), li("k"), 1, 0), # range(0, len(a)+1) + (Len(I64, "a"), add(li("k"), i(1)), 1, 0), # a[k+1] + (Len(I64, "b"), li("k"), 1, 0), # len of another array + (Len(I64, "a"), li("k"), -1, 0), # negative step + (Len(I64, "a"), li("k"), 1, -1), # negative start + (Len(I64, "a"), i(0), 1, 0), # constant index, empty array possible +]) +def test_other_indices_keep_the_runtime_check(stop, index, step, start): + function = _sum(stop, index, step=step, start=start) + if any(isinstance(n, BinOp) and n.type is I64 for n in (stop, index)): + function = dataclasses.replace(function, pure=True, may_deopt=True) + assert _proven(function) is False + + +def test_index_after_the_loop_is_not_proven(): + function = fn(params=[ArrayParam("a", F64A, stored=False)], locals_={"k": I64}, returns=F64, + body=[Assign("k", i(0)), + ForRange("k", i(0), Len(I64, "a"), i(1), ()), + Return(Index(F64, "a", li("k")))]) + assert _proven(function) is False + + +def test_a_proven_flag_from_the_input_is_not_trusted(): + function = _sum(Len(I64, "b"), li("k")) + lying = dataclasses.replace(function, body=( + function.body[0], + dataclasses.replace(function.body[1], body=( + Assign("s", add(lf("s"), Index(F64, "a", li("k"), proven=True), F64)),)), + function.body[2])) + assert _proven(lying) is False + + +def test_array_passed_to_call_is_rejected(): + g = fn("g", params=[ArrayParam("x", F64A, stored=False)], returns=F64, + body=[Return(Index(F64, "x", i(0)))]) + caller = fn("f", params=[ArrayParam("a", F64A, stored=False)], returns=F64, + body=[Return(Call(F64, "g", (Local(F64A, "a"),)))]) + assert_rejected(ARRAY_USE, caller, g) + + +def test_array_read_as_a_value_is_rejected(): + assert_rejected(ARRAY_USE, fn(params=[ArrayParam("a", F64A, stored=False)], returns=OBJ, + body=[Return(Box(OBJ, Local(F64A, "a")))])) + assert_rejected(ARRAY_USE, fn(params=[ArrayParam("a", F64A, stored=False)], returns=OBJ, + body=[Return(pycall("print", Local(F64A, "a")))])) + + +def test_assigning_an_array_param_is_rejected(): + assert_rejected(ARRAY_USE, fn(params=[ArrayParam("a", F64A, stored=False)], + locals_={"a": F64}, body=[Assign("a", f(0.0)), Return()])) + + +def test_stored_flag_must_match_the_body(): + assert_rejected(ARRAY_STORED, fn(params=[ArrayParam("a", F64A, stored=False)], + body=[StoreIndex("a", i(0), f(1.0)), Return()])) + assert_rejected(ARRAY_STORED, fn(params=[ArrayParam("a", F64A, stored=True)], returns=F64, + body=[Return(Index(F64, "a", i(0)))])) + + +def test_array_param_must_have_an_array_type(): + assert_rejected(TYPE, fn(params=[ArrayParam("a", F64, stored=False)], body=[Return()])) + assert_rejected(TYPE, fn(params=[Param("a", F64A)], body=[Return()])) + + +# --- 5. ownership and node set ------------------------------------------------------------------- + +def test_box_callobject_unbox_pipeline_is_accepted(): + assert_accepted(fn(params=[Param("x", F64), Param("o", OBJ)], locals_={"r": OBJ}, returns=F64, + body=[Assign("r", pycall("f2", Box(OBJ, lf("x")), lo("o"))), + Assign("r", BinOp(OBJ, BinOpKind.ADD, lo("r"), lo("o"))), + Return(Unbox(F64, lo("r")))])) + + +def test_obj_into_a_scalar_slot_without_unbox_is_rejected(): + assert_rejected(OWNERSHIP, fn(params=[Param("o", OBJ)], locals_={"x": F64}, + body=[Assign("x", lo("o")), Return()])) + assert_rejected(OWNERSHIP, fn(params=[Param("o", OBJ)], returns=F64, body=[Return(lo("o"))])) + assert_rejected(OWNERSHIP, fn(params=[Param("o", OBJ)], returns=OBJ, + body=[Return(Box(OBJ, lo("o")))])) + assert_rejected(OWNERSHIP, fn(params=[Param("o", OBJ)], body=[If(lo("o"), ()), Return()])) + + +def test_scalar_into_an_obj_slot_without_box_is_rejected(): + assert_rejected(OWNERSHIP, fn(params=[Param("x", F64)], returns=OBJ, body=[Return(lf("x"))])) + assert_rejected(OWNERSHIP, fn(params=[Param("x", F64)], + body=[ExprStmt(pycall("print", lf("x"))), Return()])) + + +def test_obj_from_a_node_that_does_not_create_a_reference_is_rejected(): + assert_rejected(OWNERSHIP, fn(returns=OBJ, body=[Return(Const(OBJ, None))])) + assert_rejected(OWNERSHIP, fn(params=[Param("o", OBJ)], returns=OBJ, + body=[Return(UnaryOp(OBJ, UnaryOpKind.NEG, lo("o")))])) + assert_rejected(OWNERSHIP, fn(params=[ArrayParam("a", F64A, stored=False)], returns=OBJ, + body=[Return(Index(OBJ, "a", i(0)))])) + assert_rejected(OWNERSHIP, fn(params=[Param("o", OBJ)], returns=OBJ, + body=[Return(Truth(OBJ, lo("o")))])) + assert_rejected(OWNERSHIP, fn(params=[Param("o", OBJ)], returns=OBJ, + body=[Return(ObjToFloat(OBJ, lo("o")))])) + + +@dataclass(frozen=True) +class Weird(ir.Expr): + pass + + +@dataclass(frozen=True) +class SneakyBinOp(BinOp): + pass + + +@dataclass(frozen=True) +class Free(ir.Stmt): + name: str + + +def test_unknown_node_subclasses_are_rejected(): + assert_rejected(UNKNOWN, fn(returns=I64, body=[Return(Weird(I64))])) + assert_rejected(UNKNOWN, fn(params=[Param("x", F64)], returns=F64, + body=[Return(SneakyBinOp(F64, BinOpKind.ADD, lf("x"), lf("x")))])) + assert_rejected(UNKNOWN, fn(params=[Param("o", OBJ)], body=[Free("o"), Return()])) + assert_rejected(UNKNOWN, fn(body=[ExprStmt(Return()), Return()])) + assert_rejected(UNKNOWN, fn(params=[Param("x", F64)], returns=F64, + body=[Return(BinOp(F64, "+", lf("x"), lf("x")))])) + + +# --- 6. calls ------------------------------------------------------------------------------------ + +def test_unknown_callee_is_rejected(): + assert_rejected(UNRESOLVED, fn(returns=I64, body=[Return(Call(I64, "nowhere", ()))])) + + +def test_callee_left_interpreted_by_the_front_end_is_unresolved(): + out, diags = run(fn(returns=I64, body=[Return(Call(I64, "g", ()))]), skipped={"g": "kwargs"}) + assert "f" not in kept(out) and UNRESOLVED in rules_for(diags, "f") + + +def test_caller_of_a_rejected_callee_is_rejected_transitively(): + g = fn("g", returns=I64, body=[Return(f(1.0))]) # type error + h = fn("h", returns=I64, body=[Return(Call(I64, "g", ()))]) + k = fn("k", returns=I64, body=[Return(Call(I64, "h", ()))]) + out, diags = run(k, h, g) + assert kept(out) == [] + assert UNVERIFIED in rules_for(diags, "h") and UNVERIFIED in rules_for(diags, "k") + + +def test_recursion_is_rejected(): + assert_rejected(RECURSION, fn("f", returns=I64, body=[Return(Call(I64, "f", ()))])) + g = fn("g", returns=I64, body=[Return(Call(I64, "f", ()))]) + out, diags = assert_rejected(RECURSION, fn("f", returns=I64, body=[Return(Call(I64, "g", ()))]), + g) + assert "g" not in kept(out) + + +def test_duplicate_function_names_are_rejected(): + out, diags = run(fn("f", body=[Return()]), fn("f", body=[Return()])) + assert kept(out) == [] and STRUCTURE in rules_for(diags, "f") + + +# --- "nothing slips through": seeded IR mutation ------------------------------------------------- + +def _base_functions(): + """Two correct functions exercising every node kind; the mutator breaks them.""" + pure = fn("dot", params=[ArrayParam("a", F64A, stored=False), + ArrayParam("w", F64A, stored=False), Param("n", I64)], + locals_={"s": F64, "k": I64, "c": I64, "t": OBJ}, returns=F64, pure=True, + may_deopt=True, line=3, body=[ + Assign("s", f(0.0)), + Assign("c", i(0)), + Assign("t", g_("TOLERANCE")), + ForRange("k", i(0), Len(I64, "a"), i(1), ( + If(And(BOOL, lt(li("k"), li("n")), lt(Index(F64, "a", li("k")), f(1e9))), + (Assign("s", add(lf("s"), BinOp(F64, BinOpKind.MUL, + Index(F64, "a", li("k")), + Index(F64, "w", i(0))), F64)),), + (Continue(),)), + Assign("c", add(li("c"), i(1))), + )), + Return(add(lf("s"), MathCall(F64, MathFunc.SQRT, (ToFloat(F64, li("c")),)), + F64)), + ]) + impure = fn("scale", params=[ArrayParam("a", F64A, stored=True), Param("x", F64), + Param("o", OBJ), Param("c", BOOL)], + locals_={"k": I64, "r": OBJ, "y": F64}, returns=F64, line=20, body=[ + Assign("r", call_obj(GetAttr(OBJ, g_("Modifier"), "padding"), + Box(OBJ, lf("x")), lo("o"), kw=("dp",))), + Assign("y", Unbox(F64, lo("r"))), + If(Truth(BOOL, lo("r")), (Assign("y", ObjToFloat(F64, lo("o"))),)), + ForRange("k", i(0), Len(I64, "a"), i(1), ( + StoreIndex("a", li("k"), BinOp(F64, BinOpKind.MUL, + Index(F64, "a", li("k")), lf("y"))), + )), + While(And(BOOL, Compare(BOOL, CompareKind.GT, lf("y"), f(1.0)), + CompareObj(BOOL, CompareKind.NE, lo("o"), lo("r"))), ( + Assign("y", BinOp(F64, BinOpKind.TRUEDIV, lf("y"), f(2.0))), + Assign("r", BinOp(OBJ, BinOpKind.ADD, lo("r"), lo("o"))), + If(UnaryOp(BOOL, UnaryOpKind.NOT, lt(lf("y"), f(100.0))), (Break(),)), + )), + Return(lf("y")), + ]) + return pure, impure + + +def _paths(node, pred, path=()): + """Every (path, node) below `node` whose node satisfies pred. A path is a tuple of + field names / tuple indices.""" + out = [] + if pred(node): + out.append((path, node)) + if dataclasses.is_dataclass(node) and not isinstance(node, type): + for fl in dataclasses.fields(node): + out += _paths(getattr(node, fl.name), pred, path + (fl.name,)) + elif isinstance(node, tuple): + for k, x in enumerate(node): + out += _paths(x, pred, path + (k,)) + return out + + +def _replace(node, path, new): + if not path: + return new + head, rest = path[0], path[1:] + if isinstance(node, tuple): + return node[:head] + (_replace(node[head], rest, new),) + node[head + 1:] + return dataclasses.replace(node, **{head: _replace(getattr(node, head), rest, new)}) + + +def _is_expr(n): return isinstance(n, ir.Expr) + + +def _replace_body(g, path, new): + return dataclasses.replace(g, body=_replace(g.body, path, new)) + + +def _breaking_mutations(rng, g): + """Return (description, mutant, rules): the mutant breaks a property by construction; rules + is the set of rule ids any one of which is an acceptable report.""" + exprs = _paths(g.body, _is_expr) + path, e = rng.choice(exprs) + kind = rng.randrange(14) + if kind == 0: # read of an undeclared/unassigned local of the right type + return "unassigned read", _replace_body(g, path, Local(e.type, "zz_unset")), { + DA, TYPE, STRUCTURE} + if kind == 1: # change the declared type of a node + other = rng.choice([t for t in (I64, F64, BOOL, OBJ) if t is not e.type]) + return "retype", _replace_body(g, path, dataclasses.replace(e, type=other)), { + TYPE, OWNERSHIP, STRUCTURE} + if kind == 2: # substitute an unknown node + return "unknown", _replace_body(g, path, Weird(e.type)), {UNKNOWN, STRUCTURE} + if kind == 3: # flip pure + return "flip pure", dataclasses.replace(g, pure=not g.pure), {FLAG_PURE, DEOPT, FLAG_DEOPT} + if kind == 4: # flip may_deopt + return "flip may_deopt", dataclasses.replace(g, may_deopt=not g.may_deopt), { + FLAG_DEOPT, DEOPT} + if kind == 5: # flip stored on the first array + k = next(j for j, p in enumerate(g.params) if isinstance(p, ArrayParam)) + p = g.params[k] + ps = g.params[:k] + (dataclasses.replace(p, stored=not p.stored),) + g.params[k + 1:] + return "flip stored", dataclasses.replace(g, params=ps), {ARRAY_STORED, FLAG_PURE} + if kind == 6: # change the return type + other = rng.choice([t for t in (I64, F64, BOOL, OBJ, NONE) if t is not g.returns]) + return "returns", dataclasses.replace(g, returns=other), {TYPE, OWNERSHIP, MISSING_RETURN} + if kind == 7: # drop a declared local + name = rng.choice(sorted(g.locals)) + loc = {k: v for k, v in g.locals.items() if k != name} + return "drop local", dataclasses.replace(g, locals=loc), {TYPE, STRUCTURE} + if kind == 8: # call an unknown function + return "unknown call", _replace_body(g, path, Call(e.type, "nowhere", ())), { + UNRESOLVED, STRUCTURE} + if kind == 9: # a store into the pure function, or an I64 add into the impure one + if g.pure: + st = StoreIndex(next(p.name for p in g.params if isinstance(p, ArrayParam)), i(0), + f(0.0)) + return "store in pure", dataclasses.replace(g, body=(st,) + g.body), { + FLAG_PURE, ARRAY_STORED} + st = ExprStmt(add(i(1), i(2))) + return "deopt in impure", dataclasses.replace(g, body=(st,) + g.body), {DEOPT} + if kind == 10: # a Break/Continue at function level + return "stray break", dataclasses.replace( + g, body=(rng.choice([Break(), Continue()]),) + g.body), {STRUCTURE} + if kind == 11: # an effecting object node into the pure function / a value-position CompareObj + if g.pure: + node = rng.choice([GetAttr(OBJ, lo("t"), "v"), call_obj(lo("t")), Truth(BOOL, lo("t")), + ObjToFloat(F64, lo("t")), BinOp(OBJ, BinOpKind.ADD, lo("t"), + lo("t"))]) + return "effect in pure", dataclasses.replace( + g, body=g.body[:3] + (ExprStmt(node),) + g.body[3:]), {FLAG_PURE} + cmp = CompareObj(BOOL, CompareKind.EQ, lo("o"), lo("o")) + return "compare_obj value", dataclasses.replace( + g, locals={**g.locals, "q": BOOL}, body=(Assign("q", cmp),) + g.body), {CONDITION_ONLY} + if kind == 12: # a scalar into an OBJ slot (unbox removed / box removed) + boxes = _paths(g.body, lambda n: type(n) in (Box, Unbox, ObjToFloat)) + if boxes: + bpath, bx = rng.choice(boxes) + return "drop box", _replace_body(g, bpath, bx.operand), {OWNERSHIP, TYPE} + return "drop box", _replace_body(g, path, Box(OBJ, e)) if e.type in (I64, F64, BOOL) \ + else _replace_body(g, path, Unbox(F64, e)), {OWNERSHIP, TYPE, STRUCTURE} + # kind 13: drop the first Assign (its local becomes possibly unassigned) — breaking because the + # local is read later, which holds for the first Assign of both base functions. + k = next(j for j, s in enumerate(g.body) if isinstance(s, Assign)) + return "drop assign", dataclasses.replace(g, body=g.body[:k] + g.body[k + 1:]), {DA} + + +def _preserving_mutations(rng, g): + """Rewrites that keep every property; the verifier must still accept them.""" + kind = rng.randrange(4) + if kind == 0: # wrap the body in `if True:` on both branches + return "if-true wrap", dataclasses.replace(g, body=(If(b(True), g.body, g.body),)) + if kind == 1: # an extra declared, assigned, unused local + return "dead local", dataclasses.replace( + g, locals={**g.locals, "zz": F64}, body=(Assign("zz", f(0.5)),) + g.body) + if kind == 2: # a dead statement after the return + return "dead tail", dataclasses.replace(g, body=g.body + (ExprStmt(f(1.0)),)) + # swap the operands of a Compare / CompareObj (types stay valid) + cmps = _paths(g.body, lambda n: type(n) in (Compare, CompareObj)) + path, c = rng.choice(cmps) + flipped = {CompareKind.LT: CompareKind.GT, CompareKind.GT: CompareKind.LT}.get(c.op, c.op) + return "flip compare", _replace_body(g, path, type(c)(BOOL, flipped, c.right, c.left)) + + +def test_base_functions_are_accepted(): + out = assert_accepted(*_base_functions()) + dot, scale = out.functions + assert [a.proven for a in _accesses(dot)] == [True, True, False] + assert [a.proven for a in _accesses(scale)] == [True, True] + + +@pytest.mark.parametrize("seed", [20261003]) +def test_no_breaking_mutation_slips_through(seed): + rng = random.Random(seed) + base = _base_functions() + missed, seen = [], set() + for n in range(200): + g = base[rng.randrange(2)] + desc, mutant, allowed = _breaking_mutations(rng, g) + seen.add(desc) + others = [h for h in base if h.name != g.name] + out, diags = run(*others, mutant) + got = rules_for(diags, g.name) + if g.name in kept(out) or not (got & allowed): + missed.append((n, desc, g.name, sorted(got))) + assert missed == [] + assert len(seen) >= 12 # the seed exercised most mutation kinds + + +@pytest.mark.parametrize("seed", [7]) +def test_preserving_rewrites_are_accepted(seed): + rng = random.Random(seed) + base = _base_functions() + rejected = [] + for n in range(100): + g = base[rng.randrange(2)] + desc, mutant = _preserving_mutations(rng, g) + out, diags = run(*[h for h in base if h.name != g.name], mutant) + if g.name not in kept(out): + rejected.append((n, desc, [dataclasses.astuple(d) for d in diags])) + assert rejected == [] From 9d8edcff47bc181baa639a41b5eb7375a78d0bef Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:08:49 +0900 Subject: [PATCH 15/36] Spec: Settle IR ambiguities found by the verifier CompareObj takes two OBJ operands and appears only directly as a condition; Unbox checks the exact type at run time and deopts, so it is pure-only; Global returns a new reference; call cycles are rejected (no RecursionError in C); ATAN2/HYPOT take two arguments; Function.locals excludes parameters; a value- returning function returns on every path; -2**63 % -1 handled by the runtime. Refs #41 --- .../src/main/python/typedpython/ir.py | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index 2caa21177..1e3651800 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -109,7 +109,8 @@ class BinOp(Expr): I64 (result I64, except TRUEDIV → F64): ADD/SUB/MUL — checked; overflow → deopt. - FLOORDIV/MOD — Python floor semantics (result sign follows the divisor for MOD); divisor 0 + FLOORDIV/MOD — Python floor semantics (result sign follows the divisor for MOD; the runtime + handles -2**63 % -1 == 0 without C's undefined behaviour); divisor 0 → ZeroDivisionError("integer division or modulo by zero") for FLOORDIV and ZeroDivisionError("integer modulo by zero") for MOD (CPython 3.13/3.14 messages; the runtime's differential tests pin them); -2**63 // -1 overflows → deopt. @@ -197,8 +198,9 @@ class Box(Expr): @dataclass(frozen=True) class Unbox(Expr): - """An OBJ known by the front end to be exactly int/float/bool converted to a scalar. For I64, - a value outside the i64 range → deopt (so only legal in a pure function or a guard).""" + """An OBJ converted to a scalar. The runtime checks the exact type (`tp_unbox_*`) and the i64 + range; anything else → deopt. Because it can deopt, it is legal only in a pure function (the + verifier enforces this); impure code converts objects with `ObjToFloat` instead.""" operand: Expr @@ -217,7 +219,8 @@ class MathFunc(Enum): @dataclass(frozen=True) class MathCall(Expr): - """`math.(x...)` on F64, type F64, with the stdlib `math` module's errors: + """`math.(x...)` on F64, type F64 (ATAN2 and HYPOT take exactly two arguments; other arities + stay interpreted), with the stdlib `math` module's errors: a domain error → ValueError("math domain error"), a range overflow → OverflowError("math range error") (CPython's math_1/math_2 rules on errno/inf/nan).""" @@ -229,7 +232,9 @@ class MathCall(Expr): class Call(Expr): """A direct call of another compiled function in the same module (C to C). If the callee may deopt, the caller must be pure (the deopt propagates and the outermost compiled call is - redone); the front end otherwise emits `PyCall`.""" + redone); otherwise the front end calls it as an object (`CallObject(Global(name), ...)`). + Recursion (any call cycle) is rejected by the verifier: compiled C has no recursion limit where + CPython raises RecursionError.""" function: str args: tuple[Expr, ...] @@ -237,7 +242,7 @@ class Call(Expr): @dataclass(frozen=True) class Global(Expr): - """Reading a module-level name (then builtins), looked up **at the time of the read** in the + """Reading a module-level name (then builtins) as a new reference, looked up **at the time of the read** in the module namespace, exactly as CPython's LOAD_GLOBAL: rebinding the global later is seen. Unbound → NameError("name 'x' is not defined"). Type OBJ. This is how compiled code reaches functions and classes imported from Kotlin packages through the binder (they are ordinary @@ -279,10 +284,11 @@ class Truth(Expr): @dataclass(frozen=True) class CompareObj(Expr): - """A comparison with an OBJ operand **in a condition** (If/While cond, And/Or operand): + """A comparison of two OBJ operands (a scalar side arrives through `Box`) **directly as a + condition**: an If/While cond, or an And/Or operand when that And/Or is itself the condition. PyObject_RichCompareBool semantics — the rich comparison runs, then its truth value. Type BOOL. - The front end must not use it where the comparison's own result object is kept (`x = a < b`), - because that object need not be a bool.""" + Anywhere the comparison's own result object would be kept (`x = a < b`, `r = c and a < b`, + `not (a < b)`) it is not allowed, because that object need not be a bool.""" op: CompareKind left: Expr @@ -439,13 +445,13 @@ class Function: param (`type(x) is float`, not isinstance — CPython would compute with the int or the subclass it was given), i64 range for I64, the ArrayParam checks. A failing guard → deopt (always safe: nothing has happened yet). Wrong arity or keyword use → deopt too, so CPython raises its own - TypeError. + TypeError. A function returning a value must return on every path (no falling off the end). """ name: str params: tuple[Param | ArrayParam, ...] returns: Type - locals: dict[str, Type] + locals: dict[str, Type] # non-parameter locals only (temporaries included) body: tuple[Stmt, ...] pure: bool may_deopt: bool From 7fd55475821a295fcc01b65614c854aa7284b411 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:11:08 +0900 Subject: [PATCH 16/36] Feat: Make compiled a builtin for plain CPython and the gate The typedpython wheel installs a one-line .pth that sets builtins.compiled to an identity decorator unless the runtime already set it (#42 sets it in the embedded interpreter). Pyrefly 1.3.2 reads __builtins__.pyi from a search path entry as extra builtins, so the gate adds typedpython/builtin_stubs and @compiled type-checks without an import while unknown names still fail. Red: 5 of 7 failing. Green: 7/7, suite 55. Mutation (stub dir off the search path): 2 gate tests fail. Refs #41, #42 --- docs/design/typedpython.md | 11 +- python-multiplatform-ksp/pyproject.toml | 4 + .../builtin_stubs/__builtins__.pyi | 3 + .../src/main/python/typedpython/pyrefly.py | 7 +- .../src/main/python/typedpython_builtins.pth | 1 + .../src/test/python/test_builtin_compiled.py | 100 ++++++++++++++++++ 6 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi create mode 100644 python-multiplatform-ksp/src/main/python/typedpython_builtins.pth create mode 100644 python-multiplatform-ksp/src/test/python/test_builtin_compiled.py diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 108f57523..d5412a614 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -55,7 +55,7 @@ TypedPython 의 목표는 **새 언어를 만들지 않고, 파이썬 문법 그 |---|---|---|---| | 0 | 서드파티 패키지 (의존성) | 없음 — 검사하지 않는다 | 바이트코드 (pypackpack 이 받은 그대로) | | 1 `checked` | **사용자 프로젝트의 파이썬 코드 전부 — 기본값** | 빌드 타임 타입 오류, `Any` 유출 0 (§4.2) | 바이트코드 | -| 1 `compiled` | `@compiled` 또는 모듈 선언 | `checked` + §3.1 금지 목록 | C 확장 모듈 | +| 1 `compiled` | `@compiled` (내장 이름, import 없음) 또는 모듈 선언 `# typedpython: compiled` | `checked` + §3.1 금지 목록 | C 확장 모듈 | | 2 `kernel` | `@kernel` | 객체·힙 할당 없음, 고정 dtype | SPIR-V / MSL / WGSL / PTX | - **사용자 코드는 표시하지 않아도 검사된다.** 타입 오류가 있으면 Kotlin 컴파일 에러처럼 빌드가 @@ -245,7 +245,14 @@ upstream 대비 수치 오차를 기준선으로 박아 둔다. ## 8. 미결 사항 (사용자 결정) -1. `compiled` 의 표기 — 모듈 선언 방식, 데코레이터 이름. +1. ~~`compiled` 의 표기~~ — **결정됨 (2026-10-03, 메인테이너):** `compiled` 는 **내장 이름**이다. 패키지 + 아래에 두지 않고(`@typedpython.compiled` 아님) import 없이 `@compiled` 로 쓴다. 런타임은 임베디드 + 인터프리터 시작 시 `builtins.compiled` 를 설치하고(#42), 일반 CPython 에서는 `typedpython` wheel 의 + `typedpython_builtins.pth` 가 `builtins` 에 없을 때만 항등 함수를 설치한다(런타임이 넣은 것을 덮어쓰지 + 않는다). 타입 검사기에는 `__builtins__.pyi` 로 알린다 — Pyrefly 1.3.2 는 검색 경로의 그 파일을 추가 + 내장으로 읽는다(실측). 모듈 단위 표기는 `# typedpython: compiled` 주석을 유지한다: `# pyright: strict` 와 같은 + 도구 pragma 형태이고, 실행 시 이름 조회가 필요 없어 일반 CPython 에서 부작용이 없으며, 데코레이터 없이 + 모듈 전체를 지정할 수 있다. 2. 고정폭 정수의 오버플로 의미. 3. pypackpack Cython 백엔드 슬롯의 인터페이스를 누가 정하는가. 4. GPU 착지를 torchnative 커스텀 op 로 할 것인가 (다른 레포 변경). diff --git a/python-multiplatform-ksp/pyproject.toml b/python-multiplatform-ksp/pyproject.toml index 62f6993b2..460631868 100644 --- a/python-multiplatform-ksp/pyproject.toml +++ b/python-multiplatform-ksp/pyproject.toml @@ -19,6 +19,10 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["src/main/python/typedpython"] +# `compiled` is a builtin name (design §2.3): this .pth installs it into `builtins` on plain CPython. +[tool.hatch.build.targets.wheel.force-include] +"src/main/python/typedpython_builtins.pth" = "typedpython_builtins.pth" + [project.optional-dependencies] test = ["pytest"] diff --git a/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi b/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi new file mode 100644 index 000000000..f773368b9 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi @@ -0,0 +1,3 @@ +# TypedPython's builtin names (docs/design/typedpython.md §2.3). A `__builtins__.pyi` on the type +# checker's search path declares names the interpreter provides without an import. +def compiled[T](f: T, /) -> T: ... diff --git a/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py b/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py index b077ae942..016c7b6bb 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py @@ -63,6 +63,11 @@ class Report: expression_types: dict[str, dict[tuple[int, int], str]] +# Directory holding `__builtins__.pyi`: names the interpreter provides without an import (`compiled`). +# Pyrefly 1.3.2 reads that file from any search-path entry as extra builtins. +BUILTIN_STUBS = Path(__file__).parent / "builtin_stubs" + + def command() -> list[str]: """The Pyrefly command line: `TYPEDPYTHON_PYREFLY`, or the copy installed next to this Python.""" override = os.environ.get("TYPEDPYTHON_PYREFLY") @@ -73,7 +78,7 @@ def run(paths: Sequence[Path], search_paths: Sequence[Path] = ()) -> Report: files = [str(Path(p).resolve()) for p in paths] with tempfile.TemporaryDirectory(prefix="typedpython-") as work: config = Path(work) / "pyrefly.toml" - config.write_text(_config([*search_paths, *import_roots(files)])) + config.write_text(_config([*search_paths, *import_roots(files), BUILTIN_STUBS])) report_dir = Path(work) / "pysa" argv = [ *command(), "check", *files, diff --git a/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth b/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth new file mode 100644 index 000000000..98776970a --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth @@ -0,0 +1 @@ +import builtins; builtins.compiled = getattr(builtins, "compiled", lambda f, /: f) diff --git a/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py b/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py new file mode 100644 index 000000000..71ed65011 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py @@ -0,0 +1,100 @@ +"""`compiled` is a builtin name (design §2.3): no import, on plain CPython and for the checker.""" +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path + +import pytest + +from conftest import errors, rules +from typedpython import gate + +PROJECT = Path(__file__).resolve().parents[3] +TMP = PROJECT / ".tmp" +PTH = "typedpython_builtins.pth" + + +def test_gate_accepts_compiled_without_an_import(write): + path = write("m.py", """ + @compiled + def f(x: int) -> int: + return x + 1 + + y: int = f(1) + """) + assert errors(gate.check([path])) == [] + + +def test_gate_still_rejects_an_unknown_builtin(write): + path = write("m.py", """ + @not_a_builtin + def f(x: int) -> int: + return x + """) + found = errors(gate.check([path])) + assert "pyrefly/unknown-name" in rules(found) + assert any("not_a_builtin" in d.message for d in found) + + +def test_compiled_keeps_the_decorated_type(write): + path = write("m.py", """ + @compiled + def f(x: int) -> int: + return x + + s: str = f(1) + """) + assert "pyrefly/bad-assignment" in rules(errors(gate.check([path]))) + + +@pytest.fixture(scope="module") +def installed(): + """The wheel, built and installed into a fresh venv; returns (python, wheel path).""" + root = TMP / "builtin-venv" + wheels = TMP / "builtin-wheel" + for d in (root, wheels): + shutil.rmtree(d, ignore_errors=True) + subprocess.run(["uv", "build", "--wheel", "--out-dir", str(wheels), str(PROJECT)], + check=True, capture_output=True) + wheel = next(wheels.glob("typedpython-*.whl")) + subprocess.run(["uv", "venv", "--python", "3.13", str(root)], check=True, capture_output=True) + python = root / "bin" / "python" + subprocess.run(["uv", "pip", "install", "--python", str(python), "--no-deps", str(wheel)], + check=True, capture_output=True) + return python, wheel + + +def py(python, *args): + return subprocess.run([str(python), *args], capture_output=True, text=True) + + +def test_wheel_puts_the_pth_at_its_root(installed): + _, wheel = installed + assert PTH in zipfile.ZipFile(wheel).namelist() + + +def test_decorator_works_with_no_import(installed): + python, _ = installed + r = py(python, "-c", "@compiled\ndef f(): return 1\nprint(f())") + assert (r.returncode, r.stdout.strip()) == (0, "1"), r.stderr + + +def test_compiled_is_the_identity(installed): + python, _ = installed + r = py(python, "-c", "def f(): pass\nprint(compiled(f) is f)") + assert r.stdout.strip() == "True", r.stderr + + +def test_the_pth_does_not_overwrite_the_runtime(installed): + python, _ = installed + site = next((python.parent.parent / "lib").glob("python3*/site-packages")) + code = ( + "import builtins, site\n" + "sentinel = object()\n" + "builtins.compiled = sentinel\n" + f"site.addsitedir({str(site)!r})\n" + "print(builtins.compiled is sentinel)\n" + ) + r = py(python, "-S", "-c", code) + assert r.stdout.strip() == "True", r.stderr From 191732402fd2544d5932d6d9e7f8cde6373c93ad Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:18:06 +0900 Subject: [PATCH 17/36] Feat: TypedPython C runtime (tp_runtime.h) with differential tests against CPython Header-only helpers implementing ir.py's CPython semantics: checked i64 arithmetic, Python floor division/modulo (MIN % -1 handled), exact int/float comparison, CPython's float div/floordiv/mod, math_1/math_2 error rules, exact-type boxing/unboxing, native arrays with dirty write-back, and the opaque-object helpers (global/getattr/vectorcall/truth/compare/binop/release). Exception messages follow the running interpreter (3.13 vs 3.14 differ). Green: 140/140 (default and portable-overflow builds), also under ASan+UBSan via a sanitized launcher. Mutation (floor-mod sign fix removed): 4 fail. Refs #41 --- .../python/typedpython/runtime/tp_runtime.h | 869 ++++++++++++++++ .../src/test/python/runtime_shim/tp_shim.c | 443 ++++++++ .../src/test/python/test_runtime.py | 978 ++++++++++++++++++ 3 files changed, 2290 insertions(+) create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h create mode 100644 python-multiplatform-ksp/src/test/python/runtime_shim/tp_shim.c create mode 100644 python-multiplatform-ksp/src/test/python/test_runtime.py diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h new file mode 100644 index 000000000..437462878 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -0,0 +1,869 @@ +/* tp_runtime.h -- the C runtime API generated TypedPython code may call (SPEC N-8, API.md). + * + * Header-only, `static inline`, C99 + the CPython C API (3.12+; exception messages follow CPython + * 3.13 and 3.14, which differ -- see TP_MSG_* below). Every helper implements the CPython + * behaviour named in ir.py; the differential tests in src/test/python/test_runtime.py compare each + * helper with CPython itself, bit for bit and message for message. + * + * Return convention unless stated: 0 ok, 1 deopt (no Python error set, nothing to undo), + * -1 Python error set. + */ +#ifndef TP_RUNTIME_H +#define TP_RUNTIME_H + +#include +#include +#include +#include +#include +#include + +#if PY_VERSION_HEX < 0x030C0000 +#error "tp_runtime.h needs CPython 3.12 or newer (PyErr_GetRaisedException)" +#endif + +/* ------------------------------------------------------------------------------------------ + * Exception messages that CPython changed between 3.13 and 3.14. + * 3.13: "integer division or modulo by zero" (int //), "integer modulo by zero" (int %), + * "float division by zero" / "float floor division by zero" / "float modulo by zero", + * "math domain error" for every math-domain ValueError. + * 3.14: "division by zero" for all of them; math_1 domain errors name the argument + * ("expected a nonnegative input, got -1.0", "expected a finite input, got inf", + * "expected a positive input, got 0.0"). + * ------------------------------------------------------------------------------------------ */ +#if PY_VERSION_HEX >= 0x030E0000 +#define TP_MSG_INT_FLOORDIV_ZERO "division by zero" +#define TP_MSG_INT_MOD_ZERO "division by zero" +#define TP_MSG_FLOAT_DIV_ZERO "division by zero" +#define TP_MSG_FLOAT_FLOORDIV_ZERO "division by zero" +#define TP_MSG_FLOAT_MOD_ZERO "division by zero" +#else +#define TP_MSG_INT_FLOORDIV_ZERO "integer division or modulo by zero" +#define TP_MSG_INT_MOD_ZERO "integer modulo by zero" +#define TP_MSG_FLOAT_DIV_ZERO "float division by zero" +#define TP_MSG_FLOAT_FLOORDIV_ZERO "float floor division by zero" +#define TP_MSG_FLOAT_MOD_ZERO "float modulo by zero" +#endif +#define TP_MSG_INT_TRUEDIV_ZERO "division by zero" /* long_true_divide: same in 3.13 and 3.14 */ + +#define TP_MATH_MSG_NONNEG "expected a nonnegative input, got %s" +#define TP_MATH_MSG_FINITE "expected a finite input, got %s" +#define TP_MATH_MSG_POSITIVE "expected a positive input, got %s" + +/* ir.BinOpKind order, for tp_binop_obj */ +#define TP_BINOP_ADD 0 +#define TP_BINOP_SUB 1 +#define TP_BINOP_MUL 2 +#define TP_BINOP_TRUEDIV 3 +#define TP_BINOP_FLOORDIV 4 +#define TP_BINOP_MOD 5 + +/* ------------------------------------------------------------------------------------------ + * Integers (int64_t) + * ------------------------------------------------------------------------------------------ */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(TP_RUNTIME_FORCE_PORTABLE_OVERFLOW) +#define TP_RUNTIME_PORTABLE_OVERFLOW 0 +#else +#define TP_RUNTIME_PORTABLE_OVERFLOW 1 +#endif + +static inline int tp_add_i64(int64_t a, int64_t b, int64_t *out) +{ +#if !TP_RUNTIME_PORTABLE_OVERFLOW + return __builtin_add_overflow(a, b, out) ? 1 : 0; +#else + if ((b > 0 && a > INT64_MAX - b) || (b < 0 && a < INT64_MIN - b)) { + return 1; + } + *out = a + b; + return 0; +#endif +} + +static inline int tp_sub_i64(int64_t a, int64_t b, int64_t *out) +{ +#if !TP_RUNTIME_PORTABLE_OVERFLOW + return __builtin_sub_overflow(a, b, out) ? 1 : 0; +#else + if ((b < 0 && a > INT64_MAX + b) || (b > 0 && a < INT64_MIN + b)) { + return 1; + } + *out = a - b; + return 0; +#endif +} + +static inline int tp_mul_i64(int64_t a, int64_t b, int64_t *out) +{ +#if !TP_RUNTIME_PORTABLE_OVERFLOW + return __builtin_mul_overflow(a, b, out) ? 1 : 0; +#else + if (a == 0 || b == 0) { + *out = 0; + return 0; + } + if (a == -1) { + if (b == INT64_MIN) return 1; + *out = -b; + return 0; + } + if (b == -1) { + if (a == INT64_MIN) return 1; + *out = -a; + return 0; + } + if (a > 0) { + if (b > 0 ? a > INT64_MAX / b : b < INT64_MIN / a) return 1; + } + else { + if (b > 0 ? a < INT64_MIN / b : a < INT64_MAX / b) return 1; + } + *out = a * b; + return 0; +#endif +} + +static inline int tp_neg_i64(int64_t a, int64_t *out) +{ + if (a == INT64_MIN) { + return 1; + } + *out = -a; + return 0; +} + +/* Python floor division: the quotient rounds toward negative infinity. */ +static inline int tp_floordiv_i64(int64_t a, int64_t b, int64_t *out) +{ + if (b == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, TP_MSG_INT_FLOORDIV_ZERO); + return -1; + } + if (b == -1) { + if (a == INT64_MIN) { + return 1; /* 2**63 does not fit */ + } + *out = -a; + return 0; + } + int64_t q = a / b; + int64_t r = a % b; + if (r != 0 && ((r < 0) != (b < 0))) { + q -= 1; + } + *out = q; + return 0; +} + +/* Python modulo: the result has the sign of the divisor. */ +static inline int tp_mod_i64(int64_t a, int64_t b, int64_t *out) +{ + if (b == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, TP_MSG_INT_MOD_ZERO); + return -1; + } + if (b == -1) { + *out = 0; /* also INT64_MIN % -1, which traps in C */ + return 0; + } + int64_t r = a % b; + if (r != 0 && ((r < 0) != (b < 0))) { + r += b; + } + *out = r; + return 0; +} + +/* CPython rounds int/int correctly; the C double division is the correctly rounded quotient when + * both operands are exactly representable, i.e. |x| <= 2**53. Otherwise: deopt. A zero divisor is + * an error whatever the dividend. */ +static inline int tp_truediv_i64(int64_t a, int64_t b, double *out) +{ + const int64_t lim = (int64_t)1 << 53; + if (b == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, TP_MSG_INT_TRUEDIV_ZERO); + return -1; + } + if (a > lim || a < -lim || b > lim || b < -lim) { + return 1; + } + *out = (double)a / (double)b; + return 0; +} + +/* float(int): the C conversion rounds to nearest, ties to even in the default rounding mode. */ +static inline double tp_i64_to_f64(int64_t a) +{ + return (double)a; +} + +/* ------------------------------------------------------------------------------------------ + * Floats (double) + * ------------------------------------------------------------------------------------------ */ +static inline int tp_truediv_f64(double a, double b, double *out) +{ + if (b == 0.0) { + PyErr_SetString(PyExc_ZeroDivisionError, TP_MSG_FLOAT_DIV_ZERO); + return -1; + } + *out = a / b; + return 0; +} + +/* floatobject.c: _float_div_mod */ +static inline void tp__float_div_mod(double vx, double wx, double *floordiv, double *mod) +{ + double div; + *mod = fmod(vx, wx); + div = (vx - *mod) / wx; + if (*mod) { + if ((wx < 0) != (*mod < 0)) { + *mod += wx; + div -= 1.0; + } + } + else { + *mod = copysign(0.0, wx); + } + if (div) { + *floordiv = floor(div); + if (div - *floordiv > 0.5) { + *floordiv += 1.0; + } + } + else { + *floordiv = copysign(0.0, vx / wx); + } +} + +static inline int tp_floordiv_f64(double a, double b, double *out) +{ + double mod, floordiv; + if (b == 0.0) { + PyErr_SetString(PyExc_ZeroDivisionError, TP_MSG_FLOAT_FLOORDIV_ZERO); + return -1; + } + tp__float_div_mod(a, b, &floordiv, &mod); + *out = floordiv; + return 0; +} + +/* floatobject.c: float_rem */ +static inline int tp_mod_f64(double a, double b, double *out) +{ + double mod; + if (b == 0.0) { + PyErr_SetString(PyExc_ZeroDivisionError, TP_MSG_FLOAT_MOD_ZERO); + return -1; + } + mod = fmod(a, b); + if (mod) { + if ((b < 0) != (mod < 0)) { + mod += b; + } + } + else { + mod = copysign(0.0, b); + } + *out = mod; + return 0; +} + +/* ------------------------------------------------------------------------------------------ + * Comparisons: exact int/float comparison (float_richcompare), never via (double)int. + * ------------------------------------------------------------------------------------------ */ +static inline int tp__apply_cmp(int c, int op) +{ + switch (op) { + case Py_LT: return c < 0; + case Py_LE: return c <= 0; + case Py_EQ: return c == 0; + case Py_NE: return c != 0; + case Py_GT: return c > 0; + case Py_GE: return c >= 0; + } + return 0; +} + +/* three-way compare of an int64 with a non-NaN double: -1, 0, 1 */ +static inline int tp__cmp3_i64_f64(int64_t a, double b) +{ + if (b >= 9223372036854775808.0) return -1; /* b >= 2**63 > any int64 (also +inf) */ + if (b < -9223372036854775808.0) return 1; /* b < -2**63 <= any int64 (also -inf) */ + int64_t bi = (int64_t)b; /* truncation toward zero, b in range */ + if (a < bi) return -1; + if (a > bi) return 1; + double frac = b - (double)bi; /* exact: bi is b with the fraction dropped */ + if (frac > 0.0) return -1; /* b = a + frac */ + if (frac < 0.0) return 1; + return 0; +} + +static inline int tp_cmp_i64_f64(int64_t a, double b, int op) +{ + if (b != b) { + return op == Py_NE; + } + return tp__apply_cmp(tp__cmp3_i64_f64(a, b), op); +} + +static inline int tp_cmp_f64_i64(double a, int64_t b, int op) +{ + if (a != a) { + return op == Py_NE; + } + return tp__apply_cmp(-tp__cmp3_i64_f64(b, a), op); +} + +/* ------------------------------------------------------------------------------------------ + * math (mathmodule.c: math_1 / math_2 / is_error rules) + * ------------------------------------------------------------------------------------------ */ +static inline void tp__math_domain_error(double x, const char *msg314) +{ +#if PY_VERSION_HEX >= 0x030E0000 + if (msg314) { + char *buf = PyOS_double_to_string(x, 'r', 0, Py_DTSF_ADD_DOT_0, NULL); + if (buf) { + PyErr_Format(PyExc_ValueError, msg314, buf); + PyMem_Free(buf); + } + return; + } +#else + (void)x; + (void)msg314; +#endif + PyErr_SetString(PyExc_ValueError, "math domain error"); +} + +/* mathmodule.c is_error: 1 if an exception was set, 0 if the errno is to be ignored */ +static inline int tp__math_is_error(double x) +{ + int result = 1; + if (errno == EDOM) { + PyErr_SetString(PyExc_ValueError, "math domain error"); + } + else if (errno == ERANGE) { + if (fabs(x) < 1.5) { + result = 0; + } + else { + PyErr_SetString(PyExc_OverflowError, "math range error"); + } + } + else { + PyErr_SetFromErrno(PyExc_ValueError); + } + return result; +} + +/* math_1: x is the argument, r the libm result computed with errno cleared just before */ +static inline int tp__math_1_finish(double x, double r, int can_overflow, const char *msg314, + double *out) +{ + if (isnan(r) && !isnan(x)) { + tp__math_domain_error(x, msg314); + return -1; + } + if (isinf(r) && isfinite(x)) { + if (can_overflow) { + PyErr_SetString(PyExc_OverflowError, "math range error"); + } + else { + tp__math_domain_error(x, msg314); + } + return -1; + } + if (isfinite(r) && errno && tp__math_is_error(r)) { + return -1; + } + *out = r; + return 0; +} + +/* mathmodule.c m_log */ +static inline double tp__m_log(double x) +{ + if (isfinite(x)) { + if (x > 0.0) return log(x); + errno = EDOM; + if (x == 0.0) return -HUGE_VAL; + return NAN; + } + else if (isnan(x)) { + return x; + } + else if (x > 0.0) { + return x; + } + errno = EDOM; + return NAN; +} + +#define TP__DEF_MATH1(NAME, CALL, CAN_OVERFLOW, MSG) \ + static inline int tp_math_##NAME(double x, double *out) \ + { \ + double r; \ + errno = 0; \ + r = CALL; \ + return tp__math_1_finish(x, r, CAN_OVERFLOW, MSG, out); \ + } +TP__DEF_MATH1(sqrt, sqrt(x), 0, TP_MATH_MSG_NONNEG) +TP__DEF_MATH1(exp, exp(x), 1, NULL) +TP__DEF_MATH1(log, tp__m_log(x), 0, TP_MATH_MSG_POSITIVE) +TP__DEF_MATH1(sin, sin(x), 0, TP_MATH_MSG_FINITE) +TP__DEF_MATH1(cos, cos(x), 0, TP_MATH_MSG_FINITE) +TP__DEF_MATH1(tan, tan(x), 0, TP_MATH_MSG_FINITE) +TP__DEF_MATH1(fabs, fabs(x), 0, NULL) +#undef TP__DEF_MATH1 + +/* mathmodule.c m_atan2: the special values are defined here, the rest is libm */ +static inline double tp__m_atan2(double y, double x) +{ + const double pi = 3.14159265358979323846; + if (isnan(x) || isnan(y)) return NAN; + if (isinf(y)) { + if (isinf(x)) { + if (copysign(1., x) == 1.) return copysign(0.25 * pi, y); + return copysign(0.75 * pi, y); + } + return copysign(0.5 * pi, y); + } + if (isinf(x) || y == 0.) { + if (copysign(1., x) == 1.) return copysign(0., y); + return copysign(pi, y); + } + return atan2(y, x); +} + +/* math_2 rules for atan2 */ +static inline int tp_math_atan2(double y, double x, double *out) +{ + double r; + errno = 0; + r = tp__m_atan2(y, x); + if (isnan(r)) { + errno = (!isnan(x) && !isnan(y)) ? EDOM : 0; + } + else if (isinf(r)) { + errno = (isfinite(x) && isfinite(y)) ? ERANGE : 0; + } + if (errno && tp__math_is_error(r)) { + return -1; + } + *out = r; + return 0; +} + +#define TP__DBL_MIN 2.2250738585072014e-308 /* DBL_MIN, without */ + +/* mathmodule.c vector_norm (3.13 and 3.14 are identical): the correctly rounded hypot */ +typedef struct { double hi; double lo; } tp__dl; + +static inline tp__dl tp__dl_fast_sum(double a, double b) +{ + double x = a + b; + double y = (a - x) + b; + tp__dl r = {x, y}; + return r; +} + +static inline tp__dl tp__dl_mul(double x, double y) +{ + double z = x * y; + double zz = fma(x, y, -z); + tp__dl r = {z, zz}; + return r; +} + +static inline double tp__vector_norm(Py_ssize_t n, double *vec, double max, int found_nan) +{ + double x, h, scale, csum = 1.0, frac1 = 0.0, frac2 = 0.0; + tp__dl pr, sm; + int max_e; + Py_ssize_t i; + + if (isinf(max)) return max; + if (found_nan) return NAN; + if (max == 0.0 || n <= 1) return max; + frexp(max, &max_e); + if (max_e < -1023) { + for (i = 0; i < n; i++) { + vec[i] /= TP__DBL_MIN; + } + return TP__DBL_MIN * tp__vector_norm(n, vec, max / TP__DBL_MIN, found_nan); + } + scale = ldexp(1.0, -max_e); + for (i = 0; i < n; i++) { + x = vec[i]; + x *= scale; + pr = tp__dl_mul(x, x); + sm = tp__dl_fast_sum(csum, pr.hi); + csum = sm.hi; + frac1 += pr.lo; + frac2 += sm.lo; + } + h = sqrt(csum - 1.0 + (frac1 + frac2)); + pr = tp__dl_mul(-h, h); + sm = tp__dl_fast_sum(csum, pr.hi); + csum = sm.hi; + frac1 += pr.lo; + frac2 += sm.lo; + x = csum - 1.0 + (frac1 + frac2); + h += x / (2.0 * h); + return h / scale; +} + +static inline int tp_math_hypot(double x, double y, double *out) +{ + double vec[2]; + double max = 0.0; + int found_nan; + vec[0] = fabs(x); + vec[1] = fabs(y); + found_nan = isnan(vec[0]) | isnan(vec[1]); + if (vec[0] > max) max = vec[0]; + if (vec[1] > max) max = vec[1]; + *out = tp__vector_norm(2, vec, max, found_nan); + return 0; +} + +/* ------------------------------------------------------------------------------------------ + * Boxing + * ------------------------------------------------------------------------------------------ */ +static inline PyObject *tp_box_i64(int64_t v) { return PyLong_FromLongLong((long long)v); } +static inline PyObject *tp_box_f64(double v) { return PyFloat_FromDouble(v); } +static inline PyObject *tp_box_bool(int v) { return PyBool_FromLong(v ? 1 : 0); } + +static inline int tp_unbox_i64(PyObject *o, int64_t *out) +{ + int overflow = 0; + long long v; + if (!PyLong_CheckExact(o)) { + return 1; + } + v = PyLong_AsLongLongAndOverflow(o, &overflow); + if (overflow) { + return 1; + } + if (v == -1 && PyErr_Occurred()) { + PyErr_Clear(); /* cannot happen for an exact int; deopt rather than leak an error */ + return 1; + } + *out = (int64_t)v; + return 0; +} + +static inline int tp_unbox_f64(PyObject *o, double *out) +{ + if (!PyFloat_CheckExact(o)) { + return 1; + } + *out = PyFloat_AS_DOUBLE(o); + return 0; +} + +static inline int tp_unbox_bool(PyObject *o, int *out) +{ + if (o == Py_True) { + *out = 1; + return 0; + } + if (o == Py_False) { + *out = 0; + return 0; + } + return 1; +} + +/* ------------------------------------------------------------------------------------------ + * Native arrays for list[float] / list[int] parameters (ir.ArrayParam) + * + * enter: rc 0 -> the struct holds the copy; rc 1 or -1 -> the struct is zeroed (so exit is a safe + * no-op) and nothing was changed. `list` is borrowed: the caller keeps the argument alive. + * exit: writes the dirty slots back as new objects (PyList_SetItem steals the new reference and, on + * failure, drops it), always frees, and zeroes the struct. On a write-back failure it stops, + * frees, and returns -1 with that error set (MemoryError, or IndexError if the list shrank). + * ------------------------------------------------------------------------------------------ */ +typedef struct { PyObject *list; Py_ssize_t len; double *data; unsigned char *dirty; } tp_f64_array; +typedef struct { PyObject *list; Py_ssize_t len; int64_t *data; unsigned char *dirty; } tp_i64_array; + +static inline int tp_f64_array_enter(PyObject *obj, tp_f64_array *a) +{ + Py_ssize_t n, i; + a->list = NULL; + a->len = 0; + a->data = NULL; + a->dirty = NULL; + if (!PyList_CheckExact(obj)) { + return 1; + } + n = PyList_GET_SIZE(obj); + for (i = 0; i < n; i++) { + if (!PyFloat_CheckExact(PyList_GET_ITEM(obj, i))) { + return 1; + } + } + a->data = (double *)malloc(((size_t)n ? (size_t)n : 1) * sizeof(double)); + a->dirty = (unsigned char *)calloc((size_t)n ? (size_t)n : 1, 1); + if (a->data == NULL || a->dirty == NULL) { + free(a->data); + free(a->dirty); + a->data = NULL; + a->dirty = NULL; + PyErr_NoMemory(); + return -1; + } + for (i = 0; i < n; i++) { + a->data[i] = PyFloat_AS_DOUBLE(PyList_GET_ITEM(obj, i)); + } + a->list = obj; + a->len = n; + return 0; +} + +static inline int tp_i64_array_enter(PyObject *obj, tp_i64_array *a) +{ + Py_ssize_t n, i; + a->list = NULL; + a->len = 0; + a->data = NULL; + a->dirty = NULL; + if (!PyList_CheckExact(obj)) { + return 1; + } + n = PyList_GET_SIZE(obj); + a->data = (int64_t *)malloc(((size_t)n ? (size_t)n : 1) * sizeof(int64_t)); + a->dirty = (unsigned char *)calloc((size_t)n ? (size_t)n : 1, 1); + if (a->data == NULL || a->dirty == NULL) { + free(a->data); + free(a->dirty); + a->data = NULL; + a->dirty = NULL; + PyErr_NoMemory(); + return -1; + } + for (i = 0; i < n; i++) { + if (tp_unbox_i64(PyList_GET_ITEM(obj, i), &a->data[i]) != 0) { + free(a->data); + free(a->dirty); + a->data = NULL; + a->dirty = NULL; + return 1; + } + } + a->list = obj; + a->len = n; + return 0; +} + +/* list indexing: a negative index counts from the end; out of range -> IndexError */ +static inline int tp__array_slot(Py_ssize_t len, int64_t i, int store, Py_ssize_t *slot) +{ + if (i < 0) { + if (i < -(int64_t)len) goto out_of_range; + i += (int64_t)len; + } + else if (i >= (int64_t)len) { + goto out_of_range; + } + *slot = (Py_ssize_t)i; + return 0; +out_of_range: + PyErr_SetString(PyExc_IndexError, + store ? "list assignment index out of range" : "list index out of range"); + return -1; +} + +static inline int tp_f64_array_slot(const tp_f64_array *a, int64_t i, int store, Py_ssize_t *slot) +{ + return tp__array_slot(a->len, i, store, slot); +} + +static inline int tp_i64_array_slot(const tp_i64_array *a, int64_t i, int store, Py_ssize_t *slot) +{ + return tp__array_slot(a->len, i, store, slot); +} + +static inline int tp_f64_array_exit(tp_f64_array *a) +{ + int rc = 0; + if (a->data != NULL && a->dirty != NULL && a->list != NULL) { + Py_ssize_t i; + for (i = 0; i < a->len; i++) { + PyObject *v; + if (!a->dirty[i]) { + continue; + } + v = PyFloat_FromDouble(a->data[i]); + if (v == NULL || PyList_SetItem(a->list, i, v) < 0) { + rc = -1; /* SetItem consumed v on failure; the error is set */ + break; + } + } + } + free(a->data); + free(a->dirty); + a->list = NULL; + a->len = 0; + a->data = NULL; + a->dirty = NULL; + return rc; +} + +static inline int tp_i64_array_exit(tp_i64_array *a) +{ + int rc = 0; + if (a->data != NULL && a->dirty != NULL && a->list != NULL) { + Py_ssize_t i; + for (i = 0; i < a->len; i++) { + PyObject *v; + if (!a->dirty[i]) { + continue; + } + v = PyLong_FromLongLong((long long)a->data[i]); + if (v == NULL || PyList_SetItem(a->list, i, v) < 0) { + rc = -1; + break; + } + } + } + free(a->data); + free(a->dirty); + a->list = NULL; + a->len = 0; + a->data = NULL; + a->dirty = NULL; + return rc; +} + +/* aliasing guard: 1 if any two of the n objects are the same object */ +static inline int tp_any_same(PyObject *const *objs, Py_ssize_t n) +{ + Py_ssize_t i, j; + for (i = 0; i < n; i++) { + for (j = i + 1; j < n; j++) { + if (objs[i] == objs[j]) { + return 1; + } + } + } + return 0; +} + +/* ------------------------------------------------------------------------------------------ + * Opaque objects (Kotlin interop and any Python object). All return new references or NULL with + * CPython's exception set; none deopts. + * ------------------------------------------------------------------------------------------ */ + +/* LOAD_GLOBAL: module globals, then builtins, looked up now; NameError("name 'x' is not defined") + * with .name set, as ceval's format_exc_check_arg does. `module_dict` is the module's __dict__; + * its "__builtins__" (a module or a dict) is searched next, the interpreter's builtins if absent. */ +static inline PyObject *tp_global(PyObject *module_dict, PyObject *name) +{ + PyObject *v, *builtins; + v = PyDict_GetItemWithError(module_dict, name); + if (v != NULL) { + Py_INCREF(v); + return v; + } + if (PyErr_Occurred()) { + return NULL; + } + builtins = PyDict_GetItemString(module_dict, "__builtins__"); /* borrowed, errors ignored */ + if (builtins != NULL && PyModule_Check(builtins)) { + builtins = PyModule_GetDict(builtins); + } + if (builtins == NULL || !PyDict_Check(builtins)) { + builtins = PyEval_GetBuiltins(); + } + if (builtins != NULL) { + v = PyDict_GetItemWithError(builtins, name); + if (v != NULL) { + Py_INCREF(v); + return v; + } + if (PyErr_Occurred()) { + return NULL; + } + } + { + PyObject *exc; + PyErr_Format(PyExc_NameError, "name '%.200U' is not defined", name); + exc = PyErr_GetRaisedException(); + if (exc != NULL) { + if (PyObject_SetAttrString(exc, "name", name) < 0) { + PyErr_Clear(); + } + PyErr_SetRaisedException(exc); + } + } + return NULL; +} + +static inline PyObject *tp_getattr(PyObject *obj, PyObject *name) +{ + return PyObject_GetAttr(obj, name); +} + +/* vectorcall: the last len(kwnames) of the nargs values are keyword values */ +static inline PyObject *tp_call(PyObject *callee, PyObject *const *args, size_t nargs, + PyObject *kwnames) +{ + return PyObject_Vectorcall(callee, args, nargs, kwnames); +} + +static inline int tp_truth(PyObject *obj) +{ + return PyObject_IsTrue(obj); +} + +/* A comparison in a condition, as CPython's `if a < b:` runs it: the rich comparison, then the truth + * value of its result. NOT PyObject_RichCompareBool: that one short-circuits `a is b` for == and != + * (so a NaN would equal itself), which `if x == x:` does not. */ +static inline int tp_compare_bool(PyObject *a, PyObject *b, int op) +{ + int r; + PyObject *res = PyObject_RichCompare(a, b, op); + if (res == NULL) { + return -1; + } + r = PyObject_IsTrue(res); + Py_DECREF(res); + return r; +} + +/* float(obj): PyNumber_Float, then the double of the result */ +static inline int tp_obj_to_f64(PyObject *obj, double *out) +{ + PyObject *f = PyNumber_Float(obj); + if (f == NULL) { + return -1; + } + *out = PyFloat_AS_DOUBLE(f); + Py_DECREF(f); + return 0; +} + +/* op is an ir.BinOpKind: TP_BINOP_ADD .. TP_BINOP_MOD (declaration order) */ +static inline PyObject *tp_binop_obj(PyObject *a, PyObject *b, int op) +{ + switch (op) { + case TP_BINOP_ADD: return PyNumber_Add(a, b); + case TP_BINOP_SUB: return PyNumber_Subtract(a, b); + case TP_BINOP_MUL: return PyNumber_Multiply(a, b); + case TP_BINOP_TRUEDIV: return PyNumber_TrueDivide(a, b); + case TP_BINOP_FLOORDIV: return PyNumber_FloorDivide(a, b); + case TP_BINOP_MOD: return PyNumber_Remainder(a, b); + } + PyErr_SetString(PyExc_SystemError, "tp_binop_obj: unknown operator"); + return NULL; +} + +/* Py_CLEAR: the only way generated code drops a reference */ +static inline void tp_release(PyObject **slot) +{ + Py_CLEAR(*slot); +} + +#endif /* TP_RUNTIME_H */ diff --git a/python-multiplatform-ksp/src/test/python/runtime_shim/tp_shim.c b/python-multiplatform-ksp/src/test/python/runtime_shim/tp_shim.c new file mode 100644 index 000000000..84d053365 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/runtime_shim/tp_shim.c @@ -0,0 +1,443 @@ +/* Test extension: exposes every tp_runtime.h helper to Python so test_runtime.py can compare it + * with CPython. Every function returns (rc, value); rc == -1 raises the Python error the helper + * set. A helper that breaks the return convention (rc -1 with no error set, rc 0/1 with an error + * set) raises AssertionError, so convention violations cannot hide. */ +#define PY_SSIZE_T_CLEAN +#include +#include "tp_runtime.h" + +#ifndef TP_SHIM_NAME +#define TP_SHIM_NAME tp_shim +#endif +#define TP_CAT2(a, b) a##b +#define TP_CAT(a, b) TP_CAT2(a, b) +#define TP_STR2(a) #a +#define TP_STR(a) TP_STR2(a) + +static int rc_ok(int rc) +{ + if (rc < 0 && !PyErr_Occurred()) { + PyErr_SetString(PyExc_AssertionError, "rc < 0 but no Python error is set"); + return 0; + } + if (rc < 0) { + return 0; + } + if (PyErr_Occurred()) { + PyErr_Format(PyExc_AssertionError, "rc == %d but a Python error is set", rc); + return 0; + } + return 1; +} + +static PyObject *ret_none(int rc) { return Py_BuildValue("(iO)", rc, Py_None); } + +/* ---- int64 ---- */ +#define I64_BIN(OP) \ + static PyObject *s_##OP##_i64(PyObject *self, PyObject *args) \ + { \ + long long a, b; \ + int64_t out = 0; \ + if (!PyArg_ParseTuple(args, "LL", &a, &b)) return NULL; \ + int rc = tp_##OP##_i64((int64_t)a, (int64_t)b, &out); \ + if (!rc_ok(rc)) return NULL; \ + if (rc) return ret_none(rc); \ + return Py_BuildValue("(iL)", rc, (long long)out); \ + } +I64_BIN(add) +I64_BIN(sub) +I64_BIN(mul) +I64_BIN(floordiv) +I64_BIN(mod) + +static PyObject *s_neg_i64(PyObject *self, PyObject *args) +{ + long long a; + int64_t out = 0; + if (!PyArg_ParseTuple(args, "L", &a)) return NULL; + int rc = tp_neg_i64((int64_t)a, &out); + if (!rc_ok(rc)) return NULL; + if (rc) return ret_none(rc); + return Py_BuildValue("(iL)", rc, (long long)out); +} + +static PyObject *s_truediv_i64(PyObject *self, PyObject *args) +{ + long long a, b; + double out = 0; + if (!PyArg_ParseTuple(args, "LL", &a, &b)) return NULL; + int rc = tp_truediv_i64((int64_t)a, (int64_t)b, &out); + if (!rc_ok(rc)) return NULL; + if (rc) return ret_none(rc); + return Py_BuildValue("(id)", rc, out); +} + +static PyObject *s_i64_to_f64(PyObject *self, PyObject *args) +{ + long long a; + if (!PyArg_ParseTuple(args, "L", &a)) return NULL; + return PyFloat_FromDouble(tp_i64_to_f64((int64_t)a)); +} + +/* ---- float ---- */ +#define F64_BIN(OP) \ + static PyObject *s_##OP##_f64(PyObject *self, PyObject *args) \ + { \ + double a, b, out = 0; \ + if (!PyArg_ParseTuple(args, "dd", &a, &b)) return NULL; \ + int rc = tp_##OP##_f64(a, b, &out); \ + if (!rc_ok(rc)) return NULL; \ + if (rc) return ret_none(rc); \ + return Py_BuildValue("(id)", rc, out); \ + } +F64_BIN(truediv) +F64_BIN(floordiv) +F64_BIN(mod) + +static PyObject *s_cmp_i64_f64(PyObject *self, PyObject *args) +{ + long long a; + double b; + int op; + if (!PyArg_ParseTuple(args, "Ldi", &a, &b, &op)) return NULL; + return PyLong_FromLong(tp_cmp_i64_f64((int64_t)a, b, op)); +} + +static PyObject *s_cmp_f64_i64(PyObject *self, PyObject *args) +{ + double a; + long long b; + int op; + if (!PyArg_ParseTuple(args, "dLi", &a, &b, &op)) return NULL; + return PyLong_FromLong(tp_cmp_f64_i64(a, (int64_t)b, op)); +} + +/* ---- math ---- */ +static PyObject *s_math1(PyObject *self, PyObject *args) +{ + const char *name; + double x, out = 0; + int rc; + if (!PyArg_ParseTuple(args, "sd", &name, &x)) return NULL; + if (!strcmp(name, "sqrt")) rc = tp_math_sqrt(x, &out); + else if (!strcmp(name, "exp")) rc = tp_math_exp(x, &out); + else if (!strcmp(name, "log")) rc = tp_math_log(x, &out); + else if (!strcmp(name, "sin")) rc = tp_math_sin(x, &out); + else if (!strcmp(name, "cos")) rc = tp_math_cos(x, &out); + else if (!strcmp(name, "tan")) rc = tp_math_tan(x, &out); + else if (!strcmp(name, "fabs")) rc = tp_math_fabs(x, &out); + else { PyErr_SetString(PyExc_ValueError, "unknown math1"); return NULL; } + if (!rc_ok(rc)) return NULL; + if (rc) return ret_none(rc); + return Py_BuildValue("(id)", rc, out); +} + +static PyObject *s_math2(PyObject *self, PyObject *args) +{ + const char *name; + double x, y, out = 0; + int rc; + if (!PyArg_ParseTuple(args, "sdd", &name, &x, &y)) return NULL; + if (!strcmp(name, "atan2")) rc = tp_math_atan2(x, y, &out); + else if (!strcmp(name, "hypot")) rc = tp_math_hypot(x, y, &out); + else { PyErr_SetString(PyExc_ValueError, "unknown math2"); return NULL; } + if (!rc_ok(rc)) return NULL; + if (rc) return ret_none(rc); + return Py_BuildValue("(id)", rc, out); +} + +/* ---- boxing ---- */ +static PyObject *s_box_i64(PyObject *self, PyObject *args) +{ + long long v; + if (!PyArg_ParseTuple(args, "L", &v)) return NULL; + return tp_box_i64((int64_t)v); +} +static PyObject *s_box_f64(PyObject *self, PyObject *args) +{ + double v; + if (!PyArg_ParseTuple(args, "d", &v)) return NULL; + return tp_box_f64(v); +} +static PyObject *s_box_bool(PyObject *self, PyObject *args) +{ + int v; + if (!PyArg_ParseTuple(args, "i", &v)) return NULL; + return tp_box_bool(v); +} +static PyObject *s_unbox_i64(PyObject *self, PyObject *o) +{ + int64_t out = 0; + int rc = tp_unbox_i64(o, &out); + if (!rc_ok(rc)) return NULL; + if (rc) return ret_none(rc); + return Py_BuildValue("(iL)", rc, (long long)out); +} +static PyObject *s_unbox_f64(PyObject *self, PyObject *o) +{ + double out = 0; + int rc = tp_unbox_f64(o, &out); + if (!rc_ok(rc)) return NULL; + if (rc) return ret_none(rc); + return Py_BuildValue("(id)", rc, out); +} +static PyObject *s_unbox_bool(PyObject *self, PyObject *o) +{ + int out = 0; + int rc = tp_unbox_bool(o, &out); + if (!rc_ok(rc)) return NULL; + if (rc) return ret_none(rc); + return Py_BuildValue("(ii)", rc, out); +} + +/* ---- arrays ---- + * array_run(list, script) -> (enter_rc, results, exit_rc, exit_error) + * script items: ("get", i) | ("set", i, v) | ("len",) | ("dirty",) | ("clear",) + * A failing "get"/"set" slot lookup appends the exception instance. The struct is filled with + * garbage before enter, so an enter that leaves a field unset shows up. */ +#define ARRAY_RUN(SUF, STRUCT, CTYPE, FROMC, PARSEFMT, PARSET) \ + static PyObject *s_##SUF##_array_run(PyObject *self, PyObject *args) \ + { \ + PyObject *obj, *script; \ + if (!PyArg_ParseTuple(args, "OO", &obj, &script)) return NULL; \ + STRUCT a; \ + memset(&a, 0xAB, sizeof a); \ + int erc = tp_##SUF##_array_enter(obj, &a); \ + if (!rc_ok(erc)) return NULL; \ + PyObject *results = PyList_New(0); \ + if (!results) { tp_##SUF##_array_exit(&a); return NULL; } \ + if (erc == 0) { \ + Py_ssize_t n = PyList_GET_SIZE(script); \ + for (Py_ssize_t k = 0; k < n; k++) { \ + PyObject *op = PyList_GET_ITEM(script, k); \ + const char *kind = PyUnicode_AsUTF8(PyTuple_GET_ITEM(op, 0)); \ + PyObject *item = NULL; \ + if (!strcmp(kind, "get") || !strcmp(kind, "set")) { \ + long long i = PyLong_AsLongLong(PyTuple_GET_ITEM(op, 1)); \ + int store = kind[0] == 's'; \ + Py_ssize_t slot = -7; \ + int rc = tp_##SUF##_array_slot(&a, (int64_t)i, store, &slot); \ + if (rc < 0) { \ + item = PyErr_GetRaisedException(); \ + } else if (rc != 0 || PyErr_Occurred()) { \ + PyErr_SetString(PyExc_AssertionError, "slot rc not 0 / -1"); \ + Py_DECREF(results); tp_##SUF##_array_exit(&a); return NULL; \ + } else if (!store) { \ + item = FROMC(a.data[slot]); \ + } else { \ + PARSET v; \ + if (!PyArg_Parse(PyTuple_GET_ITEM(op, 2), PARSEFMT, &v)) { \ + Py_DECREF(results); tp_##SUF##_array_exit(&a); return NULL; \ + } \ + a.data[slot] = (CTYPE)v; \ + a.dirty[slot] = 1; \ + item = Py_NewRef(Py_None); \ + } \ + } else if (!strcmp(kind, "len")) { \ + item = PyLong_FromSsize_t(a.len); \ + } else if (!strcmp(kind, "dirty")) { \ + item = PyBytes_FromStringAndSize((const char *)a.dirty, a.len); \ + } else if (!strcmp(kind, "clear")) { \ + if (PyList_SetSlice(a.list, 0, PyList_GET_SIZE(a.list), NULL) < 0) { \ + Py_DECREF(results); tp_##SUF##_array_exit(&a); return NULL; \ + } \ + item = Py_NewRef(Py_None); \ + } \ + if (!item || PyList_Append(results, item) < 0) { \ + Py_XDECREF(item); Py_DECREF(results); \ + tp_##SUF##_array_exit(&a); \ + return NULL; \ + } \ + Py_DECREF(item); \ + } \ + } \ + int xrc = tp_##SUF##_array_exit(&a); \ + PyObject *xerr = NULL; \ + if (xrc < 0) { \ + if (!PyErr_Occurred()) { \ + PyErr_SetString(PyExc_AssertionError, "exit rc -1 without error"); \ + Py_DECREF(results); return NULL; \ + } \ + xerr = PyErr_GetRaisedException(); \ + } else if (PyErr_Occurred()) { \ + Py_DECREF(results); return NULL; \ + } \ + /* the struct must be unusable-but-safe afterwards: a second exit is a no-op */ \ + int again = tp_##SUF##_array_exit(&a); \ + PyObject *ret = Py_BuildValue("(iNiNi)", erc, results, xrc, \ + xerr ? xerr : Py_NewRef(Py_None), again); \ + return ret; \ + } +ARRAY_RUN(f64, tp_f64_array, double, PyFloat_FromDouble, "d", double) +ARRAY_RUN(i64, tp_i64_array, int64_t, PyLong_FromLongLong, "L", long long) + +static PyObject *s_array_exit_zeroed(PyObject *self, PyObject *arg) +{ + int which = (int)PyLong_AsLong(arg); + int rc; + if (which == 0) { + tp_f64_array a; + memset(&a, 0, sizeof a); + rc = tp_f64_array_exit(&a); + } else { + tp_i64_array a; + memset(&a, 0, sizeof a); + rc = tp_i64_array_exit(&a); + } + if (!rc_ok(rc)) return NULL; + return PyLong_FromLong(rc); +} + +static PyObject *s_any_same(PyObject *self, PyObject *seq) +{ + PyObject *fast = PySequence_Fast(seq, "sequence expected"); + if (!fast) return NULL; + int r = tp_any_same(PySequence_Fast_ITEMS(fast), PySequence_Fast_GET_SIZE(fast)); + Py_DECREF(fast); + return PyLong_FromLong(r); +} + + +/* ---- opaque objects ---- */ +static PyObject *s_global(PyObject *self, PyObject *args) +{ + PyObject *d, *name; + if (!PyArg_ParseTuple(args, "OO", &d, &name)) return NULL; + return tp_global(d, name); +} +static PyObject *s_getattr(PyObject *self, PyObject *args) +{ + PyObject *o, *name; + if (!PyArg_ParseTuple(args, "OO", &o, &name)) return NULL; + return tp_getattr(o, name); +} +/* call(callee, args_tuple, kwnames_tuple_or_None): the trailing len(kwnames) args are keywords */ +static PyObject *s_call(PyObject *self, PyObject *args) +{ + PyObject *callee, *argt, *kw; + if (!PyArg_ParseTuple(args, "OO!O", &callee, &PyTuple_Type, &argt, &kw)) return NULL; + if (kw != Py_None && (!PyTuple_Check(kw) || PyTuple_GET_SIZE(kw) > PyTuple_GET_SIZE(argt))) { + PyErr_SetString(PyExc_ValueError, "shim: kwnames must be a tuple no longer than args"); + return NULL; + } + /* vectorcall: nargs counts the positional arguments only; the keyword values follow them */ + return tp_call(callee, &PyTuple_GET_ITEM(argt, 0), + (size_t)(PyTuple_GET_SIZE(argt) - (kw == Py_None ? 0 : PyTuple_GET_SIZE(kw))), + kw == Py_None ? NULL : kw); +} +/* call_many: n calls in a row, results dropped (for refcount stability checks) */ +static PyObject *s_call_many(PyObject *self, PyObject *args) +{ + PyObject *callee, *argt; + long n; + if (!PyArg_ParseTuple(args, "OO!l", &callee, &PyTuple_Type, &argt, &n)) return NULL; + long errors = 0; + for (long i = 0; i < n; i++) { + PyObject *r = tp_call(callee, &PyTuple_GET_ITEM(argt, 0), (size_t)PyTuple_GET_SIZE(argt), NULL); + if (r == NULL) { errors++; PyErr_Clear(); } else { tp_release(&r); } + } + return PyLong_FromLong(errors); +} +static PyObject *s_getattr_many(PyObject *self, PyObject *args) +{ + PyObject *o, *name; + long n; + if (!PyArg_ParseTuple(args, "OOl", &o, &name, &n)) return NULL; + long errors = 0; + for (long i = 0; i < n; i++) { + PyObject *r = tp_getattr(o, name); + if (r == NULL) { errors++; PyErr_Clear(); } else { tp_release(&r); } + } + return PyLong_FromLong(errors); +} +static PyObject *s_binop_many(PyObject *self, PyObject *args) +{ + PyObject *a, *b; + int op; + long n; + if (!PyArg_ParseTuple(args, "OOil", &a, &b, &op, &n)) return NULL; + long errors = 0; + for (long i = 0; i < n; i++) { + PyObject *r = tp_binop_obj(a, b, op); + if (r == NULL) { errors++; PyErr_Clear(); } else { tp_release(&r); } + } + return PyLong_FromLong(errors); +} +static PyObject *s_truth(PyObject *self, PyObject *o) +{ + int rc = tp_truth(o); + if (rc < 0) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_AssertionError, "-1 w/o error"); return NULL; } + if (PyErr_Occurred()) return NULL; + return PyLong_FromLong(rc); +} +static PyObject *s_compare_bool(PyObject *self, PyObject *args) +{ + PyObject *a, *b; + int op; + if (!PyArg_ParseTuple(args, "OOi", &a, &b, &op)) return NULL; + int rc = tp_compare_bool(a, b, op); + if (rc < 0) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_AssertionError, "-1 w/o error"); return NULL; } + if (PyErr_Occurred()) return NULL; + return PyLong_FromLong(rc); +} +static PyObject *s_obj_to_f64(PyObject *self, PyObject *o) +{ + double out = 0; + int rc = tp_obj_to_f64(o, &out); + if (!rc_ok(rc)) return NULL; + return PyFloat_FromDouble(out); +} +static PyObject *s_binop_obj(PyObject *self, PyObject *args) +{ + PyObject *a, *b; + int op; + if (!PyArg_ParseTuple(args, "OOi", &a, &b, &op)) return NULL; + PyObject *r = tp_binop_obj(a, b, op); + if (r == NULL && !PyErr_Occurred()) PyErr_SetString(PyExc_AssertionError, "NULL w/o error"); + return r; +} +/* release(obj) -> True if the slot was cleared; the shim owns one extra reference meanwhile */ +static PyObject *s_release(PyObject *self, PyObject *o) +{ + PyObject *slot = Py_NewRef(o); + tp_release(&slot); + PyObject *null_slot = NULL; + tp_release(&null_slot); /* releasing an empty slot is a no-op */ + return PyBool_FromLong(slot == NULL && null_slot == NULL); +} + +static PyObject *s_info(PyObject *self, PyObject *unused) +{ + return Py_BuildValue("{s:i,s:i}", "portable", TP_RUNTIME_PORTABLE_OVERFLOW, + "py_version_hex", (int)PY_VERSION_HEX); +} + +#define M(N, F, FL) {N, (PyCFunction)(void (*)(void))F, FL, NULL} +static PyMethodDef methods[] = { + M("add_i64", s_add_i64, METH_VARARGS), M("sub_i64", s_sub_i64, METH_VARARGS), + M("mul_i64", s_mul_i64, METH_VARARGS), M("neg_i64", s_neg_i64, METH_VARARGS), + M("floordiv_i64", s_floordiv_i64, METH_VARARGS), M("mod_i64", s_mod_i64, METH_VARARGS), + M("truediv_i64", s_truediv_i64, METH_VARARGS), M("i64_to_f64", s_i64_to_f64, METH_VARARGS), + M("truediv_f64", s_truediv_f64, METH_VARARGS), M("floordiv_f64", s_floordiv_f64, METH_VARARGS), + M("mod_f64", s_mod_f64, METH_VARARGS), + M("cmp_i64_f64", s_cmp_i64_f64, METH_VARARGS), M("cmp_f64_i64", s_cmp_f64_i64, METH_VARARGS), + M("math1", s_math1, METH_VARARGS), M("math2", s_math2, METH_VARARGS), + M("box_i64", s_box_i64, METH_VARARGS), M("box_f64", s_box_f64, METH_VARARGS), + M("box_bool", s_box_bool, METH_VARARGS), + M("unbox_i64", s_unbox_i64, METH_O), M("unbox_f64", s_unbox_f64, METH_O), + M("unbox_bool", s_unbox_bool, METH_O), + M("f64_array_run", s_f64_array_run, METH_VARARGS), M("i64_array_run", s_i64_array_run, METH_VARARGS), + M("array_exit_zeroed", s_array_exit_zeroed, METH_O), + M("global_", s_global, METH_VARARGS), M("getattr_", s_getattr, METH_VARARGS), + M("call", s_call, METH_VARARGS), M("call_many", s_call_many, METH_VARARGS), + M("getattr_many", s_getattr_many, METH_VARARGS), M("binop_many", s_binop_many, METH_VARARGS), + M("truth", s_truth, METH_O), M("compare_bool", s_compare_bool, METH_VARARGS), + M("obj_to_f64", s_obj_to_f64, METH_O), M("binop_obj", s_binop_obj, METH_VARARGS), + M("release", s_release, METH_O), + M("any_same", s_any_same, METH_O), M("info", s_info, METH_NOARGS), + {NULL, NULL, 0, NULL}}; + +static struct PyModuleDef moddef = {PyModuleDef_HEAD_INIT, TP_STR(TP_SHIM_NAME), NULL, -1, methods, + NULL, NULL, NULL, NULL}; + +PyMODINIT_FUNC TP_CAT(PyInit_, TP_SHIM_NAME)(void) { return PyModule_Create(&moddef); } diff --git a/python-multiplatform-ksp/src/test/python/test_runtime.py b/python-multiplatform-ksp/src/test/python/test_runtime.py new file mode 100644 index 000000000..d77303dd9 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_runtime.py @@ -0,0 +1,978 @@ +"""Differential tests of tp_runtime.h against CPython (design §4.3, #41). + +`runtime_shim/tp_shim.c` is a tiny extension that exposes every helper of +`typedpython/runtime/tp_runtime.h`; each test calls the helper and CPython's own operation on the +same inputs and compares the outcome exactly: + + * results bit-for-bit (floats compared by their IEEE bits, so -0.0 and 0.0 differ; NaN is + compared as "any NaN", payload and sign of a NaN are not part of the contract); + * exceptions by exact type AND message, taken from the interpreter running the test (so the + 3.13 and 3.14 messages are each checked against the real thing); + * the return convention: 0 ok / 1 deopt / -1 error. The shim turns a convention violation (-1 + without an exception, 0/1 with an exception set) into AssertionError. + +Deopt (rc 1) is allowed only where API.md names it; each test says when it expects it. + +Environment: set TP_SANITIZE=1 (with DYLD_INSERT_LIBRARIES pointing at the ASan runtime on macOS, +or LD_PRELOAD on Linux) to build the shim with AddressSanitizer + UBSan. +""" +from __future__ import annotations + +import importlib.util +import itertools +import math +import operator +import os +import random +import shlex +import subprocess +import struct +import sys +import sysconfig +from pathlib import Path + +import pytest + +HERE = Path(__file__).resolve().parent +KSP = HERE.parents[2] +REPO_ROOT = KSP.parent +RUNTIME_DIR = KSP / "src" / "main" / "python" / "typedpython" / "runtime" +SHIM_C = HERE / "runtime_shim" / "tp_shim.c" +BUILD_DIR = REPO_ROOT / ".tmp" / "runtime_shim_build" +HEAVY = REPO_ROOT.parent.parent / ".tmp" / "heavy.sh" + +I64_MIN = -(2**63) +I64_MAX = 2**63 - 1 +INF = math.inf +NAN = math.nan + + +# --- building the shim ------------------------------------------------------------------------ + +def _lock_already_held() -> bool: + """True if an ancestor process is heavy.sh (the whole pytest run then already holds the lock; + taking it again would deadlock).""" + pid = os.getpid() + for _ in range(40): + out = subprocess.run(["ps", "-o", "ppid=,command=", "-p", str(pid)], + capture_output=True, text=True).stdout.strip() + if not out: + return False + ppid, _, cmd = out.partition(" ") + if "heavy.sh" in cmd: + return True + pid = int(ppid) + if pid <= 1: + return False + return False + + +def _run_compile(cmd: list[str]) -> None: + if HEAVY.exists() and not _lock_already_held(): + cmd = [str(HEAVY)] + cmd + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode != 0: + raise RuntimeError("shim build failed:\n" + " ".join(cmd) + "\n" + proc.stdout + proc.stderr) + + +def _build(variant: str): + suffix = sysconfig.get_config_var("EXT_SUFFIX") + sanitize = os.environ.get("TP_SANITIZE") == "1" + name = f"tp_shim_{variant}" + ("_san" if sanitize else "") + BUILD_DIR.mkdir(parents=True, exist_ok=True) + out = BUILD_DIR / (name + suffix) + cc = shlex.split(sysconfig.get_config_var("CC") or "cc") + cmd = cc + ["-std=c99", "-O1", "-g", "-Wall", "-Wextra", "-Werror", "-Wno-unused-parameter", + "-shared", "-fPIC", f"-DTP_SHIM_NAME={name}", + "-I", sysconfig.get_paths()["include"], "-I", str(RUNTIME_DIR)] + if variant == "portable": + cmd.append("-DTP_RUNTIME_FORCE_PORTABLE_OVERFLOW") + if sanitize: + cmd += ["-fsanitize=address,undefined", "-fno-sanitize-recover=undefined", + "-fno-omit-frame-pointer"] + res = subprocess.run(cc + ["-print-resource-dir"], capture_output=True, text=True).stdout.strip() + if sys.platform == "darwin" and res: + cmd += ["-Wl,-rpath," + res + "/lib/darwin"] + if sys.platform == "darwin": + cmd += ["-undefined", "dynamic_lookup"] + cmd += [str(SHIM_C), "-o", str(out)] + _run_compile(cmd) + spec = importlib.util.spec_from_file_location(name, out) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +@pytest.fixture(scope="session", params=["default", "portable"]) +def rt(request): + """The compiled shim; built once per variant per session. `portable` forces the fallback for + the overflow builtins so both code paths of the header are exercised.""" + mod = _build(request.param) + info = mod.info() + assert bool(info["portable"]) == (request.param == "portable") + return mod + + +# --- comparison helpers ----------------------------------------------------------------------- + +def norm(v): + if isinstance(v, bool): + return ("b", v) + if isinstance(v, float): + return ("nan",) if v != v else ("f", struct.pack(" bool: + return I64_MIN <= v <= I64_MAX + + +INT_VALUES = sorted({0, 1, -1, 2, -2, 7, -7, 2**31, -(2**31), 2**31 - 1, 2**32, 2**53, -(2**53), + 2**53 + 1, -(2**53) - 1, 2**53 - 1, 2**62, -(2**62), 2**62 + 1, I64_MIN, + I64_MAX, I64_MIN + 1, I64_MAX - 1, 3, -3, 10, -10, 1000003, -1000003}) + +FLOAT_VALUES = [0.0, -0.0, 1.0, -1.0, 1.5, -1.5, 0.5, -0.5, 2.5, -2.5, 3.0, -3.0, 7.0, 0.1, -0.1, + INF, -INF, NAN, 5e-324, -5e-324, 2.2250738585072014e-308, 1e-300, -1e-300, 1e300, + -1e300, 1.7976931348623157e308, -1.7976931348623157e308, 2.0**53, -(2.0**53), + 2.0**63, -(2.0**63), 1e22, 4.5e15] + + +# --- ints ------------------------------------------------------------------------------------- + +@pytest.mark.parametrize("name,pyop", [("add_i64", operator.add), ("sub_i64", operator.sub), + ("mul_i64", operator.mul)]) +def test_checked_arithmetic_matches_bigint_and_deopts_exactly_on_overflow(rt, name, pyop): + fn = getattr(rt, name) + cases = [] + for a, b in itertools.product(INT_VALUES, repeat=2): + exact = pyop(a, b) + cases.append(((a, b), ("i", exact) if fits(exact) else ("deopt",), sh(fn, a, b))) + check(cases, name) + + +def test_neg_i64_deopts_only_for_int64_min(rt): + cases = [] + for a in INT_VALUES: + cases.append(((a,), ("i", -a) if fits(-a) else ("deopt",), sh(rt.neg_i64, a))) + check(cases, "neg_i64") + assert sh(rt.neg_i64, I64_MIN) == ("deopt",) + + +def test_fuzz_checked_arithmetic_against_bigint(rt): + rnd = random.Random(20261003) + cases = [] + for _ in range(4000): + bits_a, bits_b = rnd.choice([8, 31, 32, 53, 62, 63]), rnd.choice([8, 31, 32, 53, 62, 63]) + a = rnd.randint(-(2**bits_a), 2**bits_a) + b = rnd.randint(-(2**bits_b), 2**bits_b) + a, b = max(I64_MIN, min(I64_MAX, a)), max(I64_MIN, min(I64_MAX, b)) + for name, pyop in (("add_i64", operator.add), ("sub_i64", operator.sub), + ("mul_i64", operator.mul)): + exact = pyop(a, b) + cases.append(((name, a, b), ("i", exact) if fits(exact) else ("deopt",), + sh(getattr(rt, name), a, b))) + check(cases, "fuzz") + + +def test_floordiv_i64_is_python_floor_division(rt): + cases = [] + for a, b in itertools.product(INT_VALUES, repeat=2): + ref = py(operator.floordiv, a, b) + if ref[0] == "i" and not fits(ref[1]): + ref = ("deopt",) # only MIN // -1 + cases.append(((a, b), ref, sh(rt.floordiv_i64, a, b))) + check(cases, "floordiv_i64") + assert sh(rt.floordiv_i64, I64_MIN, -1) == ("deopt",) + + +def test_mod_i64_has_the_sign_of_the_divisor(rt): + cases = [((a, b), py(operator.mod, a, b), sh(rt.mod_i64, a, b)) + for a, b in itertools.product(INT_VALUES, repeat=2)] + check(cases, "mod_i64") + assert sh(rt.mod_i64, -7, 2) == ("i", 1) + assert sh(rt.mod_i64, 7, -2) == ("i", -1) + assert sh(rt.mod_i64, I64_MIN, -1) == ("i", 0) # CPython: 0 (the C % would trap) + + +def test_int_division_by_zero_messages_are_cpythons(rt): + for fn, op in ((rt.floordiv_i64, operator.floordiv), (rt.mod_i64, operator.mod), + (rt.truediv_i64, operator.truediv)): + for a in (0, 1, -5, I64_MAX, I64_MIN): + exp, got = py(op, a, 0), sh(fn, a, 0) + assert exp[0] == "exc" and exp[1] is ZeroDivisionError + assert got == exp, (op, a, exp, got) + + +def test_fuzz_floordiv_mod_i64(rt): + rnd = random.Random(7) + cases = [] + for _ in range(4000): + a = rnd.randint(-(2**rnd.choice([4, 31, 63])), 2**rnd.choice([4, 31, 63]) - 1) + b = rnd.randint(-(2**rnd.choice([2, 9, 31, 63])), 2**rnd.choice([2, 9, 31, 63]) - 1) + a, b = max(I64_MIN, min(I64_MAX, a)), max(I64_MIN, min(I64_MAX, b)) + for name, op in (("floordiv_i64", operator.floordiv), ("mod_i64", operator.mod)): + ref = py(op, a, b) + if ref[0] == "i" and not fits(ref[1]): + ref = ("deopt",) + cases.append(((name, a, b), ref, sh(getattr(rt, name), a, b))) + check(cases, "fuzz floordiv/mod") + + +TRUEDIV_INTS = INT_VALUES + [2**53 - 2, 2**53 + 2, -(2**53) + 1, 2**53 + 3, 9007199254740993, 5, + -5, 6, 49, 355, 113] + + +def test_truediv_i64_is_correctly_rounded_or_deopts_above_2_53(rt): + cases = [] + for a, b in itertools.product(TRUEDIV_INTS, repeat=2): + ref = py(operator.truediv, a, b) + if ref[0] != "exc" and (abs(a) > 2**53 or abs(b) > 2**53): + ref = ("deopt",) + cases.append(((a, b), ref, sh(rt.truediv_i64, a, b))) + check(cases, "truediv_i64") + # the boundary itself is exact-rounded by the C division + assert sh(rt.truediv_i64, 2**53, 3) == py(operator.truediv, 2**53, 3) + assert sh(rt.truediv_i64, 2**53 + 1, 3) == ("deopt",) + assert sh(rt.truediv_i64, 1, -(2**53) - 1) == ("deopt",) + assert sh(rt.truediv_i64, 0, -5) == ("f", struct.pack(" +0.0 (sign of divisor) + assert sh(rt.mod_f64, 1.0, -1.0) == ("f", nz) + assert sh(rt.mod_f64, -7.5, 2.0) == norm(-7.5 % 2.0) + assert sh(rt.floordiv_f64, -0.0, 5.0) == norm(-0.0 // 5.0) + assert sh(rt.floordiv_f64, 1.0, -INF) == norm(1.0 // -INF) + + +# --- int/float exact comparison --------------------------------------------------------------- + +OPS = [(0, operator.lt), (1, operator.le), (2, operator.eq), (3, operator.ne), (4, operator.gt), + (5, operator.ge)] +CMP_INTS = INT_VALUES + [2**53 + 2, 2**63 - 1024, -(2**63) + 1024, 2**63 - 513, 9223372036854774784, + 4503599627370497, 9007199254740992] +CMP_FLOATS = FLOAT_VALUES + [float(2**53), float(2**53 + 2), 9.223372036854776e18, + -9.223372036854776e18, 9.223372036854775e18, 9223372036854774784.0, + 0.1 + 0.2, 2.0**62, 4503599627370496.5, -4503599627370496.5, + 9007199254740993.0, 1e19, -1e19, 2.0**63 - 1024] + + +def test_int_float_comparison_is_exact_for_every_operator(rt): + assert op_name_sanity() + cases = [] + for a, b in itertools.product(CMP_INTS, CMP_FLOATS): + for code, pyop in OPS: + cases.append(((a, b, code), int(pyop(a, b)), rt.cmp_i64_f64(a, b, code))) + cases.append(((b, a, code), int(pyop(b, a)), rt.cmp_f64_i64(b, a, code))) + check(cases, "cmp") + + +def op_name_sanity(): + import _operator # noqa: F401 + return True + + +def test_int_float_comparison_named_hard_cases(rt): + # 2**53 + 1 is not equal to float(2**53), although (double)(2**53+1) == 2**53 + assert rt.cmp_i64_f64(2**53 + 1, float(2**53), 2) == 0 + assert rt.cmp_i64_f64(2**53 + 1, float(2**53), 4) == 1 + assert rt.cmp_i64_f64(I64_MAX, 9.223372036854776e18, 0) == 1 # INT64_MAX < 2**63 + assert rt.cmp_i64_f64(I64_MAX, 9.223372036854776e18, 2) == 0 + assert rt.cmp_i64_f64(I64_MIN, -9.223372036854776e18, 2) == 1 # -2**63 == -2.0**63 + for code, _ in OPS: + assert rt.cmp_i64_f64(1, NAN, code) == (1 if code == 3 else 0) + assert rt.cmp_f64_i64(NAN, 1, code) == (1 if code == 3 else 0) + assert rt.cmp_i64_f64(I64_MAX, INF, 0) == 1 and rt.cmp_i64_f64(I64_MIN, -INF, 4) == 1 + + +# --- math ------------------------------------------------------------------------------------- + +MATH1 = ["sqrt", "exp", "log", "sin", "cos", "tan", "fabs"] +MATH_VALUES = FLOAT_VALUES + [1000.0, -1000.0, 709.0, 709.782712893384, 709.7827128933841, -745.0, + -746.0, -1e5, 1e5, 0.5 * math.pi, math.pi, 1e22, 1e15, 100.0, + -100.0, 710.0] + + +@pytest.mark.parametrize("name", MATH1) +def test_math_one_argument_matches_math_module_including_messages(rt, name): + ref_fn = getattr(math, name) + cases = [((x,), py(ref_fn, x), sh(rt.math1, name, x)) for x in MATH_VALUES] + check(cases, "math." + name) + + +@pytest.mark.parametrize("name", ["sqrt", "exp", "log", "sin", "cos", "tan"]) +def test_math_one_argument_fuzz_is_bit_exact(rt, name): + rnd = random.Random(hash(name) & 0xFFFF) + ref_fn = getattr(math, name) + xs = [rnd.uniform(-1, 1) * 10.0 ** rnd.randint(-10, 10) for _ in range(3000)] + cases = [((x,), py(ref_fn, x), sh(rt.math1, name, x)) for x in xs] + check(cases, "fuzz math." + name) + + +def test_math_named_error_cases(rt): + # the rows named in the task, spelled out: type+message are whatever this interpreter says + for name, x in [("sqrt", -1.0), ("sqrt", -0.0), ("exp", 1000.0), ("log", 0.0), ("log", -1.0), + ("log", -INF), ("sin", INF), ("cos", -INF), ("tan", INF), ("sqrt", -INF)]: + exp, got = py(getattr(math, name), x), sh(rt.math1, name, x) + assert got == exp, (name, x, exp, got) + assert sh(rt.math1, "sqrt", -0.0) == ("f", struct.pack(" Date: Sat, 3 Oct 2026 09:18:39 +0900 Subject: [PATCH 18/36] Feat: TypedPython C back end (IR -> C-API extension) and build driver cgen generates one C file per module: multi-phase init that runs the module's original source in the extension's own namespace, then swaps each compiled function for a METH_FASTCALL wrapper with entry guards (exact types, i64 range, array enter + aliasing), deopt to __typedpython_interpreted__, and array write-back on every exit. One ownership rule, references dropped only through tp_release; FP contraction off. cbuild compiles with the host or a target CPython's headers via sysconfig, no setuptools or Cython. Green: 29/29 (incl. ASan+UBSan via a sanitized launcher and a compile against the embedded 3.14 headers). Mutation (aliasing guard off): 2 fail. Refs #41, #45 --- .../src/main/python/typedpython/cbuild.py | 84 ++ .../src/main/python/typedpython/cgen.py | 1065 +++++++++++++++++ .../src/test/python/test_cgen.py | 847 +++++++++++++ 3 files changed, 1996 insertions(+) create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/cbuild.py create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/cgen.py create mode 100644 python-multiplatform-ksp/src/test/python/test_cgen.py diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py b/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py new file mode 100644 index 000000000..3872a4bbe --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py @@ -0,0 +1,84 @@ +"""Build a generated C file (cgen) into a CPython extension with the platform C compiler. + +No setuptools, no Cython: one compiler invocation, `-shared -fPIC`, on macOS +`-undefined dynamic_lookup` (the extension resolves the C API from the process that loads it). +The headers and the extension suffix default to the host interpreter's (`sysconfig`), and can be +pointed at the CPython the app embeds (`python_include`, `ext_suffix`). +""" +from __future__ import annotations + +import importlib.machinery +import importlib.util +import shlex +import subprocess +import sys +import sysconfig +from pathlib import Path + +RUNTIME_DIR = Path(__file__).resolve().parent / "runtime" + +# -ffp-contract=off: a fused multiply-add rounds once where CPython rounds twice. +DEFAULT_FLAGS: tuple[str, ...] = ( + "-O2", + "-ffp-contract=off", + "-fno-fast-math", + "-Werror=implicit-function-declaration", +) + + +class CBuildError(RuntimeError): + """The C compiler rejected the generated source; the message carries its stderr.""" + + +def _python_include(python_include: Path | None) -> Path: + if python_include is None: + return Path(sysconfig.get_paths()["include"]) + p = Path(python_include) + if (p / "Python.h").is_file(): + return p + found = sorted(d for d in p.glob("python3*") if (d / "Python.h").is_file()) + if len(found) == 1: + return found[0] + raise CBuildError(f"no Python.h in {p} (nor in exactly one python3.*/ below it: {found})") + + +def compiler() -> list[str]: + return shlex.split(sysconfig.get_config_var("CC") or "cc") + + +def build(c_source: str, module_name: str, out_dir: Path, runtime_dir: Path | None = None, + flags: list[str] | tuple[str, ...] = DEFAULT_FLAGS, python_include: Path | None = None, + ext_suffix: str | None = None) -> Path: + """Compile `c_source` to `/`; return it.""" + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + stem = module_name.rpartition(".")[2] + suffix = ext_suffix if ext_suffix is not None else sysconfig.get_config_var("EXT_SUFFIX") + if not suffix: + raise CBuildError("no extension suffix (sysconfig EXT_SUFFIX is empty); pass ext_suffix") + runtime = Path(runtime_dir) if runtime_dir is not None else RUNTIME_DIR + if not (runtime / "tp_runtime.h").is_file(): + raise CBuildError(f"tp_runtime.h not found in {runtime}") + include = _python_include(python_include) + c_file = out_dir / f"{stem}.c" + c_file.write_text(c_source, encoding="utf-8") + so = out_dir / f"{stem}{suffix}" + cmd = compiler() + list(flags) + ["-fPIC", "-shared", f"-I{include}", f"-I{runtime}"] + if sys.platform == "darwin": + cmd += ["-undefined", "dynamic_lookup"] + cmd += [str(c_file), "-o", str(so)] + run = subprocess.run(cmd, capture_output=True, text=True) + if run.returncode != 0: + raise CBuildError(f"C compiler failed ({run.returncode}): {shlex.join(cmd)}\n{run.stderr}") + return so + + +def load(module_name: str, path: Path): + """Import the built extension at `path` as `module_name` (not registered in sys.modules).""" + loader = importlib.machinery.ExtensionFileLoader(module_name, str(path)) + spec = importlib.util.spec_from_file_location(module_name, str(path), loader=loader) + if spec is None: + raise ImportError(f"cannot make a spec for {path}") + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py new file mode 100644 index 000000000..6afe94196 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -0,0 +1,1065 @@ +"""TypedPython C back end: `ir.Module` -> one C source file (a CPython extension, PEP 489). + +The contract is `ir.py`; this file implements each node exactly as its docstring says, and reaches +anything with Python semantics only through the runtime helpers of `runtime/API.md` +(`tp_runtime.h`). What the generated file contains: + +Module shape (ir.py "Module shape"). Multi-phase init. The exec slot runs the module's ORIGINAL +source (read at build time and embedded as a C string literal, compiled with its own path as the +filename) in the extension module's own dict, so every name, class and global is the interpreted +one. It then stores each compiled function's interpreted original in `__typedpython_interpreted__` +(a dict), sets `__typedpython_deopts__ = 0`, and replaces the global `f` with a C function +(METH_FASTCALL | METH_KEYWORDS) bound to the module. + +Per function `f`: + `static int tp_impl_f(PyObject *tp_module, , *tp_out)` returns 0 ok, 1 deopt, + -1 error. Scalars are passed by value, OBJ params borrowed (the impl takes its own reference), + array params as `tp_*_array *`. + `tp_wrap_f` is the Python-visible function: wrong arity or any keyword -> the interpreted + function (CPython raises its own TypeError); entry guards (exact scalar types through + tp_unbox_*, the array aliasing guard tp_any_same, then tp_*_array_enter); the impl; then + tp_*_array_exit on every path (ok, error, deopt) before returning or redoing. A deopt (failed + guard, or the impl's 1 in a pure function) increments `__typedpython_deopts__` and calls + `__typedpython_interpreted__[f]` with the original arguments. + +Ownership rule (ir.py "Safety"), implemented by the `_own_*` emit helpers below and nowhere else: + 1. every OBJ local — including each OBJ parameter, which the impl copies with a new reference at + entry — owns one strong reference, or is NULL; + 2. every OBJ-typed expression evaluates into a fresh temporary that owns a new reference; + temporaries are NULL at every statement boundary; + 3. a temporary is consumed exactly once: moved into a local (the local's old reference is + released), moved into `*tp_out` by Return, or released (`tp_release`) right after the helper + that borrowed it returns; + 4. every exit of the impl — return, error, deopt — goes through one exit block that releases + every OBJ local and temporary. + The generated code never decrements a reference except through `tp_release` (API.md); the only + increment is `Py_NewRef` when an OBJ local is read or an OBJ parameter is adopted. + +Generated arithmetic is in separate C statements and the file sets `#pragma STDC FP_CONTRACT OFF` +(cbuild also passes -ffp-contract=off): a fused multiply-add would round differently from CPython. +""" +from __future__ import annotations + +from pathlib import Path + +from typedpython import ir +from typedpython.ir import Type + + +class CGenError(Exception): + """The IR cannot be lowered as specified (a contract violation by the producer of the IR).""" + + +_CT = {Type.I64: "int64_t", Type.F64: "double", Type.BOOL: "int", Type.OBJ: "PyObject *"} +_ARRAY_CT = {Type.F64_ARRAY: "tp_f64_array", Type.I64_ARRAY: "tp_i64_array"} +_ARRAY_ELEM = {Type.F64_ARRAY: Type.F64, Type.I64_ARRAY: Type.I64} +_ARRAY_PFX = {Type.F64_ARRAY: "tp_f64_array", Type.I64_ARRAY: "tp_i64_array"} +_UNBOX = {Type.I64: "tp_unbox_i64", Type.F64: "tp_unbox_f64", Type.BOOL: "tp_unbox_bool"} +_BOX = {Type.I64: "tp_box_i64", Type.F64: "tp_box_f64", Type.BOOL: "tp_box_bool"} +_PYCMP = {ir.CompareKind.EQ: "Py_EQ", ir.CompareKind.NE: "Py_NE", ir.CompareKind.LT: "Py_LT", + ir.CompareKind.LE: "Py_LE", ir.CompareKind.GT: "Py_GT", ir.CompareKind.GE: "Py_GE"} +_I64_OP = {ir.BinOpKind.ADD: "tp_add_i64", ir.BinOpKind.SUB: "tp_sub_i64", + ir.BinOpKind.MUL: "tp_mul_i64", ir.BinOpKind.FLOORDIV: "tp_floordiv_i64", + ir.BinOpKind.MOD: "tp_mod_i64", ir.BinOpKind.TRUEDIV: "tp_truediv_i64"} +_F64_OP = {ir.BinOpKind.TRUEDIV: "tp_truediv_f64", ir.BinOpKind.FLOORDIV: "tp_floordiv_f64", + ir.BinOpKind.MOD: "tp_mod_f64"} +_F64_INLINE = {ir.BinOpKind.ADD: "+", ir.BinOpKind.SUB: "-", ir.BinOpKind.MUL: "*"} +_OBJ_OP = {k: f"TP_BINOP_{k.name}" for k in ir.BinOpKind} +_MATH = {ir.MathFunc.SQRT: 1, ir.MathFunc.EXP: 1, ir.MathFunc.LOG: 1, ir.MathFunc.SIN: 1, + ir.MathFunc.COS: 1, ir.MathFunc.TAN: 1, ir.MathFunc.FABS: 1, ir.MathFunc.ATAN2: 2, + ir.MathFunc.HYPOT: 2} +# I64 operations whose helper can return 1 (API.md): MOD never overflows (MIN % -1 == 0). +_I64_DEOPT = {ir.BinOpKind.ADD, ir.BinOpKind.SUB, ir.BinOpKind.MUL, ir.BinOpKind.FLOORDIV, + ir.BinOpKind.TRUEDIV} +_INT64_MIN, _INT64_MAX = -(2 ** 63), 2 ** 63 - 1 + + +# --- C literals and identifiers ------------------------------------------------------------------ + +def c_string(data: str | bytes) -> str: + """A C string literal for `data` (UTF-8). Octal escapes only (hex escapes are greedy), `?` + escaped against trigraphs; split after each newline so long sources stay readable.""" + raw = data.encode("utf-8") if isinstance(data, str) else data + out, pieces = [], [] + for b in raw: + ch = chr(b) + if ch in '\\"?': + out.append("\\" + ch) + elif 0x20 <= b < 0x7F: + out.append(ch) + else: + out.append("\\%03o" % b) + if b == 0x0A: + pieces.append('"' + "".join(out) + '"') + out = [] + if out or not pieces: + pieces.append('"' + "".join(out) + '"') + return "\n ".join(pieces) + + +def _ident(prefix: str, name: str) -> str: + """A C identifier for a Python name: ASCII names keep their spelling, others are hex-encoded + under a distinct prefix so the two forms cannot collide.""" + if name.isascii() and name.isidentifier(): + return f"{prefix}_{name}" + return f"{prefix}x_{name.encode('utf-8').hex()}" + + +def _i64_lit(v: int) -> str: + if isinstance(v, bool) or not isinstance(v, int) or not (_INT64_MIN <= v <= _INT64_MAX): + raise CGenError(f"I64 constant out of range or not an int: {v!r}") + if v == _INT64_MIN: + return "(-INT64_C(9223372036854775807) - 1)" + return f"INT64_C({v})" + + +def _f64_lit(v: float) -> str: + v = float(v) + if v != v: + return "(-Py_NAN)" if _sign_bit(v) else "Py_NAN" + if v in (float("inf"), float("-inf")): + return "Py_HUGE_VAL" if v > 0 else "(-Py_HUGE_VAL)" + return f"({v.hex()})" + + +def _sign_bit(v: float) -> bool: + import struct + return struct.pack(">d", v)[0] >> 7 == 1 + + +# --- IR walks ------------------------------------------------------------------------------------ + +def _exprs_of_stmt(s: ir.Stmt): + if isinstance(s, ir.Assign): + yield s.value + elif isinstance(s, ir.StoreIndex): + yield s.value + yield s.index + elif isinstance(s, ir.ExprStmt): + yield s.value + elif isinstance(s, ir.If): + yield s.cond + elif isinstance(s, ir.While): + yield s.cond + elif isinstance(s, ir.ForRange): + yield s.start + yield s.stop + elif isinstance(s, ir.Return) and s.value is not None: + yield s.value + + +def _children(s: ir.Stmt): + if isinstance(s, ir.If): + return s.then + s.orelse + if isinstance(s, (ir.While, ir.ForRange)): + return s.body + return () + + +def _walk_stmts(body): + for s in body: + yield s + yield from _walk_stmts(_children(s)) + + +def _sub_exprs(e: ir.Expr): + for name in ("left", "right", "operand", "index", "obj", "callee"): + v = getattr(e, name, None) + if isinstance(v, ir.Expr): + yield v + for v in getattr(e, "args", ()) or (): + yield v + + +def _walk_exprs(e: ir.Expr): + yield e + for c in _sub_exprs(e): + yield from _walk_exprs(c) + + +def _all_exprs(f: ir.Function): + for s in _walk_stmts(f.body): + for e in _exprs_of_stmt(s): + yield from _walk_exprs(e) + + +def _can_deopt(e: ir.Expr, functions: dict[str, ir.Function]) -> bool: + if isinstance(e, ir.BinOp) and e.left.type is Type.I64 and e.op in _I64_DEOPT: + return True + if isinstance(e, ir.UnaryOp) and e.op is ir.UnaryOpKind.NEG and e.operand.type is Type.I64: + return True + if isinstance(e, ir.Unbox): + return True + if isinstance(e, ir.Call): + callee = functions.get(e.function) + return callee is not None and callee.may_deopt + return False + + +def _is_effect(e: ir.Expr) -> bool: + if isinstance(e, (ir.GetAttr, ir.CallObject, ir.Truth, ir.CompareObj, ir.ObjToFloat)): + return True + return isinstance(e, ir.BinOp) and e.type is Type.OBJ + + +# --- module -------------------------------------------------------------------------------------- + +class _ModGen: + def __init__(self, module: ir.Module): + self.module = module + self.functions = {f.name: f for f in module.functions} + if len(self.functions) != len(module.functions): + raise CGenError("duplicate compiled function names") + self.names: list[str] = [] + self.kwtuples: list[tuple[str, ...]] = [] + + def name_index(self, s: str) -> int: + if s not in self.names: + self.names.append(s) + return self.names.index(s) + + def kw_index(self, kw: tuple[str, ...]) -> int: + for k in kw: + self.name_index(k) + if kw not in self.kwtuples: + self.kwtuples.append(kw) + return self.kwtuples.index(kw) + + +def generate(module: ir.Module, source_path: Path) -> str: + """One C file implementing `module`; `source_path` is the original Python source, embedded.""" + source_path = Path(source_path) + source = source_path.read_text(encoding="utf-8") + mg = _ModGen(module) + for f in module.functions: + _check_function(f, mg.functions) + for f in module.functions: + mg.name_index(f.name) + bodies = [_FnGen(mg, f).generate() for f in module.functions] + init_name = module.name.rpartition(".")[2] + if not (init_name.isascii() and init_name.isidentifier()): + raise CGenError(f"module name {module.name!r} cannot name a PyInit_ function") + + out: list[str] = [] + w = out.append + w(f"/* Generated by typedpython.cgen from {source_path.name} — do not edit. */") + w("#define PY_SSIZE_T_CLEAN") + w("#include ") + w("#include ") + w('#include "tp_runtime.h"') + w("") + w("#if PY_VERSION_HEX < 0x030D0000") + w('#error "typedpython extensions need CPython 3.13 or newer (PyDict_GetItemRef)"') + w("#endif") + w("#pragma STDC FP_CONTRACT OFF") + w("") + w(f"static const char tp_source[] =\n {c_string(source)};") + w(f"static const char tp_source_path[] = {c_string(str(source_path))};") + w("") + n_names, n_kw = len(mg.names), len(mg.kwtuples) + w("/* Module state: interned names (globals, attributes, keywords, function names) and the") + w(" kwnames tuples of keyword calls. Owned by the module; released in tp_clear. */") + w("typedef struct {") + w(f" PyObject *names[{n_names + 1}];") + w(f" PyObject *kwnames[{n_kw + 1}];") + w("} tp_state;") + w("") + w("static const char *const tp_name_strings[] = {") + for s in mg.names: + w(f" {c_string(s)},") + w(" NULL") + w("};") + w("") + for f in module.functions: + w(_impl_signature(f) + ";") + w("") + w(_TEMPLATE_REDO) + for b in bodies: + w(b) + # method table + w("static PyMethodDef tp_methods[] = {") + for f in module.functions: + w(f" {{{c_string(f.name)}, (PyCFunction)(void (*)(void)){_ident('tp_wrap', f.name)}, " + f"METH_FASTCALL | METH_KEYWORDS, NULL}},") + w(" {NULL, NULL, 0, NULL}") + w("};") + w("") + w(_exec_function(mg)) + w(_TEMPLATE_STATE) + w("static PyModuleDef_Slot tp_slots[] = {") + w(" {Py_mod_exec, (void *)tp_exec},") + w(" {0, NULL}") + w("};") + w("") + w("static struct PyModuleDef tp_moduledef = {") + w(f" PyModuleDef_HEAD_INIT, {c_string(module.name)}, NULL, sizeof(tp_state), NULL, tp_slots,") + w(" tp_traverse, tp_clear, tp_free") + w("};") + w("") + w(f"PyMODINIT_FUNC PyInit_{init_name}(void) {{ return PyModuleDef_Init(&tp_moduledef); }}") + return "\n".join(out) + "\n" + + +def _check_function(f: ir.Function, functions: dict[str, ir.Function]) -> None: + deopting = [e for e in _all_exprs(f) if _can_deopt(e, functions)] + if deopting and not f.pure: + raise CGenError(f"{f.name}: a node that can deopt ({type(deopting[0]).__name__}) in an " + "impure function (ir.py: only a pure function may deopt after entry)") + if deopting and not f.may_deopt: + raise CGenError(f"{f.name}: may_deopt is False but a node can deopt " + f"({type(deopting[0]).__name__})") + effects = [e for e in _all_exprs(f) if _is_effect(e)] + stores = [s for s in _walk_stmts(f.body) if isinstance(s, ir.StoreIndex)] + if f.pure and (effects or stores): + what = type(effects[0]).__name__ if effects else "StoreIndex" + raise CGenError(f"{f.name}: marked pure but contains an effect ({what})") + for e in _all_exprs(f): + if isinstance(e, ir.Call): + callee = functions.get(e.function) + if callee is None: + raise CGenError(f"{f.name}: Call of {e.function!r}, which is not compiled here") + if any(isinstance(p, ir.ArrayParam) for p in callee.params): + raise CGenError(f"{f.name}: Call of {e.function!r}, which takes array params") + if callee.may_deopt and not f.pure: + raise CGenError(f"{f.name}: Call of deopting {e.function!r} from an impure caller") + for p in f.params: + if isinstance(p, ir.ArrayParam): + if p.type not in ir.ARRAYS: + raise CGenError(f"{f.name}: array param {p.name} has type {p.type}") + if p.stored and f.pure: + raise CGenError(f"{f.name}: array param {p.name} is stored into in a pure function") + elif p.type not in _CT: + raise CGenError(f"{f.name}: param {p.name} has type {p.type}") + + +def _impl_signature(f: ir.Function) -> str: + params = ["PyObject *tp_module"] + for p in f.params: + if isinstance(p, ir.ArrayParam): + params.append(f"{_ARRAY_CT[p.type]} *{_ident('l', p.name)}") + elif p.type is Type.OBJ: + params.append(f"PyObject *{_ident('p', p.name)}") + else: + params.append(f"{_CT[p.type]} {_ident('l', p.name)}") + if f.returns is not Type.NONE: + ct = _CT[f.returns] + params.append(f"{ct}{'' if ct.endswith('*') else ' '}*tp_out") + return f"static int {_ident('tp_impl', f.name)}({', '.join(params)})" + + +# --- function bodies ----------------------------------------------------------------------------- + +class _FnGen: + def __init__(self, mg: _ModGen, f: ir.Function): + self.mg, self.f = mg, f + self.lines: list[str] = [] + self.depth = 1 + self.temps: list[tuple[str, str]] = [] # (C type, name) + self.arrays = {p.name: p for p in f.params if isinstance(p, ir.ArrayParam)} + self.types: dict[str, Type] = {} + for p in f.params: + if not isinstance(p, ir.ArrayParam): + self.types[p.name] = p.type + for name, t in f.locals.items(): + if name in self.arrays or (name in self.types and self.types[name] is not t): + raise CGenError(f"{f.name}: local {name} conflicts with a parameter") + if t not in _CT: + raise CGenError(f"{f.name}: local {name} has type {t}") + self.types[name] = t + + # emit primitives + + def emit(self, line: str) -> None: + self.lines.append(" " * self.depth + line) + + def tmp(self, ctype: str) -> str: + name = f"tp_t{len(self.temps)}" + self.temps.append((ctype, name)) + return name + + def check(self, call: str) -> None: + """A helper with the 0 / 1 deopt / -1 error convention.""" + self.emit(f"tp_s = {call};") + self.emit("if (tp_s != 0) { tp_rc = tp_s; goto tp_exit; }") + + def fail(self) -> None: + self.emit("{ tp_rc = -1; goto tp_exit; }") + + # ownership helpers — the only places that create or drop references (see module docstring) + + def _own_new(self, call: str) -> str: + """A fresh temporary owning the new reference `call` returns (NULL -> error exit).""" + t = self.tmp("PyObject *") + self.emit(f"{t} = {call};") + self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def _own_copy(self, local: str) -> str: + """A fresh temporary owning a new reference to what an OBJ local holds.""" + t = self.tmp("PyObject *") + self.emit(f"{t} = Py_NewRef({local});") + return t + + def _own_move(self, dst: str, t: str) -> None: + """Move temporary `t` into local `dst`, releasing the reference `dst` held.""" + self.emit(f"{{ PyObject *tp_old = {dst}; {dst} = {t}; {t} = NULL; tp_release(&tp_old); }}") + + def _own_move_out(self, t: str) -> None: + """Move temporary `t` into the caller's `*tp_out` (Return of OBJ).""" + self.emit(f"*tp_out = {t}; {t} = NULL;") + + def _own_release(self, *ts: str) -> None: + for t in ts: + self.emit(f"tp_release(&{t});") + + def _own_release_all(self) -> list[str]: + objs = [_ident("l", n) for n, t in self.types.items() if t is Type.OBJ] + objs += [n for ct, n in self.temps if ct == "PyObject *"] + return [f" tp_release(&{n});" for n in objs] + + # expressions: return a C expression naming the value (a literal, a scalar local, or a temp); + # OBJ results are always owned temporaries. + + def ev(self, e: ir.Expr) -> str: + m = getattr(self, "ev_" + type(e).__name__, None) + if m is None: + raise CGenError(f"{self.f.name}: no lowering for {type(e).__name__}") + return m(e) + + def ev_Const(self, e: ir.Const) -> str: + if e.type is Type.BOOL: + return "1" if e.value else "0" + if e.type is Type.I64: + return _i64_lit(e.value) + if e.type is Type.F64: + return _f64_lit(e.value) + if e.type is Type.OBJ: + if e.value is None: + return self._own_copy("Py_None") + if isinstance(e.value, bool): + return self._own_copy("Py_True" if e.value else "Py_False") + raise CGenError(f"{self.f.name}: OBJ constant {e.value!r} (use Box of a scalar Const)") + raise CGenError(f"{self.f.name}: constant of type {e.type}") + + def ev_Local(self, e: ir.Local) -> str: + t = self.types.get(e.name) + if t is None: + raise CGenError(f"{self.f.name}: unknown local {e.name}") + if t is not e.type: + raise CGenError(f"{self.f.name}: local {e.name} is {t}, read as {e.type}") + if t is Type.OBJ: + return self._own_copy(_ident("l", e.name)) + return _ident("l", e.name) + + def ev_BinOp(self, e: ir.BinOp) -> str: + lt, rt = e.left.type, e.right.type + if lt is not rt: + raise CGenError(f"{self.f.name}: BinOp operands {lt} and {rt}") + a, b = self.ev(e.left), self.ev(e.right) + if lt is Type.I64: + t = self.tmp("double" if e.op is ir.BinOpKind.TRUEDIV else "int64_t") + self.check(f"{_I64_OP[e.op]}({a}, {b}, &{t})") + return t + if lt is Type.F64: + t = self.tmp("double") + if e.op in _F64_INLINE: + self.emit(f"{t} = {a} {_F64_INLINE[e.op]} {b};") + else: + self.check(f"{_F64_OP[e.op]}({a}, {b}, &{t})") + return t + if lt is Type.OBJ: + t = self.tmp("PyObject *") + self.emit(f"{t} = tp_binop_obj({a}, {b}, {_OBJ_OP[e.op]});") + self._own_release(a, b) + self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + return t + raise CGenError(f"{self.f.name}: BinOp on {lt}") + + def ev_UnaryOp(self, e: ir.UnaryOp) -> str: + ot = e.operand.type + v = self.ev(e.operand) + if e.op is ir.UnaryOpKind.NOT: + if ot is not Type.BOOL: + raise CGenError(f"{self.f.name}: NOT of {ot}") + t = self.tmp("int") + self.emit(f"{t} = !({v});") + return t + if ot is Type.I64 and e.op is ir.UnaryOpKind.NEG: + t = self.tmp("int64_t") + self.check(f"tp_neg_i64({v}, &{t})") + return t + if ot is Type.F64 and e.op is ir.UnaryOpKind.NEG: + t = self.tmp("double") + self.emit(f"{t} = -({v});") + return t + if ot in (Type.I64, Type.F64) and e.op is ir.UnaryOpKind.POS: + t = self.tmp(_CT[ot]) + self.emit(f"{t} = {v};") + return t + raise CGenError(f"{self.f.name}: {e.op} of {ot}") + + def ev_Compare(self, e: ir.Compare) -> str: + lt, rt = e.left.type, e.right.type + a, b = self.ev(e.left), self.ev(e.right) + t = self.tmp("int") + if lt is rt and lt in ir.SCALARS: + self.emit(f"{t} = ({a}) {e.op.value} ({b});") + elif (lt, rt) == (Type.I64, Type.F64): + self.emit(f"{t} = tp_cmp_i64_f64({a}, {b}, {_PYCMP[e.op]});") + elif (lt, rt) == (Type.F64, Type.I64): + self.emit(f"{t} = tp_cmp_f64_i64({a}, {b}, {_PYCMP[e.op]});") + else: + raise CGenError(f"{self.f.name}: Compare of {lt} and {rt} (OBJ uses CompareObj)") + return t + + def ev_CompareObj(self, e: ir.CompareObj) -> str: + if e.left.type is not Type.OBJ or e.right.type is not Type.OBJ: + raise CGenError(f"{self.f.name}: CompareObj operands must be OBJ (Box scalars)") + a, b = self.ev(e.left), self.ev(e.right) + t = self.tmp("int") + self.emit(f"{t} = tp_compare_bool({a}, {b}, {_PYCMP[e.op]});") + self._own_release(a, b) + self.emit(f"if ({t} < 0) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def _short_circuit(self, e, take_right_if: str) -> str: + if e.left.type is not Type.BOOL or e.right.type is not Type.BOOL: + raise CGenError(f"{self.f.name}: And/Or on non-BOOL operands") + t = self.tmp("int") + a = self.ev(e.left) + self.emit(f"{t} = {a};") + self.emit(f"if ({take_right_if}{t}) {{") + self.depth += 1 + b = self.ev(e.right) + self.emit(f"{t} = {b};") + self.depth -= 1 + self.emit("}") + return t + + def ev_And(self, e: ir.And) -> str: + return self._short_circuit(e, "") + + def ev_Or(self, e: ir.Or) -> str: + return self._short_circuit(e, "!") + + def ev_Truth(self, e: ir.Truth) -> str: + if e.operand.type is not Type.OBJ: + raise CGenError(f"{self.f.name}: Truth of {e.operand.type}") + o = self.ev(e.operand) + t = self.tmp("int") + self.emit(f"{t} = tp_truth({o});") + self._own_release(o) + self.emit(f"if ({t} < 0) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def ev_ToFloat(self, e: ir.ToFloat) -> str: + if e.operand.type is not Type.I64: + raise CGenError(f"{self.f.name}: ToFloat of {e.operand.type}") + v = self.ev(e.operand) + t = self.tmp("double") + self.emit(f"{t} = tp_i64_to_f64({v});") + return t + + def ev_ObjToFloat(self, e: ir.ObjToFloat) -> str: + if e.operand.type is not Type.OBJ: + raise CGenError(f"{self.f.name}: ObjToFloat of {e.operand.type}") + o = self.ev(e.operand) + t = self.tmp("double") + self.emit(f"tp_s = tp_obj_to_f64({o}, &{t});") + self._own_release(o) + self.emit("if (tp_s != 0) { tp_rc = -1; goto tp_exit; }") + return t + + def ev_Box(self, e: ir.Box) -> str: + ot = e.operand.type + if ot not in _BOX: + raise CGenError(f"{self.f.name}: Box of {ot}") + v = self.ev(e.operand) + return self._own_new(f"{_BOX[ot]}({v})") + + def ev_Unbox(self, e: ir.Unbox) -> str: + if e.operand.type is not Type.OBJ or e.type not in _UNBOX: + raise CGenError(f"{self.f.name}: Unbox {e.operand.type} -> {e.type}") + o = self.ev(e.operand) + t = self.tmp(_CT[e.type]) + self.emit(f"tp_s = {_UNBOX[e.type]}({o}, &{t});") + self._own_release(o) + self.emit("if (tp_s != 0) { tp_rc = tp_s; goto tp_exit; }") + return t + + def ev_MathCall(self, e: ir.MathCall) -> str: + if len(e.args) != _MATH[e.func] or any(a.type is not Type.F64 for a in e.args): + raise CGenError(f"{self.f.name}: math.{e.func.value} arguments") + args = [self.ev(a) for a in e.args] + t = self.tmp("double") + self.check(f"tp_math_{e.func.value}({', '.join(args)}, &{t})") + return t + + def ev_Call(self, e: ir.Call) -> str | None: + callee = self.mg.functions[e.function] + if len(e.args) != len(callee.params): + raise CGenError(f"{self.f.name}: Call of {e.function} with {len(e.args)} args") + args = [] + owned = [] + for p, a in zip(callee.params, e.args): + if a.type is not p.type: + raise CGenError(f"{self.f.name}: Call {e.function} arg {p.name}: {a.type} for {p.type}") + v = self.ev(a) + args.append(v) + if a.type is Type.OBJ: + owned.append(v) + t = None + if callee.returns is not Type.NONE: + t = self.tmp(_CT[callee.returns]) + args.append(f"&{t}") + # C-to-C recursion has no Python frame: guard the C stack as CPython guards its own. + self.emit('if (Py_EnterRecursiveCall(" in compiled code")) { tp_rc = -1; goto tp_exit; }') + self.emit(f"tp_s = {_ident('tp_impl', e.function)}(tp_module{''.join(', ' + x for x in args)});") + self.emit("Py_LeaveRecursiveCall();") + self._own_release(*owned) + self.emit("if (tp_s != 0) { tp_rc = tp_s; goto tp_exit; }") + return t + + def ev_Global(self, e: ir.Global) -> str: + k = self.mg.name_index(e.name) + return self._own_new(f"tp_global(tp_dict, tp_st->names[{k}])") + + def ev_GetAttr(self, e: ir.GetAttr) -> str: + if e.obj.type is not Type.OBJ: + raise CGenError(f"{self.f.name}: GetAttr on {e.obj.type}") + o = self.ev(e.obj) + k = self.mg.name_index(e.name) + t = self.tmp("PyObject *") + self.emit(f"{t} = tp_getattr({o}, tp_st->names[{k}]);") + self._own_release(o) + self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def ev_CallObject(self, e: ir.CallObject) -> str: + if e.callee.type is not Type.OBJ or any(a.type is not Type.OBJ for a in e.args): + raise CGenError(f"{self.f.name}: CallObject operands must be OBJ") + if len(e.kwnames) > len(e.args) or len(set(e.kwnames)) != len(e.kwnames): + raise CGenError(f"{self.f.name}: CallObject kwnames {e.kwnames}") + callee = self.ev(e.callee) # CPython evaluates the callee first + args = [self.ev(a) for a in e.args] + npos = len(e.args) - len(e.kwnames) + kw = f"tp_st->kwnames[{self.mg.kw_index(tuple(e.kwnames))}]" if e.kwnames else "NULL" + t = self.tmp("PyObject *") + if args: + self.emit(f"{{ PyObject *tp_av[{len(args)}] = {{{', '.join(args)}}};") + self.emit(f" {t} = tp_call({callee}, tp_av, {npos}, {kw}); }}") + else: + self.emit(f"{t} = tp_call({callee}, NULL, 0, NULL);") + self._own_release(callee, *args) + self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def _array(self, name: str) -> ir.ArrayParam: + a = self.arrays.get(name) + if a is None: + raise CGenError(f"{self.f.name}: {name} is not an array parameter") + return a + + def ev_Len(self, e: ir.Len) -> str: + self._array(e.array) + t = self.tmp("int64_t") + self.emit(f"{t} = (int64_t){_ident('l', e.array)}->len;") + return t + + def _slot(self, a: ir.ArrayParam, index: ir.Expr, proven: bool, store: int) -> str: + if index.type is not Type.I64: + raise CGenError(f"{self.f.name}: index of type {index.type}") + i = self.ev(index) + if proven: + return f"(Py_ssize_t)({i})" + slot = self.tmp("Py_ssize_t") + self.check(f"{_ARRAY_PFX[a.type]}_slot({_ident('l', a.name)}, {i}, {store}, &{slot})") + return slot + + def ev_Index(self, e: ir.Index) -> str: + a = self._array(e.array) + if e.type is not _ARRAY_ELEM[a.type]: + raise CGenError(f"{self.f.name}: Index of {a.type} typed {e.type}") + slot = self._slot(a, e.index, e.proven, 0) + t = self.tmp(_CT[e.type]) + self.emit(f"{t} = {_ident('l', a.name)}->data[{slot}];") + return t + + # statements + + def stmts(self, body) -> None: + for s in body: + m = getattr(self, "st_" + type(s).__name__, None) + if m is None: + raise CGenError(f"{self.f.name}: no lowering for {type(s).__name__}") + m(s) + + def st_Assign(self, s: ir.Assign) -> None: + t = self.types.get(s.target) + if t is None: + raise CGenError(f"{self.f.name}: assignment to unknown local {s.target}") + if s.value.type is not t: + raise CGenError(f"{self.f.name}: {s.target} is {t}, assigned {s.value.type}") + v = self.ev(s.value) + if t is Type.OBJ: + self._own_move(_ident("l", s.target), v) + else: + self.emit(f"{_ident('l', s.target)} = {v};") + + def st_StoreIndex(self, s: ir.StoreIndex) -> None: + a = self._array(s.array) + if not a.stored: + raise CGenError(f"{self.f.name}: store into {a.name}, declared stored=False") + if s.value.type is not _ARRAY_ELEM[a.type]: + raise CGenError(f"{self.f.name}: store of {s.value.type} into {a.type}") + v = self.ev(s.value) # CPython: value, then container, then index + slot = self._slot(a, s.index, s.proven, 1) + arr = _ident("l", a.name) + self.emit(f"{arr}->data[{slot}] = {v};") + self.emit(f"{arr}->dirty[{slot}] = 1;") + + def st_ExprStmt(self, s: ir.ExprStmt) -> None: + v = self.ev(s.value) + if v is None: + return + if s.value.type is Type.OBJ: + self._own_release(v) + else: + self.emit(f"(void){v};") + + def _cond(self, e: ir.Expr) -> str: + if e.type is not Type.BOOL: + raise CGenError(f"{self.f.name}: condition of type {e.type} (use Truth/CompareObj)") + return self.ev(e) + + def st_If(self, s: ir.If) -> None: + c = self._cond(s.cond) + self.emit(f"if ({c}) {{") + self.depth += 1 + self.stmts(s.then) + self.depth -= 1 + if s.orelse: + self.emit("} else {") + self.depth += 1 + self.stmts(s.orelse) + self.depth -= 1 + self.emit("}") + + def st_While(self, s: ir.While) -> None: + self.emit("for (;;) {") + self.depth += 1 + c = self._cond(s.cond) + self.emit(f"if (!({c})) break;") + self.stmts(s.body) + self.depth -= 1 + self.emit("}") + + def st_ForRange(self, s: ir.ForRange) -> None: + if self.types.get(s.var) is not Type.I64: + raise CGenError(f"{self.f.name}: range variable {s.var} must be an I64 local") + if not isinstance(s.step, ir.Const) or s.step.type is not Type.I64 or s.step.value == 0: + raise CGenError(f"{self.f.name}: range step must be a non-zero I64 Const") + if s.start.type is not Type.I64 or s.stop.type is not Type.I64: + raise CGenError(f"{self.f.name}: range bounds must be I64") + step = s.step.value + _i64_lit(step) + ts, te = self.tmp("int64_t"), self.tmp("int64_t") + a = self.ev(s.start) + self.emit(f"{ts} = {a};") # range() evaluates its arguments once + b = self.ev(s.stop) + self.emit(f"{te} = {b};") + tn, tk = self.tmp("uint64_t"), self.tmp("uint64_t") + mag = f"UINT64_C({abs(step)})" + var = _ident("l", s.var) + # Iteration count computed in unsigned arithmetic, as range's length: no overflow for any + # start/stop/step, and the variable stays between start and stop. + if step > 0: + self.emit(f"if ({ts} < {te}) {{") + self.emit(f" {tn} = ((uint64_t){te} - (uint64_t){ts} - 1u) / {mag} + 1u;") + sign = "+" + else: + self.emit(f"if ({ts} > {te}) {{") + self.emit(f" {tn} = ((uint64_t){ts} - (uint64_t){te} - 1u) / {mag} + 1u;") + sign = "-" + self.depth += 1 + self.emit(f"for ({tk} = 0; {tk} < {tn}; {tk}++) {{") + self.depth += 1 + self.emit(f"{var} = (int64_t)((uint64_t){ts} {sign} {tk} * {mag});") + self.stmts(s.body) + self.depth -= 1 + self.emit("}") + self.depth -= 1 + self.emit("}") + + def st_Return(self, s: ir.Return) -> None: + rt = self.f.returns + if rt is Type.NONE: + if s.value is not None and not (isinstance(s.value, ir.Const) and s.value.value is None): + raise CGenError(f"{self.f.name}: returns None but Return has a value") + self.emit("tp_rc = 0; goto tp_exit;") + return + if s.value is None or s.value.type is not rt: + raise CGenError(f"{self.f.name}: Return type mismatch (function returns {rt})") + v = self.ev(s.value) + if rt is Type.OBJ: + self._own_move_out(v) + else: + self.emit(f"*tp_out = {v};") + self.emit("tp_rc = 0; goto tp_exit;") + + def st_Break(self, s) -> None: + self.emit("break;") + + def st_Continue(self, s) -> None: + self.emit("continue;") + + # whole function + + def generate(self) -> str: + f = self.f + prologue = [] + for p in f.params: + if not isinstance(p, ir.ArrayParam) and p.type is Type.OBJ: + # adopt the borrowed argument: the local owns its own reference (rule 1) + prologue.append(f" {_ident('l', p.name)} = Py_NewRef({_ident('p', p.name)});") + self.stmts(f.body) + if f.returns is Type.NONE: + self.emit("tp_rc = 0;") + else: + self.emit(f'PyErr_SetString(PyExc_SystemError, {c_string("compiled function " + f.name + " ended without a return")});') + self.emit("tp_rc = -1;") + self.emit("goto tp_exit;") + + out = [_impl_signature(f), "{", + " int tp_rc = 0;", + " int tp_s = 0;", + " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);", + " PyObject *tp_dict = PyModule_GetDict(tp_module); /* borrowed: owned by the module */"] + params = {p.name for p in f.params} + for name, t in self.types.items(): + if name in params and t is not Type.OBJ: + continue + init = "NULL" if t is Type.OBJ else "0" + ct = _CT[t] + out.append(f" {ct}{'' if ct.endswith('*') else ' '}{_ident('l', name)} = {init};") + for ct, n in self.temps: + init = "NULL" if ct == "PyObject *" else "0" + sep = "" if ct.endswith("*") else " " + out.append(f" {ct}{sep}{n} = {init};") + out.append(" (void)tp_st; (void)tp_dict; (void)tp_s;" + + "".join(f" (void){_ident('l', n)};" for n in self.types if n not in params)) + out += prologue + out += self.lines + out.append("tp_exit:") + out += self._own_release_all() + out.append(" return tp_rc;") + out.append("}") + out.append("") + out.append(self.wrapper()) + return "\n".join(out) + + def wrapper(self) -> str: + f = self.f + name_k = self.mg.name_index(f.name) + n = len(f.params) + L = [f"static PyObject *{_ident('tp_wrap', f.name)}(PyObject *tp_module, PyObject *const *tp_args, " + "Py_ssize_t tp_nargs, PyObject *tp_kwnames)", "{", + " int tp_s = 0, tp_x = 0;"] + if f.returns not in (Type.NONE,): + init = "NULL" if f.returns is Type.OBJ else "0" + L.append(f" {_CT[f.returns]} tp_r = {init};") + args = [] + arrays = [] + for i, p in enumerate(f.params): + if isinstance(p, ir.ArrayParam): + L.append(f" {_ARRAY_CT[p.type]} tp_a{i} = {{0}};") + args.append(f"&tp_a{i}") + arrays.append((i, p)) + elif p.type is Type.OBJ: + args.append(f"tp_args[{i}]") # borrowed for the call's duration + else: + L.append(f" {_CT[p.type]} tp_a{i} = 0;") + args.append(f"tp_a{i}") + if f.returns is not Type.NONE: + args.append("&tp_r") + exits = [f" if ({_ARRAY_PFX[p.type]}_exit(&tp_a{i}) < 0) tp_x = -1;" for i, p in arrays] + L.append(" (void)tp_x;") + # arity / keywords -> the interpreted function raises CPython's own TypeError + L.append(f" if (tp_nargs != {n} || (tp_kwnames != NULL && PyTuple_GET_SIZE(tp_kwnames) != 0)) goto tp_deopt;") + # scalar guards: exact type (and i64 range) + for i, p in enumerate(f.params): + if not isinstance(p, ir.ArrayParam) and p.type in _UNBOX: + L.append(f" tp_s = {_UNBOX[p.type]}(tp_args[{i}], &tp_a{i});") + L.append(" if (tp_s < 0) goto tp_error;") + L.append(" if (tp_s != 0) goto tp_deopt;") + # aliasing guard, then copy-in + if len(arrays) > 1: + objs = ", ".join(f"tp_args[{i}]" for i, _ in arrays) + L.append(f" {{ PyObject *const tp_objs[{len(arrays)}] = {{{objs}}};") + L.append(f" if (tp_any_same(tp_objs, {len(arrays)})) goto tp_deopt; }}") + for i, p in arrays: + L.append(f" tp_s = {_ARRAY_PFX[p.type]}_enter(tp_args[{i}], &tp_a{i});") + L.append(" if (tp_s < 0) goto tp_error;") + L.append(" if (tp_s != 0) goto tp_deopt;") + L.append(f" tp_s = {_ident('tp_impl', f.name)}(tp_module{''.join(', ' + a for a in args)});") + if f.pure: + L.append(" if (tp_s == 1) goto tp_deopt;") + else: + # cgen proved there is no deopting node in an impure function; never redo after effects + L.append(" if (tp_s == 1) { PyErr_SetString(PyExc_SystemError, " + "\"typedpython: deopt after an effect\"); goto tp_error; }") + L.append(" if (tp_s < 0) goto tp_error;") + # ok: write back, then box + L += exits + if f.returns is Type.OBJ: + L.append(" if (tp_x < 0) { tp_release(&tp_r); return NULL; }") + L.append(" return tp_r;") + else: + L.append(" if (tp_x < 0) return NULL;") + if f.returns is Type.NONE: + L.append(" return Py_NewRef(Py_None);") + else: + L.append(f" return {_BOX[f.returns]}(tp_r);") + L.append("tp_error:") + if arrays: + # keep CPython's exception; write-back still runs (partial updates stay visible) + L.append(" { PyObject *tp_exc = PyErr_GetRaisedException();") + L += [" " + e for e in exits] + L.append(" if (tp_x < 0) { PyObject *tp_exc2 = PyErr_GetRaisedException();") + L.append(" PyException_SetContext(tp_exc2, tp_exc); /* steals tp_exc */") + L.append(" PyErr_SetRaisedException(tp_exc2); }") + L.append(" else PyErr_SetRaisedException(tp_exc); }") + L.append(" return NULL;") + L.append("tp_deopt:") + L += exits + L.append(" if (tp_x < 0) return NULL;") + L.append(f" return tp_cg_redo(tp_module, {name_k}, tp_args, tp_nargs, tp_kwnames);") + L.append("}") + L.append("") + return "\n".join(L) + + +# --- fixed parts of the module ------------------------------------------------------------------- + +_TEMPLATE_REDO = r''' +/* Deopt: count it, then call __typedpython_interpreted__[name] with the original arguments. */ +static PyObject *tp_cg_redo(PyObject *tp_module, int tp_name, PyObject *const *tp_args, + Py_ssize_t tp_nargs, PyObject *tp_kwnames) +{ + tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module); + PyObject *tp_dict = PyModule_GetDict(tp_module); /* borrowed: owned by the module */ + PyObject *tp_count = NULL, *tp_one = NULL, *tp_next = NULL, *tp_table = NULL, *tp_f = NULL; + PyObject *tp_result = NULL; + int tp_k; + tp_k = PyDict_GetItemStringRef(tp_dict, "__typedpython_deopts__", &tp_count); + if (tp_k == 0) PyErr_SetString(PyExc_RuntimeError, "typedpython: __typedpython_deopts__ is missing"); + if (tp_k <= 0) goto tp_done; + tp_one = PyLong_FromLong(1); + if (tp_one == NULL) goto tp_done; + tp_next = PyNumber_Add(tp_count, tp_one); + if (tp_next == NULL || PyDict_SetItemString(tp_dict, "__typedpython_deopts__", tp_next) < 0) goto tp_done; + tp_k = PyDict_GetItemStringRef(tp_dict, "__typedpython_interpreted__", &tp_table); + if (tp_k == 0) PyErr_SetString(PyExc_RuntimeError, "typedpython: __typedpython_interpreted__ is missing"); + if (tp_k <= 0) goto tp_done; + if (!PyDict_Check(tp_table)) { + PyErr_SetString(PyExc_TypeError, "typedpython: __typedpython_interpreted__ is not a dict"); + goto tp_done; + } + tp_k = PyDict_GetItemRef(tp_table, tp_st->names[tp_name], &tp_f); + if (tp_k == 0) PyErr_Format(PyExc_RuntimeError, "typedpython: no interpreted function %R", tp_st->names[tp_name]); + if (tp_k <= 0) goto tp_done; + tp_result = PyObject_Vectorcall(tp_f, tp_args, (size_t)tp_nargs, tp_kwnames); +tp_done: + tp_release(&tp_count); tp_release(&tp_one); tp_release(&tp_next); + tp_release(&tp_table); tp_release(&tp_f); + return tp_result; +} +''' + +_TEMPLATE_STATE = r''' +static int tp_traverse(PyObject *tp_module, visitproc visit, void *arg) +{ + tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module); + size_t i; + if (tp_st == NULL) return 0; + for (i = 0; i < sizeof(tp_st->names) / sizeof(tp_st->names[0]); i++) Py_VISIT(tp_st->names[i]); + for (i = 0; i < sizeof(tp_st->kwnames) / sizeof(tp_st->kwnames[0]); i++) Py_VISIT(tp_st->kwnames[i]); + return 0; +} + +static int tp_clear(PyObject *tp_module) +{ + tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module); + size_t i; + if (tp_st == NULL) return 0; + for (i = 0; i < sizeof(tp_st->names) / sizeof(tp_st->names[0]); i++) tp_release(&tp_st->names[i]); + for (i = 0; i < sizeof(tp_st->kwnames) / sizeof(tp_st->kwnames[0]); i++) tp_release(&tp_st->kwnames[i]); + return 0; +} + +static void tp_free(void *tp_module) { (void)tp_clear((PyObject *)tp_module); } +''' + + +def _exec_function(mg: _ModGen) -> str: + L = [ + "/* Exec slot: run the original source in this module's dict, then swap in the C functions. */", + "static int tp_exec(PyObject *tp_module)", + "{", + " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);", + " PyObject *tp_dict = PyModule_GetDict(tp_module); /* borrowed: owned by the module */", + " PyObject *tp_code = NULL, *tp_res = NULL, *tp_table = NULL, *tp_zero = NULL;", + " PyObject *tp_modname = NULL, *tp_f = NULL, *tp_cf = NULL;", + " int tp_rc = -1, tp_k;", + " size_t i;", + " for (i = 0; tp_name_strings[i] != NULL; i++) {", + " tp_st->names[i] = PyUnicode_InternFromString(tp_name_strings[i]);", + " if (tp_st->names[i] == NULL) goto tp_done;", + " }", + ] + for k, kw in enumerate(mg.kwtuples): + items = ", ".join(f"tp_st->names[{mg.name_index(x)}]" for x in kw) + L.append(f" tp_st->kwnames[{k}] = PyTuple_Pack({len(kw)}, {items});") + L.append(f" if (tp_st->kwnames[{k}] == NULL) goto tp_done;") + L += [ + " tp_k = PyDict_ContainsString(tp_dict, \"__builtins__\");", + " if (tp_k < 0) goto tp_done;", + " if (tp_k == 0) {", + " PyObject *tp_b = PyImport_ImportModule(\"builtins\");", + " if (tp_b == NULL) goto tp_done;", + " tp_k = PyDict_SetItemString(tp_dict, \"__builtins__\", PyModule_GetDict(tp_b));", + " tp_release(&tp_b);", + " if (tp_k < 0) goto tp_done;", + " }", + " tp_code = Py_CompileString(tp_source, tp_source_path, Py_file_input);", + " if (tp_code == NULL) goto tp_done;", + " tp_res = PyEval_EvalCode(tp_code, tp_dict, tp_dict);", + " if (tp_res == NULL) goto tp_done;", + " tp_table = PyDict_New();", + " if (tp_table == NULL || PyDict_SetItemString(tp_dict, \"__typedpython_interpreted__\", tp_table) < 0) goto tp_done;", + " tp_zero = PyLong_FromLong(0);", + " if (tp_zero == NULL || PyDict_SetItemString(tp_dict, \"__typedpython_deopts__\", tp_zero) < 0) goto tp_done;", + " tp_modname = PyModule_GetNameObject(tp_module);", + " if (tp_modname == NULL) goto tp_done;", + ] + for j, f in enumerate(mg.module.functions): + k = mg.name_index(f.name) + L += [ + f" tp_k = PyDict_GetItemRef(tp_dict, tp_st->names[{k}], &tp_f);", + f" if (tp_k == 0) PyErr_Format(PyExc_ImportError, \"typedpython: the module source does not define %R\", tp_st->names[{k}]);", + " if (tp_k <= 0) goto tp_done;", + f" if (PyDict_SetItem(tp_table, tp_st->names[{k}], tp_f) < 0) goto tp_done;", + f" tp_cf = PyCFunction_NewEx(&tp_methods[{j}], tp_module, tp_modname);", + f" if (tp_cf == NULL || PyDict_SetItem(tp_dict, tp_st->names[{k}], tp_cf) < 0) goto tp_done;", + " tp_release(&tp_f); tp_release(&tp_cf);", + ] + L += [ + " tp_rc = 0;", + "tp_done:", + " tp_release(&tp_code); tp_release(&tp_res); tp_release(&tp_table); tp_release(&tp_zero);", + " tp_release(&tp_modname); tp_release(&tp_f); tp_release(&tp_cf);", + " return tp_rc;", + "}", + "", + ] + return "\n".join(L) diff --git a/python-multiplatform-ksp/src/test/python/test_cgen.py b/python-multiplatform-ksp/src/test/python/test_cgen.py new file mode 100644 index 000000000..f36343614 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_cgen.py @@ -0,0 +1,847 @@ +"""TypedPython C back end: typed IR -> C source -> built CPython extension (#41). + +The IR is built by hand here (no front end). Every end-to-end test compares the compiled function +with the interpreted one that the extension keeps in `__typedpython_interpreted__`, because the +contract (ir.py, INTENT §1.4) is "compiled code computes what CPython computes". + +End-to-end tests need the C runtime header `tp_runtime.h`. It is looked up in +`$TYPEDPYTHON_RUNTIME_DIR` when that is set, otherwise in the package's own `runtime/` directory; +when it is missing those tests skip with that reason (a pre-implementation skip, not a regression). +Generation-only tests (`test_gen_*`) never need it. + +Every build happens inside the caller's `heavy.sh` lock (the whole pytest process runs under it); +build output goes to the worktree's `.tmp/cgen-tests/`, never to /tmp. +""" +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import sys +import sysconfig +import textwrap +from pathlib import Path + +import pytest + +from typedpython import ir +from typedpython.ir import ( + And, ArrayParam, Assign, BinOp, BinOpKind, Box, Call, Compare, CompareKind, Const, ExprStmt, + ForRange, Function, If, Index, Len, Local, Module, Param, Return, StoreIndex, Type, + Global, GetAttr, CallObject, Truth, CompareObj, ObjToFloat, + UnaryOp, UnaryOpKind, Unbox, While, Break, Continue, ToFloat, MathCall, MathFunc, +) + +PACKAGE = Path(__file__).resolve().parents[2] / "main" / "python" / "typedpython" +WORKTREE = Path(__file__).resolve().parents[4] +OUT_ROOT = WORKTREE / ".tmp" / "cgen-tests" + + +def runtime_dir() -> Path: + env = os.environ.get("TYPEDPYTHON_RUNTIME_DIR") + return Path(env) if env else PACKAGE / "runtime" + + +def need_runtime(): + header = runtime_dir() / "tp_runtime.h" + if not header.is_file(): + pytest.skip(f"C runtime header not available yet: {header} does not exist " + "(set TYPEDPYTHON_RUNTIME_DIR to the tp-runtime worktree's runtime/ dir)") + text = header.read_text() + missing = [h for h in OBJ_HELPERS if not re.search(rf"\b{h}\s*\(", text)] + if missing: + pytest.skip(f"{header} does not define the opaque-object helpers yet: {missing}") + + +# --- IR shorthands ------------------------------------------------------------------------------- + +I64, F64, BOOL, OBJ, NONE = Type.I64, Type.F64, Type.BOOL, Type.OBJ, Type.NONE + + +def c(v, t=None): + if t is None: + t = BOOL if isinstance(v, bool) else I64 if isinstance(v, int) else F64 + return Const(t, v) + + +def L(name, t): + return Local(t, name) + + +def binop(op, a, b, t=None): + if t is None: + t = F64 if (op is BinOpKind.TRUEDIV) else a.type + return BinOp(t, op, a, b) + + +def call(name, *args, kwnames=()): + """`name(*args)` for a module global (a binder function is an ordinary global).""" + return CallObject(OBJ, Global(OBJ, name), tuple(args), tuple(kwnames)) + + +def fn(name, params, returns, locals_, body, *, pure, may_deopt, line=1): + return Function(name=name, params=tuple(params), returns=returns, locals=dict(locals_), + body=tuple(body), pure=pure, may_deopt=may_deopt, source_line=line) + + +# The module source that every e2e module embeds; each test picks the functions it compiles. +SOURCE = textwrap.dedent('''\ + """typedpython test module — the interpreted reference.""" + calls = [] + + def record(x): + calls.append(x) + return x + + def poly(x: float, n: int) -> float: + acc = 0.0 + for i in range(n): + acc = acc * x + 0.1 + return acc + + def mul_all(a: int, n: int) -> int: + acc = 1 + for i in range(n): + acc = acc * a + return acc + + def twice_mul(a: int, n: int) -> int: + return mul_all(a, n) * 2 + + def eff(x: int) -> int: + record(x) + return x + + def scale(xs: list[float], k: float, n: int) -> None: + for i in range(n): + xs[i] = xs[i] * k + + def addfirst(dst: list[float], src: list[float]) -> None: + for i in range(len(dst)): + dst[i] = dst[i] + src[0] + + def floordiv(a: int, b: int) -> int: + return a // b + + def objmix(o, p): + q = o + p + q = record(q) + return q + o + + def count_until(limit: int, skip: int) -> int: + total = 0 + i = 0 + while True: + if i >= limit: + break + i = i + 1 + if i == skip: + continue + total = total + i + return total + + def isum(xs: list[int]) -> int: + s = 0 + for i in range(len(xs)): + s = s + xs[i] + return s + + def hyp(x: float, y: float) -> float: + return math.sqrt(x * x + y * y) + + def mixed_lt(a: int, b: float) -> bool: + return a < b and not (b < 0.0) + + def tofl(a: int) -> float: + return float(a) / 3.0 + + def neg_last(xs: list[float]) -> float: + return -xs[-1] + + # --- plain-Python stand-ins for binder proxies (a Kotlin function, a Kotlin object) --- + made = [] + + class Counter: + def __init__(self, scale): + self.scale = scale + self.reads = 0 + + @property + def factor(self): + self.reads += 1 + return self.scale + + def apply(self, v, *, offset=0.0): + return v * self.factor + offset + + def make(scale): + made.append(Counter(scale)) + return made[-1] + + def boom(x): + raise ValueError(f"bad value {x!r}") + + def target(): + return 1 + + def interop(scale: float, n: int) -> float: + obj = make(scale) + acc = 0.0 + for i in range(n): + v = obj.apply(acc, offset=0.5) + acc = float(v) * 0.5 + math.sqrt(acc + 1.0) + if obj.factor > 1.0: + acc = acc + 1.0 + return acc + float(obj.reads) + + def which(): + return target() + + def length(o): + return len(o) + + def calls_boom(o): + return boom(o) + + def truthy(o) -> bool: + if o: + return True + return False + ''') +SOURCE = "import math\n" + SOURCE + + +def f_poly(): + acc, x, n, i = L("acc", F64), L("x", F64), L("n", I64), L("i", I64) + return fn("poly", [Param("x", F64), Param("n", I64)], F64, + {"acc": F64, "i": I64}, + [Assign("acc", c(0.0)), + ForRange("i", c(0), n, c(1), + (Assign("acc", binop(BinOpKind.ADD, binop(BinOpKind.MUL, acc, x), c(0.1))),)), + Return(acc)], + pure=True, may_deopt=False, line=9) + + +def f_mul_all(): + acc, a, n = L("acc", I64), L("a", I64), L("n", I64) + return fn("mul_all", [Param("a", I64), Param("n", I64)], I64, {"acc": I64, "i": I64}, + [Assign("acc", c(1)), + ForRange("i", c(0), n, c(1), (Assign("acc", binop(BinOpKind.MUL, acc, a)),)), + Return(acc)], + pure=True, may_deopt=True) + + +def f_twice_mul(): + call = Call(I64, "mul_all", (L("a", I64), L("n", I64))) + return fn("twice_mul", [Param("a", I64), Param("n", I64)], I64, {}, + [Return(binop(BinOpKind.MUL, call, c(2)))], pure=True, may_deopt=True) + + +def f_eff(): + x = L("x", I64) + return fn("eff", [Param("x", I64)], I64, {}, + [ExprStmt(call("record", Box(OBJ, x))), Return(x)], + pure=False, may_deopt=False) + + +def f_scale(): + xs = "xs" + i = L("i", I64) + return fn("scale", [ArrayParam(xs, Type.F64_ARRAY, stored=True), Param("k", F64), Param("n", I64)], + NONE, {"i": I64}, + [ForRange("i", c(0), L("n", I64), c(1), + (StoreIndex(xs, i, binop(BinOpKind.MUL, Index(F64, xs, i), L("k", F64))),))], + pure=False, may_deopt=False) + + +def f_addfirst(): + i = L("i", I64) + return fn("addfirst", [ArrayParam("dst", Type.F64_ARRAY, stored=True), + ArrayParam("src", Type.F64_ARRAY, stored=False)], + NONE, {"i": I64}, + [ForRange("i", c(0), Len(I64, "dst"), c(1), + (StoreIndex("dst", i, binop(BinOpKind.ADD, Index(F64, "dst", i, proven=True), + Index(F64, "src", c(0)))),))], + pure=False, may_deopt=False) + + +def f_floordiv(): + return fn("floordiv", [Param("a", I64), Param("b", I64)], I64, {}, + [Return(binop(BinOpKind.FLOORDIV, L("a", I64), L("b", I64)))], pure=True, may_deopt=True) + + +def f_objmix(): + o, p, q = L("o", OBJ), L("p", OBJ), L("q", OBJ) + return fn("objmix", [Param("o", OBJ), Param("p", OBJ)], OBJ, {"q": OBJ}, + [Assign("q", binop(BinOpKind.ADD, o, p)), + Assign("q", call("record", q)), + Return(binop(BinOpKind.ADD, q, o))], + pure=False, may_deopt=False) + + +def f_count_until(): + i, total = L("i", I64), L("total", I64) + return fn("count_until", [Param("limit", I64), Param("skip", I64)], I64, {"total": I64, "i": I64}, + [Assign("total", c(0)), Assign("i", c(0)), + While(c(True), ( + If(Compare(BOOL, CompareKind.GE, i, L("limit", I64)), (Break(),)), + Assign("i", binop(BinOpKind.ADD, i, c(1))), + If(Compare(BOOL, CompareKind.EQ, i, L("skip", I64)), (Continue(),)), + Assign("total", binop(BinOpKind.ADD, total, i)), + )), + Return(total)], + pure=True, may_deopt=True) + + +def f_isum(): + s, i = L("s", I64), L("i", I64) + return fn("isum", [ArrayParam("xs", Type.I64_ARRAY, stored=False)], I64, {"s": I64, "i": I64}, + [Assign("s", c(0)), + ForRange("i", c(0), Len(I64, "xs"), c(1), + (Assign("s", binop(BinOpKind.ADD, s, Index(I64, "xs", i, proven=True))),)), + Return(s)], + pure=True, may_deopt=True) + + +def f_hyp(): + x, y = L("x", F64), L("y", F64) + s = binop(BinOpKind.ADD, binop(BinOpKind.MUL, x, x), binop(BinOpKind.MUL, y, y)) + return fn("hyp", [Param("x", F64), Param("y", F64)], F64, {}, + [Return(MathCall(F64, MathFunc.SQRT, (s,)))], pure=True, may_deopt=False) + + +def f_mixed_lt(): + a, b = L("a", I64), L("b", F64) + return fn("mixed_lt", [Param("a", I64), Param("b", F64)], BOOL, {}, + [Return(And(BOOL, Compare(BOOL, CompareKind.LT, a, b), + UnaryOp(BOOL, UnaryOpKind.NOT, Compare(BOOL, CompareKind.LT, b, c(0.0)))))], + pure=True, may_deopt=False) + + +def f_tofl(): + return fn("tofl", [Param("a", I64)], F64, {}, + [Return(binop(BinOpKind.TRUEDIV, ToFloat(F64, L("a", I64)), c(3.0)))], + pure=True, may_deopt=False) + + +def f_neg_last(): + return fn("neg_last", [ArrayParam("xs", Type.F64_ARRAY, stored=False)], F64, {}, + [Return(UnaryOp(F64, UnaryOpKind.NEG, Index(F64, "xs", c(-1))))], + pure=True, may_deopt=False) + + +def f_interop(): + obj, acc, v, i = L("obj", OBJ), L("acc", F64), L("v", OBJ), L("i", I64) + apply = CallObject(OBJ, GetAttr(OBJ, obj, "apply"), (Box(OBJ, acc), Box(OBJ, c(0.5))), ("offset",)) + step = binop(BinOpKind.ADD, binop(BinOpKind.MUL, ObjToFloat(F64, v), c(0.5)), + MathCall(F64, MathFunc.SQRT, (binop(BinOpKind.ADD, acc, c(1.0)),))) + return fn("interop", [Param("scale", F64), Param("n", I64)], F64, + {"obj": OBJ, "acc": F64, "v": OBJ, "i": I64}, + [Assign("obj", call("make", Box(OBJ, L("scale", F64)))), + Assign("acc", c(0.0)), + ForRange("i", c(0), L("n", I64), c(1), ( + Assign("v", apply), + Assign("acc", step), + If(CompareObj(BOOL, CompareKind.GT, GetAttr(OBJ, obj, "factor"), Box(OBJ, c(1.0))), + (Assign("acc", binop(BinOpKind.ADD, acc, c(1.0))),)), + )), + Return(binop(BinOpKind.ADD, acc, ObjToFloat(F64, GetAttr(OBJ, obj, "reads"))))], + pure=False, may_deopt=False) + + +def f_which(): + return fn("which", [], OBJ, {}, [Return(call("target"))], pure=False, may_deopt=False) + + +def f_length(): + return fn("length", [Param("o", OBJ)], OBJ, {}, [Return(call("len", L("o", OBJ)))], + pure=False, may_deopt=False) + + +def f_calls_boom(): + return fn("calls_boom", [Param("o", OBJ)], OBJ, {}, [Return(call("boom", L("o", OBJ)))], + pure=False, may_deopt=False) + + +def f_truthy(): + return fn("truthy", [Param("o", OBJ)], BOOL, {}, + [If(Truth(BOOL, L("o", OBJ)), (Return(c(True)),)), Return(c(False))], + pure=False, may_deopt=False) + + +INTEROP = (f_interop, f_which, f_length, f_calls_boom, f_truthy) +OBJ_HELPERS = ("tp_global", "tp_getattr", "tp_call", "tp_truth", "tp_compare_bool", "tp_obj_to_f64", + "tp_binop_obj", "tp_release") + +ALL = (f_poly, f_mul_all, f_twice_mul, f_eff, f_scale, f_addfirst, f_floordiv, f_objmix, + f_count_until, f_isum, f_hyp, f_mixed_lt, f_tofl, f_neg_last) + INTEROP + + +# --- building ------------------------------------------------------------------------------------- + +def source_file(name: str) -> Path: + d = OUT_ROOT / name + if d.exists(): + shutil.rmtree(d) + d.mkdir(parents=True) + p = d / f"{name}.py" + p.write_text(SOURCE) + return p + + +def build_module(name: str, functions=ALL, flags=None): + """Generate, build and import an extension `name` compiling `functions`.""" + from typedpython import cbuild, cgen + + need_runtime() + src = source_file(name) + module = Module(name=name, functions=tuple(f() for f in functions)) + c_source = cgen.generate(module, src) + kwargs = {} if flags is None else {"flags": flags} + so = cbuild.build(c_source, name, src.parent, runtime_dir=runtime_dir(), **kwargs) + return cbuild.load(name, so) + + +@pytest.fixture(scope="module") +def mod(): + return build_module("tp_cgen_e2e") + + +def interp(m, name): + return m.__typedpython_interpreted__[name] + + +def deopts(m): + return m.__typedpython_deopts__ + + +# --- generation only (no runtime header needed) --------------------------------------------------- + +def gen(functions=ALL, name="tp_gen_only"): + from typedpython import cgen + + src = source_file(name) + return cgen.generate(Module(name=name, functions=tuple(f() for f in functions)), src) + + +def test_gen_has_multiphase_init_and_embeds_the_source(): + text = gen() + assert "PyInit_tp_gen_only" in text and "PyModuleDef_Init" in text and "Py_mod_exec" in text + assert "__typedpython_interpreted__" in text and "__typedpython_deopts__" in text + assert "tp_impl_poly" in text and "METH_FASTCALL" in text + + +def test_gen_rejects_a_deopting_node_in_an_impure_function(): + from typedpython import cgen + + bad = fn("bad", [Param("a", I64)], I64, {}, + [ExprStmt(call("record", Box(OBJ, L("a", I64)))), + Return(binop(BinOpKind.ADD, L("a", I64), c(1)))], + pure=False, may_deopt=False) + with pytest.raises(cgen.CGenError, match="deopt"): + cgen.generate(Module("tp_bad", (bad,)), source_file("tp_bad")) + + +def test_gen_rejects_a_deopting_call_from_an_impure_caller(): + from typedpython import cgen + + caller = fn("caller", [Param("a", I64)], I64, {}, + [ExprStmt(call("record", Box(OBJ, L("a", I64)))), + Return(Call(I64, "mul_all", (L("a", I64), c(2))))], + pure=False, may_deopt=False) + with pytest.raises(cgen.CGenError, match="deopt"): + cgen.generate(Module("tp_bad2", (f_mul_all(), caller)), source_file("tp_bad2")) + + +# Every function the generated C calls must be CPython C API, a tp_runtime.h helper named in +# runtime/API.md, or a function the generated file defines itself. +C_WORDS = {"void", "if", "for", "while", "switch", "return", "sizeof", "defined", "INT64_C", "UINT64_C"} + + +def api_helpers() -> set[str]: + text = (PACKAGE / "runtime" / "API.md").read_text() + return set(re.findall(r"\b(tp_[a-z0-9_]+)\s*\(", text)) + + +def test_gen_calls_only_cpython_api_and_runtime_helpers(): + text = gen() + # strip comments and string literals so the embedded source does not count + body = re.sub(r"/\*.*?\*/", "", text, flags=re.S) + body = re.sub(r'"(?:\\.|[^"\\])*"', '""', body) + called = set(re.findall(r"\b([A-Za-z_][A-Za-z0-9_]*)\s*\(", body)) + defined = set(re.findall(r"^(?:static\s+)?[A-Za-z_][A-Za-z0-9_ \*]*?\b([A-Za-z_][A-Za-z0-9_]*)\s*\(", + body, flags=re.M)) + helpers = api_helpers() + assert "tp_add_i64" in helpers and "tp_any_same" in helpers # the scan reads API.md for real + bad = sorted(n for n in called - C_WORDS - helpers - defined + if not re.match(r"^(Py[A-Z_]|PyModuleDef_Init$)", n)) + assert bad == [], f"generated C calls functions outside the CPython API and tp_runtime.h: {bad}" + # tp_* names that are called but neither in API.md nor defined here are typos in waiting + stray = sorted(n for n in called if n.startswith("tp_") and n not in helpers and n not in defined) + assert stray == [] + # no private CPython API + assert not re.search(r"\b_Py[A-Za-z]", body) + + +# --- end to end ----------------------------------------------------------------------------------- + +def test_compiled_functions_replace_the_globals(mod): + assert type(mod.poly).__name__ == "builtin_function_or_method" + assert type(interp(mod, "poly")).__name__ == "function" + assert mod.record.__module__ == "tp_cgen_e2e" # interpreted names live in the same dict + assert interp(mod, "poly").__globals__ is mod.__dict__ + + +def test_f64_loop_returns_exactly_the_interpreted_result(mod): + before = deopts(mod) + for x, n in ((1.1, 1000), (0.37, 17), (-2.5, 9), (1e308, 3), (float("nan"), 2)): + got, want = mod.poly(x, n), interp(mod, "poly")(x, n) + assert type(got) is float + assert repr(got) == repr(want), (x, n, got, want) # bit-exact, incl. inf/nan + assert deopts(mod) == before + + +def test_i64_overflow_in_a_pure_function_deopts_to_the_big_int_result(mod): + before = deopts(mod) + assert mod.mul_all(3, 10) == 3 ** 10 + assert deopts(mod) == before + assert mod.mul_all(3, 50) == 3 ** 50 + assert deopts(mod) == before + 1 + + +def test_deopt_propagates_through_a_c_to_c_call(mod): + before = deopts(mod) + assert mod.twice_mul(7, 5) == 2 * 7 ** 5 + assert deopts(mod) == before + assert mod.twice_mul(7, 40) == 2 * 7 ** 40 + # The outermost compiled call is redone once (+1). The interpreted twice_mul then calls the + # global mul_all — the compiled one, since there is one namespace — which deopts again (+1). + assert deopts(mod) == before + 2 + + +def test_big_int_argument_takes_the_interpreted_path_before_any_effect(mod): + mod.calls.clear() + before = deopts(mod) + assert mod.eff(5) == 5 + assert mod.calls == [5] and deopts(mod) == before + big = 2 ** 70 + assert mod.eff(big) == big + assert mod.calls == [5, big] # the effect ran once, by the interpreter + assert deopts(mod) == before + 1 + assert mod.mul_all(2 ** 64, 1) == 2 ** 64 and deopts(mod) == before + 2 + + +def test_wrong_arity_raises_cpythons_own_type_error(mod): + with pytest.raises(TypeError) as want: + interp(mod, "poly")(1.0) + with pytest.raises(TypeError) as got: + mod.poly(1.0) + assert str(got.value) == str(want.value) + assert "poly() missing 1 required positional argument: 'n'" == str(got.value) + with pytest.raises(TypeError) as got3: + mod.poly(1.0, 2, 3) + assert "poly() takes 2 positional arguments but 3 were given" == str(got3.value) + + +def test_keyword_arguments_take_the_interpreted_path(mod): + before = deopts(mod) + assert repr(mod.poly(x=1.1, n=30)) == repr(interp(mod, "poly")(1.1, 30)) + assert repr(mod.poly(1.1, n=30)) == repr(interp(mod, "poly")(1.1, 30)) + assert deopts(mod) == before + 2 + with pytest.raises(TypeError) as got: + mod.poly(1.1, m=3) + with pytest.raises(TypeError) as want: + interp(mod, "poly")(1.1, m=3) + assert str(got.value) == str(want.value) + + +def test_exact_float_guard_rejects_int_and_subclass(mod): + class F(float): + def __rmul__(self, other): + return 42.0 + + before = deopts(mod) + assert mod.poly(2, 3) == interp(mod, "poly")(2, 3) # int x -> interpreted + assert mod.poly(F(2.0), 3) == interp(mod, "poly")(F(2.0), 3) == 42.1 # subclass -> interpreted + assert deopts(mod) == before + 2 + + +def test_f64_array_param_is_written_back(mod): + xs = [1.0, 2.0, 3.5] + ident = id(xs) + before = deopts(mod) + assert mod.scale(xs, 2.0, 3) is None + assert xs == [2.0, 4.0, 7.0] and id(xs) == ident + assert all(type(v) is float for v in xs) + assert deopts(mod) == before + + +def test_exception_mid_loop_keeps_the_partial_updates(mod): + xs, ys = [1.0, 2.0, 3.0], [1.0, 2.0, 3.0] + with pytest.raises(IndexError) as got: + mod.scale(xs, 10.0, 5) + with pytest.raises(IndexError) as want: + interp(mod, "scale")(ys, 10.0, 5) + assert str(got.value) == str(want.value) == "list index out of range" + assert xs == ys == [10.0, 20.0, 30.0] + + +def test_negative_index_counts_from_the_end(mod): + assert mod.neg_last([1.0, 2.5]) == -2.5 + with pytest.raises(IndexError, match="list index out of range"): + mod.neg_last([]) + + +def test_aliasing_two_array_params_takes_the_interpreted_path(mod): + a, b = [1.0, 2.0, 3.0], [1.0, 2.0, 3.0] + before = deopts(mod) + mod.addfirst(a, a) + interp(mod, "addfirst")(b, b) + assert a == b == [2.0, 4.0, 5.0] + assert deopts(mod) == before + 1 + d, s = [1.0, 2.0], [10.0] + mod.addfirst(d, s) + assert d == [11.0, 12.0] and s == [10.0] and deopts(mod) == before + 1 + + +def test_a_non_float_element_takes_the_interpreted_path(mod): + xs, ys = [1.0, 2, 3.0], [1.0, 2, 3.0] + before = deopts(mod) + mod.scale(xs, 2.0, 3) + interp(mod, "scale")(ys, 2.0, 3) + assert xs == ys and [type(v) for v in xs] == [type(v) for v in ys] + assert deopts(mod) == before + 1 + assert mod.scale((1.0,), 2.0, 0) is None and deopts(mod) == before + 2 # a tuple is not a list + + +def test_i64_array_and_its_range_guard(mod): + before = deopts(mod) + assert mod.isum([1, 2, 3, -4]) == 2 and deopts(mod) == before + assert mod.isum([2 ** 63 - 1, 1]) == 2 ** 63 and deopts(mod) == before + 1 # overflow in body + assert mod.isum([2 ** 64, 1]) == 2 ** 64 + 1 and deopts(mod) == before + 2 # element guard + assert mod.isum([True, 1]) == 2 and deopts(mod) == before + 3 # bool is not int + + +def test_integer_division_by_zero_has_cpythons_message(mod): + assert mod.floordiv(-7, 2) == -4 + with pytest.raises(ZeroDivisionError) as got: + mod.floordiv(1, 0) + with pytest.raises(ZeroDivisionError) as want: + interp(mod, "floordiv")(1, 0) + assert str(got.value) == str(want.value) + before = deopts(mod) + assert mod.floordiv(-2 ** 63, -1) == 2 ** 63 and deopts(mod) == before + 1 + + +def test_while_break_continue(mod): + for limit, skip in ((10, 3), (0, 0), (5, 9), (100, 100)): + assert mod.count_until(limit, skip) == interp(mod, "count_until")(limit, skip) + + +def test_math_and_mixed_compare_and_tofloat(mod): + for x, y in ((3.0, 4.0), (1e200, 1e200), (0.1, 0.2)): + assert repr(mod.hyp(x, y)) == repr(interp(mod, "hyp")(x, y)) + for a, b in ((2 ** 53 + 1, float(2 ** 53)), (1, 1.5), (3, 2.0), (-1, -0.5), (1, float("nan"))): + assert mod.mixed_lt(a, b) is interp(mod, "mixed_lt")(a, b), (a, b) + for a in (1, 2 ** 53 + 1, -(2 ** 62) - 3, 7): + assert repr(mod.tofl(a)) == repr(interp(mod, "tofl")(a)) + + +def test_obj_code_keeps_reference_counts_balanced(mod): + o, p = 10 ** 30, 10 ** 31 # not cached small ints + mod.calls.clear() + assert mod.objmix(o, p) == interp(mod, "objmix")(o, p) + mod.calls.clear() + ro, rp = sys.getrefcount(o), sys.getrefcount(p) + for _ in range(1000): + mod.objmix(o, p) + mod.calls.clear() + assert (sys.getrefcount(o), sys.getrefcount(p)) == (ro, rp) + with pytest.raises(TypeError) as got: + mod.objmix(1, "a") + with pytest.raises(TypeError) as want: + interp(mod, "objmix")(1, "a") + assert str(got.value) == str(want.value) + + +def test_interop_matches_the_interpreted_run_including_property_reads(mod): + for scale, n in ((1.5, 20), (0.5, 7), (2.0, 0)): + mod.made.clear() + got = mod.interop(scale, n) + got_reads = mod.made[-1].reads + want = interp(mod, "interop")(scale, n) + want_reads = mod.made[-1].reads + assert repr(got) == repr(want), (scale, n) + assert got_reads == want_reads == 2 * n # `factor` is read in apply() and in the if + assert len(mod.made) == 2 # make() ran once per call, no redo + + +def test_a_rebound_global_is_seen_and_builtins_are_found(mod): + assert mod.which() == 1 + old = mod.target + try: + mod.target = lambda: 2 + assert mod.which() == 2 + del mod.target + with pytest.raises(NameError) as got: + mod.which() + with pytest.raises(NameError) as want: + interp(mod, "which")() + assert str(got.value) == str(want.value) == "name 'target' is not defined" + finally: + mod.target = old + assert mod.length([1, 2, 3]) == 3 + + +def test_an_exception_in_the_callee_propagates_and_leaks_nothing(mod): + o = object() + with pytest.raises(ValueError) as want: + interp(mod, "calls_boom")(o) + del want + before = sys.getrefcount(o) + for _ in range(1000): + with pytest.raises(ValueError) as got: + mod.calls_boom(o) + assert str(got.value) == f"bad value {o!r}" + del got + assert sys.getrefcount(o) == before + + +def test_truth_runs_dunder_bool_and_its_errors(mod): + class B: + def __bool__(self): + raise RuntimeError("no truth") + + assert mod.truthy([1]) is True and mod.truthy([]) is False and mod.truthy(0.0) is False + with pytest.raises(RuntimeError, match="no truth"): + mod.truthy(B()) + + +def test_gen_rejects_an_effect_in_a_pure_function(): + from typedpython import cgen + + bad = fn("badpure", [Param("o", OBJ)], OBJ, {}, [Return(GetAttr(OBJ, L("o", OBJ), "x"))], + pure=True, may_deopt=False) + with pytest.raises(cgen.CGenError, match="pure"): + cgen.generate(Module("tp_bad3", (bad,)), source_file("tp_bad3")) + + +# --- sanitizers ----------------------------------------------------------------------------------- + +ASAN_SCRIPT = textwrap.dedent('''\ + import sys + sys.path.insert(0, {pkg!r}) + from typedpython import cbuild + m = cbuild.load({name!r}, {so!r}) + I = m.__typedpython_interpreted__ + assert repr(m.poly(1.1, 500)) == repr(I["poly"](1.1, 500)) + assert m.mul_all(3, 50) == 3 ** 50 + assert m.twice_mul(7, 40) == 2 * 7 ** 40 + xs = [1.0, 2.0, 3.0] + try: + m.scale(xs, 10.0, 5) + except IndexError: + pass + assert xs == [10.0, 20.0, 30.0], xs + a = [1.0, 2.0, 3.0]; m.addfirst(a, a); assert a == [2.0, 4.0, 5.0] + d = [1.0] * 1000; m.addfirst(d, [2.0]); assert d == [3.0] * 1000 + assert m.isum(list(range(10000))) == sum(range(10000)) + assert m.isum([2 ** 63 - 1, 1]) == 2 ** 63 + try: + m.floordiv(1, 0) + except ZeroDivisionError: + pass + for _ in range(200): + m.objmix(10 ** 30, 10 ** 31); m.calls.clear() + assert m.neg_last([1.0, 2.0]) == -2.0 + assert m.count_until(50, 7) == I["count_until"](50, 7) + for _ in range(50): + m.made.clear(); m.interop(1.5, 20) + try: + m.calls_boom(1) + except ValueError: + pass + print("ASAN-RUN-OK", m.__typedpython_deopts__) + ''') + + +def test_build_with_address_and_undefined_sanitizers(): + from typedpython import cbuild + + need_runtime() + cc = sysconfig.get_config_var("CC") or "cc" + if sys.platform != "darwin": + pytest.skip("sanitizer preload recipe written for macOS only") + probe = subprocess.run(cc.split() + ["-print-file-name=libclang_rt.asan_osx_dynamic.dylib"], + capture_output=True, text=True) + asan_rt = Path(probe.stdout.strip()) + if probe.returncode != 0 or not asan_rt.is_file(): + pytest.skip(f"host compiler {cc!r} has no ASan runtime ({probe.stdout.strip()!r})") + name = "tp_cgen_asan" + src = source_file(name) + from typedpython import cgen + c_source = cgen.generate(Module(name=name, functions=tuple(f() for f in ALL)), src) + flags = list(cbuild.DEFAULT_FLAGS) + ["-O1", "-g", "-fno-omit-frame-pointer", + "-fsanitize=address,undefined", + "-fno-sanitize-recover=undefined"] + so = cbuild.build(c_source, name, src.parent, runtime_dir=runtime_dir(), flags=flags) + # macOS refuses DYLD_INSERT_LIBRARIES of the sanitizer runtime into the framework python + # ("Sanitizer load violates platform policy", observed 2026-10-03 on Darwin 25.5), so the + # extension is loaded by a small sanitized launcher linked against libpython: ASan is then in + # the process from the start, as it must be. + launcher_c = src.parent / "asan_python.c" + launcher_c.write_text("#include \nint main(int argc, char **argv) " + "{ return Py_BytesMain(argc, argv); }\n") + launcher = src.parent / "asan_python" + libdir = sysconfig.get_config_var("LIBDIR") + ldver = sysconfig.get_config_var("LDVERSION") + link = subprocess.run(cc.split() + ["-fsanitize=address,undefined", "-g", + f"-I{sysconfig.get_paths()['include']}", str(launcher_c), + f"-L{libdir}", f"-lpython{ldver}", "-o", str(launcher)], + capture_output=True, text=True) + if link.returncode != 0: + pytest.skip(f"cannot link a sanitized launcher against libpython{ldver}: {link.stderr[-2000:]}") + env = dict(os.environ, PYTHONHOME=sys.base_prefix, + ASAN_OPTIONS="detect_leaks=0:abort_on_error=0:halt_on_error=1", + UBSAN_OPTIONS="halt_on_error=1:print_stacktrace=1") + env.pop("DYLD_INSERT_LIBRARIES", None) + script = ASAN_SCRIPT.format(pkg=str(PACKAGE.parent), name=name, so=str(so)) + run = subprocess.run([str(launcher), "-c", script], capture_output=True, text=True, env=env) + out = run.stdout + run.stderr + if "violates platform policy" in out or "Interceptors are not working" in out: + pytest.skip(f"ASan runtime could not be loaded: {out[-2000:]}") + assert run.returncode == 0 and "ASAN-RUN-OK" in run.stdout, out[-6000:] + linked = subprocess.run(["otool", "-L", str(so)], capture_output=True, text=True).stdout + assert "libclang_rt.asan" in linked, linked # the extension really is instrumented + assert "runtime error" not in out and "AddressSanitizer" not in out, out[-6000:] + + +# --- building against the embedded CPython ------------------------------------------------------ + +EMBEDDED_314 = (WORKTREE.parents[1] / "python-multiplatform" / "build" / "python-standalone" + / "extracted" / "macos-aarch64" / "python" / "include" / "python3.14") + + +def test_builds_against_the_embedded_cpython_314_headers(): + """Compile only: the host 3.13 interpreter cannot import a 3.14 extension.""" + from typedpython import cbuild, cgen + + need_runtime() + if not (EMBEDDED_314 / "Python.h").is_file(): + pytest.skip(f"embedded CPython 3.14 headers not extracted: {EMBEDDED_314}") + name = "tp_cgen_314" + src = source_file(name) + c_source = cgen.generate(Module(name=name, functions=tuple(f() for f in ALL)), src) + flags = list(cbuild.DEFAULT_FLAGS) + ["-Werror=deprecated-declarations", + "-Werror=incompatible-pointer-types", + "-Werror=int-conversion"] + so = cbuild.build(c_source, name, src.parent, runtime_dir=runtime_dir(), flags=flags, + python_include=EMBEDDED_314, ext_suffix=".cpython-314-darwin.so") + assert so.name == "tp_cgen_314.cpython-314-darwin.so" and so.stat().st_size > 0 + # the 3.14 headers were really used: the same source against them sees PY_VERSION_HEX 3.14 + probe = cbuild.build('#include \n#if PY_VERSION_HEX < 0x030E0000\n#error not 3.14\n#endif\n' + 'PyMODINIT_FUNC PyInit_tp_probe(void) { return NULL; }\n', + "tp_probe", src.parent, runtime_dir=runtime_dir(), + python_include=EMBEDDED_314, ext_suffix=".cpython-314-darwin.so") + assert probe.is_file() From 8f87972de6c39dc000be8f319aba45ca1b8a2a05 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:18:41 +0900 Subject: [PATCH 19/36] Feat: TypedPython front end (AST + Pyrefly types -> typed IR) Lowers @compiled functions (or the module marker) to ir.Module: scalar and array parameters, locals from annotations/range targets/Pyrefly probes, purity and may_deopt by fixpoint, an interval proof for i64 operations that cannot overflow, OBJ paths for calls into any callable (Kotlin interop), chained comparisons via temporaries. Anything outside the IR is skipped with a line-numbered reason, never partially lowered. Green: 51/51. Mutation (impure unbounded int forced to I64): 2 fail. Refs #41 --- .../src/main/python/typedpython/frontend.py | 1153 +++++++++++++++++ .../src/test/python/test_frontend.py | 611 +++++++++ 2 files changed, 1764 insertions(+) create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/frontend.py create mode 100644 python-multiplatform-ksp/src/test/python/test_frontend.py diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py new file mode 100644 index 000000000..f09bf4a70 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -0,0 +1,1153 @@ +"""Front end: Python AST + Pyrefly-inferred types -> the typed IR (`ir.py`, #41, SPEC N-7). + +`lower(path)` reads one module and returns an `ir.Module`: + +- Which functions: module-level `def`s decorated with the builtin `@compiled` (a bare name, no + import — #42), or every module-level `def` when the first non-empty line is + `# typedpython: compiled`. +- A function is lowered only when **all** of it is expressible in the IR; otherwise it is left out + and `Module.skipped[name]` says why, with the line. There is no partial lowering. +- Types: parameters and returns from their annotations (`int`/`float`/`bool`, `list[float]` and + `list[int]` parameters as native arrays, anything else an opaque object); locals from their + annotation, a `for ... in range()` target, or else the Pyrefly-inferred type of their + assignments (read back through the same probe the gate uses, `rebinding.probe`). + +Purity and deopt (ir.Function). Each function is lowered first as if pure; if the result is impure +(it stores into an array, calls an object, reads an attribute, or calls an impure function), it is +lowered again in impure mode, where nothing may deopt: an int whose i64 arithmetic is not proved +free of overflow is a CPython int object (OBJ, `PyNumber_*`), a call to a compiled function that +may deopt goes through the module global (`CallObject(Global)`), and only `for ... in range()` +targets whose every binding is such a loop stay I64. The proof is a small interval analysis: +constants, range targets (between their bounds), `len()` and the i64 range of a guarded parameter. +""" +from __future__ import annotations + +import ast +import tempfile +from collections.abc import Iterable +from dataclasses import dataclass, field +from pathlib import Path + +from typedpython import gate, ir, pyrefly, rebinding +from typedpython.ir import Type + +MARKER = "# typedpython: compiled" + +I64_MIN, I64_MAX = -2**63, 2**63 - 1 +TOP = (I64_MIN, I64_MAX) +EXACT_INT_FLOAT = 2**53 # |n| <= this: int/int true division is exact-rounded in C (ir.BinOp) + +Interval = tuple[int, int] | None # None: no value at all (not yet known to be reached) + +TYPE_NAME = { + Type.I64: "int", Type.F64: "float", Type.BOOL: "bool", Type.NONE: "None", Type.OBJ: "object", + Type.F64_ARRAY: "list[float]", Type.I64_ARRAY: "list[int]", +} +SCALAR_KIND = {"int": Type.I64, "float": Type.F64, "bool": Type.BOOL} +ARRAY_KIND = {"list[float]": Type.F64_ARRAY, "list[int]": Type.I64_ARRAY} + +BINOP = { + ast.Add: ir.BinOpKind.ADD, ast.Sub: ir.BinOpKind.SUB, ast.Mult: ir.BinOpKind.MUL, + ast.Div: ir.BinOpKind.TRUEDIV, ast.FloorDiv: ir.BinOpKind.FLOORDIV, ast.Mod: ir.BinOpKind.MOD, +} +COMPARE = { + ast.Eq: ir.CompareKind.EQ, ast.NotEq: ir.CompareKind.NE, ast.Lt: ir.CompareKind.LT, + ast.LtE: ir.CompareKind.LE, ast.Gt: ir.CompareKind.GT, ast.GtE: ir.CompareKind.GE, +} +MATH_ARITY = { + ir.MathFunc.SQRT: 1, ir.MathFunc.EXP: 1, ir.MathFunc.LOG: 1, ir.MathFunc.SIN: 1, + ir.MathFunc.COS: 1, ir.MathFunc.TAN: 1, ir.MathFunc.FABS: 1, ir.MathFunc.ATAN2: 2, + ir.MathFunc.HYPOT: 2, +} + +UNSUPPORTED = { + "Try": "try statement", "TryStar": "try statement", "With": "with statement", + "AsyncWith": "async with statement", "Match": "match statement", "Raise": "raise statement", + "Assert": "assert statement", "Global": "global declaration", + "Nonlocal": "nonlocal declaration", "FunctionDef": "nested def", + "AsyncFunctionDef": "nested async def", "ClassDef": "nested class", + "Import": "import inside a function", "ImportFrom": "import inside a function", + "Delete": "del statement", "AsyncFor": "async for", "TypeAlias": "type alias", + "Lambda": "lambda", "ListComp": "list comprehension", "SetComp": "set comprehension", + "DictComp": "dict comprehension", "GeneratorExp": "generator expression", + "List": "list display", "Tuple": "tuple display", "Dict": "dict display", + "Set": "set display", "Yield": "yield (generator)", "YieldFrom": "yield from (generator)", + "Await": "await", "IfExp": "conditional expression", "JoinedStr": "f-string", + "NamedExpr": "assignment expression", "Starred": "starred expression", "Slice": "slicing", +} + + +class Skip(Exception): + """This function cannot be expressed in the IR; it stays interpreted.""" + + def __init__(self, line: int, message: str) -> None: + super().__init__(f"line {line}: {message}") + self.reason = f"line {line}: {message}" + + +def _unsupported(node: ast.AST) -> Skip: + what = UNSUPPORTED.get(type(node).__name__, f"`{type(node).__name__}`") + return Skip(getattr(node, "lineno", 0), f"{what} is not supported") + + +def _src(node: ast.AST) -> str: + return ast.unparse(node) + + +# --- the module ---------------------------------------------------------------------------------- + +@dataclass +class _ModuleInfo: + bindings: dict[str, list[tuple[int, str]]] # module-level name -> [(line, how)] + stdlib_math: bool + + def bound(self, name: str) -> bool: + return name in self.bindings + + +@dataclass +class _Signature: + node: ast.FunctionDef + params: tuple[ir.Param | ir.ArrayParam, ...] + returns: Type + + +def lower(path: Path) -> ir.Module: + path = Path(path).resolve() + source = path.read_text() + tree = ast.parse(source, filename=str(path)) + info = _module_info(tree) + + skipped: dict[str, str] = {} + signatures: dict[str, _Signature] = {} + for node, reason in _targets(tree, source, info): + if reason is not None: + skipped[node.name] = reason + continue + assert isinstance(node, ast.FunctionDef) + try: + signatures[node.name] = _signature(node) + except Skip as e: + skipped[node.name] = e.reason + + for name, cycle in _call_cycles(signatures).items(): + skipped[name] = (f"line {signatures[name].node.lineno}: recursion " + f"({' -> '.join(cycle)}) is not compiled: C has no recursion limit " + "where CPython raises RecursionError") + del signatures[name] + + kinds = _inferred_kinds(path, source, tree, [s.node for s in signatures.values()]) + + # Optimistic start: every candidate lowered, pure, not deopting; then iterate to a fixpoint + # (purity is the greatest fixpoint through calls, may_deopt the least). + env: dict[str, ir.Function] = { + name: ir.Function(name, s.params, s.returns, {}, (), True, False, s.node.lineno) + for name, s in signatures.items() + } + results: dict[str, ir.Function | str] = {} + for _ in range(25): + new = {name: _lower_function(s, env, info, kinds) for name, s in signatures.items()} + if new == results: + break + results = new + env = {n: r for n, r in results.items() if isinstance(r, ir.Function)} + else: + results = {n: f"line {s.node.lineno}: purity/deopt analysis did not converge" + for n, s in signatures.items()} + + functions = [] + for name, result in results.items(): + if isinstance(result, ir.Function): + functions.append(result) + else: + skipped[name] = result + functions.sort(key=lambda f: f.source_line) + return ir.Module(path.stem, tuple(functions), skipped) + + +def _call_cycles(signatures: dict[str, _Signature]) -> dict[str, list[str]]: + """Candidates on a call cycle through calls by name to other candidates (syntactic, so a + call that would end up as CallObject is counted too: conservative).""" + calls = { + name: {n.func.id for n in ast.walk(s.node) if isinstance(n, ast.Call) + and isinstance(n.func, ast.Name) and n.func.id in signatures} + for name, s in signatures.items() + } + cycles: dict[str, list[str]] = {} + for start in signatures: + stack = [(start, [start])] + seen: set[str] = set() + while stack and start not in cycles: + name, path = stack.pop() + for callee in sorted(calls[name]): + if callee == start: + cycles[start] = path + [start] + break + if callee not in seen: + seen.add(callee) + stack.append((callee, path + [callee])) + return cycles + + +def _module_info(tree: ast.Module) -> _ModuleInfo: + bindings: dict[str, list[tuple[int, str]]] = {} + + def add(name: str, line: int, how: str) -> None: + bindings.setdefault(name, []).append((line, how)) + + for stmt in rebinding._own_statements(tree): + if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + add(stmt.name, stmt.lineno, "def") + elif isinstance(stmt, ast.Import): + for alias in stmt.names: + stdlib = alias.name == "math" and alias.asname is None + add(alias.asname or alias.name.split(".")[0], stmt.lineno, + "import math" if stdlib else "import") + elif isinstance(stmt, ast.ImportFrom): + for alias in stmt.names: + add(alias.asname or alias.name, stmt.lineno, "import") + else: + for target in _binding_targets(stmt): + for n in ast.walk(target): + if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store): + add(n.id, stmt.lineno, "assign") + for node in ast.walk(tree): + if isinstance(node, ast.Global): + for name in node.names: + add(name, node.lineno, "global") + + math = bindings.get("math", []) + return _ModuleInfo(bindings, len(math) == 1 and math[0][1] == "import math") + + +def _binding_targets(stmt: ast.stmt) -> list[ast.expr]: + if isinstance(stmt, ast.Assign): + return list(stmt.targets) + if isinstance(stmt, (ast.AugAssign, ast.AnnAssign, ast.For, ast.AsyncFor)): + return [stmt.target] + if isinstance(stmt, (ast.With, ast.AsyncWith)): + return [i.optional_vars for i in stmt.items if i.optional_vars is not None] + return [] + + +def _targets( + tree: ast.Module, source: str, info: _ModuleInfo, +) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]]: + first = next((line.strip() for line in source.splitlines() if line.strip()), "") + whole_module = first == MARKER + found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]] = [] + for node in tree.body: + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + marked = [d for d in node.decorator_list if isinstance(d, ast.Name) and d.id == "compiled"] + if not (whole_module or marked): + continue + reason = None + if marked and info.bound("compiled"): + line = info.bindings["compiled"][0][0] + reason = (f"line {node.lineno}: `compiled` is bound in this module (line {line}), " + "so `@compiled` is not the builtin") + elif other := [d for d in node.decorator_list if d not in marked]: + reason = (f"line {other[0].lineno}: decorator `@{_src(other[0])}` is not supported " + "(only `@compiled`)") + elif isinstance(node, ast.AsyncFunctionDef): + reason = f"line {node.lineno}: async function is not supported" + elif len(info.bindings.get(node.name, [])) > 1: + lines = ", ".join(str(line) for line, _ in info.bindings[node.name]) + reason = f"line {node.lineno}: `{node.name}` is bound more than once (lines {lines})" + found.append((node, reason)) + return found + + +def _signature(fn: ast.FunctionDef) -> _Signature: + args = fn.args + line = fn.lineno + if args.posonlyargs: + raise Skip(line, "positional-only parameters are not supported") + if args.kwonlyargs: + raise Skip(line, "keyword-only parameters are not supported") + if args.vararg is not None: + raise Skip(line, f"`*{args.vararg.arg}` is not supported") + if args.kwarg is not None: + raise Skip(line, f"`**{args.kwarg.arg}` is not supported") + if args.defaults: + raise Skip(line, "default parameter values are not supported") + params: list[ir.Param | ir.ArrayParam] = [] + for a in args.args: + if a.annotation is None: + raise Skip(a.lineno, f"parameter `{a.arg}` has no annotation") + kind = _annotation_kind(a.annotation) + if kind in ARRAY_KIND: + params.append(ir.ArrayParam(a.arg, ARRAY_KIND[kind], False)) # `stored` set later + else: + params.append(ir.Param(a.arg, SCALAR_KIND.get(kind, Type.OBJ))) + if fn.returns is None: + raise Skip(line, "no return annotation") + if isinstance(fn.returns, ast.Constant) and fn.returns.value is None: + returns = Type.NONE + else: + returns = SCALAR_KIND.get(_annotation_kind(fn.returns), Type.OBJ) + return _Signature(fn, tuple(params), returns) + + +def _annotation_kind(annotation: ast.expr) -> str: + if isinstance(annotation, ast.Name) and annotation.id in SCALAR_KIND: + return annotation.id + return _src(annotation).replace(" ", "") + + +# --- local types from Pyrefly -------------------------------------------------------------------- + +def _inferred_kinds( + path: Path, source: str, tree: ast.Module, functions: list[ast.FunctionDef], +) -> dict[str, dict[str, list[tuple[int, str]]]]: + """function -> local -> [(line, base type)] for every unannotated assignment (Pyrefly).""" + scopes = rebinding._scopes(tree) + wanted_scopes = {scopes.index(fn) for fn in functions} + wanted = [a for a in rebinding.assignments(tree) if a.scope in wanted_scopes] + if not wanted: + return {} + probed, probes = rebinding.probe(source, wanted) + with tempfile.TemporaryDirectory(prefix="typedpython-frontend-") as work: + copies = gate._mirror(Path(work), {str(path): probed}) + [copy] = copies + roots = pyrefly.import_roots([str(path)]) + report = pyrefly.run([Path(copy)], [Path(work) / "0", Path(copy).parent, *roots]) + types = report.expression_types.get(copy, {}) + + found: dict[str, dict[str, list[tuple[int, str]]]] = {} + for p in probes: + a = p.assignment + if a is None: + continue + spelled = types.get((p.line, p.column), "Unknown") + function = scopes[a.scope] + assert isinstance(function, ast.FunctionDef) + found.setdefault(function.name, {}).setdefault(p.name, []).append( + (a.line, rebinding._base(spelled))) + return found + + +# --- one function -------------------------------------------------------------------------------- + +def _lower_function( + sig: _Signature, env: dict[str, ir.Function], info: _ModuleInfo, + kinds: dict[str, dict[str, list[tuple[int, str]]]], +) -> ir.Function | str: + inferred = kinds.get(sig.node.name, {}) + pure_error: Skip | None = None + try: + fn = _lower_in_mode(sig, env, info, inferred, impure=False) + if fn.pure: + return fn + except Skip as e: + pure_error = e + try: + return _lower_in_mode(sig, env, info, inferred, impure=True) + except Skip as e: + return (pure_error or e).reason + + +def _lower_in_mode(sig, env, info, inferred, impure: bool) -> ir.Function: + # Range-target intervals: least fixpoint by iteration, widened to TOP if it does not settle. + bounds: dict[str, Interval] = {} + for _ in range(8): + lowerer = _Lowerer(sig, env, info, inferred, impure, bounds) + fn = lowerer.run() + joined = {**bounds} + for var, iv in lowerer.found_bounds.items(): + joined[var] = _join(joined.get(var), iv) + if joined == bounds: + return fn + bounds = joined + widened = {var: TOP for var in bounds} + return _Lowerer(sig, env, info, inferred, impure, widened).run() + + +def _join(a: Interval, b: Interval) -> Interval: + if a is None: + return b + if b is None: + return a + return (min(a[0], b[0]), max(a[1], b[1])) + + +@dataclass +class _Binding: + line: int + how: str # "range" (a for-over-range target), "annotation" (no value), "assign", "for" + + +class _Lowerer: + def __init__(self, sig: _Signature, env: dict[str, ir.Function], info: _ModuleInfo, + inferred: dict[str, list[tuple[int, str]]], impure: bool, + bounds: dict[str, Interval]) -> None: + self.sig = sig + self.fn = sig.node + self.env = env + self.info = info + self.inferred = inferred + self.impure_mode = impure + self.bounds = bounds + self.found_bounds: dict[str, Interval] = {} + + self.params = {p.name: p for p in sig.params} + self.arrays = {p.name: p.type for p in sig.params if isinstance(p, ir.ArrayParam)} + self.bindings: dict[str, list[_Binding]] = {} + self.annotations: dict[str, list[tuple[int, str]]] = {} + self._collect_bindings() + self.range_vars = { + n for n, bs in self.bindings.items() + if n not in self.params and any(b.how == "range" for b in bs) + and all(b.how in ("range", "annotation") for b in bs) + } + self.all_names = {n.id for n in ast.walk(self.fn) if isinstance(n, ast.Name)} \ + | set(self.params) + + self.types: dict[str, Type] = {} + self.temps: dict[str, Type] = {} + self.pre: list[ir.Stmt] = [] + self.hoist_ok = True + self.impure = False # an effect outside the locals + self.deopts = False # a node that can deopt + self.stored: set[str] = set() + + # --- entry --- + + def run(self) -> ir.Function: + body = self.block(self.fn.body) + if self.sig.returns != Type.NONE and _falls_through(body): + last = self.fn.body[-1] + raise Skip(getattr(last, "end_lineno", None) or last.lineno, + f"can reach the end without returning a {TYPE_NAME[self.sig.returns]} " + "(CPython would return None)") + locals_ = {} + for name in self.bindings: + if name not in self.params: + locals_[name] = self.local_type(name, self.bindings[name][0].line) + locals_.update(self.temps) + params = tuple( + ir.ArrayParam(p.name, p.type, p.name in self.stored) if isinstance(p, ir.ArrayParam) + else p for p in self.sig.params + ) + pure = not self.impure + assert pure or not self.deopts or not self.impure_mode, \ + "impure mode emitted a deopting node" + return ir.Function(self.fn.name, params, self.sig.returns, locals_, body, pure, + self.deopts, self.fn.lineno) + + # --- names and types --- + + def _collect_bindings(self) -> None: + def add(name: str, line: int, how: str) -> None: + self.bindings.setdefault(name, []).append(_Binding(line, how)) + + for stmt in rebinding._own_statements(self.fn): + if stmt is self.fn: + continue + if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name): + add(stmt.target.id, stmt.lineno, "assign" if stmt.value is not None else "annotation") + self.annotations.setdefault(stmt.target.id, []).append( + (stmt.lineno, _annotation_kind(stmt.annotation))) + elif isinstance(stmt, (ast.For, ast.AsyncFor)): + how = "range" if _is_range_call(stmt.iter) else "for" + for n in ast.walk(stmt.target): + if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store): + add(n.id, stmt.lineno, how) + else: + for target in _binding_targets(stmt): + for n in ast.walk(target): + if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store): + add(n.id, stmt.lineno, "assign") + + def is_local(self, name: str) -> bool: + return name in self.params or name in self.bindings or name in self.temps + + def local_type(self, name: str, line: int) -> Type: + if name in self.temps: + return self.temps[name] + if name in self.params: + return self.params[name].type + if name in self.types: + return self.types[name] + kinds: list[tuple[int, str]] = list(self.annotations.get(name, [])) + kinds += [(b.line, "int") for b in self.bindings.get(name, []) if b.how == "range"] + if name not in self.annotations: + kinds += self.inferred.get(name, []) + distinct = sorted({k for _, k in kinds}) + if not distinct: + raise Skip(self.bindings[name][0].line, f"no type is known for local `{name}`") + if len(distinct) > 1: + where = next(line for line, k in kinds if k != kinds[0][1]) + raise Skip(where, f"local `{name}` is given values of types " + + " and ".join(f"`{k}`" for k in distinct)) + kind = distinct[0] + if kind in ARRAY_KIND or kind.startswith(("list[float]", "list[int]")): + raise Skip(kinds[0][0], f"local `{name}: {kind}`: only parameters can be native " + "arrays") + if kind == "int": + t = Type.OBJ if self.impure_mode and name not in self.range_vars else Type.I64 + else: + t = SCALAR_KIND.get(kind, Type.OBJ) + self.types[name] = t + return t + + def temp(self, value: ir.Expr) -> ir.Local: + n = len(self.temps) + while f"_tp{n}" in self.all_names or f"_tp{n}" in self.temps: + n += 1 + name = f"_tp{n}" + self.temps[name] = value.type + self.pre.append(ir.Assign(name, value)) + return ir.Local(value.type, name) + + # --- statements --- + + def block(self, stmts: Iterable[ast.stmt]) -> tuple[ir.Stmt, ...]: + out: list[ir.Stmt] = [] + for s in stmts: + out += self.stmt(s) + return tuple(out) + + def start(self) -> None: + self.pre = [] + self.hoist_ok = True + + def take(self) -> list[ir.Stmt]: + pre, self.pre = self.pre, [] + return pre + + def stmt(self, node: ast.stmt) -> list[ir.Stmt]: + self.start() + line = node.lineno + if isinstance(node, ast.Pass): + return [] + if isinstance(node, ast.Break): + return [ir.Break()] + if isinstance(node, ast.Continue): + return [ir.Continue()] + if isinstance(node, ast.Expr): + if isinstance(node.value, ast.Constant): + return [] # a docstring or a bare literal: no effect + value = self.sub(node.value) + return self.take() + [ir.ExprStmt(value)] + if isinstance(node, ast.Assign): + if len(node.targets) != 1: + raise Skip(line, "chained assignment `a = b = ...` is not supported") + return self.assign(node.targets[0], node.value, node) + if isinstance(node, ast.AnnAssign): + if not isinstance(node.target, ast.Name): + raise Skip(line, f"annotated assignment to `{_src(node.target)}` is not supported") + if node.value is None: + return [] + return self.assign(node.target, node.value, node) + if isinstance(node, ast.AugAssign): + return self.aug_assign(node) + if isinstance(node, ast.If): + cond = self.condition(node.test) + pre = self.take() + return pre + [ir.If(cond, self.block(node.body), self.block(node.orelse))] + if isinstance(node, ast.While): + if node.orelse: + raise Skip(line, "`while ... else` is not supported") + cond = self.condition(node.test) + pre = self.take() + body = self.block(node.body) + if not pre: + return [ir.While(cond, body)] + # The condition needs statements (a chain's temporary): run them on every iteration. + exit_ = ir.If(cond, (), (ir.Break(),)) # cond stays directly a condition + return [ir.While(ir.Const(Type.BOOL, True), (*pre, exit_, *body))] + if isinstance(node, ast.For): + return self.for_range(node) + if isinstance(node, ast.Return): + return self.return_(node) + raise _unsupported(node) + + def assign(self, target: ast.expr, value_node: ast.expr, node: ast.stmt) -> list[ir.Stmt]: + line = node.lineno + if isinstance(target, ast.Name): + name = target.id + if name in self.arrays: + raise Skip(line, f"array parameter `{name}` is rebound") + value = self.value(value_node) + t = self.local_type(name, line) + return self.take() + [ir.Assign(name, self.coerce(value, t, value_node, + f"local `{name}`"))] + if isinstance(target, ast.Subscript): + array, elem = self.array_of(target) + # CPython evaluates the value first, then the index. + value = self.value(value_node) + value = self.coerce(value, elem, value_node, f"`{array}: {TYPE_NAME[self.arrays[array]]}`") + if not _simple(target.slice) and not _trivial(value): + value = self.temp(value) + index = self.index(target.slice) + self.stored.add(array) + self.impure = True + return self.take() + [ir.StoreIndex(array, index, value)] + if isinstance(target, ast.Attribute): + raise Skip(line, f"assignment to attribute `{_src(target)}` is not supported") + if isinstance(target, (ast.Tuple, ast.List)): + raise Skip(line, "tuple unpacking is not supported") + raise _unsupported(target) + + def aug_assign(self, node: ast.AugAssign) -> list[ir.Stmt]: + line = node.lineno + op = self.binop_kind(node.op, node) + target = node.target + if isinstance(target, ast.Name): + name = target.id + if name in self.arrays: + raise Skip(line, f"array parameter `{name}` is rebound") + if not self.is_local(name): + raise _unsupported(node) + t = self.local_type(name, line) + current = ir.Local(t, name) + result = self.arith(op, current, self.sub(node.value), node) + return self.take() + [ir.Assign(name, self.coerce(result, t, node, f"local `{name}`"))] + if isinstance(target, ast.Subscript): + array, elem = self.array_of(target) + index = self.index(target.slice) + if not isinstance(index, (ir.Local, ir.Const)): + index = self.temp(index) # `a[i] += e` evaluates `i` once + load = ir.Index(elem, array, index) + self.hoist_ok = False + result = self.arith(op, load, self.sub(node.value), node) + result = self.coerce(result, elem, node, f"`{array}: {TYPE_NAME[self.arrays[array]]}`") + self.stored.add(array) + self.impure = True + return self.take() + [ir.StoreIndex(array, index, result)] + if isinstance(target, ast.Attribute): + raise Skip(line, f"assignment to attribute `{_src(target)}` is not supported") + raise _unsupported(target) + + def for_range(self, node: ast.For) -> list[ir.Stmt]: + line = node.lineno + if node.orelse: + raise Skip(line, "`for ... else` is not supported") + if not isinstance(node.target, ast.Name): + raise Skip(line, f"loop target `{_src(node.target)}` is not supported (only a name)") + if not (_is_range_call(node.iter) and self.builtin("range")): + raise Skip(line, f"`for` over `{_src(node.iter)}` is not supported (only range())") + call = node.iter + assert isinstance(call, ast.Call) + var = node.target.id + if var in self.params: + raise Skip(line, f"loop variable `{var}` is a parameter") + for inner in ast.walk(ast.Module(body=node.body, type_ignores=[])): + for target in _binding_targets(inner) if isinstance(inner, ast.stmt) else []: + if any(isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store) and n.id == var + for n in ast.walk(target)): + raise Skip(inner.lineno, f"loop variable `{var}` is assigned in the loop body") + if self.local_type(var, line) != Type.I64: + other = next(b.line for b in self.bindings[var] if b.how != "range") + raise Skip(line, f"loop variable `{var}` is also assigned at line {other}; in an " + "impure function that makes it a CPython int, not a range index") + args = call.args + if len(args) == 3: + step = _int_literal(args[2]) + if step is None or step == 0: + raise Skip(line, f"range step `{_src(args[2])}` must be a non-zero int literal") + else: + step = 1 + if len(args) == 1: + start: ir.Expr = ir.Const(Type.I64, 0) + stop = self.range_bound(args[0]) + else: + start = self.range_bound(args[0]) + stop = self.range_bound(args[1]) + pre = self.take() + + s, e = self.interval(start), self.interval(stop) + if s is not None and e is not None: + if step > 0: + iv = (s[0], max(s[0], e[1] - 1)) + else: + iv = (min(s[1], e[0] + 1), s[1]) + self.found_bounds[var] = _join(self.found_bounds.get(var), iv) + + body = self.block(node.body) + return pre + [ir.ForRange(var, start, stop, ir.Const(Type.I64, step), body)] + + def range_bound(self, node: ast.expr) -> ir.Expr: + e = self.value(node) + if e.type != Type.I64: + why = (" (it is a CPython int here: its arithmetic is not proved free of i64 overflow " + "in an impure function)" if e.type == Type.OBJ and self.impure_mode else "") + raise Skip(node.lineno, f"range bound `{_src(node)}` is not a native int{why}") + return e + + def return_(self, node: ast.Return) -> list[ir.Stmt]: + returns = self.sig.returns + if node.value is None or (isinstance(node.value, ast.Constant) and node.value.value is None): + if returns != Type.NONE: + raise Skip(node.lineno, f"returns None from a function declared to return " + f"{TYPE_NAME[returns]}") + return [ir.Return(None)] + if returns == Type.NONE: + raise Skip(node.lineno, "returns a value from a function declared to return None") + value = self.value(node.value) + value = self.coerce(value, returns, node.value, "the return value") + return self.take() + [ir.Return(value)] + + # --- expressions --- + + def sub(self, node: ast.expr, cond: int = 0) -> ir.Expr: + """Lower a subexpression in evaluation order; after a non-trivial one, nothing later in + the statement may be hoisted in front of it.""" + e = self.expr(node, cond) + if not _trivial(e): + self.hoist_ok = False + return e + + def value(self, node: ast.expr) -> ir.Expr: + e = self.sub(node) + if e.type == Type.NONE: + raise Skip(node.lineno, f"uses the result of `{_src(node)}`, which returns None") + return e + + def condition(self, node: ast.expr) -> ir.Expr: + return self.truth(self.sub(node, cond=2), node) + + def truth(self, e: ir.Expr, node: ast.expr) -> ir.Expr: + if e.type == Type.BOOL: + return e + if e.type == Type.I64: + return ir.Compare(Type.BOOL, ir.CompareKind.NE, e, ir.Const(Type.I64, 0)) + if e.type == Type.F64: + return ir.Compare(Type.BOOL, ir.CompareKind.NE, e, ir.Const(Type.F64, 0.0)) + if e.type == Type.OBJ: + self.impure = True + return ir.Truth(Type.BOOL, e) + raise Skip(node.lineno, f"truth value of `{_src(node)}` ({TYPE_NAME[e.type]})") + + def expr(self, node: ast.expr, cond: int = 0) -> ir.Expr: + """`cond`: 2 when `node` is itself an If/While condition, 1 when it is an operand of an + And/Or that is the condition, 0 elsewhere (ir.CompareObj is allowed only at 2 and 1).""" + line = node.lineno + if isinstance(node, ast.Constant): + v = node.value + if isinstance(v, bool): + return ir.Const(Type.BOOL, v) + if isinstance(v, int): + if not I64_MIN <= v <= I64_MAX: + raise Skip(line, f"int literal {v} is outside i64") + return ir.Const(Type.I64, v) + if isinstance(v, float): + return ir.Const(Type.F64, v) + raise Skip(line, f"{type(v).__name__} constant `{_src(node)}` is not supported") + if isinstance(node, ast.Name): + return self.name(node) + if isinstance(node, ast.BinOp): + op = self.binop_kind(node.op, node) + left = self.sub(node.left) + right = self.sub(node.right) + return self.arith(op, left, right, node) + if isinstance(node, ast.UnaryOp): + return self.unary(node) + if isinstance(node, ast.Compare): + return self.compare(node, cond) + if isinstance(node, ast.BoolOp): + return self.boolop(node, cond) + if isinstance(node, ast.Call): + return self.call(node) + if isinstance(node, ast.Attribute): + if isinstance(node.value, ast.Name) and node.value.id in self.arrays: + raise self.array_misuse(node.value.id, node) + obj = self.boxed(self.sub(node.value), node.value) + self.impure = True + return ir.GetAttr(Type.OBJ, obj, node.attr) + if isinstance(node, ast.Subscript): + if isinstance(node.value, ast.Name) and node.value.id in self.arrays: + array, elem = self.array_of(node) + return ir.Index(elem, array, self.index(node.slice)) + raise Skip(line, f"subscript `{_src(node)}` of an object is not supported") + raise _unsupported(node) + + def name(self, node: ast.Name) -> ir.Expr: + name = node.id + if name in self.arrays: + raise self.array_misuse(name, node) + if self.is_local(name): + if name not in self.params and name not in self.temps and all( + b.how == "annotation" for b in self.bindings[name]): + raise Skip(node.lineno, f"`{name}` is read but never assigned") + return ir.Local(self.local_type(name, node.lineno), name) + return ir.Global(Type.OBJ, name) + + def array_misuse(self, name: str, node: ast.AST) -> Skip: + return Skip(node.lineno, f"array parameter `{name}` is used other than by indexing or " + f"len() (in `{_src(node)}`)") + + def array_of(self, node: ast.Subscript) -> tuple[str, Type]: + if not (isinstance(node.value, ast.Name) and node.value.id in self.arrays): + raise Skip(node.lineno, f"subscript `{_src(node)}` of an object is not supported") + name = node.value.id + return name, Type.F64 if self.arrays[name] == Type.F64_ARRAY else Type.I64 + + def index(self, node: ast.expr) -> ir.Expr: + if isinstance(node, ast.Slice): + raise _unsupported(node) + e = self.value(node) + if e.type != Type.I64: + why = (" (it is a CPython int here: its arithmetic is not proved free of i64 overflow " + "in an impure function)" if e.type == Type.OBJ and self.impure_mode else "") + raise Skip(node.lineno, f"index `{_src(node)}` is {TYPE_NAME[e.type]}, not a native " + f"int{why}") + return e + + def binop_kind(self, op: ast.operator, node: ast.AST) -> ir.BinOpKind: + kind = BINOP.get(type(op)) + if kind is None: + raise Skip(node.lineno, f"operator `{_src(node)}` is not supported") + return kind + + def arith(self, op: ir.BinOpKind, left: ir.Expr, right: ir.Expr, node: ast.AST) -> ir.Expr: + lt, rt = left.type, right.type + if Type.BOOL in (lt, rt) or Type.NONE in (lt, rt): + raise Skip(node.lineno, f"arithmetic on {TYPE_NAME[Type.BOOL if Type.BOOL in (lt, rt) else Type.NONE]} " + f"in `{_src(node)}` is not supported") + if Type.OBJ in (lt, rt): + return self.obj_binop(op, left, right) + if lt == Type.I64 and rt == Type.I64: + result = Type.F64 if op == ir.BinOpKind.TRUEDIV else Type.I64 + e = ir.BinOp(result, op, left, right) + if self.proved(e): + return e + if self.impure_mode: + return self.obj_binop(op, left, right) + self.deopts = True + return e + if lt == Type.I64: + left = ir.ToFloat(Type.F64, left) + if rt == Type.I64: + right = ir.ToFloat(Type.F64, right) + return ir.BinOp(Type.F64, op, left, right) + + def obj_binop(self, op: ir.BinOpKind, left: ir.Expr, right: ir.Expr) -> ir.Expr: + self.impure = True + return ir.BinOp(Type.OBJ, op, _box(left), _box(right)) + + def unary(self, node: ast.UnaryOp) -> ir.Expr: + if isinstance(node.op, ast.Not): + operand = self.truth(self.sub(node.operand), node.operand) + return ir.UnaryOp(Type.BOOL, ir.UnaryOpKind.NOT, operand) + if not isinstance(node.op, (ast.USub, ast.UAdd)): + raise Skip(node.lineno, f"operator `{_src(node)}` is not supported") + operand = self.sub(node.operand) + kind = ir.UnaryOpKind.NEG if isinstance(node.op, ast.USub) else ir.UnaryOpKind.POS + if operand.type == Type.F64: + return ir.UnaryOp(Type.F64, kind, operand) + if operand.type == Type.I64: + e = ir.UnaryOp(Type.I64, kind, operand) + if kind == ir.UnaryOpKind.POS or self.proved(e): + return e + if self.impure_mode: # -x of a CPython int is 0 - x + return self.obj_binop(ir.BinOpKind.SUB, ir.Const(Type.I64, 0), operand) + self.deopts = True + return e + raise Skip(node.lineno, f"unary `{_src(node)}` on {TYPE_NAME[operand.type]} is not " + "supported") + + def compare(self, node: ast.Compare, cond: int) -> ir.Expr: + # A chain is an And of its links: as a condition (2) that And is the condition; as an + # operand of the condition's And/Or (1) it would be nested, so only a single link there. + obj_ok = cond == 2 or (cond == 1 and len(node.ops) == 1) + left = self.sub(node.left) + links: list[ir.Expr] = [] + last = len(node.ops) - 1 + for i, (op, comparator) in enumerate(zip(node.ops, node.comparators)): + if i > 0: + self.hoist_ok = False # later links run only if the earlier ones held + hoist_ok = self.hoist_ok + right = self.expr(comparator) + if right.type == Type.NONE: + raise Skip(comparator.lineno, f"uses the result of `{_src(comparator)}`, which " + "returns None") + if i < last and not isinstance(right, (ir.Local, ir.Const)): + # The middle operand of a chain is evaluated once: bind it to a temporary. + if hoist_ok: + right = self.temp(right) + elif not _trivial(right): + raise Skip(comparator.lineno, f"chained comparison: `{_src(comparator)}` " + "cannot be evaluated once here") + if not _trivial(right): + self.hoist_ok = False + links.append(self.link(op, left, right, obj_ok, node)) + left = right + result = links[0] + for link in links[1:]: + result = ir.And(Type.BOOL, result, link) + return result + + def link(self, op: ast.cmpop, left: ir.Expr, right: ir.Expr, obj_ok: bool, + node: ast.Compare) -> ir.Expr: + kind = COMPARE.get(type(op)) + if kind is None: + raise Skip(node.lineno, f"comparison `{_src(node)}` is not supported " + "(only == != < <= > >=)") + lt, rt = left.type, right.type + if Type.OBJ in (lt, rt): + if not obj_ok: + raise Skip(node.lineno, f"comparison `{_src(node)}` with an object is not " + "directly a condition (its result need not be a bool)") + self.impure = True + return ir.CompareObj(Type.BOOL, kind, _box(left), _box(right)) + if (lt == Type.BOOL) != (rt == Type.BOOL): + raise Skip(node.lineno, f"comparison `{_src(node)}` of a bool with a number is not " + "supported") + return ir.Compare(Type.BOOL, kind, left, right) + + def boolop(self, node: ast.BoolOp, cond: int) -> ir.Expr: + word = "and" if isinstance(node.op, ast.And) else "or" + operands = [] + for i, v in enumerate(node.values): + if i > 0: + self.hoist_ok = False # evaluated only if the earlier operands did not decide + e = self.sub(v, 1 if cond == 2 else 0) + if cond == 2: + e = self.truth(e, v) + elif e.type != Type.BOOL: + raise Skip(node.lineno, f"`{word}` on a non-bool operand `{_src(v)}` outside a " + "condition (it returns the operand)") + operands.append(e) + result = operands[0] + for e in operands[1:]: + result = (ir.And if word == "and" else ir.Or)(Type.BOOL, result, e) + return result + + def builtin(self, name: str) -> bool: + return not self.info.bound(name) and not self.is_local(name) + + def call(self, node: ast.Call) -> ir.Expr: + line = node.lineno + if any(isinstance(a, ast.Starred) for a in node.args) or any( + k.arg is None for k in node.keywords): + raise Skip(line, f"`*`/`**` arguments in `{_src(node)}` are not supported") + func = node.func + plain = not node.keywords + + if isinstance(func, ast.Name): + name = func.id + if name in self.arrays: + raise self.array_misuse(name, node) + if plain and len(node.args) == 1 and name == "len" and self.builtin("len") \ + and isinstance(node.args[0], ast.Name) and node.args[0].id in self.arrays: + return ir.Len(Type.I64, node.args[0].id) + if plain and len(node.args) == 1 and name == "float" and self.builtin("float"): + arg = self.value(node.args[0]) + if arg.type == Type.I64: + return ir.ToFloat(Type.F64, arg) + if arg.type == Type.F64: + return arg # float(x) of an exact float is x itself + self.impure = True + return ir.ObjToFloat(Type.F64, _box(arg)) + if not self.is_local(name) and name in self.env: + direct = self.direct_call(name, node) + if direct is not None: + return direct + + if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name) \ + and func.value.id == "math" and self.info.stdlib_math \ + and not self.is_local("math") and plain: + math = self.math_call(func.attr, node) + if math is not None: + return math + + if isinstance(func, ast.Name) and func.id in self.arrays: + raise self.array_misuse(func.id, node) + callee = self.boxed(self.sub(func), func) + self.hoist_ok = False + args = [self.boxed(self.value(a), a) for a in node.args] + args += [self.boxed(self.value(k.value), k.value) for k in node.keywords] + self.impure = True + return ir.CallObject(Type.OBJ, callee, tuple(args), + tuple(k.arg for k in node.keywords if k.arg is not None)) + + def direct_call(self, name: str, node: ast.Call) -> ir.Expr | None: + """`Call` of another compiled function, else `CallObject` of its module global; None when + the generic path should lower the call (decided before any argument is lowered).""" + callee = self.env[name] + if node.keywords or len(node.args) != len(callee.params): + return None # CPython's own TypeError, through the interpreted path + if any(isinstance(p, ir.ArrayParam) for p in callee.params): + return None + self.hoist_ok = False + args = [self.value(a) for a in node.args] + direct = not (callee.may_deopt and self.impure_mode) # else a deopt could not be redone + for p, e in zip(callee.params, args): + if p.type != Type.OBJ and e.type != p.type: + direct = False # the callee's guard would deopt; the global keeps CPython's result + if not direct: + self.impure = True + return ir.CallObject(Type.OBJ, ir.Global(Type.OBJ, name), + tuple(self.boxed(e, a) for e, a in zip(args, node.args))) + if not callee.pure: + self.impure = True + if callee.may_deopt: + self.deopts = True + typed = tuple(self.boxed(e, a) if p.type == Type.OBJ else e + for p, e, a in zip(callee.params, args, node.args)) + return ir.Call(callee.returns, name, typed) + + def math_call(self, attr: str, node: ast.Call) -> ir.Expr | None: + try: + func = ir.MathFunc(attr) + except ValueError: + return None + if len(node.args) != MATH_ARITY[func]: + return None + self.hoist_ok = False + args = [self.value(a) for a in node.args] + if all(e.type in (Type.I64, Type.F64) for e in args): + return ir.MathCall(Type.F64, func, tuple( + ir.ToFloat(Type.F64, e) if e.type == Type.I64 else e for e in args)) + # math's own conversion of an object (or a bool): call `math.` as an object. + self.impure = True + callee = ir.GetAttr(Type.OBJ, ir.Global(Type.OBJ, "math"), attr) + return ir.CallObject(Type.OBJ, callee, tuple(self.boxed(e, a) + for e, a in zip(args, node.args))) + + def boxed(self, e: ir.Expr, node: ast.AST) -> ir.Expr: + if e.type == Type.NONE: + raise Skip(node.lineno, f"uses the result of `{_src(node)}`, which returns None") + return _box(e) + + def coerce(self, value: ir.Expr, target: Type, node: ast.AST, what: str) -> ir.Expr: + if value.type == target: + return value + if target == Type.OBJ and value.type in ir.SCALARS: + return ir.Box(Type.OBJ, value) + extra = "" + if value.type == Type.OBJ: + extra = " (CPython would keep the object as it is; there is no conversion to make)" + elif value.type == Type.I64 and target == Type.F64: + extra = " (CPython keeps an int there, it does not convert it to float)" + raise Skip(getattr(node, "lineno", 0), + f"`{_src(node)}` is {TYPE_NAME[value.type]}, but {what} is " + f"{TYPE_NAME[target]}{extra}") + + # --- intervals --- + + def interval(self, e: ir.Expr) -> Interval: + if isinstance(e, ir.Const) and e.type == Type.I64: + return (e.value, e.value) # type: ignore[return-value] + if isinstance(e, ir.Local) and e.type == Type.I64 and e.name in self.range_vars: + return self.bounds.get(e.name) + if isinstance(e, ir.Len): + return (0, I64_MAX) + if isinstance(e, (ir.BinOp, ir.UnaryOp)) and e.type == Type.I64 and self.proved(e): + return self._result_interval(e) + return TOP + + def proved(self, e: ir.BinOp | ir.UnaryOp) -> bool: + """The i64 operation provably does not deopt (no overflow, exact division).""" + if isinstance(e, ir.UnaryOp): + if e.op == ir.UnaryOpKind.POS: + return True + a = self.interval(e.operand) + return a is None or a[0] > I64_MIN + a, b = self.interval(e.left), self.interval(e.right) + if a is None or b is None: + return True # no value reaches here (yet) + if e.op == ir.BinOpKind.TRUEDIV: + return all(-EXACT_INT_FLOAT <= v <= EXACT_INT_FLOAT for v in (*a, *b)) + if e.op == ir.BinOpKind.MOD: + return True # the runtime computes -2**63 % -1 == 0 (ir.BinOp); only a raise remains + if e.op == ir.BinOpKind.FLOORDIV: + return not (a[0] == I64_MIN and b[0] <= -1 <= b[1]) + r = self._result_interval(e) + return r is None or (r[0] >= I64_MIN and r[1] <= I64_MAX) + + def _result_interval(self, e: ir.BinOp | ir.UnaryOp) -> Interval: + if isinstance(e, ir.UnaryOp): + a = self.interval(e.operand) + if a is None: + return None + return a if e.op == ir.UnaryOpKind.POS else (-a[1], -a[0]) + a, b = self.interval(e.left), self.interval(e.right) + if a is None or b is None: + return None + if e.op == ir.BinOpKind.ADD: + return (a[0] + b[0], a[1] + b[1]) + if e.op == ir.BinOpKind.SUB: + return (a[0] - b[1], a[1] - b[0]) + if e.op == ir.BinOpKind.MUL: + products = [x * y for x in a for y in b] + return (min(products), max(products)) + if e.op == ir.BinOpKind.FLOORDIV: + m = max(abs(a[0]), abs(a[1])) + return (max(-m, I64_MIN), min(m, I64_MAX)) + if e.op == ir.BinOpKind.MOD: + d = max(abs(b[0]), abs(b[1]), 1) + return (-(d - 1), d - 1) + return TOP + + +# --- helpers ------------------------------------------------------------------------------------- + +def _box(e: ir.Expr) -> ir.Expr: + return ir.Box(Type.OBJ, e) if e.type in ir.SCALARS else e + + +def _trivial(e: ir.Expr) -> bool: + """No effect, cannot raise, cannot deopt: may be evaluated in any order, or twice.""" + if isinstance(e, (ir.Const, ir.Local, ir.Len)): + return True + if isinstance(e, ir.ToFloat): + return _trivial(e.operand) + if isinstance(e, ir.UnaryOp): + return (e.op == ir.UnaryOpKind.NOT or e.type == Type.F64) and _trivial(e.operand) + if isinstance(e, ir.BinOp): + return e.type == Type.F64 and e.op in (ir.BinOpKind.ADD, ir.BinOpKind.SUB, + ir.BinOpKind.MUL) \ + and _trivial(e.left) and _trivial(e.right) + if isinstance(e, (ir.Compare, ir.And, ir.Or)): + return _trivial(e.left) and _trivial(e.right) + return False + + +def _simple(node: ast.expr) -> bool: + return isinstance(node, (ast.Name, ast.Constant)) + + +def _is_range_call(node: ast.expr) -> bool: + return isinstance(node, ast.Call) and isinstance(node.func, ast.Name) \ + and node.func.id == "range" and not node.keywords and 1 <= len(node.args) <= 3 \ + and not any(isinstance(a, ast.Starred) for a in node.args) + + +def _int_literal(node: ast.expr) -> int | None: + if isinstance(node, ast.Constant) and type(node.value) is int: + return node.value + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub) \ + and isinstance(node.operand, ast.Constant) and type(node.operand.value) is int: + return -node.operand.value + return None + + +def _falls_through(body: tuple[ir.Stmt, ...]) -> bool: + if not body: + return True + last = body[-1] + if isinstance(last, ir.Return): + return False + if isinstance(last, ir.If): + return _falls_through(last.then) or _falls_through(last.orelse) + if isinstance(last, ir.While): + forever = isinstance(last.cond, ir.Const) and last.cond.value is True + return not forever or _breaks(last.body) + return True + + +def _breaks(body: tuple[ir.Stmt, ...]) -> bool: + """A `break` of this loop (not of a nested one).""" + for s in body: + if isinstance(s, ir.Break): + return True + if isinstance(s, ir.If) and (_breaks(s.then) or _breaks(s.orelse)): + return True + return False + + +__all__ = ["lower", "MARKER"] diff --git a/python-multiplatform-ksp/src/test/python/test_frontend.py b/python-multiplatform-ksp/src/test/python/test_frontend.py new file mode 100644 index 000000000..be2906421 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_frontend.py @@ -0,0 +1,611 @@ +"""TypedPython front end: Python AST + Pyrefly types -> typed IR (#41, SPEC N-7). + +Each test writes a small module and asserts on the IR `frontend.lower` produces, or on the reason +a function was left interpreted. The IR's own docstrings (typedpython/ir.py) are the contract: +mixed int/float gets `ToFloat`, chains evaluate the middle operand once, an impure function has +no node that can deopt. +""" +import shutil +import textwrap +from pathlib import Path + +import pytest + +from typedpython import frontend +from typedpython.ir import ( + And, ArrayParam, Assign, BinOp, BinOpKind, Box, Break, Call, CallObject, Compare, CompareKind, + CompareObj, Const, ExprStmt, While, ForRange, GetAttr, Global, If, Index, Len, Local, MathCall, + MathFunc, ObjToFloat, Param, Return, StoreIndex, ToFloat, Truth, Type, +) + +I64, F64, BOOL, OBJ = Type.I64, Type.F64, Type.BOOL, Type.OBJ +BENCH = Path("/Volumes/macMini/thisisthepy/PythonMultiplatform/.worktrees/typedpython-bench" + "/benchmarks/typedpython/py") + + +def lower(tmp_path, source, name="mod.py"): + path = tmp_path / name + path.write_text(textwrap.dedent(source).lstrip("\n")) + return frontend.lower(path) + + +def function(module, name): + found = [f for f in module.functions if f.name == name] + assert found, f"{name} was not lowered: {module.skipped.get(name)!r}" + return found[0] + + +# --- the basic shape ----------------------------------------------------------------------------- + +def test_scalar_loop_lowers_to_for_range(tmp_path): + m = lower(tmp_path, """ + # typedpython: compiled + def total(n: int, x: float) -> float: + acc: float = 0.0 + for k in range(n): + acc += x + return acc + """) + assert m.skipped == {} + f = function(m, "total") + assert f.params == (Param("n", I64), Param("x", F64)) + assert f.returns == F64 + assert f.locals == {"acc": F64, "k": I64} + assert f.body == ( + Assign("acc", Const(F64, 0.0)), + ForRange("k", Const(I64, 0), Local(I64, "n"), Const(I64, 1), ( + Assign("acc", BinOp(F64, BinOpKind.ADD, Local(F64, "acc"), Local(F64, "x"))), + )), + Return(Local(F64, "acc")), + ) + assert f.pure and not f.may_deopt + assert f.source_line == 2 + + +def test_int_locals_in_a_pure_function_are_i64_and_may_deopt(tmp_path): + # `c` is unannotated: its type comes from Pyrefly. + m = lower(tmp_path, """ + @compiled + def f(a: int, b: int) -> int: + c = a + b + return c * 2 + """) + f = function(m, "f") + assert f.locals == {"c": I64} + assert f.body[0] == Assign("c", BinOp(I64, BinOpKind.ADD, Local(I64, "a"), Local(I64, "b"))) + assert f.pure and f.may_deopt + + +def test_unannotated_local_with_two_types_is_skipped(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(a: int) -> float: + c = a + c = 1.5 + return c + """) + assert m.functions == () + assert "`c`" in m.skipped["f"] and "line " in m.skipped["f"] + + +def test_impure_function_keeps_an_unbounded_int_local_as_a_python_int(tmp_path): + # Impure: a deopt could not be redone, so `m` is a CPython int object, not a checked i64. + m = lower(tmp_path, """ + @compiled + def g(a: list[float], n: int) -> None: + m: int = n + m += 1 + a[0] = 1.0 + """) + f = function(m, "g") + assert f.locals == {"m": OBJ} + assert f.body[:2] == ( + Assign("m", Box(OBJ, Local(I64, "n"))), + Assign("m", BinOp(OBJ, BinOpKind.ADD, Local(OBJ, "m"), Box(OBJ, Const(I64, 1)))), + ) + assert not f.pure and not f.may_deopt + + +def test_impure_function_with_bounded_range_indices_lowers_without_deopt(tmp_path): + m = lower(tmp_path, """ + @compiled + def h(a: list[float], n: int) -> None: + for i in range(n): + for j in range(i + 1, n): + a[j] += a[i] + """) + f = function(m, "h") + assert not f.pure and not f.may_deopt + assert f.params == (ArrayParam("a", Type.F64_ARRAY, True), Param("n", I64)) + + +def test_impure_function_with_unproved_int_arithmetic_is_skipped(tmp_path): + m = lower(tmp_path, """ + @compiled + def h(a: list[float], n: int) -> None: + a[n * 2] = 1.0 + """) + # `n * 2` can overflow i64 and the function is impure, so it is an int object: no native index. + assert m.functions == () + assert "line 3" in m.skipped["h"] and "n * 2" in m.skipped["h"] + + +# --- conversions --------------------------------------------------------------------------------- + +def test_mixed_int_float_arithmetic_inserts_to_float(tmp_path): + m = lower(tmp_path, """ + @compiled + def mix(n: int, x: float) -> float: + return n * x + 1 + """) + f = function(m, "mix") + assert f.body == (Return(BinOp( + F64, BinOpKind.ADD, + BinOp(F64, BinOpKind.MUL, ToFloat(F64, Local(I64, "n")), Local(F64, "x")), + ToFloat(F64, Const(I64, 1)), + )),) + + +def test_int_value_into_a_float_local_is_skipped(tmp_path): + # CPython keeps the int: `y` would be an int at run time, not a float. + m = lower(tmp_path, """ + @compiled + def f(n: int) -> float: + y: float = n + return y + """) + assert m.functions == () + assert "line 3" in m.skipped["f"] + + +def test_int_true_division_gives_float(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(a: int, b: int) -> float: + return a / b + """) + assert function(m, "f").body == ( + Return(BinOp(F64, BinOpKind.TRUEDIV, Local(I64, "a"), Local(I64, "b"))),) + + +def test_float_of_int_is_to_float(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(a: int) -> float: + return float(a) + """) + assert function(m, "f").body == (Return(ToFloat(F64, Local(I64, "a"))),) + + +# --- chained comparison -------------------------------------------------------------------------- + +def test_chained_compare_of_locals(tmp_path): + m = lower(tmp_path, """ + @compiled + def between(lo: int, x: int, hi: int) -> bool: + return lo < x <= hi + """) + assert function(m, "between").body == (Return(And( + BOOL, + Compare(BOOL, CompareKind.LT, Local(I64, "lo"), Local(I64, "x")), + Compare(BOOL, CompareKind.LE, Local(I64, "x"), Local(I64, "hi")), + )),) + + +def test_chained_compare_evaluates_the_middle_once(tmp_path): + m = lower(tmp_path, """ + @compiled + def sq(v: float) -> float: + return v * v + + @compiled + def within(lo: float, x: float, hi: float) -> bool: + return lo < sq(x) < hi + """) + f = function(m, "within") + assert len(f.body) == 2 + first, ret = f.body + assert isinstance(first, Assign) and first.value == Call(F64, "sq", (Local(F64, "x"),)) + t = first.target + assert f.locals[t] == F64 and t not in ("lo", "x", "hi") + assert ret == Return(And( + BOOL, + Compare(BOOL, CompareKind.LT, Local(F64, "lo"), Local(F64, t)), + Compare(BOOL, CompareKind.LT, Local(F64, t), Local(F64, "hi")), + )) + + +# --- arrays -------------------------------------------------------------------------------------- + +def test_augmented_store_into_an_array_element(tmp_path): + m = lower(tmp_path, """ + @compiled + def bump(a: list[float], i: int, e: float) -> None: + a[i] += e + """) + assert function(m, "bump").body == (StoreIndex( + "a", Local(I64, "i"), + BinOp(F64, BinOpKind.ADD, Index(F64, "a", Local(I64, "i")), Local(F64, "e")), + ),) + + +def test_augmented_store_evaluates_a_computed_index_once(tmp_path): + m = lower(tmp_path, """ + @compiled + def bump_last(a: list[float], e: float) -> None: + a[len(a) - 1] += e + """) + f = function(m, "bump_last") + first, store = f.body + assert isinstance(first, Assign) + assert first.value == BinOp(I64, BinOpKind.SUB, Len(I64, "a"), Const(I64, 1)) + t = first.target + assert store == StoreIndex( + "a", Local(I64, t), + BinOp(F64, BinOpKind.ADD, Index(F64, "a", Local(I64, t)), Local(F64, "e")), + ) + assert not f.pure and not f.may_deopt + + +def test_list_params_become_array_params_with_stored(tmp_path): + m = lower(tmp_path, """ + @compiled + def scale(a: list[float], b: list[float], k: float, c: list[int]) -> None: + for i in range(len(a)): + a[i] = b[i] * k + """) + f = function(m, "scale") + assert f.params == ( + ArrayParam("a", Type.F64_ARRAY, True), ArrayParam("b", Type.F64_ARRAY, False), + Param("k", F64), ArrayParam("c", Type.I64_ARRAY, False), + ) + assert f.body[0].stop == Len(I64, "a") + + +def test_array_param_used_whole_is_skipped(tmp_path): + m = lower(tmp_path, """ + @compiled + def first(a: list[float]) -> float: + return a[0] + + @compiled + def caller(a: list[float]) -> float: + return first(a) + """) + assert [f.name for f in m.functions] == ["first"] + assert "`a`" in m.skipped["caller"] and "line 7" in m.skipped["caller"] + + +def test_int_stored_into_a_float_array_is_skipped(tmp_path): + m = lower(tmp_path, """ + @compiled + def zero(a: list[float]) -> None: + a[0] = 0 + """) + assert m.functions == () and "line 3" in m.skipped["zero"] + + +# --- math ---------------------------------------------------------------------------------------- + +def test_math_sqrt_of_stdlib_math(tmp_path): + m = lower(tmp_path, """ + import math + + @compiled + def norm(x: float, y: float) -> float: + return math.sqrt(x * x + y * y) + """) + assert function(m, "norm").body == (Return(MathCall(F64, MathFunc.SQRT, (BinOp( + F64, BinOpKind.ADD, + BinOp(F64, BinOpKind.MUL, Local(F64, "x"), Local(F64, "x")), + BinOp(F64, BinOpKind.MUL, Local(F64, "y"), Local(F64, "y")), + ),)),),) + + +@pytest.mark.parametrize("binding", ["import cmath as math", "import math\nmath = 3"]) +def test_shadowed_math_is_skipped(tmp_path, binding): + m = lower(tmp_path, binding + """ + +@compiled +def norm(x: float) -> float: + return math.sqrt(x) +""") + assert m.functions == () + assert "math" in m.skipped["norm"] + + +# --- calls and purity ---------------------------------------------------------------------------- + +def test_impure_caller_of_a_deopting_function_is_skipped(tmp_path): + m = lower(tmp_path, """ + @compiled + def tri(i: int, j: int) -> float: + return 1.0 / ((i + j) * (i + j + 1) // 2 + i + 1) + + @compiled + def fill(n: int, out: list[float]) -> None: + for i in range(n): + out[i] = tri(i, i) + """) + tri = function(m, "tri") + assert tri.pure and tri.may_deopt + assert "tri" in m.skipped["fill"] and "line 8" in m.skipped["fill"] + + +def test_pure_caller_of_a_deopting_function_lowers(tmp_path): + m = lower(tmp_path, """ + @compiled + def add(i: int, j: int) -> int: + return i + j + + @compiled + def twice(i: int) -> int: + return add(i, i) + """) + twice = function(m, "twice") + assert twice.body == (Return(Call(I64, "add", (Local(I64, "i"), Local(I64, "i")))),) + assert twice.pure and twice.may_deopt + + +def test_caller_of_an_interpreted_function_calls_it_as_an_object(tmp_path): + m = lower(tmp_path, """ + @compiled + def bad(x: float) -> float: + try: + return x + except ValueError: + return 0.0 + + @compiled + def good(x: float) -> object: + return bad(x) + + @compiled + def typed(x: float) -> float: + return bad(x) + """) + assert [f.name for f in m.functions] == ["good"] + assert "line 3" in m.skipped["bad"] and "try" in m.skipped["bad"] + assert function(m, "good").body == ( + Return(CallObject(OBJ, Global(OBJ, "bad"), (Box(OBJ, Local(F64, "x")),))),) + # The result is an object; a float return would need a conversion CPython does not make. + assert "bad(x)" in m.skipped["typed"] and "line 14" in m.skipped["typed"] + + +# --- what is (not) lowered ----------------------------------------------------------------------- + +@pytest.mark.parametrize("body, what", [ + (" try:\n pass\n except ValueError:\n pass\n", "try"), + (" with x:\n pass\n", "with"), + (" yield x\n", "yield"), + (" global q\n", "global"), + (" y = [x]\n", "list"), + (" y = (lambda: x)\n", "lambda"), +]) +def test_unsupported_construct_is_skipped_with_its_line(tmp_path, body, what): + m = lower(tmp_path, "@compiled\ndef f(x: float) -> None:\n" + body) + assert m.functions == () + assert "line 3" in m.skipped["f"] and what in m.skipped["f"] + + +@pytest.mark.parametrize("signature", [ + "def f(x: float = 1.0) -> float:", "def f(*x: float) -> float:", "def f(x) -> float:", + "def f(x: float):", "def f(*, x: float) -> float:", "def f(x: float, /) -> float:", + "@staticmethod\ndef f(x: float) -> float:", +]) +def test_unsupported_signature_is_skipped(tmp_path, signature): + m = lower(tmp_path, f"@compiled\n{signature}\n return 1.0\n") + assert m.functions == () and "f" in m.skipped + + +def test_global_read_is_a_global_node(tmp_path): + m = lower(tmp_path, """ + K: float = 2.0 + + @compiled + def f(x: float) -> object: + return x * K + + @compiled + def g(x: float) -> float: + return x * K + """) + f = function(m, "f") + assert f.body == (Return( + BinOp(OBJ, BinOpKind.MUL, Box(OBJ, Local(F64, "x")), Global(OBJ, "K"))),) + assert not f.pure and not f.may_deopt + assert "x * K" in m.skipped["g"] + + +def test_module_marker_compiles_every_module_level_def(tmp_path): + m = lower(tmp_path, """ + + # typedpython: compiled + def a(x: float) -> float: + return x + + def b(x: float) -> float: + return -x + + @staticmethod + def c(x: float) -> float: + return x + """) + assert [f.name for f in m.functions] == ["a", "b"] + assert set(m.skipped) == {"c"} and "decorator" in m.skipped["c"] + + +def test_without_marker_only_compiled_functions_are_lowered(tmp_path): + m = lower(tmp_path, """ + x: int = 1 + # typedpython: compiled + def a(x: float) -> float: + return x + + @compiled + def b(x: float) -> float: + return -x + """) + assert [f.name for f in m.functions] == ["b"] + assert m.skipped == {} + + +# --- objects: Kotlin interop and any Python value ------------------------------------------------- + +def test_compiled_code_calls_imported_callables_around_native_float_math(tmp_path): + (tmp_path / "kotlinlib.py").write_text(textwrap.dedent(""" + class Canvas: + pass + + def make_canvas(n: int) -> Canvas: + return Canvas() + + def draw(c: Canvas, x: float, alpha: float) -> None: + pass + """)) + m = lower(tmp_path, """ + import math + from kotlinlib import draw, make_canvas + + @compiled + def render(n: int, scale: float) -> float: + canvas = make_canvas(n) + total: float = 0.0 + hits: int = 0 + for i in range(n): + x: float = i * scale + total += math.sqrt(x) + draw(canvas, x, alpha=0.5) + hits += 1 + if hits > n: + break + return total + """) + f = function(m, "render") + assert not f.pure and not f.may_deopt + assert f.locals == {"canvas": OBJ, "total": F64, "hits": OBJ, "i": I64, "x": F64} + assert f.body[0] == Assign("canvas", CallObject( + OBJ, Global(OBJ, "make_canvas"), (Box(OBJ, Local(I64, "n")),))) + loop = f.body[3] + assert isinstance(loop, ForRange) and loop.var == "i" + assert loop.body == ( + Assign("x", BinOp(F64, BinOpKind.MUL, ToFloat(F64, Local(I64, "i")), Local(F64, "scale"))), + Assign("total", BinOp(F64, BinOpKind.ADD, Local(F64, "total"), + MathCall(F64, MathFunc.SQRT, (Local(F64, "x"),)))), + ExprStmt(CallObject(OBJ, Global(OBJ, "draw"), ( + Local(OBJ, "canvas"), Box(OBJ, Local(F64, "x")), Box(OBJ, Const(F64, 0.5)), + ), ("alpha",))), + Assign("hits", BinOp(OBJ, BinOpKind.ADD, Local(OBJ, "hits"), Box(OBJ, Const(I64, 1)))), + If(CompareObj(BOOL, CompareKind.GT, Local(OBJ, "hits"), Box(OBJ, Local(I64, "n"))), + (Break(),)), + ) + + +def test_object_truth_float_and_method_calls(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(o: object, x: float) -> float: + if o: + return float(o.scale(x)) + return x + """) + assert function(m, "f").body == ( + If(Truth(BOOL, Local(OBJ, "o")), (Return(ObjToFloat(F64, CallObject( + OBJ, GetAttr(OBJ, Local(OBJ, "o"), "scale"), (Box(OBJ, Local(F64, "x")),)))),)), + Return(Local(F64, "x")), + ) + + +def test_object_comparison_outside_a_condition_is_skipped(tmp_path): + # `a < b` on objects returns whatever __lt__ returns, which need not be a bool. + m = lower(tmp_path, """ + @compiled + def f(o: object, x: float) -> bool: + return o < x + """) + assert m.functions == () and "line 3" in m.skipped["f"] + + +@pytest.mark.parametrize("body", ["return not (o < x)", "r: bool = c and o < x\n return r"]) +def test_object_comparison_not_directly_a_condition_is_skipped(tmp_path, body): + m = lower(tmp_path, f"@compiled\ndef f(o: object, x: float, c: bool) -> bool:\n {body}\n") + assert m.functions == () and "line 3" in m.skipped["f"] + + +def test_object_comparison_as_an_operand_of_a_condition(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(o: object, x: float, c: bool) -> int: + if c and o < x: + return 1 + return 0 + """) + assert function(m, "f").body[0].cond == And( + BOOL, Local(BOOL, "c"), + CompareObj(BOOL, CompareKind.LT, Local(OBJ, "o"), Box(OBJ, Local(F64, "x")))) + + +def test_while_condition_with_a_temporary_reevaluates_it_each_iteration(tmp_path): + m = lower(tmp_path, """ + @compiled + def sq(v: float) -> float: + return v * v + + @compiled + def shrink(lo: float, x: float, hi: float) -> float: + while lo < sq(x) < hi: + x = x * 0.5 + return x + """) + loop = function(m, "shrink").body[0] + assert isinstance(loop, While) and loop.cond == Const(BOOL, True) + temp, test = loop.body[0], loop.body[1] + assert isinstance(temp, Assign) and temp.value == Call(F64, "sq", (Local(F64, "x"),)) + # The condition stays an If condition (never under `not`), and exits through its else. + assert isinstance(test, If) and isinstance(test.cond, And) + assert test.then == () and test.orelse == (Break(),) + + +@pytest.mark.parametrize("source", [ + "@compiled\ndef f(n: int) -> int:\n return f(n)\n", + "@compiled\ndef f(n: int) -> int:\n return g(n)\n\n" + "@compiled\ndef g(n: int) -> int:\n return f(n)\n", +]) +def test_recursion_is_skipped(tmp_path, source): + m = lower(tmp_path, source) + assert m.functions == () and "recurs" in m.skipped["f"] + + +def test_int_modulo_never_deopts(tmp_path): + m = lower(tmp_path, """ + @compiled + def put(a: list[float], i: int, n: int) -> None: + a[i % n] = 1.0 + """) + f = function(m, "put") + assert not f.pure and not f.may_deopt + + +def test_locals_exclude_parameters(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(n: int) -> int: + n = n + 1 + return n + """) + assert function(m, "f").locals == {} + + +# --- benchmarks ---------------------------------------------------------------------------------- + +@pytest.mark.skipif(not BENCH.exists(), reason="benchmark worktree not present") +def test_nbody_kernels_lower(tmp_path): + path = tmp_path / "nbody.py" + shutil.copy(BENCH / "nbody.py", path) + path.write_text("# typedpython: compiled\n" + path.read_text()) + m = frontend.lower(path) + for name in ("advance", "energy"): + function(m, name) + advance = function(m, "advance") + assert not advance.pure and not advance.may_deopt + assert function(m, "energy").pure From 14ccf6a21fb1f1d525bacaf62a45d9bbcc423e0a Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:20:16 +0900 Subject: [PATCH 20/36] Docs: The compile slot needs no separate .py fallback Refs #41 --- docs/design/typedpython.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 5012a5f55..d726cbeb5 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -285,7 +285,7 @@ upstream 대비 수치 오차를 기준선으로 박아 둔다. 도구 pragma 형태이고, 실행 시 이름 조회가 필요 없어 일반 CPython 에서 부작용이 없으며, 데코레이터 없이 모듈 전체를 지정할 수 있다. 2. ~~고정폭 정수의 오버플로 의미~~ — **결정됨:** `int` 는 i64 로 컴파일하고 오버플로 시 인터프리터 재실행으로 큰 정수로 승격(의미 보존). -3. ~~pypackpack Cython 슬롯 인터페이스 소유~~ — **결정됨:** pypackpack 이 정한다. TypedPython 은 생성한 C 소스, 빌드 플래그, 모듈 목록, 모듈별 `.py` 폴백을 넘긴다(Cython 폐기 후 갱신). +3. ~~pypackpack Cython 슬롯 인터페이스 소유~~ — **결정됨:** pypackpack 이 정한다. TypedPython 은 생성한 C 소스, 빌드 플래그, C 를 낸 모듈 목록을 넘긴다. 확장이 원본 소스를 품으므로 별도 `.py` 폴백은 없다(Cython 폐기 후 갱신). 4. GPU 착지를 torchnative 커스텀 op 로 할 것인가 (다른 레포 변경). 5. ~~TypedPython 을 별도 레포로 분리할 것인가~~ — **결정됨 (2026-10-02):** PythonMultiplatform 의 기본 동작. 검사는 기본, 컴파일은 opt-in. From 8f6b87c08c196b8115a5f480680cf9ab52a022e7 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:25:15 +0900 Subject: [PATCH 21/36] Feat: TypedPython pipeline and demo on the IR-to-C back end pipeline.compile_module runs the gate (compiled mode), frontend, verifier, cgen and cbuild; functions outside the IR or unproved stay interpreted with a reason, and a module with nothing left produces no C. python -m typedpython demo builds a module, runs main() both ways and fails on any difference. Red: 5/5 NotImplementedError. Green: 5/5 (incl. fact(30) promoting past i64 and demo identical output). Whole integration suite: 372 + 5. Refs #41 --- .../src/main/python/typedpython/__main__.py | 66 ++++++++++++ .../src/main/python/typedpython/pipeline.py | 70 ++++++++++++ .../src/test/python/test_pipeline.py | 102 ++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 python-multiplatform-ksp/src/main/python/typedpython/pipeline.py create mode 100644 python-multiplatform-ksp/src/test/python/test_pipeline.py diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py index 7f3cc1503..38dffde8c 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py @@ -23,8 +23,18 @@ def main(argv: list[str]) -> int: check.add_argument("--format", choices=["text", "json"], default="text") check.add_argument("--search-path", action="append", type=Path, default=[], help="directory to resolve imports from (repeatable)") + demo = commands.add_parser( + "demo", help="compile a module, run its main() interpreted and compiled, and compare", + ) + demo.add_argument("module", type=Path) + demo.add_argument("argv", nargs="*", help="sys.argv[1:] for the module's main()") + demo.add_argument("--out", type=Path, default=Path("build/typedpython"), + help="where the extension is built (default: build/typedpython)") args = parser.parse_args(argv) + if args.command == "demo": + return _demo(args.module, args.argv, args.out) + try: found = gate.check(python_files(args.paths), mode=args.mode, search_paths=args.search_path) except PyreflyError as e: @@ -40,6 +50,62 @@ def main(argv: list[str]) -> int: return EXIT_ERRORS if any(d.severity == "error" for d in found) else EXIT_CLEAN +def _demo(module: Path, module_argv: list[str], out: Path) -> int: + import contextlib + import importlib.util + import io + import time + + from typedpython import pipeline + + try: + result = pipeline.compile_module(module, out / module.stem) + except pipeline.CompileError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_ERRORS + except PyreflyError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_FAILURE + + def interpreted() -> object: + spec = importlib.util.spec_from_file_location(module.stem + "_interpreted", module) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + def run(load: object) -> tuple[str, float]: + mod = load() # type: ignore[operator] + saved = sys.argv + sys.argv = [str(module), *module_argv] + captured = io.StringIO() + try: + with contextlib.redirect_stdout(captured): + start = time.perf_counter() + mod.main() + elapsed = time.perf_counter() - start + finally: + sys.argv = saved + return captured.getvalue(), elapsed + + compiled_names = [f.name for f in result.module.functions] + print(f"compiled functions: {', '.join(compiled_names) or '(none)'}") + for name, reason in sorted(result.skipped.items()): + print(f" interpreted: {name} — {reason}") + out_i, t_i = run(interpreted) + out_c, t_c = run(lambda: pipeline.load(result)) + print(f"interpreted: {t_i * 1000:.1f} ms compiled: {t_c * 1000:.1f} ms " + f"speedup {t_i / t_c:.1f}x") + if out_i != out_c: + print("output differs:") + print(f" interpreted: {out_i!r}") + print(f" compiled: {out_c!r}") + return EXIT_ERRORS + print("identical output:") + print(out_c, end="") + return EXIT_CLEAN + + def cli() -> None: """Console-script entry point (`typedpython check ...`).""" sys.exit(main(sys.argv[1:])) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py new file mode 100644 index 000000000..a0d7de7d9 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py @@ -0,0 +1,70 @@ +"""Source → gate → frontend → verify → cgen → cbuild (design §4.3, SPEC N-7/N-8, #41). + +`compile_module` is the one entry point the CLI, the demo and the build slot use: + +1. The gate in `compiled` mode: a module with type errors is not compiled at all. +2. `frontend.lower`: functions outside the IR are left interpreted (`skipped`, with reasons). +3. `verify.verify`: functions the verifier cannot prove are left interpreted too (SPEC N-8 — + unsafe code is never generated silently). +4. If nothing is left to compile, no C is produced (`extension is None`): the module ships as + ordinary Python. +5. `cgen.generate` + `cbuild.build`, against the host's CPython or a target's headers. +""" +from dataclasses import dataclass, field +from pathlib import Path +from types import ModuleType + +from typedpython import cbuild, cgen, frontend, gate, ir, verify + + +class CompileError(RuntimeError): + """The module cannot be compiled at all (it fails the gate).""" + + +@dataclass +class Result: + source: Path + module: ir.Module # what was compiled (verified functions only) + extension: Path | None # None: nothing to compile, ship the .py as is + skipped: dict[str, str] = field(default_factory=dict) # function -> why it stays interpreted + diagnostics: list[verify.Diagnostic] = field(default_factory=list) + + +def compile_module( + path: Path, + out_dir: Path, + *, + python_include: Path | None = None, + ext_suffix: str | None = None, + runtime_dir: Path | None = None, +) -> Result: + path = Path(path).resolve() + errors = [d for d in gate.check([path], mode="compiled") if d.severity == "error"] + if errors: + listing = "\n".join(f"{d.path}:{d.line}:{d.column}: {d.message} [{d.rule}]" for d in errors) + raise CompileError(f"{path.name} does not pass the gate in compiled mode:\n{listing}") + + lowered = frontend.lower(path) + proved, diagnostics = verify.verify(lowered) + skipped = dict(proved.skipped) + if not proved.functions: + return Result(path, proved, None, skipped, diagnostics) + + c_source = cgen.generate(proved, path) + extension = cbuild.build( + c_source, path.stem, Path(out_dir), + runtime_dir=runtime_dir, python_include=python_include, ext_suffix=ext_suffix, + ) + return Result(path, proved, extension, skipped, diagnostics) + + +def load(result: Result) -> ModuleType: + """Import the built extension (or, with nothing compiled, the source) from its path.""" + if result.extension is None: + import importlib.util + spec = importlib.util.spec_from_file_location(result.source.stem, result.source) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + return cbuild.load(result.source.stem, result.extension) diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline.py b/python-multiplatform-ksp/src/test/python/test_pipeline.py new file mode 100644 index 000000000..d1a8b7299 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_pipeline.py @@ -0,0 +1,102 @@ +"""End to end: source -> frontend -> verify -> cgen -> cbuild -> import (SPEC N-7, #41).""" +import importlib.util +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from typedpython import pipeline + +SRC = Path(__file__).parents[2] / "main" / "python" + + +def interpreted(path: Path): + spec = importlib.util.spec_from_file_location(path.stem + "_interp", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +SCALAR = """\ +# typedpython: compiled + + +def total(n: int) -> float: + acc: float = 0.0 + for k in range(n): + acc += k * 0.5 + return acc + + +def fact(n: int) -> int: + r: int = 1 + for k in range(2, n + 1): + r = r * k + return r +""" + + +def test_scalar_module_compiles_and_matches_cpython(write, tmp_path): + path = write("scalar.py", SCALAR) + result = pipeline.compile_module(path, tmp_path / "out") + assert result.extension is not None, result.skipped + assert {f.name for f in result.module.functions} >= {"total", "fact"}, result.skipped + compiled = pipeline.load(result) + ref = interpreted(path) + assert compiled.total(1000) == ref.total(1000) + assert compiled.fact(30) == ref.fact(30) # promotes past i64 + assert compiled.__typedpython_deopts__ >= 1 + + +def test_unsupported_functions_stay_interpreted_with_a_reason(write, tmp_path): + path = write("mixed.py", SCALAR + """ + +def greet(name: str) -> str: + return "hi " + name +""") + result = pipeline.compile_module(path, tmp_path / "out") + assert "greet" in result.skipped and result.skipped["greet"] + assert pipeline.load(result).greet("x") == "hi x" + + +def test_a_module_with_nothing_to_compile_produces_no_extension(write, tmp_path): + path = write("plain.py", "def f(x: int) -> int:\n return x\n") + result = pipeline.compile_module(path, tmp_path / "out") + assert result.extension is None + + +def test_gate_errors_block_compilation(write, tmp_path): + path = write("bad.py", "# typedpython: compiled\n\ndef f(x):\n return x\n") + with pytest.raises(pipeline.CompileError, match="implicit-any-parameter"): + pipeline.compile_module(path, tmp_path / "out") + + +DEMO = """\ +# typedpython: compiled +import sys + + +def total(n: int) -> float: + acc: float = 0.0 + for k in range(n): + acc += k * 0.5 + return acc + + +def main() -> None: + print(f"{total(int(sys.argv[1])):.3f}") +""" + + +def test_demo_reports_identical_output(write, tmp_path): + path = write("demo_mod.py", DEMO) + env = dict(os.environ, PYTHONPATH=str(SRC)) + result = subprocess.run( + [sys.executable, "-m", "typedpython", "demo", "--out", str(tmp_path / "b"), str(path), "1000"], + capture_output=True, text=True, env=env, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "identical output" in result.stdout and "249750.000" in result.stdout + assert "compiled functions: total" in result.stdout From 957315ea55e38809e848ad753a4f9a73042e3a93 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:26:06 +0900 Subject: [PATCH 22/36] Spec: Proven i64 ops, callee-only redo, entry snapshots of globals Found by running the pipeline on nbody/spectral-norm: the front end proves some i64 operations cannot overflow but the IR could not say so, so the verifier rejected advance/energy. New: BinOp/UnaryOp.proven (re-proved by the verifier), Call.redo (an impure caller may call a pure may_deopt callee; only the callee is redone, F64/BOOL/NONE results only), Function.entry_globals (scalar module globals read once at entry in closed functions). Also: CompareObj is RichCompare+truth without the identity shortcut; exception messages follow the running interpreter (3.14 changed some). Refs #41 --- .../src/main/python/typedpython/ir.py | 30 ++++++++++++++++--- .../main/python/typedpython/runtime/API.md | 7 +++-- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index 1e3651800..df77bc3e5 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -111,9 +111,10 @@ class BinOp(Expr): ADD/SUB/MUL — checked; overflow → deopt. FLOORDIV/MOD — Python floor semantics (result sign follows the divisor for MOD; the runtime handles -2**63 % -1 == 0 without C's undefined behaviour); divisor 0 - → ZeroDivisionError("integer division or modulo by zero") for FLOORDIV and - ZeroDivisionError("integer modulo by zero") for MOD (CPython 3.13/3.14 messages; the - runtime's differential tests pin them); -2**63 // -1 overflows → deopt. + → ZeroDivisionError with the running interpreter's message (3.13: "integer division or + modulo by zero" / "integer modulo by zero"; 3.14: "division by zero" for every case — the + runtime selects by version and its differential tests pin both); -2**63 // -1 overflows → + deopt. Every ZeroDivisionError/math-domain message below follows the same rule. TRUEDIV — CPython rounds int/int correctly; when both |operands| <= 2**53 the C double division is exact-rounded too, otherwise → deopt. Divisor 0 → ZeroDivisionError("division by zero"). @@ -129,6 +130,11 @@ class BinOp(Expr): op: BinOpKind left: Expr right: Expr + # I64 ADD/SUB/MUL only: set by the front end when an interval proof shows the result cannot + # leave the i64 range; the verifier re-proves it (never trusts it) and a proven op cannot + # deopt, so it is allowed in an impure function. The back end still uses the checked helper + # and treats an overflow there as an internal error (SystemError), never as a deopt. + proven: bool = False class UnaryOpKind(Enum): @@ -143,6 +149,7 @@ class UnaryOp(Expr): op: UnaryOpKind operand: Expr + proven: bool = False # NEG of I64: as BinOp.proven class CompareKind(Enum): @@ -238,6 +245,13 @@ class Call(Expr): function: str args: tuple[Expr, ...] + # Set when an impure caller calls a may_deopt (therefore pure) callee: on a deopt inside the + # callee, only the callee is redone by its interpreted function — unobservable, because the + # callee is pure — and the caller continues. Allowed only for callees returning F64, BOOL or + # NONE: an F64-typed IR expression is a Python float in CPython too (ints reach floats only + # through explicit ToFloat or float arithmetic), so the redone result is exactly a float; an + # I64 result could be a big int and is not allowed here. + redo: bool = False @dataclass(frozen=True) @@ -286,7 +300,9 @@ class Truth(Expr): class CompareObj(Expr): """A comparison of two OBJ operands (a scalar side arrives through `Box`) **directly as a condition**: an If/While cond, or an And/Or operand when that And/Or is itself the condition. - PyObject_RichCompareBool semantics — the rich comparison runs, then its truth value. Type BOOL. + Semantics of `if a == b:` in CPython: the rich comparison runs, then the truth value of its + result — **without** PyObject_RichCompareBool's identity shortcut (a NaN float compared with + itself is False in CPython's `if x == x`). Type BOOL. Anywhere the comparison's own result object would be kept (`x = a < b`, `r = c and a < b`, `not (a < b)`) it is not allowed, because that object need not be a bool.""" @@ -452,6 +468,12 @@ class Function: params: tuple[Param | ArrayParam, ...] returns: Type locals: dict[str, Type] # non-parameter locals only (temporaries included) + # Module globals of a scalar type read ONCE at entry, as part of the guards: each must be + # exactly that type (else deopt, before any effect), and the body reads it as `Local(name)`. + # Legal only for a *closed* function — one that runs no user code (no GetAttr, CallObject, + # Truth, CompareObj, ObjToFloat, OBJ BinOp; Calls only to closed functions) — so nothing can + # rebind the global between entry and use, and reading it once equals reading it at each use. + entry_globals: tuple[Param, ...] = () body: tuple[Stmt, ...] pure: bool may_deopt: bool diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index 24be459a4..daa38260f 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -3,7 +3,7 @@ Generated C (cgen.py) reaches memory and `PyObject`s **only** through these helpers (SPEC N-8). Header-only, `static inline`, C99 + CPython C API, no other dependencies. Every helper implements the CPython behaviour named in `ir.py`. Return convention unless stated: `0` ok, `1` deopt (no -Python error set, nothing to undo), `-1` Python error set (CPython's exception type and message). +Python error set, nothing to undo), `-1` Python error set (CPython's exception type and message — **of the running interpreter**: 3.14 changed several ZeroDivisionError and math-domain messages, and the runtime selects them by PY_VERSION_HEX). ## Integers (int64_t) int tp_add_i64(int64_t a, int64_t b, int64_t *out); /* 0 / 1 on overflow */ @@ -65,9 +65,10 @@ Python error set, nothing to undo), `-1` Python error set (CPython's exception t All return new references or NULL with CPython's exception set; none deopts. PyObject *tp_global(PyObject *module_dict, PyObject *name); /* globals, then builtins; NameError("name 'x' is not defined") */ PyObject *tp_getattr(PyObject *obj, PyObject *name); /* PyObject_GetAttr */ - PyObject *tp_call(PyObject *callee, PyObject *const *args, size_t nargs, PyObject *kwnames); /* PyObject_Vectorcall */ + PyObject *tp_call(PyObject *callee, PyObject *const *args, size_t nargs, PyObject *kwnames); /* PyObject_Vectorcall: nargs = positional count; keyword values follow at args[nargs..] */ + /* tp_binop_obj op codes: TP_BINOP_ADD..TP_BINOP_MOD = 0..5 in ir.BinOpKind declaration order */ int tp_truth(PyObject *obj); /* 0/1, -1 error (PyObject_IsTrue) */ - int tp_compare_bool(PyObject *a, PyObject *b, int op); /* 0/1, -1 error (PyObject_RichCompareBool) */ + int tp_compare_bool(PyObject *a, PyObject *b, int op); /* 0/1, -1 error: PyObject_RichCompare then PyObject_IsTrue (no identity shortcut, as `if a == b`) */ int tp_obj_to_f64(PyObject *obj, double *out); /* 0 / -1 (PyNumber_Float, then exact double) */ PyObject *tp_binop_obj(PyObject *a, PyObject *b, int op); /* PyNumber_Add/... for ir.BinOpKind on OBJ */ void tp_release(PyObject **slot); /* Py_CLEAR: the only way generated code drops a reference */ From ac366e9e34d92175c48b130bbd19cc3a09a6d498 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:38:43 +0900 Subject: [PATCH 23/36] Fix: Put Function.entry_globals after the fields without defaults 957315ea declared the defaulted entry_globals before body/pure/may_deopt/ source_line, so ir.py failed to import (TypeError: non-default argument follows default argument) and the whole package was unusable. Found by all three agents working against it. Refs #41 --- .../src/main/python/typedpython/ir.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index df77bc3e5..806e125a4 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -468,16 +468,16 @@ class Function: params: tuple[Param | ArrayParam, ...] returns: Type locals: dict[str, Type] # non-parameter locals only (temporaries included) + body: tuple[Stmt, ...] + pure: bool + may_deopt: bool + source_line: int # Module globals of a scalar type read ONCE at entry, as part of the guards: each must be # exactly that type (else deopt, before any effect), and the body reads it as `Local(name)`. # Legal only for a *closed* function — one that runs no user code (no GetAttr, CallObject, # Truth, CompareObj, ObjToFloat, OBJ BinOp; Calls only to closed functions) — so nothing can # rebind the global between entry and use, and reading it once equals reading it at each use. entry_globals: tuple[Param, ...] = () - body: tuple[Stmt, ...] - pure: bool - may_deopt: bool - source_line: int @dataclass(frozen=True) From 4c26827c4e4192f49e3d81e33ff7e2dd37ed301d Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:38:57 +0900 Subject: [PATCH 24/36] Feat: Front end records proven i64 ops, redo calls and entry globals Interval-proved I64 ADD/SUB/MUL/NEG carry proven=True (no deopt, allowed in impure code); an impure caller calls a pure may_deopt F64/BOOL/NONE callee with redo=True; scalar module constants read only in closed functions become entry_globals. nbody advance/energy/offset_momentum and spectral-norm eval_a/mul_av/mul_atv now lower. Red: 10 failed / 60 passed. Green: 70/70. Mutation (proven never set): 3 fail. Refs #41 --- .../src/main/python/typedpython/frontend.py | 139 ++++++++- .../src/test/python/test_frontend.py | 265 +++++++++++++++++- 2 files changed, 379 insertions(+), 25 deletions(-) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index f09bf4a70..06d85cd7e 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -19,10 +19,21 @@ may deopt goes through the module global (`CallObject(Global)`), and only `for ... in range()` targets whose every binding is such a loop stay I64. The proof is a small interval analysis: constants, range targets (between their bounds), `len()` and the i64 range of a guarded parameter. +It is recorded in the IR: an I64 ADD/SUB/MUL/NEG it proves cannot overflow is `proven=True` (the +verifier re-proves it; a proven op does not deopt). Division and modulo of ints cannot be recorded, +so they count as deopting. An impure caller of a pure callee that may deopt and returns F64, BOOL +or NONE emits `Call(redo=True)`: only the callee is redone on a deopt. + +Entry globals (ir.Function.entry_globals). A module-level name assigned once at top level, of type +float/int/bool (annotated, or literals and arithmetic of such names), never rebound or deleted in +the module, and read in a *closed* function (one that runs no user code, decided after lowering) +is read once at entry: a `Param` of the function and `Local(name)` at every read. Elsewhere it is a +`Global` (OBJ). """ from __future__ import annotations import ast +import dataclasses import tempfile from collections.abc import Iterable from dataclasses import dataclass, field @@ -100,6 +111,7 @@ def _src(node: ast.AST) -> str: class _ModuleInfo: bindings: dict[str, list[tuple[int, str]]] # module-level name -> [(line, how)] stdlib_math: bool + scalar_globals: dict[str, Type] = field(default_factory=dict) # entry-global candidates def bound(self, name: str) -> bool: return name in self.bindings @@ -141,7 +153,8 @@ def lower(path: Path) -> ir.Module: # Optimistic start: every candidate lowered, pure, not deopting; then iterate to a fixpoint # (purity is the greatest fixpoint through calls, may_deopt the least). env: dict[str, ir.Function] = { - name: ir.Function(name, s.params, s.returns, {}, (), True, False, s.node.lineno) + name: ir.Function(name=name, params=s.params, returns=s.returns, locals={}, body=(), pure=True, + may_deopt=False, source_line=s.node.lineno) for name, s in signatures.items() } results: dict[str, ir.Function | str] = {} @@ -217,7 +230,55 @@ def add(name: str, line: int, how: str) -> None: add(name, node.lineno, "global") math = bindings.get("math", []) - return _ModuleInfo(bindings, len(math) == 1 and math[0][1] == "import math") + return _ModuleInfo(bindings, len(math) == 1 and math[0][1] == "import math", + _scalar_globals(tree, bindings)) + + +def _scalar_globals(tree: ast.Module, bindings: dict[str, list[tuple[int, str]]]) -> dict[str, Type]: + """Module-level names that are assigned exactly once, at top level, with a float/int/bool + value, and are never rebound (`global`, `del`, `:=`, a loop target ...) anywhere in the module.""" + spoiled = {n.id for n in ast.walk(tree) if isinstance(n, ast.Name) + and isinstance(n.ctx, ast.Del)} + spoiled |= {n.target.id for n in ast.walk(tree) + if isinstance(n, ast.NamedExpr) and isinstance(n.target, ast.Name)} + found: dict[str, Type] = {} + for stmt in tree.body: + if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name) \ + and stmt.value is not None: + name, annotated = stmt.target.id, SCALAR_KIND.get(_annotation_kind(stmt.annotation)) + t = annotated + elif isinstance(stmt, ast.Assign) and len(stmt.targets) == 1 \ + and isinstance(stmt.targets[0], ast.Name): + name, t = stmt.targets[0].id, _literal_type(stmt.value, found) + else: + continue + if t is not None and len(bindings.get(name, [])) == 1 and name not in spoiled: + found[name] = t + return found + + +def _literal_type(node: ast.expr, known: dict[str, Type]) -> Type | None: + """The type of an expression made only of literals, arithmetic and `known` globals.""" + if isinstance(node, ast.Constant): + v = node.value + if isinstance(v, bool): + return Type.BOOL + if isinstance(v, int): + return Type.I64 if I64_MIN <= v <= I64_MAX else None + return Type.F64 if isinstance(v, float) else None + if isinstance(node, ast.Name): + return known.get(node.id) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)): + t = _literal_type(node.operand, known) + return t if t in (Type.I64, Type.F64) else None + if isinstance(node, ast.BinOp) and type(node.op) in BINOP: + a, b = _literal_type(node.left, known), _literal_type(node.right, known) + if a not in (Type.I64, Type.F64) or b not in (Type.I64, Type.F64): + return None + if a == b == Type.I64 and not isinstance(node.op, ast.Div): + return Type.I64 + return Type.F64 + return None def _binding_targets(stmt: ast.stmt) -> list[ast.expr]: @@ -349,10 +410,20 @@ def _lower_function( def _lower_in_mode(sig, env, info, inferred, impure: bool) -> ir.Function: + fn = _lower_with_bounds(sig, env, info, inferred, impure, info.scalar_globals) + if fn.entry_globals and not _closed(fn, env): + # A global read once at entry is only the same as a read at each use if nothing between + # can run user code: here something can, so the reads stay `Global`. + fn = _lower_with_bounds(sig, env, info, inferred, impure, {}) + return fn + + +def _lower_with_bounds(sig, env, info, inferred, impure: bool, + candidates: dict[str, Type]) -> ir.Function: # Range-target intervals: least fixpoint by iteration, widened to TOP if it does not settle. bounds: dict[str, Interval] = {} for _ in range(8): - lowerer = _Lowerer(sig, env, info, inferred, impure, bounds) + lowerer = _Lowerer(sig, env, info, inferred, impure, bounds, candidates) fn = lowerer.run() joined = {**bounds} for var, iv in lowerer.found_bounds.items(): @@ -361,7 +432,7 @@ def _lower_in_mode(sig, env, info, inferred, impure: bool) -> ir.Function: return fn bounds = joined widened = {var: TOP for var in bounds} - return _Lowerer(sig, env, info, inferred, impure, widened).run() + return _Lowerer(sig, env, info, inferred, impure, widened, candidates).run() def _join(a: Interval, b: Interval) -> Interval: @@ -381,8 +452,10 @@ class _Binding: class _Lowerer: def __init__(self, sig: _Signature, env: dict[str, ir.Function], info: _ModuleInfo, inferred: dict[str, list[tuple[int, str]]], impure: bool, - bounds: dict[str, Interval]) -> None: + bounds: dict[str, Interval], candidates: dict[str, Type]) -> None: self.sig = sig + self.candidates = candidates + self.entry_used: dict[str, Type] = {} self.fn = sig.node self.env = env self.info = info @@ -433,8 +506,10 @@ def run(self) -> ir.Function: pure = not self.impure assert pure or not self.deopts or not self.impure_mode, \ "impure mode emitted a deopting node" - return ir.Function(self.fn.name, params, self.sig.returns, locals_, body, pure, - self.deopts, self.fn.lineno) + return ir.Function( + name=self.fn.name, params=params, returns=self.sig.returns, locals=locals_, body=body, + pure=pure, may_deopt=self.deopts, source_line=self.fn.lineno, + entry_globals=tuple(ir.Param(n, t) for n, t in self.entry_used.items())) # --- names and types --- @@ -773,6 +848,9 @@ def name(self, node: ast.Name) -> ir.Expr: b.how == "annotation" for b in self.bindings[name]): raise Skip(node.lineno, f"`{name}` is read but never assigned") return ir.Local(self.local_type(name, node.lineno), name) + if name in self.candidates: + self.entry_used.setdefault(name, self.candidates[name]) + return ir.Local(self.candidates[name], name) return ir.Global(Type.OBJ, name) def array_misuse(self, name: str, node: ast.AST) -> Skip: @@ -812,8 +890,8 @@ def arith(self, op: ir.BinOpKind, left: ir.Expr, right: ir.Expr, node: ast.AST) if lt == Type.I64 and rt == Type.I64: result = Type.F64 if op == ir.BinOpKind.TRUEDIV else Type.I64 e = ir.BinOp(result, op, left, right) - if self.proved(e): - return e + if self.recordable(e) and self.proved(e): + return dataclasses.replace(e, proven=True) if self.impure_mode: return self.obj_binop(op, left, right) self.deopts = True @@ -840,8 +918,10 @@ def unary(self, node: ast.UnaryOp) -> ir.Expr: return ir.UnaryOp(Type.F64, kind, operand) if operand.type == Type.I64: e = ir.UnaryOp(Type.I64, kind, operand) - if kind == ir.UnaryOpKind.POS or self.proved(e): + if kind == ir.UnaryOpKind.POS: return e + if self.proved(e): + return dataclasses.replace(e, proven=True) if self.impure_mode: # -x of a CPython int is 0 - x return self.obj_binop(ir.BinOpKind.SUB, ir.Const(Type.I64, 0), operand) self.deopts = True @@ -974,7 +1054,10 @@ def direct_call(self, name: str, node: ast.Call) -> ir.Expr | None: return None self.hoist_ok = False args = [self.value(a) for a in node.args] - direct = not (callee.may_deopt and self.impure_mode) # else a deopt could not be redone + redo = callee.may_deopt and self.impure_mode and callee.pure \ + and callee.returns in (Type.F64, Type.BOOL, Type.NONE) + # else a deopt could not be redone (an I64 result of a redo could be a big int) + direct = redo or not (callee.may_deopt and self.impure_mode) for p, e in zip(callee.params, args): if p.type != Type.OBJ and e.type != p.type: direct = False # the callee's guard would deopt; the global keeps CPython's result @@ -984,11 +1067,11 @@ def direct_call(self, name: str, node: ast.Call) -> ir.Expr | None: tuple(self.boxed(e, a) for e, a in zip(args, node.args))) if not callee.pure: self.impure = True - if callee.may_deopt: + if callee.may_deopt and not redo: self.deopts = True typed = tuple(self.boxed(e, a) if p.type == Type.OBJ else e for p, e, a in zip(callee.params, args, node.args)) - return ir.Call(callee.returns, name, typed) + return ir.Call(callee.returns, name, typed, redo=redo) def math_call(self, attr: str, node: ast.Call) -> ir.Expr | None: try: @@ -1040,6 +1123,11 @@ def interval(self, e: ir.Expr) -> Interval: return self._result_interval(e) return TOP + @staticmethod + def recordable(e: ir.BinOp) -> bool: + """`proven` exists for ADD/SUB/MUL only; the verifier cannot re-prove the others.""" + return e.op in (ir.BinOpKind.ADD, ir.BinOpKind.SUB, ir.BinOpKind.MUL) + def proved(self, e: ir.BinOp | ir.UnaryOp) -> bool: """The i64 operation provably does not deopt (no overflow, exact division).""" if isinstance(e, ir.UnaryOp): @@ -1086,6 +1174,31 @@ def _result_interval(self, e: ir.BinOp | ir.UnaryOp) -> Interval: # --- helpers ------------------------------------------------------------------------------------- +def _walk(x): + if isinstance(x, (tuple, list)): + for y in x: + yield from _walk(y) + elif dataclasses.is_dataclass(x) and not isinstance(x, type): + yield x + for f in dataclasses.fields(x): + yield from _walk(getattr(x, f.name)) + + +def _closed(fn: ir.Function, env: dict[str, ir.Function], seen: frozenset[str] = frozenset()) -> bool: + """`fn` runs no user code: no GetAttr, CallObject, Truth, CompareObj, ObjToFloat or OBJ + BinOp, and calls only closed functions.""" + for n in _walk(fn.body): + if isinstance(n, (ir.GetAttr, ir.CallObject, ir.Truth, ir.CompareObj, ir.ObjToFloat)): + return False + if isinstance(n, ir.BinOp) and n.type == Type.OBJ: + return False + if isinstance(n, ir.Call): + callee = env.get(n.function) + if callee is None or n.function in seen or not _closed(callee, env, seen | {fn.name}): + return False + return True + + def _box(e: ir.Expr) -> ir.Expr: return ir.Box(Type.OBJ, e) if e.type in ir.SCALARS else e diff --git a/python-multiplatform-ksp/src/test/python/test_frontend.py b/python-multiplatform-ksp/src/test/python/test_frontend.py index be2906421..c7da7043f 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend.py @@ -5,6 +5,7 @@ mixed int/float gets `ToFloat`, chains evaluate the middle operand once, an impure function has no node that can deopt. """ +import dataclasses import shutil import textwrap from pathlib import Path @@ -15,7 +16,7 @@ from typedpython.ir import ( And, ArrayParam, Assign, BinOp, BinOpKind, Box, Break, Call, CallObject, Compare, CompareKind, CompareObj, Const, ExprStmt, While, ForRange, GetAttr, Global, If, Index, Len, Local, MathCall, - MathFunc, ObjToFloat, Param, Return, StoreIndex, ToFloat, Truth, Type, + MathFunc, ObjToFloat, Param, Return, StoreIndex, ToFloat, Truth, Type, UnaryOp, UnaryOpKind, ) I64, F64, BOOL, OBJ = Type.I64, Type.F64, Type.BOOL, Type.OBJ @@ -35,6 +36,20 @@ def function(module, name): return found[0] +def nodes(root, cls): + """Every IR node of class `cls` under `root` (a Function, statement or expression).""" + found, stack = [], [root] + while stack: + x = stack.pop() + if isinstance(x, (tuple, list)): + stack.extend(x) + elif dataclasses.is_dataclass(x) and not isinstance(x, type): + if isinstance(x, cls): + found.append(x) + stack.extend(getattr(x, f.name) for f in dataclasses.fields(x)) + return found + + # --- the basic shape ----------------------------------------------------------------------------- def test_scalar_loop_lowers_to_for_range(tmp_path): @@ -130,6 +145,61 @@ def h(a: list[float], n: int) -> None: assert "line 3" in m.skipped["h"] and "n * 2" in m.skipped["h"] +def test_range_index_arithmetic_is_proven(tmp_path): + # ir.BinOp.proven / UnaryOp.proven: the interval proof is recorded in the node. + m = lower(tmp_path, """ + @compiled + def shift(a: list[float]) -> None: + for i in range(4): + a[i + 1] = a[i] - float(-i) + float(i * 2 - 1) + """) + f = function(m, "shift") + assert not f.pure and not f.may_deopt + ints = [b for b in nodes(f, BinOp) if b.type == I64] + assert ints and all(b.proven for b in ints) + assert BinOp(I64, BinOpKind.ADD, Local(I64, "i"), Const(I64, 1), proven=True) in ints + assert nodes(f, UnaryOp) == [UnaryOp(I64, UnaryOpKind.NEG, Local(I64, "i"), proven=True)] + + +def test_unbounded_int_arithmetic_is_not_proven(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(a: int, b: int) -> int: + return -(a + b) + """) + f = function(m, "f") + assert f.pure and f.may_deopt + assert f.body == (Return(UnaryOp(I64, UnaryOpKind.NEG, BinOp( + I64, BinOpKind.ADD, Local(I64, "a"), Local(I64, "b")))),) + assert not any(b.proven for b in nodes(f, BinOp)) and not nodes(f, UnaryOp)[0].proven + + +def test_float_arithmetic_is_never_marked_proven(tmp_path): + # `proven` is for I64 ADD/SUB/MUL/NEG only (F64 never deopts). + m = lower(tmp_path, """ + @compiled + def f(x: float) -> float: + return -(x + 1.0) + """) + f = function(m, "f") + assert not any(n.proven for n in nodes(f, BinOp) + nodes(f, UnaryOp)) + + +@pytest.mark.parametrize("expr", ["i // 2", "i / 2"]) +def test_int_division_cannot_be_recorded_as_proven_so_it_deopts(tmp_path, expr): + # ir.BinOp.proven exists for ADD/SUB/MUL only; the verifier cannot re-prove // and / on i64, + # so they count as deopting: in a pure function may_deopt, in an impure one a CPython int. + m = lower(tmp_path, f""" +@compiled +def f(n: int) -> float: + t: float = 0.0 + for i in range(10): + t += {expr} + return t +""") + assert function(m, "f").may_deopt + + # --- conversions --------------------------------------------------------------------------------- def test_mixed_int_float_arithmetic_inserts_to_float(tmp_path): @@ -238,7 +308,7 @@ def bump_last(a: list[float], e: float) -> None: f = function(m, "bump_last") first, store = f.body assert isinstance(first, Assign) - assert first.value == BinOp(I64, BinOpKind.SUB, Len(I64, "a"), Const(I64, 1)) + assert first.value == BinOp(I64, BinOpKind.SUB, Len(I64, "a"), Const(I64, 1), proven=True) t = first.target assert store == StoreIndex( "a", Local(I64, t), @@ -316,7 +386,8 @@ def norm(x: float) -> float: # --- calls and purity ---------------------------------------------------------------------------- -def test_impure_caller_of_a_deopting_function_is_skipped(tmp_path): +def test_impure_caller_redoes_a_deopting_pure_callee(tmp_path): + # ir.Call.redo: on a deopt only the (pure) callee is redone; the caller does not deopt. m = lower(tmp_path, """ @compiled def tri(i: int, j: int) -> float: @@ -329,7 +400,66 @@ def fill(n: int, out: list[float]) -> None: """) tri = function(m, "tri") assert tri.pure and tri.may_deopt - assert "tri" in m.skipped["fill"] and "line 8" in m.skipped["fill"] + fill = function(m, "fill") + assert not fill.pure and not fill.may_deopt + assert nodes(fill, Call) == [Call(F64, "tri", (Local(I64, "i"), Local(I64, "i")), redo=True)] + assert nodes(fill, CallObject) == [] + + +@pytest.mark.parametrize("returns, body, use", [ + ("bool", "return i + j > 0", "if tri(i, i):\n out[i] = 1.0"), + ("None", "k = i + j", "tri(i, i)\n out[i] = 1.0"), +]) +def test_redo_call_for_bool_and_none_callees(tmp_path, returns, body, use): + m = lower(tmp_path, f""" +@compiled +def tri(i: int, j: int) -> {returns}: + {body} + +@compiled +def fill(n: int, out: list[float]) -> None: + for i in range(n): + {use} +""") + assert function(m, "tri").may_deopt + fill = function(m, "fill") + assert not fill.pure and not fill.may_deopt + [call] = nodes(fill, Call) + assert call.function == "tri" and call.redo + + +def test_impure_caller_of_an_int_returning_deopting_callee_calls_the_global(tmp_path): + # An I64 result of a redo could be a big int: such a callee is still called as an object. + m = lower(tmp_path, """ + @compiled + def add(i: int, j: int) -> int: + return i + j + + @compiled + def fill(n: int, out: list[float]) -> None: + for i in range(n): + out[i] = float(add(i, i)) + """) + fill = function(m, "fill") + assert not fill.pure and not fill.may_deopt + assert nodes(fill, Call) == [] + assert nodes(fill, CallObject) == [ + CallObject(OBJ, Global(OBJ, "add"), (Box(OBJ, Local(I64, "i")), Box(OBJ, Local(I64, "i"))))] + + +def test_pure_caller_of_a_deopting_callee_does_not_redo(tmp_path): + m = lower(tmp_path, """ + @compiled + def tri(i: int, j: int) -> float: + return 1.0 / (i + j) + + @compiled + def twice(i: int) -> float: + return tri(i, i) * 2.0 + """) + twice = function(m, "twice") + assert twice.pure and twice.may_deopt + assert [c.redo for c in nodes(twice, Call)] == [False] def test_pure_caller_of_a_deopting_function_lowers(tmp_path): @@ -398,25 +528,107 @@ def test_unsupported_signature_is_skipped(tmp_path, signature): assert m.functions == () and "f" in m.skipped -def test_global_read_is_a_global_node(tmp_path): +def test_global_read_in_a_function_that_runs_user_code_is_a_global_node(tmp_path): + # `print(x)` runs user code, which could rebind K: K is read where it is used (LOAD_GLOBAL). m = lower(tmp_path, """ K: float = 2.0 @compiled def f(x: float) -> object: + print(x) return x * K @compiled def g(x: float) -> float: + print(x) return x * K """) f = function(m, "f") - assert f.body == (Return( - BinOp(OBJ, BinOpKind.MUL, Box(OBJ, Local(F64, "x")), Global(OBJ, "K"))),) + assert f.body[1] == Return( + BinOp(OBJ, BinOpKind.MUL, Box(OBJ, Local(F64, "x")), Global(OBJ, "K"))) + assert f.entry_globals == () assert not f.pure and not f.may_deopt assert "x * K" in m.skipped["g"] +def test_scalar_globals_in_a_closed_function_are_entry_globals(tmp_path): + m = lower(tmp_path, """ + PI: float = 3.141592653589793 + SOLAR_MASS: float = 4 * PI * PI + DAYS = 365.24 + N = 3 + ON: bool = True + + @compiled + def scale(a: list[float]) -> None: + for i in range(N): + if ON: + a[i] = a[i] / SOLAR_MASS * DAYS + + @compiled + def k(x: float) -> float: + return x * SOLAR_MASS + """) + scale = function(m, "scale") + assert scale.entry_globals == (Param("N", I64), Param("ON", BOOL), Param("SOLAR_MASS", F64), + Param("DAYS", F64)) + assert nodes(scale, Global) == [] + assert Local(F64, "SOLAR_MASS") in nodes(scale, Local) + assert not scale.pure and not scale.may_deopt + k = function(m, "k") + assert k.entry_globals == (Param("SOLAR_MASS", F64),) + assert k.body == (Return(BinOp(F64, BinOpKind.MUL, Local(F64, "x"), Local(F64, "SOLAR_MASS"))),) + assert k.pure and not k.may_deopt + assert "SOLAR_MASS" not in k.locals + + +def test_closedness_follows_calls(tmp_path): + m = lower(tmp_path, """ + K: float = 2.0 + + @compiled + def noisy(x: float) -> float: + print(x) + return x + + @compiled + def quiet(x: float) -> float: + return x + 1.0 + + @compiled + def via_noisy(x: float) -> object: + return noisy(x) * K + + @compiled + def via_quiet(x: float) -> float: + return quiet(x) * K + """) + assert function(m, "via_noisy").entry_globals == () + assert Global(OBJ, "K") in nodes(function(m, "via_noisy"), Global) + assert function(m, "via_quiet").entry_globals == (Param("K", F64),) + + +@pytest.mark.parametrize("module", [ + "K: float = 2.0\nK = 3.0\n", # assigned twice + "K: float = 2.0\n\ndef reset() -> None:\n global K\n K = 1.0\n", # `global K` elsewhere + "import os\nif os.sep:\n K = 2.0\n", # not at top level + "for K in (1.0,):\n pass\n", # a loop target + "import os\nK = os.cpu_count()\n", # neither annotated nor literal + "K: str = 'a'\n", # not a scalar + "K: float = 2.0\ndel K\n", # deleted +]) +def test_a_global_that_may_be_rebound_is_never_an_entry_global(tmp_path, module): + m = lower(tmp_path, module + """ + +@compiled +def f(x: float) -> object: + return x * K +""") + f = function(m, "f") + assert f.entry_globals == () + assert Global(OBJ, "K") in nodes(f, Global) + + def test_module_marker_compiles_every_module_level_def(tmp_path): m = lower(tmp_path, """ @@ -576,14 +788,22 @@ def test_recursion_is_skipped(tmp_path, source): assert m.functions == () and "recurs" in m.skipped["f"] -def test_int_modulo_never_deopts(tmp_path): +def test_int_modulo_cannot_be_recorded_as_proven_so_it_deopts(tmp_path): + # ir.BinOp.proven covers ADD/SUB/MUL only and the verifier counts an unproven i64 `%` as a + # deopting node: in a pure function that is may_deopt, in an impure one a CPython int. m = lower(tmp_path, """ + @compiled + def pure_mod(i: int, n: int) -> int: + return i % n + @compiled def put(a: list[float], i: int, n: int) -> None: a[i % n] = 1.0 """) - f = function(m, "put") - assert not f.pure and not f.may_deopt + f = function(m, "pure_mod") + assert f.pure and f.may_deopt + assert not any(b.proven for b in nodes(f, BinOp)) + assert "i % n" in m.skipped["put"] def test_locals_exclude_parameters(tmp_path): @@ -604,8 +824,29 @@ def test_nbody_kernels_lower(tmp_path): shutil.copy(BENCH / "nbody.py", path) path.write_text("# typedpython: compiled\n" + path.read_text()) m = frontend.lower(path) - for name in ("advance", "energy"): + for name in ("advance", "energy", "offset_momentum"): function(m, name) advance = function(m, "advance") assert not advance.pure and not advance.may_deopt - assert function(m, "energy").pure + assert BinOp(I64, BinOpKind.ADD, Local(I64, "i"), Const(I64, 1), proven=True) \ + in nodes(advance, BinOp) + energy = function(m, "energy") + assert energy.pure and not energy.may_deopt + offset = function(m, "offset_momentum") + assert not offset.pure and not offset.may_deopt + assert offset.entry_globals == (Param("SOLAR_MASS", F64),) + assert nodes(offset, Global) == [] + + +@pytest.mark.skipif(not BENCH.exists(), reason="benchmark worktree not present") +def test_spectral_norm_kernels_lower(tmp_path): + path = tmp_path / "spectral_norm.py" + path.write_text("# typedpython: compiled\n" + (BENCH / "spectral_norm.py").read_text()) + m = frontend.lower(path) + eval_a = function(m, "eval_a") + assert eval_a.pure and eval_a.may_deopt + for name in ("mul_av", "mul_atv"): + f = function(m, name) + assert not f.pure and not f.may_deopt + [call] = nodes(f, Call) + assert call.function == "eval_a" and call.redo From f34e23850ab6a153c72d4620d993778e43d2ba5b Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:38:57 +0900 Subject: [PATCH 25/36] Feat: Verifier re-proves proven ops, redo calls and entry globals Exact interval arithmetic re-proves every proven I64 op (never trusts the flag); redo is legal only from an impure caller to a pure may_deopt F64/BOOL/NONE callee; entry globals need a closed function (fixpoint over the call graph). A call of a function with entry globals counts as deopting in the caller unless it is a legal redo or the caller snapshots the same globals. Red: 42 failed. Green: 135/135; 300 seeded mutants of 16 kinds all rejected. Mutation (interval-fit check off): 12 fail. Refs #41 --- .../src/main/python/typedpython/verify.py | 223 +++++++- .../src/test/python/test_verify.py | 475 +++++++++++++++++- 2 files changed, 691 insertions(+), 7 deletions(-) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/verify.py b/python-multiplatform-ksp/src/main/python/typedpython/verify.py index 949c0bb2c..954d9bfb5 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/verify.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/verify.py @@ -33,7 +33,60 @@ operand that is itself in a condition): its rich-comparison result need not be a bool, so only its truth value may be used verify/structure break/continue outside a loop, the ForRange var assigned in its body, - a zero or non-constant step, duplicate names, bad declarations + a zero or non-constant step, duplicate names, bad declarations, an + entry global that collides with a parameter/local/other entry global or + is assigned in the body + verify/proven-overflow `proven` is set on an op the verifier cannot prove stays inside i64 (or + on a node `proven` does not apply to, or it is not a bool) + verify/redo `Call.redo` where it is not legal (see below) + verify/entry-global-open a function with entry_globals is not closed + +Proven i64 operations. The verifier never trusts `BinOp.proven` / `UnaryOp.proven`: for every I64 +expression it computes an interval [lo, hi] (exact Python ints) and accepts `proven=True` only on an +I64 ADD/SUB/MUL (BinOp) or NEG (UnaryOp) whose exact result interval lies inside [-2**63, 2**63-1]; +anything else (a different op, a non-I64 node, a result that may leave i64, a non-bool flag) is +rejected, and so is the whole function. A proven op is not a deopting node (an impure function may +contain it; a pure one does not count it toward `may_deopt`); every unproven I64 ADD/SUB/MUL/NEG +still deopts. The interval rules, exactly: + + Const v [v, v] + Local, a ForRange var inside its own loop body: + step > 0 [lo(start), max(lo(start), hi(stop) - 1)] + step < 0 [min(hi(start), lo(stop) + 1), hi(start)] + (the bounds are the intervals of the start/stop expressions, both + evaluated before the loop; then intersected with the i64 range) + Len(array) [0, 2**62] + ADD / SUB / MUL interval arithmetic on the operand intervals (for MUL the min/max of + the four endpoint products); a proven op must fit i64 exactly, an + unproven one is clamped to i64 (a result outside it deopts, so only + in-range values flow on) + NEG / POS [-hi, -lo] / [lo, hi] + everything else the full i64 range: I64 params and entry globals, every local that is + not the ForRange var of an enclosing loop (the verifier does no flow + analysis, so a local assigned a constant is still "any i64"), a loop + var after its loop, Index elements, Unbox, Call results, FLOORDIV, + MOD, TRUEDIV + +What this cannot prove: anything that needs knowing a value through assignments (accumulators, +`x = 1; x + 1`), relations between variables (`i < n` does not bound `i + n`), bounds that come from +guards or conditions (`if n < 100`), values through array elements, calls or Unbox, the result of +`//` and `%` (even by a constant), a loop var after the loop, or `range` bounds that are themselves +full-range (`range(n)` bounds the var only by i64). A function needing such a proof keeps its +checked, deopting ops (and is then legal only if pure). + +Call.redo. Legal only if the caller is impure, the callee is pure and may_deopt, and the callee +returns F64, BOOL or NONE (`verify/redo` otherwise). A redo call is not a deopting node in the +caller: the callee alone is redone, unobservably, since it is pure. + +Function.entry_globals. Scalar `Param`s (I64/F64/BOOL), read once at entry as part of the guards; +assigned at entry for definite assignment; the body reads them as `Local(name)` of that type and may +not assign them; their names collide with no parameter, local or other entry global. The function +must be *closed*: no GetAttr, CallObject, Truth, CompareObj, ObjToFloat or OBJ BinOp, and Calls only +to closed functions (a fixpoint over the call graph), else `verify/entry-global-open`. Beyond that +rule (conservative, not in ir.py): a Call of a function that has entry globals can fail the callee's +entry guard in the middle of the caller, which is a deopt there unless the caller is itself guarded +the same way (it lists the same name and type as an entry global, and being closed cannot see it +rebound) or the call is a legal `redo`. """ from __future__ import annotations @@ -63,6 +116,9 @@ RECURSION = "verify/recursion" STRUCTURE = "verify/structure" CONDITION_ONLY = "verify/condition-only" +PROVEN = "verify/proven-overflow" +REDO = "verify/redo" +ENTRY_OPEN = "verify/entry-global-open" @dataclass(frozen=True) @@ -90,6 +146,9 @@ class Diagnostic: _I64_DEOPT_OPS = frozenset({BinOpKind.ADD, BinOpKind.SUB, BinOpKind.MUL, BinOpKind.FLOORDIV, BinOpKind.MOD, BinOpKind.TRUEDIV}) _I64_MIN, _I64_MAX = -2 ** 63, 2 ** 63 - 1 +_FULL = (_I64_MIN, _I64_MAX) +_LEN_MAX = 2 ** 62 +_PROVABLE_OPS = frozenset({BinOpKind.ADD, BinOpKind.SUB, BinOpKind.MUL}) # Definite assignment uses `None` for "unreachable" (bottom: every name counts as assigned). _State = frozenset[str] | None @@ -107,6 +166,7 @@ class _Loop: def __init__(self, var: str | None, bounds_array: str | None): self.var = var # ForRange var (None for While) self.bounds_array = bounds_array # array whose indices by `var` are in bounds + self.interval = _FULL # the values `var` takes inside the body self.breaks: list[_State] = [] @@ -125,6 +185,8 @@ def __init__(self, function: Function, signatures: dict[str, Function]): self.loops: list[_Loop] = [] self.scalars: dict[str, Type] = {} # name -> type of every scalar/OBJ variable self.arrays: dict[str, Type] = {} # name -> array type of every ArrayParam + self.entry: dict[str, Type] = {} # name -> type of every entry global + self.opens: list[str] = [] # reasons the function is not closed def error(self, rule: str, message: str) -> None: d = Diagnostic(self.fn.name, rule, message, self.fn.source_line) @@ -156,11 +218,35 @@ def declarations(self) -> frozenset[str]: if p.name in params: self.error(STRUCTURE, f"parameter '{p.name}' appears twice") params.add(p.name) + eg = fn.entry_globals + if not isinstance(eg, tuple): + self.error(STRUCTURE, "entry_globals is not a tuple") + eg = () + for p in eg: + if type(p) is ArrayParam: + self.error(TYPE, f"entry global '{p.name}' is an array") + continue + if type(p) is not Param: + self.error(UNKNOWN, f"entry global {p!r} is not an ir Param") + continue + if p.type not in ir.SCALARS: + self.error(TYPE, f"entry global '{p.name}' has type {p.type}; only i64, f64 and " + f"bool globals can be read at entry") + continue + if p.name in params or p.name in self.entry: + self.error(STRUCTURE, f"entry global '{p.name}' collides with a parameter or " + f"another entry global") + continue + self.entry[p.name] = p.type + self.scalars[p.name] = p.type + params |= set(self.entry) if not isinstance(fn.locals, dict): self.error(STRUCTURE, "locals is not a dict") return frozenset(params) for name, t in fn.locals.items(): - if name in self.arrays: + if name in self.entry: + self.error(STRUCTURE, f"local '{name}' collides with an entry global") + elif name in self.arrays: self.error(ARRAY_USE, f"local '{name}' shadows array parameter '{name}'") elif t not in _VALUE_TYPES: self.error(TYPE, f"local '{name}' has type {t}") @@ -242,12 +328,56 @@ def e_Local(self, e: Local, state: _State) -> ir.Expr: self.error(DA, f"local '{name}' may be read before it is assigned") return e + def interval(self, e: ir.Expr) -> tuple[int, int]: + """The interval rules in the module docstring: [lo, hi] bounding every value `e` (an I64 + expression, already checked) can take when it completes without deopting.""" + cls = type(e) + if getattr(e, "type", None) is not Type.I64: + return _FULL + if cls is Const: + return (e.value, e.value) if type(e.value) is int else _FULL + if cls is Local: + for loop in reversed(self.loops): + if loop.var == e.name: + return loop.interval + return _FULL + if cls is Len: + return (0, _LEN_MAX) + if cls is BinOp and e.op in _PROVABLE_OPS: + lo, hi = _exact(e.op, self.interval(e.left), self.interval(e.right)) + return (max(lo, _I64_MIN), min(hi, _I64_MAX)) + if cls is UnaryOp and e.op in (UnaryOpKind.NEG, UnaryOpKind.POS): + lo, hi = self.interval(e.operand) + lo, hi = (-hi, -lo) if e.op is UnaryOpKind.NEG else (lo, hi) + return (max(lo, _I64_MIN), min(hi, _I64_MAX)) + return _FULL + + def check_proven(self, shape_ok: bool, interval: tuple[int, int], what: str) -> bool: + """`proven=True` (already known to be a bool) is accepted only if re-proved. Returns + whether the node counts as non-deopting.""" + if not shape_ok: + self.error(PROVEN, f"`proven` is set on {what}, which has no i64 overflow to prove") + elif not (_I64_MIN <= interval[0] and interval[1] <= _I64_MAX): + self.error(PROVEN, f"`proven` is set on {what} but its result interval " + f"[{interval[0]}, {interval[1]}] is not inside i64") + else: + return True + return False + def e_BinOp(self, e: BinOp, state: _State) -> ir.Expr: if type(e.op) is not BinOpKind: self.error(UNKNOWN, f"BinOp op {e.op!r} is not a BinOpKind") return e left, right = self.expr(e.left, state), self.expr(e.right, state) lt, rt = getattr(left, "type", None), getattr(right, "type", None) + proven = False + if type(e.proven) is not bool: + self.error(TYPE, f"BinOp `proven` {e.proven!r} is not a bool") + elif e.proven: + shape = (e.type is Type.I64 and lt is Type.I64 and rt is Type.I64 + and e.op in _PROVABLE_OPS) + exact = _exact(e.op, self.interval(left), self.interval(right)) if shape else _FULL + proven = self.check_proven(shape, exact, f"{e.type.value} {e.op.value}") if lt not in (Type.I64, Type.F64, Type.OBJ): self.error(TYPE if lt is not Type.OBJ else OWNERSHIP, f"BinOp {e.op.value} on {getattr(lt, 'value', lt)} operands") @@ -258,10 +388,11 @@ def e_BinOp(self, e: BinOp, state: _State) -> ir.Expr: if e.type is not want: self.error(TYPE, f"{lt.value} {e.op.value} {rt.value} is {want.value}, node says " f"{e.type.value}") - if lt is Type.I64 and e.op in _I64_DEOPT_OPS: + if lt is Type.I64 and e.op in _I64_DEOPT_OPS and not proven: self.can_deopt = True if lt is Type.OBJ: self.effects.append(f"OBJ {e.op.value} (runs user methods)") + self.opens.append(f"OBJ {e.op.value} (runs user methods)") return dataclasses.replace(e, left=left, right=right) def e_UnaryOp(self, e: UnaryOp, state: _State) -> ir.Expr: @@ -270,6 +401,14 @@ def e_UnaryOp(self, e: UnaryOp, state: _State) -> ir.Expr: return e operand = self.expr(e.operand, state) ot = getattr(operand, "type", None) + proven = False + if type(e.proven) is not bool: + self.error(TYPE, f"UnaryOp `proven` {e.proven!r} is not a bool") + elif e.proven: + shape = e.type is Type.I64 and ot is Type.I64 and e.op is UnaryOpKind.NEG + lo, hi = self.interval(operand) if shape else _FULL + proven = self.check_proven(shape, (-hi, -lo) if shape else _FULL, + f"unary {e.op.value} of {getattr(ot, 'value', ot)}") if e.op is UnaryOpKind.NOT: self.slot(operand, Type.BOOL, "operand of not") if e.type is not Type.BOOL: @@ -282,7 +421,7 @@ def e_UnaryOp(self, e: UnaryOp, state: _State) -> ir.Expr: elif e.type is not ot: self.error(TYPE, f"unary {e.op.value} of {ot.value} is {ot.value}, node says " f"{e.type.value}") - elif ot is Type.I64 and e.op is UnaryOpKind.NEG: + elif ot is Type.I64 and e.op is UnaryOpKind.NEG and not proven: self.can_deopt = True return dataclasses.replace(e, operand=operand) @@ -387,12 +526,39 @@ def e_Call(self, e: Call, state: _State) -> ir.Expr: if e.type is not callee.returns: self.error(TYPE, f"'{e.function}' returns {callee.returns.value}, node says " f"{e.type.value}") - if callee.may_deopt is True: + redo = False + if type(e.redo) is not bool: + self.error(TYPE, f"Call `redo` {e.redo!r} is not a bool") + elif e.redo: + if self.fn.pure is True: + self.error(REDO, f"redo call of '{e.function}' in a pure function (a pure caller " + f"propagates the deopt instead)") + elif callee.pure is not True or callee.may_deopt is not True: + self.error(REDO, f"redo call of '{e.function}', which is not a pure may_deopt " + f"function") + elif callee.returns not in (Type.F64, Type.BOOL, Type.NONE): + self.error(REDO, f"redo call of '{e.function}' returning " + f"{getattr(callee.returns, 'value', callee.returns)}: only f64, " + f"bool and none results are exactly reproducible") + else: + redo = True + if callee.may_deopt is True and not redo: + self.can_deopt = True + if not redo and not self.covers(callee): self.can_deopt = True if callee.pure is not True: self.effects.append(f"calls impure '{e.function}'") return dataclasses.replace(e, args=args) + def covers(self, callee: Function) -> bool: + """Every entry global of `callee` is also an entry global of this function with the same + type: the callee's entry guard cannot fail mid-caller (this function, being closed, + cannot see the global rebound).""" + eg = callee.entry_globals + if not isinstance(eg, tuple): + return False + return all(type(p) is Param and self.entry.get(p.name) is p.type for p in eg) + def obj_operand(self, e: object, state: _State, what: str) -> ir.Expr: x = self.expr(e, state) self.slot(x, Type.OBJ, what) @@ -414,6 +580,7 @@ def e_GetAttr(self, e: GetAttr, state: _State) -> ir.Expr: self.error(TYPE, f"GetAttr name {e.name!r} is not a str") self.result(e, Type.OBJ) self.effects.append(f"GetAttr .{e.name} (a property may have effects)") + self.opens.append(f"GetAttr .{e.name}") return dataclasses.replace(e, obj=obj) def e_CallObject(self, e: CallObject, state: _State) -> ir.Expr: @@ -429,12 +596,14 @@ def e_CallObject(self, e: CallObject, state: _State) -> ir.Expr: f"argument(s) (they must name distinct trailing arguments)") self.result(e, Type.OBJ) self.effects.append("CallObject") + self.opens.append("CallObject") return dataclasses.replace(e, callee=callee, args=args) def e_Truth(self, e: Truth, state: _State) -> ir.Expr: operand = self.obj_operand(e.operand, state, "operand of Truth") self.result(e, Type.BOOL) self.effects.append("Truth (runs __bool__/__len__)") + self.opens.append("Truth") return dataclasses.replace(e, operand=operand) def e_CompareObj(self, e: CompareObj, state: _State) -> ir.Expr: @@ -450,12 +619,14 @@ def e_CompareObj(self, e: CompareObj, state: _State) -> ir.Expr: self.error(TYPE, f"CompareObj operand of type {getattr(t, 'value', t)}") self.result(e, Type.BOOL) self.effects.append("CompareObj (runs rich comparison)") + self.opens.append("CompareObj") return dataclasses.replace(e, left=left, right=right) def e_ObjToFloat(self, e: ObjToFloat, state: _State) -> ir.Expr: operand = self.obj_operand(e.operand, state, "operand of ObjToFloat") self.result(e, Type.F64) self.effects.append("ObjToFloat (runs __float__/__index__)") + self.opens.append("ObjToFloat") return dataclasses.replace(e, operand=operand) def array(self, name: str, node: str) -> Type | None: @@ -525,6 +696,8 @@ def s_Assign(self, s: Assign, state: _State): self.slot(value, self.scalars[target], f"assignment to '{target}'") if getattr(value, "type", None) is Type.NONE: self.error(TYPE, f"None assigned to '{target}'") + if target in self.entry: + self.error(STRUCTURE, f"entry global '{target}' is assigned in the body") for loop in self.loops: if loop.var == target: self.error(STRUCTURE, f"loop variable '{target}' is assigned in its loop body") @@ -582,6 +755,8 @@ def s_ForRange(self, s: ForRange, state: _State): self.error(ARRAY_USE, f"loop variable '{var}' is an array parameter") elif self.scalars.get(var) is not Type.I64: self.error(TYPE, f"loop variable '{var}' must be a declared i64 local") + if var in self.entry: + self.error(STRUCTURE, f"entry global '{var}' is assigned by the loop") for outer in self.loops: if outer.var == var: self.error(STRUCTURE, f"loop variable '{var}' is assigned in its loop body " @@ -592,6 +767,16 @@ def s_ForRange(self, s: ForRange, state: _State): and stop.array in self.arrays): bounds = stop.array loop = _Loop(var, bounds) + if (step_ok and getattr(start, "type", None) is Type.I64 + and getattr(stop, "type", None) is Type.I64): + (slo, shi), (elo, ehi) = self.interval(start), self.interval(stop) + if step.value > 0: + lo = slo + hi = max(lo, ehi - 1) + else: + hi = shi + lo = min(hi, elo + 1) + loop.interval = (max(lo, _I64_MIN), min(hi, _I64_MAX)) self.loops.append(loop) body, _ = self.block(s.body, None if state is None else state | {var}) self.loops.pop() @@ -653,6 +838,16 @@ def run(self) -> Function: return dataclasses.replace(fn, body=body) +def _exact(op: BinOpKind, a: tuple[int, int], b: tuple[int, int]) -> tuple[int, int]: + """The exact (unbounded) result interval of ADD/SUB/MUL on operand intervals.""" + if op is BinOpKind.ADD: + return a[0] + b[0], a[1] + b[1] + if op is BinOpKind.SUB: + return a[0] - b[1], a[1] - b[0] + products = (a[0] * b[0], a[0] * b[1], a[1] * b[0], a[1] * b[1]) + return min(products), max(products) + + def _cycles(graph: dict[str, set[str]]) -> set[str]: """Names on a cycle of `graph` (Tarjan's strongly connected components).""" index: dict[str, int] = {} @@ -721,6 +916,7 @@ def reject(fn_name: str, line: int, rule: str, message: str) -> None: checked: dict[str, Function] = {} callees: dict[str, set[str]] = {} + open_why: dict[str, str] = {} # a function that runs user code -> why for name, g in sigs.items(): c = _Checker(g, sigs) rebuilt = c.run() @@ -728,6 +924,23 @@ def reject(fn_name: str, line: int, rule: str, message: str) -> None: reject(d.function, d.source_line, d.rule, d.message) checked[name] = rebuilt callees[name] = c.callees + if c.opens: + open_why[name] = c.opens[0] + + # Closedness (entry_globals): a fixpoint over the call graph. + changed = True + while changed: + changed = False + for name, cs in callees.items(): + if name not in open_why: + bad = sorted(c for c in cs if c in open_why) + if bad: + open_why[name] = f"calls '{bad[0]}', which runs user code ({open_why[bad[0]]})" + changed = True + for name, g in sigs.items(): + if name in open_why and isinstance(g.entry_globals, tuple) and g.entry_globals: + reject(name, g.source_line, ENTRY_OPEN, + f"has entry globals but is not closed: {open_why[name]}") for name in sorted(_cycles(callees)): reject(name, sigs[name].source_line, RECURSION, diff --git a/python-multiplatform-ksp/src/test/python/test_verify.py b/python-multiplatform-ksp/src/test/python/test_verify.py index 903b06f77..662110338 100644 --- a/python-multiplatform-ksp/src/test/python/test_verify.py +++ b/python-multiplatform-ksp/src/test/python/test_verify.py @@ -42,6 +42,9 @@ RECURSION = "verify/recursion" STRUCTURE = "verify/structure" CONDITION_ONLY = "verify/condition-only" +PROVEN = "verify/proven-overflow" +REDO = "verify/redo" +ENTRY_OPEN = "verify/entry-global-open" def _verify(): @@ -67,10 +70,18 @@ def call_obj(callee, *args, kw=()): return CallObject(OBJ, callee, tuple(args), def pycall(name, *args): return call_obj(g_(name), *args) +def padd(l, r): return BinOp(I64, BinOpKind.ADD, l, r, proven=True) +def psub(l, r): return BinOp(I64, BinOpKind.SUB, l, r, proven=True) +def pmul(l, r): return BinOp(I64, BinOpKind.MUL, l, r, proven=True) +def pneg(x): return UnaryOp(I64, UnaryOpKind.NEG, x, proven=True) +IMPURE = ExprStmt(CallObject(OBJ, Global(OBJ, "print"), (), ())) # makes a function impure + + def fn(name="f", body=(), params=(), locals_=None, returns=NONE, pure=False, may_deopt=False, - line=10): + line=10, entry_globals=()): return Function(name=name, params=tuple(params), returns=returns, locals=dict(locals_ or {}), - body=tuple(body), pure=pure, may_deopt=may_deopt, source_line=line) + body=tuple(body), pure=pure, may_deopt=may_deopt, source_line=line, + entry_globals=tuple(entry_globals)) def run(*functions, skipped=None): @@ -704,6 +715,320 @@ def test_duplicate_function_names_are_rejected(): assert kept(out) == [] and STRUCTURE in rules_for(diags, "f") +# --- 7. proven i64 operations (BinOp.proven / UnaryOp.proven) ------------------------------------ +# +# The verifier never trusts `proven`: it computes an interval for every I64 expression and accepts +# a proven op only when the exact result interval fits in i64. Intervals (verify.py documents them): +# Const [v, v]; a ForRange var inside its own loop [lo(start), hi(stop)-1] (step > 0) or +# [lo(stop)+1, hi(start)] (step < 0); Len [0, 2**60-1]; ADD/SUB/MUL/NEG/POS by interval arithmetic; +# MOD by a divisor whose interval is positive [0, hi(divisor)-1]; FLOORDIV by a positive divisor +# interval by its endpoints; everything else (params, entry globals, other locals, Index, Unbox, +# Call) the full i64 range. + +def _ranged(op, start=0, stop=5, step=1, impure=True, pure=False, may_deopt=False, params=()): + """for k in range(start, stop, step): t = in an (impure) function""" + body = ([IMPURE] if impure else []) + [ + ForRange("k", i(start) if isinstance(start, int) else start, + i(stop) if isinstance(stop, int) else stop, i(step), (Assign("t", op(li("k"))),)), + Return(), + ] + return fn(params=params, locals_={"k": I64, "t": I64}, pure=pure, may_deopt=may_deopt, + body=body) + + +def test_proven_add_of_a_small_range_var_is_accepted_in_an_impure_function(): + assert_accepted(_ranged(lambda k: padd(k, i(1)))) + + +def test_proven_add_of_two_i64_params_is_rejected(): + ps = [Param("a", I64), Param("b", I64)] + assert_rejected(PROVEN, fn(params=ps, returns=I64, body=[IMPURE, Return(padd(li("a"), li("b")))])) + assert_rejected(PROVEN, fn(params=ps, returns=I64, pure=True, + body=[Return(padd(li("a"), li("b")))])) + + +def test_unproven_i64_ops_still_deopt_in_an_impure_function(): + assert_rejected(DEOPT, _ranged(lambda k: add(k, i(1)))) + assert_rejected(DEOPT, _ranged(lambda k: UnaryOp(I64, UnaryOpKind.NEG, k))) + + +def test_a_proven_op_is_not_a_deopting_node(): + # pure function whose only i64 op is proven: it cannot deopt, so may_deopt must be False + ok = _ranged(lambda k: padd(k, i(1)), impure=False, pure=True, may_deopt=False) + assert_accepted(ok) + assert_rejected(FLAG_DEOPT, dataclasses.replace(ok, may_deopt=True)) + + +@pytest.mark.parametrize("op", [ + lambda k: psub(k, i(1)), + lambda k: pmul(k, k), + lambda k: pneg(k), + lambda k: UnaryOp(I64, UnaryOpKind.NEG, k, proven=True), + lambda k: padd(padd(k, i(1)), pmul(k, i(3))), # proven ops nest +]) +def test_other_provable_ops_are_accepted(op): + assert_accepted(_ranged(op)) + + +def test_proven_neg_of_a_param_is_rejected(): + # -(-2**63) overflows + assert_rejected(PROVEN, fn(params=[Param("n", I64)], returns=I64, body=[IMPURE, Return(pneg(li("n")))])) + + +def test_proven_on_a_node_that_has_no_overflow_is_rejected(): + ps = [Param("x", F64), Param("n", I64)] + assert_rejected(PROVEN, fn(params=ps, returns=F64, + body=[Return(BinOp(F64, BinOpKind.ADD, lf("x"), lf("x"), proven=True))])) + assert_rejected(PROVEN, fn(params=ps, returns=I64, pure=True, may_deopt=True, body=[ + Return(BinOp(I64, BinOpKind.FLOORDIV, li("n"), i(2), proven=True))])) + assert_rejected(PROVEN, fn(params=ps, returns=I64, body=[ + Return(UnaryOp(I64, UnaryOpKind.POS, li("n"), proven=True))])) + + +def test_a_non_bool_proven_flag_is_rejected(): + out, diags = run(_ranged(lambda k: BinOp(I64, BinOpKind.ADD, k, i(1), proven=1))) + assert "f" not in kept(out) + + +def test_range_var_interval_uses_the_stop_bound(): + assert_accepted(_ranged(lambda k: padd(k, i(2 ** 63 - 5)), stop=5)) # k <= 4 + assert_rejected(PROVEN, _ranged(lambda k: padd(k, i(2 ** 63 - 4)), stop=5)) # 4 + (2**63-4) + + +def test_range_var_interval_uses_the_start_bound(): + # range(-5, 5): k >= -5 + assert_accepted(_ranged(lambda k: psub(k, i(2 ** 63 - 5)), start=-5)) # -5 - (2**63-5) + assert_rejected(PROVEN, _ranged(lambda k: psub(k, i(2 ** 63 - 4)), start=-5)) + + +def test_negative_step_bounds(): + # range(10, -1, -1): k in [0, 10] + assert_accepted(_ranged(lambda k: psub(k, i(2 ** 63 - 1)), start=10, stop=-1, step=-1)) + assert_rejected(PROVEN, _ranged(lambda k: padd(k, i(2 ** 63 - 10)), start=10, stop=-1, step=-1)) + # range(n, 0, -1) with n a param: k in [1, 2**63-1]: k - 1 is provable, k + 1 is not + ps = [Param("n", I64)] + assert_accepted(_ranged(lambda k: psub(k, i(1)), start=li("n"), stop=0, step=-1, params=ps)) + assert_rejected(PROVEN, _ranged(lambda k: padd(k, i(1)), start=li("n"), stop=0, step=-1, + params=ps)) + + +def test_range_up_to_a_param_bounds_the_var_by_the_param_range(): + # range(0, n): k <= 2**63-2, so k + 1 fits but k + 2 may not + ps = [Param("n", I64)] + assert_accepted(_ranged(lambda k: padd(k, i(1)), stop=li("n"), params=ps)) + assert_rejected(PROVEN, _ranged(lambda k: padd(k, i(2)), stop=li("n"), params=ps)) + + +def test_len_bound_is_zero_to_two_to_the_62(): + a = [ArrayParam("a", F64A, stored=False)] + L = Len(I64, "a") + assert_accepted(fn(params=a, returns=I64, body=[IMPURE, Return(padd(L, i(2 ** 62 - 1)))])) + assert_rejected(PROVEN, fn(params=a, returns=I64, + body=[IMPURE, Return(padd(L, i(2 ** 62)))])) + assert_rejected(PROVEN, fn(params=a, returns=I64, body=[IMPURE, Return(pmul(L, i(2)))])) + assert_accepted(fn(params=a, returns=I64, body=[IMPURE, Return(pmul(L, i(-2)))])) # -2**63 + assert_rejected(PROVEN, fn(params=a, returns=I64, body=[IMPURE, Return(pmul(L, i(-3)))])) + + +def test_mod_and_floordiv_are_full_range_so_nothing_is_provable_from_them(): + ps = [Param("n", I64)] + m = BinOp(I64, BinOpKind.MOD, li("n"), i(7)) + assert_rejected(PROVEN, fn(params=ps, returns=I64, pure=True, may_deopt=True, + body=[Return(padd(m, i(1)))])) + + +def test_what_the_intervals_cannot_prove(): + # a local assigned a constant is still full range (locals other than loop vars are not tracked) + assert_rejected(PROVEN, fn(locals_={"x": I64}, returns=I64, + body=[IMPURE, Assign("x", i(1)), Return(padd(li("x"), i(1)))])) + # the loop var after its loop is full range + assert_rejected(PROVEN, fn(locals_={"k": I64}, returns=I64, body=[ + IMPURE, Assign("k", i(0)), ForRange("k", i(0), i(5), i(1), ()), Return(padd(li("k"), i(1)))])) + # an i64 array element is full range + assert_rejected(PROVEN, fn(params=[ArrayParam("a", I64A, stored=False)], returns=I64, + body=[IMPURE, Return(padd(Index(I64, "a", i(0)), i(1)))])) + + +def test_proven_flag_survives_verification(): + out = assert_accepted(_ranged(lambda k: padd(k, i(1)))) + loop = out.functions[0].body[1] + assert loop.body[0].value.proven is True + + +# --- 8. Call.redo --------------------------------------------------------------------------------- + +def _callee(name="g", returns=F64, pure=True, may_deopt=True): + """g(n: int) — deopts on n * 2 overflow when may_deopt; returns `returns`.""" + prod = BinOp(I64, BinOpKind.MUL, li("n"), i(2)) if may_deopt else li("n") + value = {F64: ToFloat(F64, prod), I64: prod, BOOL: lt(prod, i(0)), NONE: None}[returns] + body = ([ExprStmt(prod), Return()] if returns is NONE else [Return(value)]) + if not pure: + body = [IMPURE] + body + return fn(name, params=[Param("n", I64)], returns=returns, pure=pure, may_deopt=may_deopt, + body=body) + + +def _redo_caller(returns=F64, pure=False, may_deopt=False, redo=True): + call = Call(returns, "g", (i(3),), redo=redo) + body = [ExprStmt(call), Return()] + if not pure: + body = [IMPURE] + body + return fn("f", pure=pure, may_deopt=may_deopt, body=body) + + +@pytest.mark.parametrize("returns", [F64, BOOL, NONE]) +def test_redo_from_an_impure_caller_to_a_pure_may_deopt_callee_is_accepted(returns): + assert_accepted(_redo_caller(returns), _callee(returns=returns)) + + +def test_redo_to_an_i64_returning_callee_is_rejected(): + out, _ = assert_rejected(REDO, _redo_caller(I64), _callee(returns=I64)) + assert "g" in kept(out) + + +def test_redo_in_a_pure_caller_is_rejected(): + assert_rejected(REDO, _redo_caller(pure=True, may_deopt=True), _callee()) + assert_rejected(REDO, _redo_caller(pure=True, may_deopt=False), _callee()) + + +def test_redo_to_a_callee_that_cannot_deopt_is_rejected(): + assert_rejected(REDO, _redo_caller(), _callee(may_deopt=False)) + + +def test_redo_to_an_impure_callee_is_rejected(): + assert_rejected(REDO, _redo_caller(), _callee(pure=False, may_deopt=False)) + + +def test_redo_does_not_make_the_caller_may_deopt(): + assert_rejected(FLAG_DEOPT, _redo_caller(may_deopt=True), _callee()) + + +def test_the_same_call_without_redo_is_a_deopt_in_an_impure_caller(): + assert_rejected(DEOPT, _redo_caller(redo=False), _callee()) + + +def test_a_non_bool_redo_flag_is_rejected(): + out, _ = run(dataclasses.replace(_redo_caller(), body=( + IMPURE, ExprStmt(Call(F64, "g", (i(3),), redo=1)), Return())), _callee()) + assert "f" not in kept(out) + + +# --- 9. Function.entry_globals -------------------------------------------------------------------- + +def _closed(entry=(Param("N", I64),), extra=(), returns=F64, name="f"): + """A closed, impure function: for k in range(0, N): a[k] = a[k] * SCALE (SCALE an entry global + when listed).""" + a = ArrayParam("a", F64A, stored=True) + return fn(name, params=[a], entry_globals=entry, locals_={"k": I64}, returns=returns, body=[ + *extra, + ForRange("k", i(0), Len(I64, "a"), i(1), ( + StoreIndex("a", li("k"), BinOp(F64, BinOpKind.MUL, Index(F64, "a", li("k")), + ToFloat(F64, li("N")))),)), + Return(f(0.0)) if returns is F64 else Return(), + ]) + + +def test_entry_global_in_a_closed_function_is_accepted(): + assert_accepted(_closed()) + + +def test_entry_globals_are_assigned_at_entry(): + assert_accepted(fn(entry_globals=[Param("G", F64), Param("B", BOOL)], returns=F64, + body=[If(lb("B"), (Return(lf("G")),)), Return(f(1.0))])) + + +def test_entry_global_in_a_pure_function_is_accepted(): + assert_accepted(fn(entry_globals=[Param("G", F64)], returns=F64, pure=True, + body=[Return(lf("G"))])) + + +@pytest.mark.parametrize("node", [ + ExprStmt(CallObject(OBJ, Global(OBJ, "print"), (), ())), + ExprStmt(GetAttr(OBJ, Global(OBJ, "m"), "x")), + If(Truth(BOOL, Global(OBJ, "m")), ()), + If(CompareObj(BOOL, CompareKind.EQ, Global(OBJ, "m"), Box(OBJ, Const(I64, 1))), ()), + ExprStmt(ObjToFloat(F64, Global(OBJ, "m"))), + ExprStmt(BinOp(OBJ, BinOpKind.ADD, Global(OBJ, "m"), Global(OBJ, "m"))), +]) +def test_entry_global_in_a_function_running_user_code_is_rejected(node): + assert_rejected(ENTRY_OPEN, _closed(extra=[node])) + + +def test_the_same_open_node_without_entry_globals_is_accepted(): + # The open node alone is fine: only entry globals need the function closed. + a = ArrayParam("a", F64A, stored=False) + assert_accepted(fn(params=[a], returns=F64, body=[ + ExprStmt(CallObject(OBJ, Global(OBJ, "print"), (), ())), Return(Index(F64, "a", i(0)))])) + + +def test_global_read_and_box_keep_a_function_closed(): + assert_accepted(_closed(extra=[ExprStmt(Global(OBJ, "m")), ExprStmt(Box(OBJ, li("N")))])) + + +def test_entry_global_needs_every_callee_closed_transitively(): + h_open = fn("h", body=[IMPURE, Return()]) + h_closed = fn("h", body=[Return()]) + g = fn("g", body=[ExprStmt(Call(NONE, "h", ())), Return()]) + caller = _closed(extra=[ExprStmt(Call(NONE, "g", ()))]) + assert_accepted(caller, g, h_closed) + out, _ = assert_rejected(ENTRY_OPEN, caller, g, h_open) + assert "g" in kept(out) and "h" in kept(out) + + +def test_entry_global_must_be_a_scalar_param(): + assert_rejected(TYPE, _closed(entry=(Param("N", OBJ),))) + assert_rejected(TYPE, fn(entry_globals=[Param("A", F64A)], body=[Return()])) + assert_rejected(TYPE, fn(entry_globals=[Param("A", NONE)], body=[Return()])) + out, _ = run(fn(entry_globals=[ArrayParam("A", F64A, stored=False)], body=[Return()])) + assert "f" not in kept(out) + out, _ = run(fn(entry_globals=["N"], body=[Return()])) + assert "f" not in kept(out) + + +def test_entry_global_names_must_not_collide(): + assert_rejected(STRUCTURE, _closed(entry=(Param("a", I64),))) # an array param + assert_rejected(STRUCTURE, fn(params=[Param("N", I64)], entry_globals=[Param("N", I64)], + body=[Return()])) + assert_rejected(STRUCTURE, fn(locals_={"N": I64}, entry_globals=[Param("N", I64)], + body=[Return()])) + assert_rejected(STRUCTURE, fn(entry_globals=[Param("N", I64), Param("N", I64)], + body=[Return()])) + + +def test_entry_global_is_read_as_a_local_of_its_type(): + assert_rejected(TYPE, fn(entry_globals=[Param("G", F64)], returns=I64, + body=[Return(li("G"))])) + + +def test_assigning_an_entry_global_is_rejected(): + assert_rejected(STRUCTURE, fn(entry_globals=[Param("G", F64)], + body=[Assign("G", f(1.0)), Return()])) + assert_rejected(STRUCTURE, fn(entry_globals=[Param("N", I64)], body=[ + ForRange("N", i(0), i(3), i(1), ()), Return()])) + + +def test_entry_global_i64_is_full_range(): + assert_rejected(PROVEN, _closed(extra=[ExprStmt(padd(li("N"), i(1)))])) + + +def test_call_of_a_function_with_entry_globals_from_an_impure_caller(): + # The callee's entry guard can fail in the middle of the caller: that is a deopt there, + # unless the caller already guarded the same global (and, being closed, cannot rebind it). + g = fn("g", entry_globals=[Param("G", F64)], returns=F64, pure=True, body=[Return(lf("G"))]) + impure = fn("f", returns=F64, body=[IMPURE, Return(Call(F64, "g", ()))]) + assert_rejected(DEOPT, impure, g) + covered = fn("f", params=[ArrayParam("a", F64A, stored=True)], + entry_globals=[Param("G", F64)], returns=F64, + body=[StoreIndex("a", i(0), Call(F64, "g", ())), Return(lf("G"))]) + assert_accepted(covered, g) + wrong_type = dataclasses.replace(covered, entry_globals=(Param("G", I64),), + body=(StoreIndex("a", i(0), Call(F64, "g", ())), + Return(f(0.0)))) + assert_rejected(DEOPT, wrong_type, g) + assert_accepted(fn("f", returns=F64, pure=True, may_deopt=True, + body=[Return(Call(F64, "g", ()))]), g) + + # --- "nothing slips through": seeded IR mutation ------------------------------------------------- def _base_functions(): @@ -902,3 +1227,149 @@ def test_preserving_rewrites_are_accepted(seed): if g.name not in kept(out): rejected.append((n, desc, [dataclasses.astuple(d) for d in diags])) assert rejected == [] + + +# --- mutation of proven / redo / entry_globals --------------------------------------------------- + +def _proof_functions(): + """Correct functions exercising proven ops, redo and entry globals; the mutator breaks them.""" + h = fn("h", params=[Param("n", I64)], returns=F64, pure=True, may_deopt=True, line=1, + body=[Return(ToFloat(F64, BinOp(I64, BinOpKind.MUL, li("n"), i(2))))]) + q = fn("q", params=[Param("n", I64)], returns=I64, pure=True, may_deopt=True, line=2, + body=[Return(add(li("n"), i(1)))]) + pc = fn("pc", returns=F64, pure=True, may_deopt=True, line=3, + body=[Return(Call(F64, "h", (i(3),)))]) + fill = fn("fill", params=[ArrayParam("a", F64A, stored=True)], + entry_globals=[Param("N", I64), Param("S", F64)], + locals_={"k": I64, "t": I64, "v": I64, "u": F64}, returns=F64, line=4, body=[ + ForRange("k", i(0), Len(I64, "a"), i(1), ( + Assign("t", psub(pmul(li("k"), i(2)), i(1))), + Assign("v", pneg(padd(li("k"), i(1)))), + StoreIndex("a", li("k"), BinOp(F64, BinOpKind.MUL, Index(F64, "a", li("k")), + ToFloat(F64, li("t")))), + )), + Assign("u", Call(F64, "h", (i(3),), redo=True)), + Return(BinOp(F64, BinOpKind.ADD, BinOp(F64, BinOpKind.ADD, lf("u"), lf("S")), + ToFloat(F64, li("N")))), + ]) + talk = fn("talk", returns=NONE, line=5, body=[IMPURE, Return()]) + return {x.name: x for x in (h, q, pc, fill, talk)} + + +def _proof_mutations(rng, base): + """(description, mutated functions by name, the function that must be rejected, the rules any + one of which is an acceptable report).""" + b = dict(base) + kind = rng.randrange(14) + if kind == 0: # widen a proven op's operand to a full-range entry global + paths = _paths(b["fill"].body, lambda n: type(n) is BinOp and n.proven) + path, node = rng.choice(paths) + wide = BinOp(I64, rng.choice([BinOpKind.ADD, BinOpKind.MUL]), node.left, li("N"), + proven=True) + b["fill"] = _replace_body(b["fill"], path, wide) + return "widen proven", b, "fill", {PROVEN} + if kind == 1: # proven set on an op that can overflow + b["q"] = _replace_body(b["q"], (0, "value"), + BinOp(I64, BinOpKind.ADD, li("n"), i(1), proven=True)) + return "proven on param add", b, "q", {PROVEN} + if kind == 2: # proven set on a node that has no i64 overflow + node = rng.choice([BinOp(F64, BinOpKind.ADD, f(1.0), f(2.0), proven=True), + BinOp(I64, BinOpKind.FLOORDIV, i(4), i(2), proven=True), + UnaryOp(I64, UnaryOpKind.POS, i(1), proven=True), + BinOp(I64, BinOpKind.ADD, i(1), i(2), proven=1)]) + t = Type.F64 if node.type is F64 else Type.I64 + b["q"] = dataclasses.replace(b["q"], returns=t, may_deopt=node.type is I64 and + getattr(node, "op", None) is BinOpKind.FLOORDIV, + body=(Return(node),)) + return "proven on wrong node", b, "q", {PROVEN, TYPE} + if kind == 3: # un-prove a proven op in the impure function: it deopts + paths = _paths(b["fill"].body, lambda n: type(n) in (BinOp, UnaryOp) and n.proven) + path, node = rng.choice(paths) + b["fill"] = _replace_body(b["fill"], path, dataclasses.replace(node, proven=False)) + return "unprove", b, "fill", {DEOPT} + if kind == 4: # un-redo + paths = _paths(b["fill"].body, lambda n: type(n) is Call) + path, node = paths[0] + b["fill"] = _replace_body(b["fill"], path, dataclasses.replace(node, redo=False)) + return "drop redo", b, "fill", {DEOPT} + if kind == 5: # redo from a pure caller + b["pc"] = dataclasses.replace(b["pc"], body=(Return(Call(F64, "h", (i(3),), redo=True)),)) + return "redo in pure", b, "pc", {REDO} + if kind == 6: # the callee stops being may_deopt / pure / F64-or-BOOL-or-NONE + how = rng.randrange(3) + h = b["h"] + if how == 0: + b["h"] = dataclasses.replace(h, may_deopt=False) + return "redo callee not may_deopt", b, "fill", {REDO} + if how == 1: + b["h"] = dataclasses.replace(h, pure=False, may_deopt=False, body=(IMPURE,) + ( + Return(ToFloat(F64, li("n"))),)) + return "redo callee impure", b, "fill", {REDO} + b["h"] = dataclasses.replace(h, returns=I64, body=(Return(BinOp( + I64, BinOpKind.MUL, li("n"), i(2))),)) + b["fill"] = _replace_body(b["fill"], (1, "value"), Call(I64, "h", (i(3),), + redo=True)) + return "redo to i64", b, "fill", {REDO, TYPE} + if kind == 7: # an open node in the function with entry globals + node = rng.choice([GetAttr(OBJ, g_("m"), "x"), pycall("f"), Truth(BOOL, g_("m")), + ObjToFloat(F64, g_("m")), BinOp(OBJ, BinOpKind.ADD, g_("m"), g_("m")), + CompareObj(BOOL, CompareKind.EQ, g_("m"), g_("m"))]) + node = ExprStmt(node) if type(node) is not CompareObj else If(node, ()) + b["fill"] = dataclasses.replace(b["fill"], body=(node,) + b["fill"].body) + return "open node", b, "fill", {ENTRY_OPEN} + if kind == 8: # entry globals on a function that is open + b["talk"] = dataclasses.replace(b["talk"], entry_globals=(Param("N", I64),)) + return "entry global on open", b, "talk", {ENTRY_OPEN} + if kind == 9: # a callee that runs user code + b["h"] = dataclasses.replace(b["h"], pure=False, may_deopt=False, body=(IMPURE,) + ( + Return(ToFloat(F64, li("n"))),)) + return "open callee", b, "fill", {ENTRY_OPEN} + if kind == 10: # entry global removed while the body reads it + keep = rng.choice([("N",), ("S",), ()]) + eg = tuple(p for p in b["fill"].entry_globals if p.name in keep) + b["fill"] = dataclasses.replace(b["fill"], entry_globals=eg) + return "drop entry global", b, "fill", {TYPE} + if kind == 11: # a bad entry global: wrong type / collision / duplicate / array + bad = rng.choice([(Param("N", OBJ), TYPE), (Param("a", I64), STRUCTURE), + (Param("k", I64), STRUCTURE), (Param("N", I64), STRUCTURE), + (Param("W", F64A), TYPE), (Param("W", NONE), TYPE)]) + b["fill"] = dataclasses.replace(b["fill"], entry_globals=b["fill"].entry_globals + (bad[0],)) + return "bad entry global", b, "fill", {bad[1]} + if kind == 12: # assign an entry global + tgt = rng.choice(["N", "S"]) + v = i(1) if tgt == "N" else f(1.0) + b["fill"] = dataclasses.replace(b["fill"], body=(Assign(tgt, v),) + b["fill"].body) + return "assign entry global", b, "fill", {STRUCTURE} + # kind 13: a function with entry globals called, unguarded, from an impure caller + caller = fn("fill2", params=[ArrayParam("a", F64A, stored=True)], returns=F64, line=6, + body=[StoreIndex("a", i(0), f(1.0)), Return(Call(F64, "pc2", ()))]) + callee = fn("pc2", entry_globals=[Param("S", F64)], returns=F64, pure=True, line=7, + body=[Return(lf("S"))]) + b["fill2"], b["pc2"] = caller, callee + return "unguarded entry-global call", b, "fill2", {DEOPT} + + +def test_proof_base_functions_are_accepted(): + assert_accepted(*_proof_functions().values()) + + +@pytest.mark.parametrize("seed", [20261004]) +def test_no_proof_breaking_mutation_slips_through(seed): + rng = random.Random(seed) + base = _proof_functions() + missed, seen = [], set() + for n in range(300): + desc, funcs, target, allowed = _proof_mutations(rng, base) + seen.add(desc) + out, diags = run(*funcs.values()) + got = rules_for(diags, target) + if target in kept(out) or not (got & allowed): + missed.append((n, desc, target, sorted(got))) + assert missed == [] + + +def test_each_proof_mutation_kind_was_exercised(): + rng = random.Random(20261004) + base = _proof_functions() + kinds = {_proof_mutations(rng, base)[0] for _ in range(300)} + assert len(kinds) == 16, sorted(kinds) # 14 kinds; kind 6 has three variants From 8e650c1213e80783144cc1f478d9164e5a8e24f8 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:38:57 +0900 Subject: [PATCH 26/36] Feat: C back end for proven ops, redo calls and entry globals Proven ops keep the checked helper and raise SystemError if one overflows; a redo call reruns only the callee's interpreted function and continues the caller (result type checked exactly); entry globals are read, type-guarded and released in the wrapper before the impl runs, so a mismatch redoes the whole call before any effect. Green: 39 passed, 1 skipped (incl. ASan/UBSan). Mutation (redo the caller instead of the callee): 5 fail. Refs #41 --- .../src/main/python/typedpython/cgen.py | 182 +++++++++- .../src/test/python/test_cgen.py | 317 +++++++++++++++++- 2 files changed, 488 insertions(+), 11 deletions(-) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index 6afe94196..4adf5153e 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -185,14 +185,17 @@ def _all_exprs(f: ir.Function): def _can_deopt(e: ir.Expr, functions: dict[str, ir.Function]) -> bool: if isinstance(e, ir.BinOp) and e.left.type is Type.I64 and e.op in _I64_DEOPT: - return True + return not e.proven # a proven op cannot deopt (overflow = SystemError) if isinstance(e, ir.UnaryOp) and e.op is ir.UnaryOpKind.NEG and e.operand.type is Type.I64: - return True + return not e.proven if isinstance(e, ir.Unbox): return True if isinstance(e, ir.Call): + if e.redo: + return False # the callee's deopt is absorbed by its interpreted redo callee = functions.get(e.function) - return callee is not None and callee.may_deopt + # a callee's entry-global guard runs at the call (`_FnGen.ev_Call`) and can fail there + return callee is not None and (callee.may_deopt or bool(callee.entry_globals)) return False @@ -314,14 +317,31 @@ def _check_function(f: ir.Function, functions: dict[str, ir.Function]) -> None: what = type(effects[0]).__name__ if effects else "StoreIndex" raise CGenError(f"{f.name}: marked pure but contains an effect ({what})") for e in _all_exprs(f): + if isinstance(e, (ir.BinOp, ir.UnaryOp)) and e.proven: + checked = (e.op in (ir.BinOpKind.ADD, ir.BinOpKind.SUB, ir.BinOpKind.MUL) + if isinstance(e, ir.BinOp) else e.op is ir.UnaryOpKind.NEG) + operand = e.left if isinstance(e, ir.BinOp) else e.operand + if not checked or operand.type is not Type.I64: + raise CGenError(f"{f.name}: proven is set on {e.op.name} of {operand.type} " + "(ir.py: I64 ADD/SUB/MUL and NEG only)") if isinstance(e, ir.Call): callee = functions.get(e.function) if callee is None: raise CGenError(f"{f.name}: Call of {e.function!r}, which is not compiled here") if any(isinstance(p, ir.ArrayParam) for p in callee.params): raise CGenError(f"{f.name}: Call of {e.function!r}, which takes array params") - if callee.may_deopt and not f.pure: - raise CGenError(f"{f.name}: Call of deopting {e.function!r} from an impure caller") + if e.redo: + if callee.returns not in (Type.F64, Type.BOOL, Type.NONE): + raise CGenError(f"{f.name}: redo Call of {e.function!r}, which returns " + f"{callee.returns} (ir.py: F64, BOOL or NONE only)") + if not callee.pure: + raise CGenError(f"{f.name}: redo Call of impure {e.function!r} " + "(redoing it could repeat an effect)") + elif (callee.may_deopt or callee.entry_globals) and not f.pure: + raise CGenError(f"{f.name}: Call of deopting {e.function!r} from an impure caller " + "without redo") + if f.entry_globals: + _check_entry_globals(f, functions) for p in f.params: if isinstance(p, ir.ArrayParam): if p.type not in ir.ARRAYS: @@ -332,6 +352,43 @@ def _check_function(f: ir.Function, functions: dict[str, ir.Function]) -> None: raise CGenError(f"{f.name}: param {p.name} has type {p.type}") +def _is_closed(f: ir.Function, functions: dict[str, ir.Function], seen=None) -> bool: + """ir.Function.entry_globals: runs no user code — no effect node, Calls only to closed + functions — so nothing can rebind a module global while it runs.""" + seen = set() if seen is None else seen + if f.name in seen: + return True + seen.add(f.name) + for e in _all_exprs(f): + if _is_effect(e): + return False + if isinstance(e, ir.Call): + callee = functions.get(e.function) + if callee is None or not _is_closed(callee, functions, seen): + return False + return True + + +def _check_entry_globals(f: ir.Function, functions: dict[str, ir.Function]) -> None: + names = [g.name for g in f.entry_globals] + if len(set(names)) != len(names): + raise CGenError(f"{f.name}: entry_globals lists a name twice") + taken = {p.name for p in f.params} | set(f.locals) + for g in f.entry_globals: + if not isinstance(g, ir.Param) or g.type not in ir.SCALARS: + raise CGenError(f"{f.name}: entry global {g.name} must be a scalar Param, not {g.type}") + if g.name in taken: + raise CGenError(f"{f.name}: entry global {g.name} is also a parameter or local") + for s in _walk_stmts(f.body): + if isinstance(s, ir.Assign) and s.target in names: + raise CGenError(f"{f.name}: assignment to entry global {s.target}") + if isinstance(s, ir.ForRange) and s.var in names: + raise CGenError(f"{f.name}: entry global {s.var} used as a range variable") + if not _is_closed(f, functions): + raise CGenError(f"{f.name}: entry_globals in a function that is not closed (it can run " + "user code, which could rebind the global between entry and use)") + + def _impl_signature(f: ir.Function) -> str: params = ["PyObject *tp_module"] for p in f.params: @@ -341,6 +398,8 @@ def _impl_signature(f: ir.Function) -> str: params.append(f"PyObject *{_ident('p', p.name)}") else: params.append(f"{_CT[p.type]} {_ident('l', p.name)}") + for g in f.entry_globals: # read and unboxed by the caller (wrapper or Call) + params.append(f"{_CT[g.type]} {_ident('l', g.name)}") if f.returns is not Type.NONE: ct = _CT[f.returns] params.append(f"{ct}{'' if ct.endswith('*') else ' '}*tp_out") @@ -360,6 +419,8 @@ def __init__(self, mg: _ModGen, f: ir.Function): for p in f.params: if not isinstance(p, ir.ArrayParam): self.types[p.name] = p.type + for g in f.entry_globals: # the body reads each as Local(name) + self.types[g.name] = g.type for name, t in f.locals.items(): if name in self.arrays or (name in self.types and self.types[name] is not t): raise CGenError(f"{f.name}: local {name} conflicts with a parameter") @@ -382,6 +443,14 @@ def check(self, call: str) -> None: self.emit(f"tp_s = {call};") self.emit("if (tp_s != 0) { tp_rc = tp_s; goto tp_exit; }") + def check_proven(self, call: str) -> None: + """A checked I64 helper on a `proven` op (ir.BinOp.proven): it can only return 0 or 1, and + 1 means the front end's and the verifier's proof was wrong — an internal error, never a + deopt (a proven op may sit after an effect).""" + self.emit(f"tp_s = {call};") + self.emit("if (tp_s != 0) { PyErr_SetString(PyExc_SystemError, " + "\"typedpython: proven operation overflowed\"); tp_rc = -1; goto tp_exit; }") + def fail(self) -> None: self.emit("{ tp_rc = -1; goto tp_exit; }") @@ -458,7 +527,10 @@ def ev_BinOp(self, e: ir.BinOp) -> str: a, b = self.ev(e.left), self.ev(e.right) if lt is Type.I64: t = self.tmp("double" if e.op is ir.BinOpKind.TRUEDIV else "int64_t") - self.check(f"{_I64_OP[e.op]}({a}, {b}, &{t})") + if e.proven: + self.check_proven(f"{_I64_OP[e.op]}({a}, {b}, &{t})") + else: + self.check(f"{_I64_OP[e.op]}({a}, {b}, &{t})") return t if lt is Type.F64: t = self.tmp("double") @@ -486,7 +558,10 @@ def ev_UnaryOp(self, e: ir.UnaryOp) -> str: return t if ot is Type.I64 and e.op is ir.UnaryOpKind.NEG: t = self.tmp("int64_t") - self.check(f"tp_neg_i64({v}, &{t})") + if e.proven: + self.check_proven(f"tp_neg_i64({v}, &{t})") + else: + self.check(f"tp_neg_i64({v}, &{t})") return t if ot is Type.F64 and e.op is ir.UnaryOpKind.NEG: t = self.tmp("double") @@ -608,18 +683,90 @@ def ev_Call(self, e: ir.Call) -> str | None: args.append(v) if a.type is Type.OBJ: owned.append(v) + values = list(args) # the evaluated arguments, for a redo t = None + self.emit("tp_s = 0;") + # The callee's entry globals (ir.Function.entry_globals) are its entry guards: read them + # here, at the callee's entry. A mismatch is the callee's deopt (tp_s = 1), handled below + # exactly as a deopt inside the callee. + for g in callee.entry_globals: + args.append(self._read_entry_global(g, "tp_s = 1;")) if callee.returns is not Type.NONE: t = self.tmp(_CT[callee.returns]) args.append(f"&{t}") + self.emit("if (tp_s == 0) {") + self.depth += 1 # C-to-C recursion has no Python frame: guard the C stack as CPython guards its own. self.emit('if (Py_EnterRecursiveCall(" in compiled code")) { tp_rc = -1; goto tp_exit; }') self.emit(f"tp_s = {_ident('tp_impl', e.function)}(tp_module{''.join(', ' + x for x in args)});") self.emit("Py_LeaveRecursiveCall();") + self.depth -= 1 + self.emit("}") + if e.redo: + self._redo_call(callee, e.args, values, t) self._own_release(*owned) self.emit("if (tp_s != 0) { tp_rc = tp_s; goto tp_exit; }") return t + def _read_entry_global(self, g: ir.Param, on_mismatch: str, on_error: str = "tp_rc = -1; goto tp_exit;", + guard: str = "tp_s == 0") -> str: + """Emit the entry read of module global `g` (tp_global, exact-type unbox, release) guarded + by `tp_s == 0`; on a wrong type or an unbound name run `on_mismatch` (a deopt: the + interpreted function then sees the same binding and raises CPython's own NameError where + CPython would). Returns the C scalar holding the value.""" + k = self.mg.name_index(g.name) + gv, go = self.tmp(_CT[g.type]), self.tmp("PyObject *") + self.emit(f"if ({guard}) {{") + self.emit(f" {go} = tp_global(tp_dict, tp_st->names[{k}]);") + self.emit(f" if ({go} == NULL) {{") + self.emit(f" if (!PyErr_ExceptionMatches(PyExc_NameError)) {{ {on_error} }}") + self.emit(f" PyErr_Clear(); {on_mismatch}") + self.emit(" } else {") + self.emit(f" tp_s = {_UNBOX[g.type]}({go}, &{gv});") + self.emit(f" tp_release(&{go});") + self.emit(f" if (tp_s < 0) {{ {on_error} }}") + self.emit(f" if (tp_s != 0) {{ {on_mismatch} }}") + self.emit(" }") + self.emit("}") + return gv + + def _redo_call(self, callee: ir.Function, arg_exprs, values: list[str], t: str | None) -> None: + """ir.Call.redo: on the callee's deopt (tp_s == 1) run `__typedpython_interpreted__[callee]` + on the already-evaluated arguments (boxed), count the deopt, convert the result back, and + continue in the caller. The callee is pure, so the redo is unobservable.""" + k = self.mg.name_index(callee.name) + self.emit("if (tp_s == 1) {") + self.depth += 1 + boxes = [] + av = [] + for a, v in zip(arg_exprs, values): + if a.type is Type.OBJ: + av.append(v) # still owned by the caller; released after + else: + b = self._own_new(f"{_BOX[a.type]}({v})") + boxes.append(b) + av.append(b) + rr = self.tmp("PyObject *") + if av: + self.emit(f"{{ PyObject *tp_av[{len(av)}] = {{{', '.join(av)}}};") + self.emit(f" {rr} = tp_cg_redo(tp_module, {k}, tp_av, {len(av)}, NULL); }}") + else: + self.emit(f"{rr} = tp_cg_redo(tp_module, {k}, NULL, 0, NULL);") + self._own_release(*boxes) + self.emit(f"if ({rr} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + rt = callee.returns + if rt is Type.NONE: + self.emit(f"tp_s = ({rr} == Py_None) ? 0 : 1;") + what = "None" + else: + self.emit(f"tp_s = {_UNBOX[rt]}({rr}, &{t});") + what = {Type.F64: "a float", Type.BOOL: "True or False"}[rt] + self._own_release(rr) + msg = c_string(f"typedpython: the interpreted redo of {callee.name} did not return {what}") + self.emit(f"if (tp_s != 0) {{ PyErr_SetString(PyExc_SystemError, {msg}); tp_rc = -1; goto tp_exit; }}") + self.depth -= 1 + self.emit("}") + def ev_Global(self, e: ir.Global) -> str: k = self.mg.name_index(e.name) return self._own_new(f"tp_global(tp_dict, tp_st->names[{k}])") @@ -835,7 +982,7 @@ def generate(self) -> str: " int tp_s = 0;", " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);", " PyObject *tp_dict = PyModule_GetDict(tp_module); /* borrowed: owned by the module */"] - params = {p.name for p in f.params} + params = {p.name for p in f.params} | {g.name for g in f.entry_globals} for name, t in self.types.items(): if name in params and t is not Type.OBJ: continue @@ -901,6 +1048,25 @@ def wrapper(self) -> str: L.append(f" tp_s = {_ARRAY_PFX[p.type]}_enter(tp_args[{i}], &tp_a{i});") L.append(" if (tp_s < 0) goto tp_error;") L.append(" if (tp_s != 0) goto tp_deopt;") + gnames = [] + if f.entry_globals: + L.insert(3, " PyObject *tp_dict = PyModule_GetDict(tp_module); /* borrowed */") + L.insert(3, " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);") + for j, g in enumerate(f.entry_globals): + # entry read of a module global: exact type or deopt (still before any effect) + k = self.mg.name_index(g.name) + L.append(f" {_CT[g.type]} tp_g{j} = 0;") + L.append(f" {{ PyObject *tp_go = tp_global(tp_dict, tp_st->names[{k}]);") + L.append(" if (tp_go == NULL) {") + L.append(" if (!PyErr_ExceptionMatches(PyExc_NameError)) goto tp_error;") + L.append(" PyErr_Clear(); goto tp_deopt; }") + L.append(f" tp_s = {_UNBOX[g.type]}(tp_go, &tp_g{j}); Py_DECREF(tp_go); }}") + L.append(" if (tp_s < 0) goto tp_error;") + L.append(" if (tp_s != 0) goto tp_deopt;") + gnames.append(f"tp_g{j}") + if gnames: + at = len(args) - (1 if f.returns is not Type.NONE else 0) + args[at:at] = gnames L.append(f" tp_s = {_ident('tp_impl', f.name)}(tp_module{''.join(', ' + a for a in args)});") if f.pure: L.append(" if (tp_s == 1) goto tp_deopt;") diff --git a/python-multiplatform-ksp/src/test/python/test_cgen.py b/python-multiplatform-ksp/src/test/python/test_cgen.py index f36343614..da069327b 100644 --- a/python-multiplatform-ksp/src/test/python/test_cgen.py +++ b/python-multiplatform-ksp/src/test/python/test_cgen.py @@ -80,9 +80,10 @@ def call(name, *args, kwnames=()): return CallObject(OBJ, Global(OBJ, name), tuple(args), tuple(kwnames)) -def fn(name, params, returns, locals_, body, *, pure, may_deopt, line=1): +def fn(name, params, returns, locals_, body, *, pure, may_deopt, line=1, entry_globals=()): return Function(name=name, params=tuple(params), returns=returns, locals=dict(locals_), - body=tuple(body), pure=pure, may_deopt=may_deopt, source_line=line) + body=tuple(body), pure=pure, may_deopt=may_deopt, source_line=line, + entry_globals=tuple(entry_globals)) # The module source that every e2e module embeds; each test picks the functions it compiles. @@ -208,6 +209,59 @@ def truthy(o) -> bool: if o: return True return False + + # --- 957315ea: proven ops, callee-only redo, entry snapshots of globals, CompareObj --- + def proven_eff(x: int) -> int: + record(x) + return x + 1 + + def growth(n: int) -> float: + acc = 1 + for i in range(n): + acc = acc * 3 + return float(acc) + + def growth_lie(n: int) -> float: + acc = 1 + for i in range(n): + acc = acc * 3 + return acc # the IR below claims float(acc): a contract violation the redo catches + + def fill(xs: list[float], n: int) -> float: + for i in range(len(xs)): + xs[i] = xs[i] * 2.0 + g = growth(n) + xs[0] = xs[0] + g + return g + + def fill_lie(xs: list[float], n: int) -> float: + xs[0] = xs[0] * 2.0 + return growth_lie(n) + + def big(n: int) -> bool: + acc = 1 + for i in range(n): + acc = acc * 3 + return acc > 1000 + + def flag(xs: list[float], n: int) -> float: + xs[0] = xs[0] + 1.0 + if big(n): + return 1.0 + return 0.0 + + def selfeq(x) -> bool: + if x == x: + return True + return False + + SCALE = 2.5 + + def scaled(x: float) -> float: + return x * SCALE + + def twice_scaled(x: float) -> float: + return scaled(x) * 2.0 ''') SOURCE = "import math\n" + SOURCE @@ -370,12 +424,92 @@ def f_truthy(): pure=False, may_deopt=False) +def f_proven_eff(): + x = L("x", I64) + return fn("proven_eff", [Param("x", I64)], I64, {}, + [ExprStmt(call("record", Box(OBJ, x))), + Return(BinOp(I64, BinOpKind.ADD, x, c(1), proven=True))], + pure=False, may_deopt=False) + + +def _growth_ir(name, result): + acc = L("acc", I64) + return fn(name, [Param("n", I64)], result, {"acc": I64, "i": I64}, + [Assign("acc", c(1)), + ForRange("i", c(0), L("n", I64), c(1), (Assign("acc", binop(BinOpKind.MUL, acc, c(3))),)), + Return(ToFloat(F64, acc) if result is F64 else Compare(BOOL, CompareKind.GT, acc, c(1000)))], + pure=True, may_deopt=True) + + +def f_growth(): + return _growth_ir("growth", F64) + + +def f_growth_lie(): + return _growth_ir("growth_lie", F64) + + +def f_big(): + return _growth_ir("big", BOOL) + + +def f_fill(): + i, g = L("i", I64), L("g", F64) + return fn("fill", [ArrayParam("xs", Type.F64_ARRAY, stored=True), Param("n", I64)], F64, + {"i": I64, "g": F64}, + [ForRange("i", c(0), Len(I64, "xs"), c(1), + (StoreIndex("xs", i, binop(BinOpKind.MUL, Index(F64, "xs", i, proven=True), c(2.0)), + proven=True),)), + Assign("g", Call(F64, "growth", (L("n", I64),), redo=True)), + StoreIndex("xs", c(0), binop(BinOpKind.ADD, Index(F64, "xs", c(0)), g)), + Return(g)], + pure=False, may_deopt=False) + + +def f_fill_lie(): + return fn("fill_lie", [ArrayParam("xs", Type.F64_ARRAY, stored=True), Param("n", I64)], F64, {}, + [StoreIndex("xs", c(0), binop(BinOpKind.MUL, Index(F64, "xs", c(0)), c(2.0))), + Return(Call(F64, "growth_lie", (L("n", I64),), redo=True))], + pure=False, may_deopt=False) + + +def f_flag(): + return fn("flag", [ArrayParam("xs", Type.F64_ARRAY, stored=True), Param("n", I64)], F64, {}, + [StoreIndex("xs", c(0), binop(BinOpKind.ADD, Index(F64, "xs", c(0)), c(1.0))), + If(Call(BOOL, "big", (L("n", I64),), redo=True), (Return(c(1.0)),)), + Return(c(0.0))], + pure=False, may_deopt=False) + + +def f_selfeq(): + x = L("x", OBJ) + return fn("selfeq", [Param("x", OBJ)], BOOL, {}, + [If(CompareObj(BOOL, CompareKind.EQ, x, x), (Return(c(True)),)), Return(c(False))], + pure=False, may_deopt=False) + + +def f_scaled(): + return fn("scaled", [Param("x", F64)], F64, {}, + [Return(binop(BinOpKind.MUL, L("x", F64), L("SCALE", F64)))], + pure=True, may_deopt=False, entry_globals=[Param("SCALE", F64)]) + + +def f_twice_scaled(): + # The callee's entry-global guard can fail at the call: that is a deopt of the callee, which a + # pure caller propagates (so this caller may deopt). + return fn("twice_scaled", [Param("x", F64)], F64, {}, + [Return(binop(BinOpKind.MUL, Call(F64, "scaled", (L("x", F64),)), c(2.0)))], + pure=True, may_deopt=True) + + INTEROP = (f_interop, f_which, f_length, f_calls_boom, f_truthy) +CONTRACT_957315EA = (f_proven_eff, f_growth, f_growth_lie, f_big, f_fill, f_fill_lie, f_flag, + f_selfeq, f_scaled, f_twice_scaled) OBJ_HELPERS = ("tp_global", "tp_getattr", "tp_call", "tp_truth", "tp_compare_bool", "tp_obj_to_f64", "tp_binop_obj", "tp_release") ALL = (f_poly, f_mul_all, f_twice_mul, f_eff, f_scale, f_addfirst, f_floordiv, f_objmix, - f_count_until, f_isum, f_hyp, f_mixed_lt, f_tofl, f_neg_last) + INTEROP + f_count_until, f_isum, f_hyp, f_mixed_lt, f_tofl, f_neg_last) + INTEROP + CONTRACT_957315EA # --- building ------------------------------------------------------------------------------------- @@ -727,6 +861,176 @@ def test_gen_rejects_an_effect_in_a_pure_function(): cgen.generate(Module("tp_bad3", (bad,)), source_file("tp_bad3")) +# --- 957315ea: proven ops, Call.redo, entry_globals, CompareObj without identity shortcut ---------- + +def test_gen_accepts_proven_ops_in_an_impure_function(): + from typedpython import cgen + + a = L("a", I64) + ok = fn("ok", [Param("a", I64)], I64, {}, + [ExprStmt(call("record", Box(OBJ, a))), + Return(UnaryOp(I64, UnaryOpKind.NEG, BinOp(I64, BinOpKind.SUB, a, c(1), proven=True), + proven=True))], + pure=False, may_deopt=False) + text = cgen.generate(Module("tp_proven_gen", (ok,)), source_file("tp_proven_gen")) + assert "typedpython: proven operation overflowed" in text + + +def test_gen_rejects_proven_on_an_op_that_is_not_checked_i64_arithmetic(): + from typedpython import cgen + + bad = fn("badproven", [Param("a", I64)], I64, {}, + [Return(BinOp(I64, BinOpKind.FLOORDIV, L("a", I64), c(2), proven=True))], + pure=True, may_deopt=True) + with pytest.raises(cgen.CGenError, match="proven"): + cgen.generate(Module("tp_bad4", (bad,)), source_file("tp_bad4")) + + +def test_gen_rejects_redo_of_an_i64_callee(): + from typedpython import cgen + + caller = fn("caller", [Param("a", I64)], I64, {}, + [ExprStmt(call("record", Box(OBJ, L("a", I64)))), + Return(Call(I64, "mul_all", (L("a", I64), c(2)), redo=True))], + pure=False, may_deopt=False) + with pytest.raises(cgen.CGenError, match="redo"): + cgen.generate(Module("tp_bad5", (f_mul_all(), caller)), source_file("tp_bad5")) + + +def test_gen_rejects_entry_globals_in_a_function_that_runs_user_code(): + from typedpython import cgen + + bad = fn("notclosed", [Param("o", OBJ)], F64, {}, + [ExprStmt(call("record", L("o", OBJ))), Return(L("SCALE", F64))], + pure=False, may_deopt=False, entry_globals=[Param("SCALE", F64)]) + with pytest.raises(cgen.CGenError, match="closed"): + cgen.generate(Module("tp_bad6", (bad,)), source_file("tp_bad6")) + + +def test_proven_op_in_an_impure_function_computes_the_interpreted_result(mod): + mod.calls.clear() + before = deopts(mod) + assert mod.proven_eff(5) == interp(mod, "proven_eff")(5) == 6 + assert mod.calls == [5, 5] and deopts(mod) == before + mod.calls.clear() + # A proof the IR producer got wrong: the overflow is an internal error, never a deopt (the + # effect has happened already, so a redo would run it twice). + with pytest.raises(SystemError, match="typedpython: proven operation overflowed"): + mod.proven_eff(2 ** 63 - 1) + assert mod.calls == [2 ** 63 - 1] and deopts(mod) == before + + +def test_redo_reruns_only_the_pure_callee_and_the_callers_effects_happen_once(mod): + xs, ys = [1.0, 2.0], [1.0, 2.0] + before = deopts(mod) + got = mod.fill(xs, 10) + assert deopts(mod) == before # 3**10 fits: no redo + want = interp(mod, "fill")(ys, 10) + assert repr(got) == repr(want) and xs == ys == [2.0 + 3.0 ** 10, 4.0] + + xs, ys = [1.0, 2.0], [1.0, 2.0] + before = deopts(mod) + got = mod.fill(xs, 45) # 3**40 leaves i64 inside growth + assert deopts(mod) == before + 1 # growth redone once, fill not redone + want = interp(mod, "fill")(ys, 45) + assert type(got) is float and repr(got) == repr(want) == repr(float(3 ** 45)) + # effects exactly once: xs[1] doubled once, xs[0] doubled once then grown once + assert xs == ys == [2.0 + float(3 ** 45), 4.0] + assert [type(v) for v in xs] == [float, float] + + +def test_redo_of_a_bool_callee(mod): + for n, want_r in ((3, 0.0), (7, 1.0), (45, 1.0)): + xs, ys = [1.0], [1.0] + before = deopts(mod) + got = mod.flag(xs, n) + assert deopts(mod) == before + (1 if n >= 40 else 0) + assert got == interp(mod, "flag")(ys, n) == want_r + assert xs == ys == [2.0] + + +def test_redo_result_of_the_wrong_type_is_a_system_error_naming_the_callee(mod): + xs = [1.0] + assert mod.fill_lie(xs, 10) == float(3 ** 10) and xs == [2.0] + xs = [1.0] + with pytest.raises(SystemError, match="growth_lie"): + mod.fill_lie(xs, 45) + assert xs == [2.0] # the caller's store is written back once + + +def test_redo_releases_its_temporaries(mod): + # The redo creates a boxed argument, the interpreted function object reference and a result + # float per call; a leak of any one is >= 24 bytes per call, so 2000 calls would grow by + # >= 48 KB. (Floats are not GC-tracked and are fresh objects, so refcounts cannot show this.) + import tracemalloc + + def run(k): + for _ in range(k): + xs = [1.0] + mod.fill(xs, 45) + mod.flag(xs, 45) + try: + mod.fill_lie(xs, 45) + except SystemError: + pass + + run(200) # warm free lists and caches + tracemalloc.start() + try: + base = tracemalloc.get_traced_memory()[0] + run(2000) + grown = tracemalloc.get_traced_memory()[0] - base + finally: + tracemalloc.stop() + assert grown < 16 * 1024, grown + n = 10 ** 30 + before = sys.getrefcount(n) + for _ in range(1000): + mod.selfeq(n) + assert sys.getrefcount(n) == before + + +def test_entry_global_is_read_at_each_call(mod): + old = mod.SCALE + try: + before = deopts(mod) + assert repr(mod.scaled(3.0)) == repr(interp(mod, "scaled")(3.0)) == repr(7.5) + mod.SCALE = 4.0 # rebinding between calls is seen + assert mod.scaled(3.0) == interp(mod, "scaled")(3.0) == 12.0 + assert mod.twice_scaled(3.0) == interp(mod, "twice_scaled")(3.0) == 24.0 + assert deopts(mod) == before + mod.SCALE = 3 # an int: the whole call is interpreted + got = mod.scaled(3.0) + assert deopts(mod) == before + 1 + assert type(got) is float and got == interp(mod, "scaled")(3.0) == 9.0 + d = deopts(mod) + assert mod.twice_scaled(3.0) == 18.0 + # the pure caller is redone (+1); the interpreted caller calls the compiled scaled, whose + # guard fails again (+1) + assert deopts(mod) == d + 2 + del mod.SCALE # unbound: interpreted path raises NameError + with pytest.raises(NameError) as got_e: + mod.scaled(3.0) + with pytest.raises(NameError) as want_e: + interp(mod, "scaled")(3.0) + assert str(got_e.value) == str(want_e.value) == "name 'SCALE' is not defined" + finally: + mod.SCALE = old + + +def test_compare_obj_has_no_identity_shortcut(mod): + nan = float("nan") + assert mod.selfeq(nan) is interp(mod, "selfeq")(nan) is False + + class Never: + def __eq__(self, other): + return False + + o = Never() + assert mod.selfeq(o) is interp(mod, "selfeq")(o) is False + assert mod.selfeq(1.0) is True + + # --- sanitizers ----------------------------------------------------------------------------------- ASAN_SCRIPT = textwrap.dedent('''\ @@ -756,6 +1060,13 @@ def test_gen_rejects_an_effect_in_a_pure_function(): m.objmix(10 ** 30, 10 ** 31); m.calls.clear() assert m.neg_last([1.0, 2.0]) == -2.0 assert m.count_until(50, 7) == I["count_until"](50, 7) + for _ in range(50): + xs = [1.0, 2.0]; m.fill(xs, 45); m.flag(xs, 45) + try: + m.fill_lie([1.0], 45) + except SystemError: + pass + assert m.scaled(2.0) == 5.0 and m.selfeq(float("nan")) is False for _ in range(50): m.made.clear(); m.interop(1.5, 20) try: From 43638b292b730e4db8f896cf6a875d5598e3016c Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:41:37 +0900 Subject: [PATCH 27/36] Spec: Record the proven/redo/entry-globals rules the verifier enforces (N-8) Including the rule the verifier added: a call of a function with entry globals is a deopt point in its caller (its entry guard can fail mid-call) unless it is a legal redo or the caller snapshots the same globals. Refs #41 --- docs/SPEC.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/SPEC.md b/docs/SPEC.md index 0ac409a8c..2d327cb0d 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -270,7 +270,9 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. | Array access in bounds, or a runtime bounds check kept | the check stays (never an unchecked access) | | Array parameters only indexed, stored, `len`-ed; no aliasing (entry guard) | not compiled; diagnostic | | OBJ references follow one ownership rule; no IR form for free, borrowed alias or NULL read | not compiled; diagnostic | - | Integer overflow checked and promoted (deopt) | — (every I64 operation is checked by construction) | + | Integer overflow checked and promoted (deopt); an op marked `proven` must re-prove its interval fits i64 | not compiled; diagnostic `verify/proven-overflow` | + | `Call.redo` only from an impure caller to a pure, may-deopt callee returning F64/BOOL/NONE | not compiled; diagnostic `verify/redo` | + | `entry_globals` only in a closed function (no user code runs); a call of a function with entry globals is a deopt point in its caller unless it is a legal redo or the caller snapshots the same globals | not compiled; diagnostic `verify/entry-global-open` / `verify/deopt-impure` | Generated C reaches memory and `PyObject`s only through the runtime helpers. ASan/UBSan CI, differential tests against the interpreter and fuzzing back the proof. `Status: planned`. From 513f10f2eaf58f4893cd1a9a4a5f2fa6d83d0ea3 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:45:17 +0900 Subject: [PATCH 28/36] Spec: Local fixed-size arrays and tuple returns in the IR (fannkuch, M2) NewArray ([fill]*n, list(range(n))), CopyArray (a[:]) and Tuple. A local array is owned by the function and never escapes, so storing into it is not an effect and its length never changes (bounds proofs against Len hold). Runtime API gains new/iota/copy/free for local arrays and tp_tuple. Leader decision: fannkuch's local list[int] and tuple return are in M2. Refs #41 --- .../src/main/python/typedpython/ir.py | 38 ++++++++++++++++++- .../main/python/typedpython/runtime/API.md | 12 ++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py index 806e125a4..78fae7b96 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/ir.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -64,8 +64,8 @@ class Type(Enum): BOOL = "bool" # a Python bool NONE = "none" # the value None (only as a return type) OBJ = "obj" # any Python object, held as a strong reference (PyObject*) - F64_ARRAY = "f64[]" # a list[float] parameter lowered to a native array (see ArrayParam) - I64_ARRAY = "i64[]" # a list[int] parameter lowered to a native array (see ArrayParam) + F64_ARRAY = "f64[]" # a list[float] parameter (ArrayParam) or a local array (NewArray/CopyArray) + I64_ARRAY = "i64[]" # a list[int] parameter (ArrayParam) or a local array (NewArray/CopyArray) SCALARS = (Type.I64, Type.F64, Type.BOOL) @@ -319,6 +319,33 @@ class ObjToFloat(Expr): operand: Expr +@dataclass(frozen=True) +class NewArray(Expr): + """A fresh **local** array (type F64_ARRAY / I64_ARRAY), owned by the function: + `[fill] * length` (`iota=False`), or `list(range(length))` (`iota=True`, I64 only, elements + 0..length-1). A negative length gives an empty array, as in CPython. Allocation failure → + MemoryError. Only legal as the value of an `Assign` to a local of the same array type.""" + + length: Expr # I64 + fill: Expr | None = None # element-typed; None when iota + iota: bool = False + + +@dataclass(frozen=True) +class CopyArray(Expr): + """`src[:]` of a local or parameter array: a fresh local array with the same elements. + Only legal as the value of an `Assign` to a local of the same array type.""" + + src: str + + +@dataclass(frozen=True) +class Tuple(Expr): + """`(a, b, ...)` of scalars/OBJs — a new tuple (type OBJ). Scalar elements arrive through Box.""" + + elements: tuple[Expr, ...] + + @dataclass(frozen=True) class Len(Expr): """`len(array)` of an array parameter. Type I64.""" @@ -447,6 +474,13 @@ class Function: and decorators other than `compiled` are not compiled in this stage (the function stays interpreted; `frontend` records why). + Local arrays (fannkuch, M2): a local of an array type holds a native array the function owns, + created only by NewArray/CopyArray. It never escapes — used only by Index/StoreIndex/Len and + as a CopyArray source; never returned, passed, boxed or captured — so stores into it are not + effects (a function whose only stores are into local arrays can be pure), and no list object + ever exists for it. Reassigning the local frees the old array; every exit frees all of them. + Its length never changes after creation, so a bounds proof against Len(local) holds. + `pure`: no effect outside its own locals — no stores into anything but locals, no CallObject, no GetAttr (a property can have effects), no Truth/CompareObj/ObjToFloat/OBJ BinOp (they run user methods), only `Call`s to pure functions. diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md index daa38260f..0fb469e40 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -72,3 +72,15 @@ All return new references or NULL with CPython's exception set; none deopts. int tp_obj_to_f64(PyObject *obj, double *out); /* 0 / -1 (PyNumber_Float, then exact double) */ PyObject *tp_binop_obj(PyObject *a, PyObject *b, int op); /* PyNumber_Add/... for ir.BinOpKind on OBJ */ void tp_release(PyObject **slot); /* Py_CLEAR: the only way generated code drops a reference */ + +## Local arrays (ir.NewArray / CopyArray; owned by the function, no list object) +Same structs as above with `list == NULL` and `dirty == NULL`. 0 ok, -1 MemoryError. + int tp_i64_array_new(tp_i64_array *a, int64_t n, int64_t fill); /* n < 0 → empty */ + int tp_i64_array_iota(tp_i64_array *a, int64_t n); /* 0..n-1; n < 0 → empty */ + int tp_f64_array_new(tp_f64_array *a, int64_t n, double fill); + int tp_i64_array_copy(tp_i64_array *dst, const tp_i64_array *src); + int tp_f64_array_copy(tp_f64_array *dst, const tp_f64_array *src); + void tp_i64_array_free(tp_i64_array *a); /* no write-back; zeroes; idempotent */ + void tp_f64_array_free(tp_f64_array *a); + /* tp_*_array_slot works unchanged on local arrays (same IndexError messages as list). */ + PyObject *tp_tuple(PyObject *const *items, Py_ssize_t n); /* new tuple; steals nothing */ From fc03e833a1782e049bad637d8180329f1d1c8ed8 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:52:30 +0900 Subject: [PATCH 29/36] Feat: Incremental TypedPython builds (pipeline.compile_project, typedpython build) compile_project runs the gate, frontend and verifier as BuildCache's lower step (so interface hashes describe what was actually compiled), cgen and cbuild as generate/compile_c. COMPILER_VERSION is a content hash of the typedpython package; platform is sysconfig platform + EXT_SUFFIX. CLI: python -m typedpython build [--cache DIR] [--out DIR] PATHS... Red: 7/7 missing compile_project. Green: 7/7 with real Pyrefly and C builds; unchanged rebuild of 3 modules 0.002 s with 0 lower/compile calls; a body-only change rebuilds that module only. Mutation (hash the unverified module): test 7 fails. Refs #41 --- .../src/main/python/typedpython/__main__.py | 40 ++++ .../src/main/python/typedpython/pipeline.py | 116 +++++++++++- .../test/python/test_pipeline_incremental.py | 171 ++++++++++++++++++ 3 files changed, 322 insertions(+), 5 deletions(-) create mode 100644 python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py index 38dffde8c..6534afebf 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py @@ -30,8 +30,16 @@ def main(argv: list[str]) -> int: demo.add_argument("argv", nargs="*", help="sys.argv[1:] for the module's main()") demo.add_argument("--out", type=Path, default=Path("build/typedpython"), help="where the extension is built (default: build/typedpython)") + build = commands.add_parser("build", help="compile modules incrementally (cached per module)") + build.add_argument("paths", nargs="+", type=Path, help="files, or directories to build recursively") + build.add_argument("--cache", type=Path, default=Path("build/typedpython-cache"), + help="incremental cache directory (default: build/typedpython-cache)") + build.add_argument("--out", type=Path, default=None, + help="copy the built extensions here (default: leave them in the cache)") args = parser.parse_args(argv) + if args.command == "build": + return _build(args.paths, args.cache, args.out) if args.command == "demo": return _demo(args.module, args.argv, args.out) @@ -50,6 +58,38 @@ def main(argv: list[str]) -> int: return EXIT_ERRORS if any(d.severity == "error" for d in found) else EXIT_CLEAN +def _build(paths: list[Path], cache: Path, out: Path | None) -> int: + import os + import shutil + + from typedpython import pipeline + + sources = python_files(paths) + if not sources: + print("typedpython: no Python files to build", file=sys.stderr) + return EXIT_FAILURE + dirs = [p.resolve() for p in paths if p.is_dir()] or [p.resolve().parent for p in paths] + root = Path(os.path.commonpath([str(d) for d in dirs])) + try: + result = pipeline.compile_project(sources, root, cache) + except pipeline.CompileError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_ERRORS + except PyreflyError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_FAILURE + for m in result.modules: + why = f" ({', '.join(m.reasons)})" if m.reasons else "" + what = m.extension.name if m.extension else "no extension (interpreted)" + print(f"{m.status:7} {m.name}: {what}{why}") + for name, reason in sorted(m.skipped.items()): + print(f" interpreted: {name} — {reason}") + if out is not None and m.extension is not None: + out.mkdir(parents=True, exist_ok=True) + shutil.copy2(m.extension, out / m.extension.name) + return EXIT_CLEAN + + def _demo(module: Path, module_argv: list[str], out: Path) -> int: import contextlib import importlib.util diff --git a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py index a0d7de7d9..848d630ae 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py @@ -10,11 +10,14 @@ ordinary Python. 5. `cgen.generate` + `cbuild.build`, against the host's CPython or a target's headers. """ +import hashlib +import json +import sysconfig from dataclasses import dataclass, field from pathlib import Path from types import ModuleType -from typedpython import cbuild, cgen, frontend, gate, ir, verify +from typedpython import cbuild, cgen, frontend, gate, incremental, ir, verify class CompileError(RuntimeError): @@ -30,6 +33,13 @@ class Result: diagnostics: list[verify.Diagnostic] = field(default_factory=list) +def _gate_or_raise(path: Path) -> None: + errors = [d for d in gate.check([path], mode="compiled") if d.severity == "error"] + if errors: + listing = "\n".join(f"{d.path}:{d.line}:{d.column}: {d.message} [{d.rule}]" for d in errors) + raise CompileError(f"{path.name} does not pass the gate in compiled mode:\n{listing}") + + def compile_module( path: Path, out_dir: Path, @@ -39,10 +49,7 @@ def compile_module( runtime_dir: Path | None = None, ) -> Result: path = Path(path).resolve() - errors = [d for d in gate.check([path], mode="compiled") if d.severity == "error"] - if errors: - listing = "\n".join(f"{d.path}:{d.line}:{d.column}: {d.message} [{d.rule}]" for d in errors) - raise CompileError(f"{path.name} does not pass the gate in compiled mode:\n{listing}") + _gate_or_raise(path) lowered = frontend.lower(path) proved, diagnostics = verify.verify(lowered) @@ -68,3 +75,102 @@ def load(result: Result) -> ModuleType: spec.loader.exec_module(module) return module return cbuild.load(result.source.stem, result.extension) + + +# --- incremental project build (SPEC N-9) ---------------------------------------------------------- + +def _tree_hash(*dirs: Path) -> str: + h = hashlib.sha256() + for d in dirs: + for f in sorted(p for p in Path(d).rglob("*") if p.is_file() and "__pycache__" not in p.parts): + h.update(f.relative_to(d).as_posix().encode() + b"\0" + f.read_bytes() + b"\0") + return h.hexdigest() + + +_PACKAGE = Path(__file__).resolve().parent + +# What the compiler is: every file of this package (front end, verifier, C back end, runtime header, +# builtin stubs). Any change to any of them can change the C produced for an unchanged source, so it +# invalidates the whole cache. Content-based, not a release number: it needs no manual bump and is +# right for a working tree; the cost is a rebuild after any edit to the compiler, which is correct. +COMPILER_VERSION = _tree_hash(_PACKAGE) + +_NO_EXTENSION = ".noext" # cache artefact of a module with nothing compiled + + +@dataclass +class ModuleResult: + path: Path + name: str + extension: Path | None # None: nothing compiled, ship the .py as is + status: str # "rebuilt" | "reused" + reasons: list[str] = field(default_factory=list) # why it was rebuilt (empty if reused) + skipped: dict[str, str] = field(default_factory=dict) # function -> why it stays interpreted + interface: str = "" + + +@dataclass +class ProjectResult: + modules: list[ModuleResult] + + @property + def rebuilt(self) -> list[ModuleResult]: + return [m for m in self.modules if m.status == "rebuilt"] + + +def compile_project( + sources: list[Path], + project_root: Path, + cache_dir: Path, + *, + python_include: Path | None = None, + ext_suffix: str | None = None, + runtime_dir: Path | None = None, +) -> ProjectResult: + sources = [Path(s).resolve() for s in sources] + cache_dir = Path(cache_dir) + verified: dict[Path, ir.Module] = {} + + def lower(path: Path) -> ir.Module: + _gate_or_raise(path) + proved, _ = verify.verify(frontend.lower(path)) + verified[path] = proved # the VERIFIED module: its signatures are what gets compiled + return proved + + def generate(module: ir.Module, path: Path) -> str: + return cgen.generate(module, path) if module.functions else "" + + def compile_c(c_source: str, name: str, out_dir: Path) -> Path: + if not c_source: + marker = Path(out_dir) / (name.rpartition(".")[2] + _NO_EXTENSION) + marker.write_text("nothing compiled\n") + return marker + return cbuild.build(c_source, name, out_dir, runtime_dir=runtime_dir, flags=cbuild.DEFAULT_FLAGS, + python_include=python_include, ext_suffix=ext_suffix) + + version = COMPILER_VERSION + if runtime_dir is not None: + version += ":" + _tree_hash(Path(runtime_dir)) + suffix = ext_suffix if ext_suffix is not None else sysconfig.get_config_var("EXT_SUFFIX") + include = cbuild._python_include(python_include) + flags = [*cbuild.DEFAULT_FLAGS, f"-I{include}"] + + cache = incremental.BuildCache(cache_dir) + report = cache.build_project( + sources, project_root, lower=lower, generate=generate, compile_c=compile_c, + compiler_version=version, platform=f"{sysconfig.get_platform()} {suffix}", flags=flags) + + side = cache_dir / "skipped" + side.mkdir(exist_ok=True) + out: list[ModuleResult] = [] + for m in report.modules: + record = side / f"{m.key}.json" + if m.status == "rebuilt": + record.write_text(json.dumps(dict(verified[m.path].skipped))) + try: + skipped = json.loads(record.read_text()) + except (OSError, ValueError): + skipped = {} + ext = None if m.artifact is None or m.artifact.suffix == _NO_EXTENSION else m.artifact + out.append(ModuleResult(m.path, m.name, ext, m.status, list(m.reasons), skipped, m.interface)) + return ProjectResult(out) diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py b/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py new file mode 100644 index 000000000..55d453fa6 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py @@ -0,0 +1,171 @@ +"""compile_project: the incremental cache wired into the real pipeline (SPEC N-9, #41). + +Real Pyrefly, real C builds. The three-module project: `a` imports `b`; `c` is independent. +""" +import importlib.util +import textwrap +import time +from pathlib import Path + +import pytest + +from typedpython import cbuild, frontend, pipeline + +HEADER = "# typedpython: compiled\n" + +B = HEADER + textwrap.dedent("""\ + def twice(n: int) -> int: + return n * 2 + """) +A = HEADER + textwrap.dedent("""\ + import b + + + def inc(n: int) -> int: + return n + 1 + """) +C = HEADER + textwrap.dedent("""\ + def half(x: float) -> float: + return x * 0.5 + """) + + +def interpreted(path: Path): + spec = importlib.util.spec_from_file_location(path.stem + "_interp", path) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + return m + + +@pytest.fixture +def project(tmp_path): + root = tmp_path / "proj" + root.mkdir() + for name, src in (("a", A), ("b", B), ("c", C)): + (root / f"{name}.py").write_text(src) + return root + + +@pytest.fixture +def counts(monkeypatch): + """Count calls into the compile steps (the work an unchanged rebuild must not do).""" + n = {"lower": 0, "build": 0} + real_lower, real_build = frontend.lower, cbuild.build + + def lower(*a, **k): + n["lower"] += 1 + return real_lower(*a, **k) + + def build(*a, **k): + n["build"] += 1 + return real_build(*a, **k) + + monkeypatch.setattr(frontend, "lower", lower) + monkeypatch.setattr(cbuild, "build", build) + return n + + +def run(project, tmp_path): + return pipeline.compile_project( + [project / "a.py", project / "b.py", project / "c.py"], project, tmp_path / "cache") + + +def status(result): + return {m.path.stem: m.status for m in result.modules} + + +def test_first_build_compiles_all_and_unchanged_rebuild_does_nothing(project, tmp_path, counts): + first = run(project, tmp_path) + assert status(first) == {"a": "rebuilt", "b": "rebuilt", "c": "rebuilt"} + assert all(m.extension is not None for m in first.modules) + assert counts == {"lower": 3, "build": 3} + + counts.update(lower=0, build=0) + start = time.perf_counter() + second = run(project, tmp_path) + elapsed = time.perf_counter() - start + print(f"\nunchanged rebuild of 3 modules: {elapsed:.3f} s (compile steps skipped)") + assert status(second) == {"a": "reused", "b": "reused", "c": "reused"} + assert counts == {"lower": 0, "build": 0} + assert [m.extension for m in second.modules] == [m.extension for m in first.modules] + + +def test_body_change_rebuilds_that_module_only(project, tmp_path, counts): + run(project, tmp_path) + (project / "b.py").write_text(B.replace("n * 2", "n * 3")) + r = run(project, tmp_path) + assert status(r) == {"a": "reused", "b": "rebuilt", "c": "reused"} + assert "source" in next(m for m in r.modules if m.path.stem == "b").reasons + + +def test_signature_change_rebuilds_dependents(project, tmp_path, counts): + run(project, tmp_path) + (project / "b.py").write_text(B.replace("n: int) -> int", "n: int, k: int) -> int") + .replace("n * 2", "n * k")) + r = run(project, tmp_path) + assert status(r) == {"a": "rebuilt", "b": "rebuilt", "c": "reused"} + a = next(m for m in r.modules if m.path.stem == "a") + assert any("dependency interface" in x for x in a.reasons), a.reasons + + +def test_compiler_version_change_rebuilds_everything(project, tmp_path, counts, monkeypatch): + run(project, tmp_path) + monkeypatch.setattr(pipeline, "COMPILER_VERSION", "something-else") + r = run(project, tmp_path) + assert status(r) == {"a": "rebuilt", "b": "rebuilt", "c": "rebuilt"} + assert all("compiler_version" in m.reasons for m in r.modules) + + +def test_rebuilt_and_reused_extensions_compute_the_interpreted_results(project, tmp_path, monkeypatch): + monkeypatch.syspath_prepend(str(project)) # a's `import b` runs at load + run(project, tmp_path) + (project / "b.py").write_text(B.replace("n * 2", "n * 3")) + r = run(project, tmp_path) + mods = {m.path.stem: m for m in r.modules} + b = cbuild.load("b", mods["b"].extension) + assert b.twice(7) == interpreted(project / "b.py").twice(7) == 21 + a = cbuild.load("a", mods["a"].extension) # reused + assert a.inc(4) == 5 + c = cbuild.load("c", mods["c"].extension) + assert c.half(5.0) == interpreted(project / "c.py").half(5.0) + + +def test_module_with_nothing_compiled_has_no_extension_and_is_reused(tmp_path): + root = tmp_path / "proj" + root.mkdir() + (root / "plain.py").write_text(HEADER + "def f(s: str) -> str:\n return s + 'x'\n") + cache = tmp_path / "cache" + first = pipeline.compile_project([root / "plain.py"], root, cache) + m = first.modules[0] + assert m.extension is None and m.status == "rebuilt" and m.skipped + second = pipeline.compile_project([root / "plain.py"], root, cache) + assert second.modules[0].status == "reused" and second.modules[0].reasons == [] + assert second.modules[0].extension is None + + +def test_interface_describes_the_verified_module_not_the_lowered_one(tmp_path, monkeypatch): + """A function the verifier rejects stays interpreted, so it must not be in the interface.""" + import dataclasses + + from typedpython import incremental, ir + + root = tmp_path / "proj" + root.mkdir() + (root / "dep.py").write_text(HEADER + "def ok(n: int) -> int:\n return n + 1\n") + real_lower = frontend.lower + seen = {} + + def lower_with_a_bad_function(path): + m = real_lower(path) + good = m.functions[0] + bad = dataclasses.replace(good, name="bad", returns=ir.Type.F64) # verify/type: wrong return + seen["lowered"] = dataclasses.replace(m, functions=(*m.functions, bad)) + return seen["lowered"] + + monkeypatch.setattr(frontend, "lower", lower_with_a_bad_function) + r = pipeline.compile_project([root / "dep.py"], root, tmp_path / "cache") + dep = r.modules[0] + assert "bad" in dep.skipped, dep.skipped + assert [f.name for f in seen["lowered"].functions] == ["ok", "bad"] + unverified = incremental.interface_hash(seen["lowered"]) + assert dep.interface != unverified From d163293095cb356fc609fa6eee2da020076c2ac6 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:52:30 +0900 Subject: [PATCH 30/36] CI: Run the TypedPython C tests under ASan+UBSan A sanitizers job (ubuntu) builds every TypedPython extension with -fsanitize=address,undefined and runs the cgen, runtime and pipeline tests with libasan preloaded. It first runs a self-check that requires ASan to report a planted heap overflow and UBSan a planted signed overflow, so a setup that silently detects nothing cannot pass. Verified locally on macOS via the sanitized launcher: self-check OK, 185 passed. The Linux route is verified only by the first CI run. Refs #41 --- .github/workflows/typedpython.yml | 37 ++++ .../src/test/python/conftest.py | 21 ++ python-multiplatform-ksp/tools/sanitizers.py | 184 ++++++++++++++++++ 3 files changed, 242 insertions(+) create mode 100644 python-multiplatform-ksp/tools/sanitizers.py diff --git a/.github/workflows/typedpython.yml b/.github/workflows/typedpython.yml index 84dba3f04..762b12fe2 100644 --- a/.github/workflows/typedpython.yml +++ b/.github/workflows/typedpython.yml @@ -38,3 +38,40 @@ jobs: - name: Measurements (recorded, sanity bounds only) # A shared runner is not an idle machine (AGENTS.md §8): read ratios, not absolute times. run: .venv/bin/python -m pytest -q -s -m measurement + + sanitizers: + # SPEC N-8: ASan/UBSan runs are the safety net outside the proof. Every extension TypedPython + # builds in these tests is compiled with -fsanitize=address,undefined (TP_SANITIZE=1, see + # conftest.py) and the interpreter runs with the ASan runtime preloaded; any report fails the run. + # Linux only: macOS refuses to load the ASan runtime into the framework Python (the cgen + # sanitizer test has a launcher-based macOS path of its own). + name: TypedPython tests under ASan + UBSan (ubuntu-latest) + runs-on: ubuntu-latest + defaults: + run: + working-directory: python-multiplatform-ksp + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + + - name: Create the environment + run: | + uv venv .venv --python 3.13 + uv pip install --python .venv/bin/python "pyrefly==1.3.2" pytest + + - name: Relax ASLR entropy (ASan shadow mapping on newer kernels) + run: sudo sysctl -w vm.mmap_rnd_bits=28 || true + + - name: Self-check, the sanitizers must catch planted faults + # Builds a deliberately faulty extension (heap out-of-bounds read, signed overflow) and + # REQUIRES both reports. If this step passes, the setup below can detect faults. + run: .venv/bin/python tools/sanitizers.py selfcheck + + - name: cgen, runtime and pipeline tests, sanitized + # detect_leaks=0: CPython itself leaks at exit. LD_PRELOAD is applied to this one command + # only (tools/sanitizers.py env prints it); the gate's pyrefly runs without it. + run: | + env $(.venv/bin/python tools/sanitizers.py env) \ + .venv/bin/python -m pytest -q -m "not measurement" \ + src/test/python/test_cgen.py src/test/python/test_runtime.py src/test/python/test_pipeline.py diff --git a/python-multiplatform-ksp/src/test/python/conftest.py b/python-multiplatform-ksp/src/test/python/conftest.py index 65b1e5f0a..f04ff2974 100644 --- a/python-multiplatform-ksp/src/test/python/conftest.py +++ b/python-multiplatform-ksp/src/test/python/conftest.py @@ -25,3 +25,24 @@ def rules(diagnostics): def errors(diagnostics): return [d for d in diagnostics if d.severity == "error"] + + +# TP_SANITIZE=1 (tools/sanitizers.py): every extension TypedPython builds gets ASan + UBSan, and a +# sanitizer report fails the run (-fno-sanitize-recover=undefined; ASan halts). Nothing else changes. +import os # noqa: E402 + +if os.environ.get("TP_SANITIZE") == "1": + import sys as _sys + from typedpython import cbuild as _cbuild + + _SAN = ["-fsanitize=address,undefined", "-fno-sanitize-recover=undefined", "-g", + "-fno-omit-frame-pointer"] + _orig_compiler = _cbuild.compiler + + def _sanitized_compiler(): + cmd = _orig_compiler() + _SAN + if _sys.platform.startswith("linux"): # keep the ASan preload out of the compiler + cmd = ["env", "-u", "LD_PRELOAD"] + cmd + return cmd + + _cbuild.compiler = _sanitized_compiler diff --git a/python-multiplatform-ksp/tools/sanitizers.py b/python-multiplatform-ksp/tools/sanitizers.py new file mode 100644 index 000000000..124edd361 --- /dev/null +++ b/python-multiplatform-ksp/tools/sanitizers.py @@ -0,0 +1,184 @@ +"""ASan + UBSan setup for the TypedPython tests (SPEC N-8, issue #41). + + python tools/sanitizers.py env print KEY=VALUE lines (append to $GITHUB_ENV) + python tools/sanitizers.py selfcheck build a deliberately faulty extension and REQUIRE that + ASan and UBSan each report it; exit 1 otherwise + +Linux: the sanitizer runtime is LD_PRELOADed into the (unsanitized) interpreter. +macOS refuses to load the runtime into the framework Python, so selfcheck runs through a small +sanitized launcher (Py_BytesMain) instead -- the same recipe as test_cgen.py. CI uses Linux only; +the macOS path exists so the self-check logic can be exercised on a developer machine. + +TP_SANITIZE=1 makes conftest.py add the sanitizer flags to every extension TypedPython builds +(cbuild.compiler) and makes test_runtime.py build its shim sanitized. +""" +from __future__ import annotations + +import os +import shlex +import subprocess +import sys +import sysconfig +import tempfile +import textwrap +from pathlib import Path + +SAN_FLAGS = ["-fsanitize=address,undefined", "-fno-sanitize-recover=undefined", "-g", "-O1", + "-fno-omit-frame-pointer"] +# detect_leaks=0: CPython itself leaks at exit. abort_on_error=0 keeps a plain nonzero exit code. +RUN_ENV = { + "ASAN_OPTIONS": "detect_leaks=0:abort_on_error=0:halt_on_error=1:allocator_may_return_null=1", + "UBSAN_OPTIONS": "halt_on_error=1:print_stacktrace=1", +} + + +def cc() -> list[str]: + return shlex.split(sysconfig.get_config_var("CC") or "cc") + + +def asan_runtime() -> str: + """Path of the shared ASan runtime belonging to the compiler that builds the extensions.""" + compiler = cc() + is_clang = "clang" in Path(compiler[0]).name + if sys.platform == "darwin": + names = ["libclang_rt.asan_osx_dynamic.dylib"] + elif is_clang: + import platform + names = [f"libclang_rt.asan-{platform.machine()}.so"] + else: + names = ["libasan.so"] + for name in names: + out = subprocess.run(compiler + [f"-print-file-name={name}"], capture_output=True, + text=True).stdout.strip() + if out and Path(out).is_file(): + return str(Path(out).resolve()) + raise SystemExit(f"no ASan runtime {names} found for compiler {compiler}") + + +def run_env() -> dict[str, str]: + env = dict(RUN_ENV) + if sys.platform.startswith("linux"): + env["LD_PRELOAD"] = asan_runtime() + return env + + +def pyrefly_wrapper() -> str: + """Pyrefly is a prebuilt Rust binary: keep the ASan preload out of it (and out of the launcher + case, where sys.executable is not the venv python). The gate honours TYPEDPYTHON_PYREFLY.""" + binary = Path(sys.prefix) / "bin" / "pyrefly" + if not binary.is_file(): + raise SystemExit(f"{binary} not found: install the test environment first") + wrapper = Path(tempfile.mkdtemp(prefix="tp-san-")) / "pyrefly" + wrapper.write_text(f'#!/bin/sh\nunset LD_PRELOAD DYLD_INSERT_LIBRARIES\nexec "{binary}" "$@"\n') + wrapper.chmod(0o755) + return str(wrapper) + + +def cmd_env() -> None: + env = {"TP_SANITIZE": "1", "TYPEDPYTHON_PYREFLY": pyrefly_wrapper(), **run_env()} + for key, value in env.items(): + print(f"{key}={value}") + + +FAULTY_C = textwrap.dedent('''\ + #include + #include + #include + + static PyObject *oob_read(PyObject *self, PyObject *arg) { + volatile long i = PyLong_AsLong(arg); /* runtime index: the read cannot be folded away */ + char *p = (char *)malloc(8); + p[0] = 1; + volatile char c = p[i]; /* heap-buffer-overflow when i >= 8 */ + free(p); + return PyLong_FromLong(c); + } + + static PyObject *overflow(PyObject *self, PyObject *arg) { + volatile int a = INT_MAX, b = (int)PyLong_AsLong(arg); + int r = a + b; /* signed-integer-overflow when b >= 1 */ + return PyLong_FromLong(r); + } + + static PyMethodDef methods[] = { + {"oob_read", oob_read, METH_O, ""}, {"overflow", overflow, METH_O, ""}, {NULL, NULL, 0, NULL}}; + static struct PyModuleDef moddef = {PyModuleDef_HEAD_INIT, "tp_faulty", NULL, -1, methods}; + PyMODINIT_FUNC PyInit_tp_faulty(void) { return PyModule_Create(&moddef); } + ''') + +DRIVER = textwrap.dedent('''\ + import importlib.machinery, importlib.util, sys + loader = importlib.machinery.ExtensionFileLoader("tp_faulty", {so!r}) + spec = importlib.util.spec_from_file_location("tp_faulty", {so!r}, loader=loader) + m = importlib.util.module_from_spec(spec); loader.exec_module(m) + print("LOADED", flush=True) + {call} + print("RETURNED-WITHOUT-REPORT", flush=True) + ''') + + +def python_command(workdir: Path) -> list[str]: + if sys.platform != "darwin": + return [sys.executable] + launcher_c = workdir / "asan_python.c" + launcher_c.write_text("#include \nint main(int argc, char **argv) " + "{ return Py_BytesMain(argc, argv); }\n") + launcher = workdir / "asan_python" + cmd = cc() + SAN_FLAGS + [f"-I{sysconfig.get_paths()['include']}", str(launcher_c), + f"-L{sysconfig.get_config_var('LIBDIR')}", + f"-lpython{sysconfig.get_config_var('LDVERSION')}", "-o", str(launcher)] + subprocess.run(cmd, check=True) + return [str(launcher)] + + +def cmd_selfcheck() -> int: + work = Path(tempfile.mkdtemp(prefix="tp-san-selfcheck-")) + src = work / "tp_faulty.c" + src.write_text(FAULTY_C) + so = work / ("tp_faulty" + sysconfig.get_config_var("EXT_SUFFIX")) + build = cc() + SAN_FLAGS + ["-fPIC", "-shared", f"-I{sysconfig.get_paths()['include']}", str(src), + "-o", str(so)] + if sys.platform == "darwin": + build += ["-undefined", "dynamic_lookup"] + subprocess.run(build, check=True) + py = python_command(work) + env = dict(os.environ, **run_env()) + if sys.platform == "darwin": + env["PYTHONHOME"] = sys.base_prefix + env.pop("DYLD_INSERT_LIBRARIES", None) + failures = [] + cases = [ + ("clean call is silent", "m.oob_read(0); m.overflow(0)", None, 0), + ("ASan reports heap overflow", "m.oob_read(64)", "AddressSanitizer: heap-buffer-overflow", None), + ("UBSan reports signed overflow", "m.overflow(1)", "runtime error: signed integer overflow", None), + ] + for name, call, needle, want_rc in cases: + run = subprocess.run(py + ["-c", DRIVER.format(so=str(so), call=call)], capture_output=True, + text=True, env=env) + out = run.stdout + run.stderr + if "violates platform policy" in out or "Interceptors are not working" in out: + failures.append(f"{name}: sanitizer runtime could not be loaded:\n{out[-1500:]}") + continue + if needle is None: + ok = run.returncode == 0 and "RETURNED-WITHOUT-REPORT" in out and "Sanitizer" not in out \ + and "runtime error" not in out + else: + ok = needle in out and run.returncode != 0 and "RETURNED-WITHOUT-REPORT" not in out + print(f"{'ok ' if ok else 'FAIL'} {name} (rc={run.returncode})") + if not ok: + failures.append(f"{name}:\n{out[-1500:]}") + if failures: + print("\n".join(failures), file=sys.stderr) + print("SELF-CHECK FAILED: the sanitizer setup does not detect a known fault", file=sys.stderr) + return 1 + print("SELF-CHECK OK: ASan and UBSan both report the planted faults") + return 0 + + +if __name__ == "__main__": + if sys.argv[1:] == ["env"]: + cmd_env() + elif sys.argv[1:] == ["selfcheck"]: + sys.exit(cmd_selfcheck()) + else: + sys.exit(__doc__) From eb4dcb8219088a355b815af2e9588fe3b6ed8264 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:53:59 +0900 Subject: [PATCH 31/36] Feat: Verifier proves local arrays and tuples NewArray/CopyArray only as the value of an Assign to a matching array local; a local array never escapes (any value use is verify/array-escape); stores into local arrays are not effects; single-assignment local arrays carry a length interval for bounds proofs; Tuple elements are OBJ and it creates a reference. Red: 67 failed / 144 passed. Green: 211/211; 400 seeded mutants of 18 variants all rejected. Mutation (escape check off): 16 fail. Refs #41 --- .../src/main/python/typedpython/verify.py | 216 +++++- .../src/test/python/test_verify.py | 632 +++++++++++++++++- 2 files changed, 822 insertions(+), 26 deletions(-) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/verify.py b/python-multiplatform-ksp/src/main/python/typedpython/verify.py index 954d9bfb5..787e40b25 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/verify.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/verify.py @@ -21,9 +21,13 @@ verify/deopt-impure a node that can deopt appears in an impure function verify/flag-may-deopt `may_deopt` differs from what the body computes verify/flag-pure `pure` is set but the body has an effect - verify/array-use an array is used other than by Index/StoreIndex/Len, or those nodes - name something that is not an array parameter + verify/array-use an array parameter is used other than by Index/StoreIndex/Len (or as a + CopyArray source), or those nodes name something that is not an array verify/array-stored `ArrayParam.stored` differs from the body + verify/array-escape a local array is used other than by Index/StoreIndex/Len and as a + CopyArray source (returned, passed, boxed, stored in a tuple, aliased, + compared...), or NewArray/CopyArray stand anywhere but as the value of + an Assign to a local array verify/unknown-node a node (or enum field) that `ir` does not define verify/call-unresolved a Call names a function that is not compiled in this module verify/call-unverified a Call names a function that the verifier rejected @@ -67,6 +71,34 @@ var after its loop, Index elements, Unbox, Call results, FLOORDIV, MOD, TRUEDIV +Local arrays (NewArray / CopyArray, ir.Function "Local arrays"). A local of an array type holds a +native array the function owns. The verifier proves, never trusting the front end: + + * NewArray/CopyArray are the value of an Assign to a local declared with the same array type, and + nowhere else (`verify/array-escape` elsewhere, `verify/type` on a type mismatch). NewArray: length + I64; fill absent iff `iota` (a bool), `iota` only for I64_ARRAY; a fill has the element type. + CopyArray: the source is a local or parameter array of the same type. + * No escape: a local array is read only by Index/StoreIndex/Len and as a CopyArray source. Any + `Local` naming it (a return value, a Call/CallObject/Box/Tuple operand, another assignment...) is + `verify/array-escape`. It cannot be a loop variable, a parameter, or an entry global. + * Purity: a StoreIndex into a local array is not an effect (and does not set `stored`, which is + for ArrayParams); a function whose only stores are into local arrays can be `pure`. A store into + an array parameter still is. `pure` and `may_deopt` are recomputed as before and compared. + * Definite assignment: the allocation's operands are read before the array is assigned, and every + Index/StoreIndex/Len/CopyArray source of a local array needs it assigned on every path. + * Bounds. (1) The ForRange rule (var over range(c >= 0, Len(a), step > 0)) now also covers a local + array, but only if the loop body never assigns that array (a reassignment may change its + length; the stop is evaluated once). (2) A local array assigned by exactly one Assign statement + in the whole body has a known length interval [lo, hi]: for NewArray the interval of the length + expression clamped at 0 below (a negative length is an empty array), for CopyArray the interval + of the source if the source is itself such a local (otherwise [0, 2**62]). Then Len(a) has that + interval, and an Index/StoreIndex whose index interval lies inside [0, lo - 1] is proven. That + proves constant indices and bounded index expressions; a negative index is never proven (the + `proven` flag means 0 <= index < len). An array assigned twice or more has no known length. + +Tuple. `Tuple(OBJ, elements)`: a tuple of OBJ expressions (a scalar needs Box), result OBJ, a +reference creator in the ownership rule. Building one is not an effect. + What this cannot prove: anything that needs knowing a value through assignments (accumulators, `x = 1; x + 1`), relations between variables (`i < n` does not bound `i + n`), bounds that come from guards or conditions (`if n < 100`), values through array elements, calls or Unbox, the result of @@ -96,9 +128,9 @@ from . import ir from .ir import ( And, ArrayParam, Assign, BinOp, BinOpKind, Box, Break, Call, CallObject, Compare, CompareKind, - CompareObj, Const, Continue, ExprStmt, ForRange, Function, GetAttr, Global, If, Index, Len, - Local, MathCall, MathFunc, Module, ObjToFloat, Or, Param, Return, StoreIndex, ToFloat, Truth, - Type, Unbox, UnaryOp, UnaryOpKind, While, + CompareObj, Const, Continue, CopyArray, ExprStmt, ForRange, Function, GetAttr, Global, If, Index, Len, + Local, MathCall, MathFunc, Module, NewArray, ObjToFloat, Or, Param, Return, StoreIndex, ToFloat, + Truth, Tuple, Type, Unbox, UnaryOp, UnaryOpKind, While, ) DA = "verify/definite-assignment" @@ -110,6 +142,7 @@ FLAG_PURE = "verify/flag-pure" ARRAY_USE = "verify/array-use" ARRAY_STORED = "verify/array-stored" +ESCAPE = "verify/array-escape" UNKNOWN = "verify/unknown-node" UNRESOLVED = "verify/call-unresolved" UNVERIFIED = "verify/call-unverified" @@ -132,13 +165,14 @@ class Diagnostic: # Exact classes, not isinstance: a subclass of an ir node is not an ir node (its meaning is not # the one the back end implements). _EXPRS = frozenset({Const, Local, BinOp, UnaryOp, Compare, And, Or, ToFloat, Box, Unbox, MathCall, - Call, Global, GetAttr, CallObject, Truth, CompareObj, ObjToFloat, Len, Index}) + Call, Global, GetAttr, CallObject, Truth, CompareObj, ObjToFloat, Len, Index, + NewArray, CopyArray, Tuple}) _STMTS = frozenset({Assign, StoreIndex, ExprStmt, If, While, ForRange, Return, Break, Continue}) # Nodes whose OBJ result is a new reference (ir.py "ownership": Box, Global, GetAttr, CallObject, -# OBJ BinOp), plus reads of an OBJ local (the local owns its reference; the back end increfs or +# OBJ BinOp, Tuple), plus reads of an OBJ local (the local owns its reference; the back end increfs or # borrows under that rule) and Calls of a compiled function returning OBJ (which returns a new # reference by the same rule). -_OBJ_PRODUCERS = frozenset({Box, Global, GetAttr, CallObject, BinOp, Local, Call}) +_OBJ_PRODUCERS = frozenset({Box, Global, GetAttr, CallObject, BinOp, Local, Call, Tuple}) _VALUE_TYPES = (Type.I64, Type.F64, Type.BOOL, Type.OBJ) # what a local may hold _RETURN_TYPES = _VALUE_TYPES + (Type.NONE,) _ELEMENT = {Type.F64_ARRAY: Type.F64, Type.I64_ARRAY: Type.I64} @@ -185,6 +219,9 @@ def __init__(self, function: Function, signatures: dict[str, Function]): self.loops: list[_Loop] = [] self.scalars: dict[str, Type] = {} # name -> type of every scalar/OBJ variable self.arrays: dict[str, Type] = {} # name -> array type of every ArrayParam + self.larrays: dict[str, Type] = {} # name -> array type of every local array + self.assigns: dict[str, int] = {} # local array -> number of Assign statements to it + self.lens: dict[str, tuple[int, int]] = {} # single-assignment local array -> length self.entry: dict[str, Type] = {} # name -> type of every entry global self.opens: list[str] = [] # reasons the function is not closed @@ -248,6 +285,11 @@ def declarations(self) -> frozenset[str]: self.error(STRUCTURE, f"local '{name}' collides with an entry global") elif name in self.arrays: self.error(ARRAY_USE, f"local '{name}' shadows array parameter '{name}'") + elif t in ir.ARRAYS: + if name in self.scalars: + self.error(TYPE, f"local array '{name}' collides with a scalar parameter") + else: + self.larrays[name] = t elif t not in _VALUE_TYPES: self.error(TYPE, f"local '{name}' has type {t}") elif name in self.scalars and self.scalars[name] is not t: @@ -293,7 +335,8 @@ def expr(self, e: object, state: _State, cond: bool = False) -> ir.Expr: return e if t is Type.OBJ and cls not in _OBJ_PRODUCERS: self.error(OWNERSHIP, f"{cls.__name__} cannot produce an OBJ (only Box, Global, GetAttr, CallObject, OBJ " - f"BinOp, OBJ locals and OBJ-returning Calls create references)") + f"BinOp, Tuple, OBJ locals and OBJ-returning Calls create " + f"references)") if cls is CompareObj and not cond: self.error(CONDITION_ONLY, "CompareObj outside a condition: the rich comparison's " "result object would be kept, and it need not be a bool") @@ -318,6 +361,10 @@ def e_Local(self, e: Local, state: _State) -> ir.Expr: self.error(ARRAY_USE, f"array parameter '{name}' is used as a value (only Index, " f"StoreIndex and Len may use it)") return e + if name in self.larrays: + self.error(ESCAPE, f"local array '{name}' is used as a value (only Index, StoreIndex, " + f"Len and CopyArray may use it)") + return e declared = self.scalars.get(name) if declared is None: self.error(TYPE, f"local '{name}' is not declared") @@ -342,7 +389,7 @@ def interval(self, e: ir.Expr) -> tuple[int, int]: return loop.interval return _FULL if cls is Len: - return (0, _LEN_MAX) + return self.lens.get(e.array, (0, _LEN_MAX)) if cls is BinOp and e.op in _PROVABLE_OPS: lo, hi = _exact(e.op, self.interval(e.left), self.interval(e.right)) return (max(lo, _I64_MIN), min(hi, _I64_MAX)) @@ -629,21 +676,28 @@ def e_ObjToFloat(self, e: ObjToFloat, state: _State) -> ir.Expr: self.opens.append("ObjToFloat") return dataclasses.replace(e, operand=operand) - def array(self, name: str, node: str) -> Type | None: + def array(self, name: str, node: str, state: _State = None) -> Type | None: + """The array type of `name` (an array parameter or a local array); a local array must be + definitely assigned.""" t = self.arrays.get(name) + if t is not None: + return t + t = self.larrays.get(name) if t is None: - self.error(ARRAY_USE, f"{node} of '{name}', which is not an array parameter") + self.error(ARRAY_USE, f"{node} of '{name}', which is not an array") + elif state is not None and name not in state: + self.error(DA, f"local array '{name}' may be used ({node}) before it is assigned") return t def e_Len(self, e: Len, state: _State) -> ir.Expr: - self.array(e.array, "len") + self.array(e.array, "len", state) if e.type is not Type.I64: self.error(TYPE, f"len gives i64, node says {e.type.value}") return e def index(self, array: str, index: object, state: _State, node: str ) -> tuple[ir.Expr, Type | None, bool]: - at = self.array(array, node) + at = self.array(array, node, state) idx = self.expr(index, state) self.slot(idx, Type.I64, f"index of {node} '{array}'") return idx, (_ELEMENT[at] if at else None), self.in_bounds(array, idx) @@ -657,15 +711,91 @@ def e_Index(self, e: Index, state: _State) -> ir.Expr: def in_bounds(self, array: str, idx: ir.Expr) -> bool: """0 <= idx < len(array): idx is the var of an enclosing ForRange over - range(c >= 0, len(array), step > 0). The var is never assigned in the body (checked), the - stop is evaluated once, and no IR node changes an array's length.""" - if type(idx) is not Local or array not in self.arrays: + range(c >= 0, len(array), step > 0) (for a local array, one whose body never assigns it), or + -- for a local array of known length -- idx has an interval inside [0, minimum length - 1]. + The var is never assigned in the body (checked), the stop is evaluated once, and no IR node + changes an array's length.""" + if array in self.lens and getattr(idx, "type", None) is Type.I64: + lo, hi = self.interval(idx) + if 0 <= lo and hi < self.lens[array][0]: + return True + if type(idx) is not Local or not (array in self.arrays or array in self.larrays): return False for loop in reversed(self.loops): if loop.var == idx.name: return loop.bounds_array == array return False + # --- tuples and local arrays -------------------------------------------------------------- + + def e_Tuple(self, e: Tuple, state: _State) -> ir.Expr: + if not isinstance(e.elements, tuple): + self.error(STRUCTURE, "Tuple elements are not a tuple") + return e + elements = tuple(self.obj_operand(x, state, f"element {k + 1} of Tuple") + for k, x in enumerate(e.elements)) + self.result(e, Type.OBJ) + return dataclasses.replace(e, elements=elements) + + def e_NewArray(self, e: NewArray, state: _State) -> ir.Expr: + return self.stray(e, state) + + def e_CopyArray(self, e: CopyArray, state: _State) -> ir.Expr: + return self.stray(e, state) + + def stray(self, e: NewArray | CopyArray, state: _State) -> ir.Expr: + self.error(ESCAPE, f"{type(e).__name__} is only legal as the value of an Assign to a local " + f"array") + return e + + def allocation(self, e: NewArray | CopyArray, target: str, state: _State) -> ir.Expr: + """`e` is the value of `Assign(target, e)`. Checks the operands, and that the node's type + is an array type equal to the target's declared one.""" + name = type(e).__name__ + want = self.larrays.get(target) + if not isinstance(e.type, Type) or e.type not in ir.ARRAYS: + self.error(TYPE, f"{name} gives an array, node says " + f"{getattr(e.type, 'value', e.type)}") + elif want is not None and e.type is not want: + self.error(TYPE, f"{name} gives {e.type.value} but local array '{target}' is " + f"{want.value}") + element = _ELEMENT.get(e.type) + if type(e) is CopyArray: + src = self.arrays.get(e.src) or self.larrays.get(e.src) + if not isinstance(e.src, str) or src is None: + self.error(ARRAY_USE, f"CopyArray of '{e.src}', which is not an array") + else: + self.array(e.src, "copy", state) + if src is not e.type: + self.error(TYPE, f"CopyArray of {src.value} '{e.src}' gives {e.type.value}") + return e + length = self.expr(e.length, state) + self.slot(length, Type.I64, "length of NewArray") + fill = e.fill + if type(e.iota) is not bool: + self.error(TYPE, f"NewArray `iota` {e.iota!r} is not a bool") + elif e.iota: + if e.type is not Type.I64_ARRAY: + self.error(TYPE, "NewArray iota is only for i64 arrays") + if fill is not None: + self.error(TYPE, "NewArray iota takes no fill") + elif fill is None: + self.error(TYPE, "NewArray needs a fill unless it is iota") + if fill is not None: + fill = self.expr(fill, state) + if element is not None: + self.slot(fill, element, "fill of NewArray") + return dataclasses.replace(e, length=length, fill=fill) + + def length_of(self, e: NewArray | CopyArray) -> tuple[int, int]: + """The interval of the length of the array `e` creates (the operands are checked).""" + if type(e) is CopyArray: + return self.lens.get(e.src, (0, _LEN_MAX)) + if getattr(e.length, "type", None) is not Type.I64: + return (0, _LEN_MAX) + lo, hi = self.interval(e.length) + return (min(max(lo, 0), _LEN_MAX), min(max(hi, 0), _LEN_MAX)) + # --- statements --------------------------------------------------------------------------- def block(self, body: object, state: _State) -> tuple[tuple[ir.Stmt, ...], _State]: @@ -686,13 +816,29 @@ def stmt(self, s: object, state: _State) -> tuple[ir.Stmt, _State]: return getattr(self, "s_" + cls.__name__)(s, state) def s_Assign(self, s: Assign, state: _State): - value = self.expr(s.value, state) target = s.target - if target in self.arrays: + alloc = type(s.value) in (NewArray, CopyArray) + if alloc: + if target in self.larrays: + value = self.allocation(s.value, target, state) + if self.assigns.get(target) == 1: + self.lens[target] = self.length_of(value) + else: + value = self.allocation(s.value, target, state) + self.error(ESCAPE, f"{type(s.value).__name__} is assigned to '{target}', which is " + f"not a local array") + else: + value = self.expr(s.value, state) + if target in self.larrays: + self.error(TYPE, f"local array '{target}' may only be assigned NewArray or " + f"CopyArray") + if target in self.larrays: + pass + elif target in self.arrays: self.error(ARRAY_USE, f"assignment to array parameter '{target}'") elif target not in self.scalars: self.error(TYPE, f"assignment to undeclared local '{target}'") - else: + elif not alloc: self.slot(value, self.scalars[target], f"assignment to '{target}'") if getattr(value, "type", None) is Type.NONE: self.error(TYPE, f"None assigned to '{target}'") @@ -708,8 +854,9 @@ def s_StoreIndex(self, s: StoreIndex, state: _State): value = self.expr(s.value, state) if element is not None: self.slot(value, element, f"value stored into '{s.array}'") - self.stored.add(s.array) - self.effects.append(f"stores into '{s.array}'") + if s.array not in self.larrays: # a local array is the function's own: not an effect + self.stored.add(s.array) + self.effects.append(f"stores into '{s.array}'") return dataclasses.replace(s, index=idx, value=value, proven=proven), state def s_ExprStmt(self, s: ExprStmt, state: _State): @@ -764,7 +911,9 @@ def s_ForRange(self, s: ForRange, state: _State): bounds = None if (step_ok and step.value > 0 and type(start) is Const and start.type is Type.I64 and type(start.value) is int and start.value >= 0 and type(stop) is Len - and stop.array in self.arrays): + and (stop.array in self.arrays + or (stop.array in self.larrays + and stop.array not in _assigned(s.body)))): bounds = stop.array loop = _Loop(var, bounds) if (step_ok and getattr(start, "type", None) is Type.I64 @@ -814,6 +963,10 @@ def _jump(self, s: Break | Continue, state: _State): def run(self) -> Function: fn = self.fn params = self.declarations() + if self.larrays: + counts: dict[str, int] = {} + _assigned(fn.body, counts) + self.assigns = {n: counts.get(n, 0) for n in self.larrays} body, end = self.block(fn.body, params) if end is not None and fn.returns is not Type.NONE: self.error(MISSING_RETURN, f"the end of '{fn.name}' is reachable but it returns " @@ -838,6 +991,23 @@ def run(self) -> Function: return dataclasses.replace(fn, body=body) +def _assigned(body: object, counts: dict[str, int] | None = None) -> dict[str, int]: + """Every name assigned by an Assign statement anywhere in `body` (nested blocks included), with + the number of Assign statements. Tolerates a malformed body (the checker reports it).""" + counts = {} if counts is None else counts + if isinstance(body, tuple): + for st in body: + cls = type(st) + if cls is Assign: + counts[st.target] = counts.get(st.target, 0) + 1 + elif cls is If: + _assigned(st.then, counts) + _assigned(st.orelse, counts) + elif cls in (While, ForRange): + _assigned(st.body, counts) + return counts + + def _exact(op: BinOpKind, a: tuple[int, int], b: tuple[int, int]) -> tuple[int, int]: """The exact (unbounded) result interval of ADD/SUB/MUL on operand intervals.""" if op is BinOpKind.ADD: diff --git a/python-multiplatform-ksp/src/test/python/test_verify.py b/python-multiplatform-ksp/src/test/python/test_verify.py index 662110338..b7b3c8354 100644 --- a/python-multiplatform-ksp/src/test/python/test_verify.py +++ b/python-multiplatform-ksp/src/test/python/test_verify.py @@ -19,9 +19,9 @@ from typedpython import ir from typedpython.ir import ( And, ArrayParam, Assign, BinOp, BinOpKind, Box, Break, Call, CallObject, Compare, CompareKind, - CompareObj, Const, Continue, ExprStmt, ForRange, Function, GetAttr, Global, If, Index, Len, - Local, MathCall, MathFunc, Module, ObjToFloat, Or, Param, Return, StoreIndex, ToFloat, Truth, - Type, Unbox, UnaryOp, UnaryOpKind, While, + CompareObj, Const, Continue, CopyArray, ExprStmt, ForRange, Function, GetAttr, Global, If, Index, + Len, Local, MathCall, MathFunc, Module, NewArray, ObjToFloat, Or, Param, Return, StoreIndex, + ToFloat, Truth, Tuple, Type, Unbox, UnaryOp, UnaryOpKind, While, ) I64, F64, BOOL, NONE, OBJ = Type.I64, Type.F64, Type.BOOL, Type.NONE, Type.OBJ @@ -45,6 +45,7 @@ PROVEN = "verify/proven-overflow" REDO = "verify/redo" ENTRY_OPEN = "verify/entry-global-open" +ESCAPE = "verify/array-escape" def _verify(): @@ -1373,3 +1374,628 @@ def test_each_proof_mutation_kind_was_exercised(): base = _proof_functions() kinds = {_proof_mutations(rng, base)[0] for _ in range(300)} assert len(kinds) == 16, sorted(kinds) # 14 kinds; kind 6 has three variants + + +# --- 10. local arrays and tuples (NewArray / CopyArray / Tuple; issue #41, N-8) ------------------- + +def na(t, n, fill=None, iota=False): return NewArray(t, n, fill, iota) +def la(name, t=F64A): return Local(t, name) +def tup(*elements): return Tuple(OBJ, tuple(elements)) + + +def _local_sum(body=None, pure=True, may_deopt=False, params=(), locals_=None, returns=F64): + """t = [0.0] * 8; for k in range(len(t)): t[k] = 1.0; s += t[k]; return s""" + body = body if body is not None else [ + Assign("t", na(F64A, i(8), f(0.0))), + Assign("s", f(0.0)), + ForRange("k", i(0), Len(I64, "t"), i(1), ( + StoreIndex("t", li("k"), f(1.0)), + Assign("s", add(lf("s"), Index(F64, "t", li("k")), F64)), + )), + Return(lf("s")), + ] + return fn(params=params, locals_={"t": F64A, "s": F64, "k": I64, **(locals_ or {})}, + returns=returns, pure=pure, may_deopt=may_deopt, body=body) + + +def _flags(function): + out = assert_accepted(function) + return [a.proven for a in _accesses(out.functions[0])] + + +# 10.1 what a local array may be and where an allocation may stand + +def test_a_pure_function_with_a_local_array_is_accepted(): + assert_accepted(_local_sum()) + + +def test_storing_into_a_local_array_is_not_an_effect(): + assert_accepted(_local_sum(pure=True)) + assert_accepted(_local_sum(pure=False)) # a conservative flag is also fine + + +def test_storing_into_an_array_param_still_makes_a_function_impure(): + g = _local_sum(params=[ArrayParam("a", F64A, stored=True)], body=[ + Assign("t", na(F64A, i(8), f(0.0))), + StoreIndex("t", i(0), f(1.0)), + StoreIndex("a", i(0), f(1.0)), + Return(f(0.0)), + ]) + assert_rejected(FLAG_PURE, g) + assert_accepted(dataclasses.replace(g, pure=False)) + + +def test_reading_a_local_array_does_not_make_the_stored_flag_wrong(): + assert_accepted(_local_sum(params=[ArrayParam("a", F64A, stored=False)])) + + +def test_may_deopt_flag_is_recomputed_with_local_arrays(): + deopting = _local_sum(body=[ + Assign("t", na(I64A, i(4), i(0))), + Assign("x", add(Index(I64, "t", i(0)), i(1))), + Return(ToFloat(F64, li("x"))), + ], locals_={"t": I64A, "x": I64}, may_deopt=True) + assert_accepted(deopting) + assert_rejected(FLAG_DEOPT, dataclasses.replace(deopting, may_deopt=False)) + assert_rejected(FLAG_DEOPT, dataclasses.replace(_local_sum(), may_deopt=True)) + assert_rejected(DEOPT, dataclasses.replace(deopting, pure=False)) + + +def test_each_allocation_form_is_accepted(): + for t, alloc, deopts in [(F64A, na(F64A, i(3), f(1.5)), False), + (I64A, na(I64A, i(3), i(7)), False), + (I64A, na(I64A, li("n"), None, True), False), + (F64A, na(F64A, add(li("n"), i(1)), f(0.0)), True)]: + assert_accepted(fn(params=[Param("n", I64)], locals_={"t": t}, returns=NONE, pure=True, + may_deopt=deopts, body=[Assign("t", alloc), Return()])) + + +@pytest.mark.parametrize("place", ["expr-stmt", "return", "call-arg", "box", "tuple", "fill", + "length", "condition-free-assign"]) +def test_an_allocation_anywhere_but_an_assign_to_a_local_array_is_rejected(place): + alloc = na(F64A, i(2), f(0.0)) + stmt = { + "expr-stmt": ExprStmt(alloc), + "return": Return(alloc), + "call-arg": ExprStmt(Call(NONE, "g", (alloc,))), + "box": Assign("o", Box(OBJ, alloc)), + "tuple": Assign("o", tup(alloc)), + "fill": Assign("t", na(F64A, i(2), alloc)), + "length": Assign("t", na(F64A, alloc, f(0.0))), + "condition-free-assign": Assign("o", alloc), + }[place] + g = fn(locals_={"t": F64A, "o": OBJ}, returns=NONE, body=[stmt, Return()] if place != "return" + else [stmt]) + out, diags = run(g, fn("g", params=[Param("x", OBJ)], returns=NONE, body=[Return()])) + assert "f" not in kept(out) + assert ESCAPE in rules_for(diags, "f"), [dataclasses.astuple(d) for d in diags] + + +def test_a_copy_anywhere_but_an_assign_to_a_local_array_is_rejected(): + for stmt in [ExprStmt(CopyArray(F64A, "a")), Return(CopyArray(F64A, "a")), + Assign("o", CopyArray(F64A, "a")), Assign("x", CopyArray(F64A, "a"))]: + g = fn(params=[ArrayParam("a", F64A, stored=False)], locals_={"o": OBJ, "x": F64}, + returns=NONE, body=[stmt, Return()]) + out, diags = run(g) + assert "f" not in kept(out) + assert ESCAPE in rules_for(diags, "f"), (stmt, [dataclasses.astuple(d) for d in diags]) + + +def test_an_allocation_into_an_array_param_is_rejected(): + assert_rejected(ARRAY_USE, fn(params=[ArrayParam("a", F64A, stored=True)], returns=NONE, body=[ + Assign("a", na(F64A, i(2), f(0.0))), Return()])) + + +def test_the_allocation_type_must_match_the_declared_local(): + assert_rejected(TYPE, fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", na(F64A, i(2), f(0.0))), Return()])) + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", CopyArray(I64A, "a")), Return()], + params=[ArrayParam("a", I64A, stored=False)])) + + +def test_the_node_type_must_be_an_array_type(): + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", NewArray(I64, i(2), i(0))), Return()])) + + +def test_a_local_array_may_only_be_assigned_an_allocation(): + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", f(1.0)), Return()])) + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", Const(NONE, None)), Return()])) + + +def test_new_array_length_must_be_i64(): + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", na(F64A, f(2.0), f(0.0))), Return()])) + assert_rejected(OWNERSHIP, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", na(F64A, g_("N"), f(0.0))), Return()])) + + +def test_new_array_fill_must_match_the_element_type(): + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", na(F64A, i(2), i(0))), Return()])) + assert_rejected(TYPE, fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", na(I64A, i(2), f(0.0))), Return()])) + assert_rejected(OWNERSHIP, fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", na(I64A, i(2), g_("zero"))), Return()])) + + +def test_new_array_fill_is_absent_exactly_when_iota(): + assert_rejected(TYPE, fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", na(I64A, i(2))), Return()])) + assert_rejected(TYPE, fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", na(I64A, i(2), i(0), True)), Return()])) + + +def test_iota_is_only_for_i64_arrays_and_must_be_a_bool(): + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", na(F64A, i(2), None, True)), Return()])) + assert_rejected(TYPE, fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", NewArray(I64A, i(2), None, 1)), Return()])) + + +def test_copy_array_source_must_be_an_array_of_the_same_type(): + ok = fn(params=[ArrayParam("a", F64A, stored=False)], locals_={"t": F64A}, returns=NONE, + pure=True, body=[Assign("t", CopyArray(F64A, "a")), Return()]) + assert_accepted(ok) + assert_rejected(ARRAY_USE, dataclasses.replace(ok, params=(Param("a", F64),))) + assert_rejected(ARRAY_USE, fn(locals_={"t": F64A, "x": F64}, returns=NONE, body=[ + Assign("x", f(1.0)), Assign("t", CopyArray(F64A, "x")), Return()])) + assert_rejected(ARRAY_USE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", CopyArray(F64A, "nowhere")), Return()])) + assert_rejected(TYPE, dataclasses.replace( + ok, params=(ArrayParam("a", I64A, stored=False),))) + + +def test_copy_of_a_local_array_is_accepted(): + assert_accepted(fn(locals_={"t": I64A, "u": I64A}, returns=NONE, pure=True, body=[ + Assign("t", na(I64A, i(4), None, True)), Assign("u", CopyArray(I64A, "t")), Return()])) + + +def test_local_array_declarations_are_checked(): + assert_rejected(TYPE, fn(params=[Param("t", I64)], locals_={"t": I64A}, returns=NONE, + body=[Return()])) + assert_rejected(ARRAY_USE, fn(params=[ArrayParam("t", I64A, stored=False)], + locals_={"t": I64A}, returns=NONE, body=[Return()])) + assert_rejected(STRUCTURE, fn(entry_globals=[Param("t", I64)], locals_={"t": I64A}, + returns=NONE, body=[Return()])) + assert_rejected(TYPE, fn(locals_={"t": Type.NONE}, returns=NONE, body=[Return()])) + + +# 10.2 no escape + +def _escapes(use, ret=NONE): + """f(): t = [0] * 2; ; with t an i64 local array; g takes an OBJ.""" + t = la("t", I64A) + stmt = use(t) + g = fn(locals_={"t": I64A, "o": OBJ, "x": I64, "a": F64A, "u": I64A}, returns=ret, body=[ + Assign("t", na(I64A, i(2), i(0))), stmt] + ([Return()] if ret is NONE else [])) + out, diags = run(g, fn("g", params=[Param("x", OBJ)], returns=NONE, body=[Return()])) + assert "f" not in kept(out) + assert ESCAPE in rules_for(diags, "f"), [dataclasses.astuple(d) for d in diags] + + +@pytest.mark.parametrize("use", [ + lambda t: ExprStmt(t), + lambda t: ExprStmt(Call(NONE, "g", (t,))), + lambda t: ExprStmt(call_obj(g_("h"), t)), + lambda t: ExprStmt(call_obj(t)), + lambda t: Assign("o", Box(OBJ, t)), + lambda t: Assign("o", tup(t)), + lambda t: Assign("o", tup(Box(OBJ, i(1)), t)), + lambda t: Assign("u", t), + lambda t: Assign("o", t), + lambda t: Assign("x", Unbox(I64, t)), + lambda t: Assign("o", GetAttr(OBJ, t, "x")), + lambda t: If(Truth(BOOL, t), ()), + lambda t: If(CompareObj(BOOL, CompareKind.EQ, t, lo("o")), ()), +]) +def test_a_local_array_used_as_a_value_is_an_escape(use): + _escapes(use) + + +def test_returning_a_local_array_is_an_escape(): + for returns in (OBJ, NONE): + g = fn(locals_={"t": I64A}, returns=returns, body=[ + Assign("t", na(I64A, i(2), i(0))), Return(la("t", I64A))]) + out, diags = run(g) + assert "f" not in kept(out) + assert ESCAPE in rules_for(diags, "f") + + +def test_a_local_array_is_not_a_loop_variable_or_an_array_param_argument(): + assert_rejected(TYPE, fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", na(I64A, i(2), i(0))), + ForRange("t", i(0), i(2), i(1), ()), Return()])) + callee = fn("g", params=[ArrayParam("a", I64A, stored=False)], returns=NONE, body=[Return()]) + caller = fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", na(I64A, i(2), i(0))), ExprStmt(Call(NONE, "g", (la("t", I64A),))), + Return()]) + out, diags = run(callee, caller) + assert "f" not in kept(out) and ESCAPE in rules_for(diags, "f") + + +def test_index_store_len_and_copy_source_are_the_only_uses_and_are_accepted(): + assert_accepted(fn(locals_={"t": I64A, "u": I64A, "x": I64}, returns=I64, pure=True, + may_deopt=True, body=[ + Assign("t", na(I64A, i(4), i(0))), + StoreIndex("t", i(1), i(5)), + Assign("u", CopyArray(I64A, "t")), + Assign("x", add(Index(I64, "u", i(1)), Len(I64, "t"))), + Return(li("x")), + ])) + + +def test_index_and_store_name_a_declared_array(): + assert_rejected(ARRAY_USE, fn(locals_={"x": I64}, returns=I64, body=[ + Assign("x", Index(I64, "nowhere", i(0))), Return(li("x"))])) + assert_rejected(ARRAY_USE, fn(locals_={"x": I64}, returns=NONE, body=[ + Assign("x", i(1)), StoreIndex("x", i(0), i(1)), Return()])) + + +def test_store_value_must_match_the_local_element_type(): + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", na(F64A, i(2), f(0.0))), StoreIndex("t", i(0), i(1)), Return()])) + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", na(F64A, i(2), f(0.0))), StoreIndex("t", f(0.0), f(1.0)), Return()])) + + +# 10.3 bounds + +def test_loop_over_len_of_a_local_array_is_proven(): + assert _flags(_local_sum()) == [True, True] + + +def test_loop_over_len_of_a_local_array_of_unknown_length_is_proven(): + g = _local_sum(params=[Param("n", I64)], body=[ + Assign("t", na(F64A, li("n"), f(0.0))), + ForRange("k", i(0), Len(I64, "t"), i(1), (StoreIndex("t", li("k"), f(1.0)),)), + Return(f(0.0)), + ]) + assert _flags(g) == [True] + + +def test_other_indices_into_an_array_of_unknown_length_are_unproven(): + g = _local_sum(params=[Param("n", I64)], body=[ + Assign("t", na(F64A, li("n"), f(0.0))), + StoreIndex("t", i(0), f(1.0)), + ForRange("k", i(0), li("n"), i(1), (StoreIndex("t", li("k"), f(1.0)),)), + Return(f(0.0)), + ]) + assert _flags(g) == [False, False] + + +def test_reassigning_the_array_inside_the_loop_defeats_the_loop_proof(): + g = _local_sum(body=[ + Assign("t", na(F64A, i(8), f(0.0))), + Assign("s", f(0.0)), + ForRange("k", i(0), Len(I64, "t"), i(1), ( + Assign("t", na(F64A, i(1), f(0.0))), + Assign("s", add(lf("s"), Index(F64, "t", li("k")), F64)), + )), + Return(lf("s")), + ]) + assert _flags(g) == [False] + + +def test_reassigning_the_array_in_a_nested_statement_also_defeats_it(): + g = _local_sum(params=[Param("c", BOOL)], body=[ + Assign("t", na(F64A, i(8), f(0.0))), + Assign("s", f(0.0)), + ForRange("k", i(0), Len(I64, "t"), i(1), ( + If(lb("c"), (Assign("t", na(F64A, i(1), f(0.0))),)), + Assign("s", add(lf("s"), Index(F64, "t", li("k")), F64)), + )), + Return(lf("s")), + ]) + assert _flags(g) == [False] + + +def _const_index(length, *indices, extra=()): + """t = [0.0] * length; x = t[i0] + t[i1] + ... -> the proven flags in order""" + reads = [Index(F64, "t", i(ix)) for ix in indices] + total = reads[0] + for r in reads[1:]: + total = add(total, r, F64) + return fn(locals_={"t": F64A}, returns=F64, pure=True, body=[ + *extra, Assign("t", na(F64A, i(length), f(0.0))), Return(total)]) + + +def test_constant_indices_below_a_known_length_are_proven(): + assert _flags(_const_index(8, 0, 7)) == [True, True] + + +def test_constant_indices_outside_a_known_length_are_not_proven(): + assert _flags(_const_index(8, 8, 100, -1, -8)) == [False, False, False, False] + assert _flags(_const_index(0, 0)) == [False] + assert _flags(_const_index(-3, 0)) == [False] # a negative length is an empty array + + +def test_an_index_expression_with_a_bounded_interval_is_proven(): + g = fn(locals_={"t": F64A, "s": F64, "k": I64}, returns=F64, pure=True, may_deopt=False, body=[ + Assign("t", na(F64A, i(10), f(0.0))), + Assign("s", f(0.0)), + ForRange("k", i(0), i(9), i(1), ( + Assign("s", add(lf("s"), Index(F64, "t", padd(li("k"), i(1))), F64)), + Assign("s", add(lf("s"), Index(F64, "t", psub(li("k"), i(1))), F64)), + )), + Return(lf("s")), + ]) + assert _flags(g) == [True, False] # k + 1 in [1, 9]; k - 1 in [-1, 7] may be negative + + +def test_the_length_interval_comes_from_the_length_expression(): + g = fn(locals_={"t": F64A, "j": I64, "s": F64}, returns=F64, pure=True, body=[ + Assign("s", f(0.0)), + ForRange("j", i(2), i(6), i(1), ( + Assign("t", na(F64A, li("j"), f(0.0))), + Assign("s", add(lf("s"), Index(F64, "t", i(0)), F64)), + Assign("s", add(lf("s"), Index(F64, "t", i(1)), F64)), + Assign("s", add(lf("s"), Index(F64, "t", i(2)), F64)), + )), + Return(lf("s")), + ]) + assert _flags(g) == [True, True, False] # j in [2, 5] -> the length is at least 2 + + +def test_a_twice_assigned_array_has_no_known_length(): + g = fn(params=[Param("c", BOOL)], locals_={"t": F64A}, returns=F64, pure=True, body=[ + Assign("t", na(F64A, i(8), f(0.0))), + If(lb("c"), (Assign("t", na(F64A, i(1), f(0.0))),)), + Return(Index(F64, "t", i(5))), + ]) + assert _flags(g) == [False] + + +def test_a_copy_inherits_the_length_of_a_known_source_only(): + g = fn(params=[ArrayParam("a", F64A, stored=False)], locals_={"t": F64A, "u": F64A, "v": F64A}, + returns=F64, pure=True, body=[ + Assign("t", na(F64A, i(4), f(0.0))), + Assign("u", CopyArray(F64A, "t")), + Assign("v", CopyArray(F64A, "a")), + Return(add(add(Index(F64, "u", i(3)), Index(F64, "u", i(4)), F64), + Index(F64, "v", i(0)), F64)), + ]) + assert _flags(g) == [True, False, False] + + +def test_a_proven_flag_from_the_input_is_not_trusted_for_local_arrays(): + g = fn(params=[Param("n", I64)], locals_={"t": F64A}, returns=F64, pure=True, body=[ + Assign("t", na(F64A, li("n"), f(0.0))), + Return(Index(F64, "t", i(0), proven=True)), + ]) + assert _flags(g) == [False] + + +def test_len_of_a_local_array_has_the_length_interval(): + def doubled(length): + return fn(params=[Param("n", I64)], locals_={"t": F64A, "x": I64}, returns=I64, pure=True, + body=[Assign("t", na(F64A, length, f(0.0))), + Assign("x", pmul(Len(I64, "t"), i(2 ** 60))), Return(li("x"))]) + assert_accepted(doubled(i(5))) # len == 5: 5 * 2**60 fits i64 (only if known) + assert_rejected(PROVEN, doubled(li("n"))) # unknown length: [0, 2**62] * 2**60 does not + assert_rejected(PROVEN, doubled(i(20))) # len == 20: 20 * 2**60 does not fit + + +# 10.4 definite assignment + +@pytest.mark.parametrize("use", [ + lambda: Assign("x", Index(F64, "t", i(0))), + lambda: StoreIndex("t", i(0), f(1.0)), + lambda: Assign("n", Len(I64, "t")), + lambda: Assign("u", CopyArray(F64A, "t")), + lambda: ForRange("k", i(0), Len(I64, "t"), i(1), ()), +]) +def test_a_local_array_must_be_assigned_before_any_use(use): + g = fn(params=[Param("c", BOOL)], locals_={"t": F64A, "u": F64A, "x": F64, "n": I64, "k": I64}, + returns=NONE, body=[ + If(lb("c"), (Assign("t", na(F64A, i(2), f(0.0))),)), + use(), Return()]) + assert_rejected(DA, g) + fixed = dataclasses.replace(g, body=(Assign("t", na(F64A, i(2), f(0.0))),) + g.body[1:]) + assert_accepted(dataclasses.replace(fixed, pure=True)) + + +def test_assigned_on_both_branches_a_local_array_is_defined(): + assert_accepted(fn(params=[Param("c", BOOL)], locals_={"t": F64A}, returns=F64, pure=True, + body=[If(lb("c"), (Assign("t", na(F64A, i(2), f(0.0))),), + (Assign("t", na(F64A, i(3), f(0.0))),)), + Return(Index(F64, "t", i(0)))])) + + +def test_a_local_array_assigned_in_a_loop_is_not_defined_after_it(): + assert_rejected(DA, fn(locals_={"t": F64A, "k": I64}, returns=F64, body=[ + ForRange("k", i(0), i(3), i(1), (Assign("t", na(F64A, i(2), f(0.0))),)), + Return(Index(F64, "t", i(0)))])) + + +def test_the_allocation_operands_are_read_before_the_array_is_assigned(): + assert_rejected(DA, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", na(F64A, Len(I64, "t"), f(0.0))), Return()])) + + +# 10.5 tuples + +def test_a_tuple_of_boxed_scalars_and_objects_is_accepted(): + assert_accepted(fn(params=[Param("o", OBJ), Param("x", F64)], locals_={"r": OBJ}, returns=OBJ, + pure=True, body=[ + Assign("r", tup(Box(OBJ, lf("x")), lo("o"), Box(OBJ, i(3)))), Return(lo("r"))])) + + +def test_a_tuple_can_be_returned_directly_and_may_be_empty(): + assert_accepted(fn(returns=OBJ, pure=True, body=[Return(tup())])) + assert_accepted(fn(params=[Param("x", I64)], returns=OBJ, pure=True, + body=[Return(tup(Box(OBJ, li("x"))))])) + + +def test_a_tuple_of_local_array_elements_is_accepted_through_index(): + assert_accepted(fn(locals_={"t": I64A}, returns=OBJ, pure=True, body=[ + Assign("t", na(I64A, i(3), i(1))), + Return(tup(Box(OBJ, Index(I64, "t", i(0))), Box(OBJ, Len(I64, "t"))))])) + + +def test_a_scalar_tuple_element_needs_box(): + assert_rejected(OWNERSHIP, fn(params=[Param("x", F64)], returns=OBJ, body=[ + Return(tup(lf("x")))])) + assert_rejected(OWNERSHIP, fn(returns=OBJ, body=[Return(tup(i(1)))])) + + +def test_a_tuple_is_an_obj_and_only_an_obj(): + assert_rejected(TYPE, fn(returns=I64, body=[Return(Tuple(I64, ()))])) + assert_rejected(OWNERSHIP, fn(returns=I64, body=[Return(tup())])) + assert_rejected(OWNERSHIP, fn(locals_={"x": F64}, returns=NONE, body=[ + Assign("x", tup()), Return()])) + assert_rejected(TYPE, fn(returns=BOOL, body=[Return(Tuple(BOOL, ()))])) + + +def test_a_tuple_creates_a_reference(): + # an OBJ local assigned from a tuple is an owned reference, like Box/CallObject + assert_accepted(fn(locals_={"r": OBJ, "q": OBJ}, returns=OBJ, pure=True, body=[ + Assign("r", tup()), Assign("q", tup(lo("r"))), Return(lo("q"))])) + + +def test_tuple_elements_must_be_a_tuple_of_checked_expressions(): + assert_rejected(STRUCTURE, fn(returns=OBJ, body=[Return(Tuple(OBJ, [Box(OBJ, i(1))]))])) + assert_rejected(DA, fn(locals_={"o": OBJ}, returns=OBJ, body=[Return(tup(lo("o")))])) + assert_rejected(TYPE, fn(returns=OBJ, body=[Return(tup(Const(F64, 1)))])) + + +def test_a_tuple_is_not_an_effect_but_unboxing_its_elements_is_not_available(): + # a tuple creation is pure; the function may keep pure=True with a deopt-free body + assert_accepted(fn(returns=OBJ, pure=True, may_deopt=False, body=[Return(tup())])) + + +# 10.6 mutation: nothing slips through + +def _array_functions(): + """Correct functions with local arrays and tuples; the mutator breaks them.""" + fk = fn("fk", params=[Param("n", I64), ArrayParam("a", I64A, stored=False)], + locals_={"p": I64A, "q": I64A, "k": I64, "s": I64, "t": OBJ, "o": OBJ}, + returns=OBJ, pure=True, may_deopt=True, line=3, body=[ + Assign("p", na(I64A, i(8), None, True)), + Assign("q", CopyArray(I64A, "p")), + Assign("s", i(0)), + Assign("o", g_("OBJECT")), + ForRange("k", i(0), Len(I64, "p"), i(1), ( + StoreIndex("q", li("k"), add(Index(I64, "p", li("k")), i(1))), + Assign("s", add(li("s"), Index(I64, "q", li("k")))), + )), + Assign("t", tup(Box(OBJ, li("s")), Box(OBJ, Index(I64, "q", i(0))), lo("o"))), + Return(lo("t")), + ]) + ip = fn("ip", params=[ArrayParam("a", F64A, stored=True)], + locals_={"z": F64A, "k": I64, "r": OBJ}, returns=NONE, line=9, body=[ + Assign("z", na(F64A, Len(I64, "a"), f(0.0))), + ForRange("k", i(0), Len(I64, "a"), i(1), ( + StoreIndex("z", li("k"), Index(F64, "a", li("k"))), + StoreIndex("a", li("k"), BinOp(F64, BinOpKind.ADD, Index(F64, "z", li("k")), + f(1.0))), + )), + Assign("r", call_obj(g_("print"), Box(OBJ, Index(F64, "z", i(0))))), + Return(), + ]) + sink = fn("sink", params=[Param("x", OBJ)], returns=NONE, line=12, body=[Return()]) + return {x.name: x for x in (fk, ip, sink)} + + +def _array_mutations(rng, base): + """(description, functions by name, the function that must be rejected, the rules any one of + which is an acceptable report).""" + b = dict(base) + fk, ip = b["fk"], b["ip"] + kind = rng.randrange(16) + arr = rng.choice([("p", I64A), ("q", I64A)]) + leak = la(*arr) + insert = lambda g, *stmts: dataclasses.replace(g, body=g.body[:-1] + tuple(stmts) + g.body[-1:]) + if kind == 0: # return the array + b["fk"] = dataclasses.replace(fk, body=fk.body[:-1] + (Return(leak),)) + return "return array", b, "fk", {ESCAPE} + if kind == 1: # pass it to a compiled Call + b["fk"] = insert(fk, ExprStmt(Call(NONE, "sink", (leak,)))) + return "call array", b, "fk", {ESCAPE} + if kind == 2: # pass it to CallObject + b["fk"] = insert(fk, Assign("o", call_obj(g_("keep"), leak))) + return "callobject array", b, "fk", {ESCAPE} + if kind == 3: # box it + b["fk"] = insert(fk, Assign("o", Box(OBJ, leak))) + return "box array", b, "fk", {ESCAPE} + if kind == 4: # put it in a tuple + b["fk"] = insert(fk, Assign("o", tup(Box(OBJ, i(1)), leak))) + return "tuple array", b, "fk", {ESCAPE} + if kind == 5: # alias it + b["fk"] = insert(fk, Assign("q" if arr[0] == "p" else "p", leak)) + return "alias array", b, "fk", {ESCAPE} + if kind == 6: # an allocation where it may not stand + alloc = rng.choice([na(I64A, i(2), i(0)), CopyArray(I64A, "p")]) + stmt = rng.choice([ExprStmt(alloc), Assign("o", alloc), Assign("s", alloc), + Assign("o", Box(OBJ, alloc))]) + b["fk"] = insert(fk, stmt) + return "stray allocation", b, "fk", {ESCAPE} + if kind == 7: # pure flag set on an impure function / an extra effect in a pure one + if rng.random() < 0.5: + b["ip"] = dataclasses.replace(ip, pure=True) + return "pure on impure", b, "ip", {FLAG_PURE} + b["fk"] = insert(fk, StoreIndex("a", i(0), i(1))) + return "param store in pure", b, "fk", {FLAG_PURE} + if kind == 8: # pure flag cleared on a function that deopts / may_deopt flipped + if rng.random() < 0.5: + b["fk"] = dataclasses.replace(fk, pure=False) + return "drop pure", b, "fk", {DEOPT} + b["fk"] = dataclasses.replace(fk, may_deopt=False) + return "drop may_deopt", b, "fk", {FLAG_DEOPT} + if kind == 9: # an impurity added to the pure function + stmt = rng.choice([ExprStmt(pycall("print")), + Assign("o", GetAttr(OBJ, g_("m"), "x"))]) + b["fk"] = insert(fk, stmt) + return "impurity in pure", b, "fk", {FLAG_PURE} + if kind == 10: # the allocation is removed: every use reads an unassigned array + keep = [s for s in fk.body if not (type(s) is Assign and s.target == arr[0])] + b["fk"] = dataclasses.replace(fk, body=tuple(keep)) + return "drop allocation", b, "fk", {DA} + if kind == 11: # a malformed allocation + bad = rng.choice([na(I64A, i(8)), na(I64A, i(8), i(0), True), na(I64A, f(8.0), i(0)), + na(I64A, i(8), f(0.0)), NewArray(I64A, i(8), None, 1), + na(F64A, i(8), f(0.0)), NewArray(I64, i(8), i(0))]) + b["fk"] = dataclasses.replace(fk, body=(Assign("p", bad),) + fk.body[1:]) + return "bad allocation", b, "fk", {TYPE} + if kind == 12: # a bad copy source + src = rng.choice([("n", {ARRAY_USE}), ("nowhere", {ARRAY_USE})]) + b["fk"] = dataclasses.replace(fk, body=(fk.body[0], Assign("q", CopyArray(I64A, src[0])) + ) + fk.body[2:]) + return "bad copy source", b, "fk", src[1] | {DA} + if kind == 13: # a scalar tuple element without Box + b["fk"] = insert(fk, Assign("o", tup(li("s")))) + return "unboxed tuple element", b, "fk", {OWNERSHIP} + if kind == 14: # the local's declared type disagrees with the allocation + b["fk"] = dataclasses.replace(fk, locals={**fk.locals, "p": F64A}) + return "declared type", b, "fk", {TYPE} + # kind 15: a tuple typed as something else + b["fk"] = insert(fk, Assign("s", Tuple(I64, ()))) + return "tuple not obj", b, "fk", {TYPE, OWNERSHIP} + + +def test_array_base_functions_are_accepted(): + assert_accepted(*_array_functions().values()) + + +@pytest.mark.parametrize("seed", [20261005]) +def test_no_array_breaking_mutation_slips_through(seed): + rng = random.Random(seed) + base = _array_functions() + missed = [] + for n in range(400): + desc, funcs, target, allowed = _array_mutations(rng, base) + out, diags = run(*funcs.values()) + got = rules_for(diags, target) + if target in kept(out) or not (got & allowed): + missed.append((n, desc, target, sorted(got))) + assert missed == [] + + +def test_each_array_mutation_kind_was_exercised(): + rng = random.Random(20261005) + base = _array_functions() + kinds = {_array_mutations(rng, base)[0] for _ in range(400)} + assert len(kinds) == 18, sorted(kinds) # 16 kinds; kinds 7 and 8 have two variants From ae67e09e789b0dab148162e0d745bf1e042ab305 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:56:22 +0900 Subject: [PATCH 32/36] Feat: Front end lowers local arrays and tuple returns (fannkuch) list(range(n)), [v]*n and a[:] become NewArray/CopyArray of an I64/F64 array local; any other use of a local array skips the function with the statement named. Stores into local arrays are not effects, so such a function stays pure. return a, b lowers to a Tuple of boxed values (returns OBJ) when the annotation matches. fannkuch lowers: pure, may_deopt. Red: 25 failed / 71 passed. Green: 96/96. Mutation (local stores as effects): 2 fail. Refs #41 --- .../src/main/python/typedpython/frontend.py | 205 +++++++++++++++++- .../src/test/python/test_frontend.py | 203 ++++++++++++++++- 2 files changed, 398 insertions(+), 10 deletions(-) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py index 06d85cd7e..852ca0f90 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -122,6 +122,10 @@ class _Signature: node: ast.FunctionDef params: tuple[ir.Param | ir.ArrayParam, ...] returns: Type + # `-> tuple[a, b]`: the element kinds (a scalar type, else OBJ); None when not a tuple annotation. + tuple_returns: tuple[Type, ...] | None = None + tuple_ellipsis: bool = False # `tuple[int, ...]`: a variable length + return_annotation: str = "" def lower(path: Path) -> ir.Module: @@ -348,7 +352,20 @@ def _signature(fn: ast.FunctionDef) -> _Signature: returns = Type.NONE else: returns = SCALAR_KIND.get(_annotation_kind(fn.returns), Type.OBJ) - return _Signature(fn, tuple(params), returns) + elements, ellipsis = _tuple_annotation(fn.returns) + return _Signature(fn, tuple(params), returns, elements, ellipsis, _src(fn.returns)) + + +def _tuple_annotation(annotation: ast.expr) -> tuple[tuple[Type, ...] | None, bool]: + """`tuple[int, float]` -> ((I64, F64), False); `tuple[int, ...]` -> (None, True).""" + if not (isinstance(annotation, ast.Subscript) and isinstance(annotation.value, ast.Name) + and annotation.value.id == "tuple"): + return None, False + inner = annotation.slice + items = list(inner.elts) if isinstance(inner, ast.Tuple) else [inner] + if any(isinstance(i, ast.Constant) and i.value is Ellipsis for i in items): + return None, True + return tuple(SCALAR_KIND.get(_annotation_kind(i), Type.OBJ) for i in items), False def _annotation_kind(annotation: ast.expr) -> str: @@ -469,6 +486,9 @@ def __init__(self, sig: _Signature, env: dict[str, ir.Function], info: _ModuleIn self.bindings: dict[str, list[_Binding]] = {} self.annotations: dict[str, list[tuple[int, str]]] = {} self._collect_bindings() + # Local arrays (ir.NewArray / CopyArray): name -> array type; also in `self.arrays`. + self.local_arrays = self._find_local_arrays() + self.arrays.update(self.local_arrays) self.range_vars = { n for n, bs in self.bindings.items() if n not in self.params and any(b.how == "range" for b in bs) @@ -484,6 +504,7 @@ def __init__(self, sig: _Signature, env: dict[str, ir.Function], info: _ModuleIn self.impure = False # an effect outside the locals self.deopts = False # a node that can deopt self.stored: set[str] = set() + self.current: ast.stmt | None = None # the statement being lowered (for messages) # --- entry --- @@ -535,6 +556,68 @@ def add(name: str, line: int, how: str) -> None: if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store): add(n.id, stmt.lineno, "assign") + def _find_local_arrays(self) -> dict[str, Type]: + """Locals created by `list(range(n))`, `[v] * n` or `src[:]` (src an array). Every binding + of such a name must be one of those creations: a local array is never rebound to anything + else, so it has one element type for its whole life.""" + creations: dict[str, list[tuple[ast.stmt, str, ast.expr]]] = {} + for stmt in rebinding._own_statements(self.fn): + if isinstance(stmt, ast.Assign) and len(stmt.targets) == 1 \ + and isinstance(stmt.targets[0], ast.Name): + name, value = stmt.targets[0].id, stmt.value + elif isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name) \ + and stmt.value is not None: + name, value = stmt.target.id, stmt.value + else: + continue + kind = _creation_kind(value) + if kind is not None and name not in self.params: + creations.setdefault(name, []).append((stmt, kind, value)) + known = set(self.arrays) | {n for n, cs in creations.items() + if any(k != "copy" for _, k, _ in cs)} + while True: + more = {n for n, cs in creations.items() if n not in known and any( + k == "copy" and v.value.id in known for _, k, v in cs)} # type: ignore[attr-defined] + if not more: + break + known |= more + found: dict[str, Type] = {} + for name, cs in creations.items(): + if name not in known: + continue + cs = [c for c in cs if c[1] != "copy" or c[2].value.id in known] # type: ignore[attr-defined] + lines = {c[0].lineno for c in cs} + other = [b for b in self.bindings[name] if b.how != "annotation" and b.line not in lines] + if other: + raise Skip(other[0].line, f"local `{name}` is a native array (created at line " + f"{cs[0][0].lineno}) but is also bound by something " + "that does not create an array") + found[name] = self._array_type(name, cs, found) + return found + + def _array_type(self, name: str, cs, found: dict[str, Type]) -> Type: + line = cs[0][0].lineno + kinds: list[tuple[int, str]] = list(self.annotations.get(name, [])) + if name not in self.annotations: + kinds += self.inferred.get(name, []) + distinct = sorted({k for _, k in kinds}) + if len(distinct) > 1: + raise Skip(line, f"local `{name}` is given values of types " + + " and ".join(f"`{k}`" for k in distinct)) + if distinct: + if distinct[0] not in ARRAY_KIND: + raise Skip(line, f"local `{name}: {distinct[0]}`: a native array's element type " + "must be exactly int or float") + return ARRAY_KIND[distinct[0]] + for _, kind, value in cs: # unknown to Pyrefly: a copy takes its source's type + if kind == "copy": + src = value.value.id + if src in self.arrays: + return self.arrays[src] + if src in found: + return found[src] + raise Skip(line, f"no type is known for local `{name}`") + def is_local(self, name: str) -> bool: return name in self.params or name in self.bindings or name in self.temps @@ -543,6 +626,8 @@ def local_type(self, name: str, line: int) -> Type: return self.temps[name] if name in self.params: return self.params[name].type + if name in self.local_arrays: + return self.local_arrays[name] if name in self.types: return self.types[name] kinds: list[tuple[int, str]] = list(self.annotations.get(name, [])) @@ -558,8 +643,8 @@ def local_type(self, name: str, line: int) -> Type: + " and ".join(f"`{k}`" for k in distinct)) kind = distinct[0] if kind in ARRAY_KIND or kind.startswith(("list[float]", "list[int]")): - raise Skip(kinds[0][0], f"local `{name}: {kind}`: only parameters can be native " - "arrays") + raise Skip(kinds[0][0], f"local `{name}: {kind}`: a list local is a native array only " + "when created by `list(range(n))`, `[v] * n` or `src[:]`") if kind == "int": t = Type.OBJ if self.impure_mode and name not in self.range_vars else Type.I64 else: @@ -594,6 +679,7 @@ def take(self) -> list[ir.Stmt]: def stmt(self, node: ast.stmt) -> list[ir.Stmt]: self.start() + self.current = node line = node.lineno if isinstance(node, ast.Pass): return [] @@ -643,6 +729,8 @@ def assign(self, target: ast.expr, value_node: ast.expr, node: ast.stmt) -> list line = node.lineno if isinstance(target, ast.Name): name = target.id + if name in self.local_arrays: + return self.new_array(name, value_node, node) if name in self.arrays: raise Skip(line, f"array parameter `{name}` is rebound") value = self.value(value_node) @@ -657,8 +745,7 @@ def assign(self, target: ast.expr, value_node: ast.expr, node: ast.stmt) -> list if not _simple(target.slice) and not _trivial(value): value = self.temp(value) index = self.index(target.slice) - self.stored.add(array) - self.impure = True + self.store_effect(array) return self.take() + [ir.StoreIndex(array, index, value)] if isinstance(target, ast.Attribute): raise Skip(line, f"assignment to attribute `{_src(target)}` is not supported") @@ -666,6 +753,53 @@ def assign(self, target: ast.expr, value_node: ast.expr, node: ast.stmt) -> list raise Skip(line, "tuple unpacking is not supported") raise _unsupported(target) + def array_word(self, name: str) -> str: + return "local array" if name in self.local_arrays else "array parameter" + + def store_effect(self, array: str) -> None: + """A store into a parameter list is an effect (write-back); into a local array it is not.""" + if array not in self.local_arrays: + self.stored.add(array) + self.impure = True + + def new_array(self, name: str, value_node: ast.expr, node: ast.stmt) -> list[ir.Stmt]: + line = node.lineno + t = self.local_arrays[name] + elem = Type.F64 if t == Type.F64_ARRAY else Type.I64 + what = f"local array `{name}: {TYPE_NAME[t]}`" + kind = _creation_kind(value_node) + if kind == "iota": + call = value_node.args[0] # type: ignore[attr-defined] + if not (self.builtin("list") and self.builtin("range")): + raise Skip(line, f"`{_src(value_node)}` with `list` or `range` rebound") + if elem != Type.I64: + raise Skip(line, f"`{_src(value_node)}` is a list of int, but {what} is " + f"{TYPE_NAME[t]}") + length = self.range_bound(call.args[0]) + return self.take() + [ir.Assign(name, ir.NewArray(t, length, None, True))] + if kind == "fill": + assert isinstance(value_node, ast.BinOp) and isinstance(value_node.left, ast.List) + fill = self.coerce(self.value(value_node.left.elts[0]), elem, + value_node.left.elts[0], what) + if not _trivial(fill): + fill = self.temp(fill) # `[v] * n` evaluates v before n + length = self.value(value_node.right) + if length.type != Type.I64: + why = (" (it is a CPython int here: its arithmetic is not proved free of i64 " + "overflow in an impure function)" if length.type == Type.OBJ + and self.impure_mode else "") + raise Skip(line, f"array length `{_src(value_node.right)}` is " + f"{TYPE_NAME[length.type]}, not a native int{why}") + return self.take() + [ir.Assign(name, ir.NewArray(t, length, fill))] + if kind == "copy": + assert isinstance(value_node, ast.Subscript) and isinstance(value_node.value, ast.Name) + src = value_node.value.id + if self.arrays[src] != t: + raise Skip(line, f"`{_src(value_node)}` is {TYPE_NAME[self.arrays[src]]}, but " + f"{what} is {TYPE_NAME[t]}") + return [ir.Assign(name, ir.CopyArray(t, src))] + raise Skip(line, f"{what} is rebound to `{_src(value_node)}`, which does not create an array") + def aug_assign(self, node: ast.AugAssign) -> list[ir.Stmt]: line = node.lineno op = self.binop_kind(node.op, node) @@ -673,7 +807,7 @@ def aug_assign(self, node: ast.AugAssign) -> list[ir.Stmt]: if isinstance(target, ast.Name): name = target.id if name in self.arrays: - raise Skip(line, f"array parameter `{name}` is rebound") + raise Skip(line, f"{self.array_word(name)} `{name}` is rebound") if not self.is_local(name): raise _unsupported(node) t = self.local_type(name, line) @@ -689,8 +823,7 @@ def aug_assign(self, node: ast.AugAssign) -> list[ir.Stmt]: self.hoist_ok = False result = self.arith(op, load, self.sub(node.value), node) result = self.coerce(result, elem, node, f"`{array}: {TYPE_NAME[self.arrays[array]]}`") - self.stored.add(array) - self.impure = True + self.store_effect(array) return self.take() + [ir.StoreIndex(array, index, result)] if isinstance(target, ast.Attribute): raise Skip(line, f"assignment to attribute `{_src(target)}` is not supported") @@ -702,6 +835,8 @@ def for_range(self, node: ast.For) -> list[ir.Stmt]: raise Skip(line, "`for ... else` is not supported") if not isinstance(node.target, ast.Name): raise Skip(line, f"loop target `{_src(node.target)}` is not supported (only a name)") + if isinstance(node.iter, ast.Name) and node.iter.id in self.arrays: + raise self.array_misuse(node.iter.id, node.iter) if not (_is_range_call(node.iter) and self.builtin("range")): raise Skip(line, f"`for` over `{_src(node.iter)}` is not supported (only range())") call = node.iter @@ -761,10 +896,35 @@ def return_(self, node: ast.Return) -> list[ir.Stmt]: return [ir.Return(None)] if returns == Type.NONE: raise Skip(node.lineno, "returns a value from a function declared to return None") + if isinstance(node.value, ast.Tuple): + return self.return_tuple(node.value) value = self.value(node.value) value = self.coerce(value, returns, node.value, "the return value") return self.take() + [ir.Return(value)] + def return_tuple(self, node: ast.Tuple) -> list[ir.Stmt]: + sig = self.sig + if any(isinstance(e, ast.Starred) for e in node.elts): + raise Skip(node.lineno, f"`return {_src(node)}` with a starred element is not supported") + if sig.tuple_returns is None: + why = ("a variable-length `tuple[..., ...]`" if sig.tuple_ellipsis + else f"`{sig.return_annotation}`") + raise Skip(node.lineno, f"returns `{_src(node)}`, but the declared return type is " + f"{why}, not a fixed `tuple[...]`") + kinds = sig.tuple_returns + if len(kinds) != len(node.elts): + raise Skip(node.lineno, f"returns {len(node.elts)} values, but the return type " + f"`{sig.return_annotation}` declares {len(kinds)}") + elements = [] + for kind, e_node in zip(kinds, node.elts): + e = self.value(e_node) + if e.type in ir.SCALARS and kind in ir.SCALARS and e.type != kind: + raise Skip(e_node.lineno, f"`{_src(e_node)}` is {TYPE_NAME[e.type]}, but the " + f"return type `{sig.return_annotation}` declares " + f"{TYPE_NAME[kind]} there") + elements.append(self.boxed(e, e_node)) + return self.take() + [ir.Return(ir.Tuple(Type.OBJ, tuple(elements)))] + # --- expressions --- def sub(self, node: ast.expr, cond: int = 0) -> ir.Expr: @@ -834,6 +994,10 @@ def expr(self, node: ast.expr, cond: int = 0) -> ir.Expr: return ir.GetAttr(Type.OBJ, obj, node.attr) if isinstance(node, ast.Subscript): if isinstance(node.value, ast.Name) and node.value.id in self.arrays: + if isinstance(node.slice, ast.Slice): + raise Skip(line, f"`{_src(node)}` slices {self.array_word(node.value.id)} " + f"`{node.value.id}`: only `y = {node.value.id}[:]` as the whole " + "right-hand side of an assignment is a native copy") array, elem = self.array_of(node) return ir.Index(elem, array, self.index(node.slice)) raise Skip(line, f"subscript `{_src(node)}` of an object is not supported") @@ -854,8 +1018,13 @@ def name(self, node: ast.Name) -> ir.Expr: return ir.Global(Type.OBJ, name) def array_misuse(self, name: str, node: ast.AST) -> Skip: + where = self.current if isinstance(self.current, ( + ast.Assign, ast.AnnAssign, ast.AugAssign, ast.Return, ast.Expr)) else node + if name in self.local_arrays: + return Skip(node.lineno, f"local array `{name}` is used other than by indexing, len() " + f"or as a `[:]` source (in `{_src(where)}`)") return Skip(node.lineno, f"array parameter `{name}` is used other than by indexing or " - f"len() (in `{_src(node)}`)") + f"len() (in `{_src(where)}`)") def array_of(self, node: ast.Subscript) -> tuple[str, Type]: if not (isinstance(node.value, ast.Name) and node.value.id in self.arrays): @@ -1220,6 +1389,24 @@ def _trivial(e: ir.Expr) -> bool: return False +def _creation_kind(node: ast.expr) -> str | None: + """How a statement's value creates a native array: "iota" `list(range(n))`, "fill" `[v] * n`, + "copy" `name[:]` (whether `name` is an array is decided by the caller), else None.""" + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "list" \ + and not node.keywords and len(node.args) == 1 and _is_range_call(node.args[0]) \ + and len(node.args[0].args) == 1: + return "iota" + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Mult) \ + and isinstance(node.left, ast.List) and len(node.left.elts) == 1 \ + and not isinstance(node.left.elts[0], ast.Starred): + return "fill" + if isinstance(node, ast.Subscript) and isinstance(node.value, ast.Name) \ + and isinstance(node.slice, ast.Slice) and node.slice.lower is None \ + and node.slice.upper is None and node.slice.step is None: + return "copy" + return None + + def _simple(node: ast.expr) -> bool: return isinstance(node, (ast.Name, ast.Constant)) diff --git a/python-multiplatform-ksp/src/test/python/test_frontend.py b/python-multiplatform-ksp/src/test/python/test_frontend.py index c7da7043f..7ecf0ccb8 100644 --- a/python-multiplatform-ksp/src/test/python/test_frontend.py +++ b/python-multiplatform-ksp/src/test/python/test_frontend.py @@ -15,7 +15,7 @@ from typedpython import frontend from typedpython.ir import ( And, ArrayParam, Assign, BinOp, BinOpKind, Box, Break, Call, CallObject, Compare, CompareKind, - CompareObj, Const, ExprStmt, While, ForRange, GetAttr, Global, If, Index, Len, Local, MathCall, + CompareObj, Const, CopyArray, NewArray, Tuple, ExprStmt, While, ForRange, GetAttr, Global, If, Index, Len, Local, MathCall, MathFunc, ObjToFloat, Param, Return, StoreIndex, ToFloat, Truth, Type, UnaryOp, UnaryOpKind, ) @@ -850,3 +850,204 @@ def test_spectral_norm_kernels_lower(tmp_path): assert not f.pure and not f.may_deopt [call] = nodes(f, Call) assert call.function == "eval_a" and call.redo + + +# --- local fixed-size arrays and tuple returns (ir.NewArray / CopyArray / Tuple, #41 M2) --------- + +def test_list_range_local_is_an_iota_array(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(n: int) -> int: + x: list[int] = list(range(n)) + return x[0] + """) + f = function(m, "f") + assert f.locals == {"x": Type.I64_ARRAY} + assert f.body == ( + Assign("x", NewArray(Type.I64_ARRAY, Local(I64, "n"), None, True)), + Return(Index(I64, "x", Const(I64, 0))), + ) + assert f.pure and not f.may_deopt + + +def test_fill_locals_are_new_arrays_of_their_element_type(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(n: int) -> float: + a: list[int] = [0] * n + b = [1.5] * n + return b[a[0]] + """) + f = function(m, "f") + assert f.locals == {"a": Type.I64_ARRAY, "b": Type.F64_ARRAY} + assert f.body[:2] == ( + Assign("a", NewArray(Type.I64_ARRAY, Local(I64, "n"), Const(I64, 0))), + Assign("b", NewArray(Type.F64_ARRAY, Local(I64, "n"), Const(F64, 1.5))), + ) + assert f.pure + + +def test_slice_copy_of_a_local_and_of_a_parameter(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(p: list[int], n: int) -> int: + x: list[int] = list(range(n)) + y = x[:] + z: list[int] = p[:] + return y[0] + z[0] + """) + f = function(m, "f") + assert f.locals == {"x": Type.I64_ARRAY, "y": Type.I64_ARRAY, "z": Type.I64_ARRAY} + assert f.body[1] == Assign("y", CopyArray(Type.I64_ARRAY, "x")) + assert f.body[2] == Assign("z", CopyArray(Type.I64_ARRAY, "p")) + assert f.params[0] == ArrayParam("p", Type.I64_ARRAY, False) + + +@pytest.mark.parametrize("what, source, snippet", [ + ("returned", "return x", "return x"), + ("passed to a call", "print(x)\n return 0", "print(x)"), + ("compared", "return x == y", "x == y"), + ("iterated over", "for v in x:\n pass\n return 0", "x"), + ("sliced other than [:]", "w = x[1:]\n return 0", "x[1:]"), + ("appended to", "x.append(1)\n return 0", "x.append(1)"), + ("aliased", "w = x\n return 0", "w = x"), + ("copied outside an assignment", "return len(x[:])", "x[:]"), + ("rebound by a non-array value", "x = 5\n return 0", "x"), +]) +def test_local_array_escapes_skip_the_function(tmp_path, what, source, snippet): + source = source.replace("\n ", "\n ") # the f-string below is dedented + m = lower(tmp_path, f""" + @compiled + def f(n: int) -> int: + x: list[int] = list(range(n)) + y: list[int] = [0] * n + {source} + """) + assert "f" not in [fn.name for fn in m.functions], what + reason = m.skipped["f"] + assert snippet in reason, (what, reason) + assert "array" in reason, (what, reason) + + +@pytest.mark.parametrize("annotation, value", [ + ("list[str]", '["a"] * n'), + ("list[bool]", "[True] * n"), + ("list[list[int]]", "[[0]] * n"), +]) +def test_local_array_element_type_must_be_exactly_int_or_float(tmp_path, annotation, value): + m = lower(tmp_path, f""" + @compiled + def f(n: int) -> int: + x: {annotation} = {value} + return 0 + """) + assert "f" in m.skipped + assert "exactly int or float" in m.skipped["f"], m.skipped["f"] + + +def test_int_fill_of_a_float_array_is_skipped(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(n: int) -> int: + x: list[float] = [0] * n + return 0 + """) + assert "f" in m.skipped and "float" in m.skipped["f"] + + +def test_function_with_only_local_array_stores_is_pure(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(n: int) -> int: + x: list[int] = [0] * n + for i in range(n): + x[i] = i * i + x[i] += 1 + return x[n - 1] * n + """) + f = function(m, "f") + assert nodes(f, StoreIndex) and f.pure + # pure, so the unbounded int arithmetic is native with deopt, not a CPython int + assert f.may_deopt + assert f.locals["x"] == Type.I64_ARRAY and f.returns == I64 + + +def test_function_storing_into_an_array_param_is_still_impure(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(p: list[int], n: int) -> int: + x: list[int] = [0] * n + p[0] = 1 + x[0] = 2 + return x[0] + """) + f = function(m, "f") + assert not f.pure and not f.may_deopt + assert f.params[0] == ArrayParam("p", Type.I64_ARRAY, True) + + +def test_storing_a_local_array_into_nothing_but_locals_keeps_calls_impure(tmp_path): + # An object call is an effect whatever the arrays do. + m = lower(tmp_path, """ + @compiled + def f(n: int, g: object) -> int: + x: list[int] = [0] * n + x[0] = 1 + g(1) + return x[0] + """) + assert not function(m, "f").pure + + +def test_tuple_return_boxes_each_element(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(a: int, b: float) -> tuple[int, float]: + return a, b + """) + f = function(m, "f") + assert f.returns == OBJ + assert f.body == (Return(Tuple(OBJ, (Box(OBJ, Local(I64, "a")), Box(OBJ, Local(F64, "b"))))),) + assert f.pure + + +def test_tuple_return_with_object_element_and_deopting_arithmetic(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(a: int, b: int, o: object) -> tuple[int, object]: + return a // b, o + """) + f = function(m, "f") + assert f.pure and f.may_deopt + [ret] = nodes(f, Return) + assert isinstance(ret.value, Tuple) and ret.value.elements[1] == Local(OBJ, "o") + + +@pytest.mark.parametrize("annotation, value, snippet", [ + ("tuple[int, float]", "a, b", "float"), + ("object", "a, b", "tuple["), + ("tuple[int, int, int]", "a, b", "declares 3"), + ("tuple[int, ...]", "a, b", "..."), +]) +def test_tuple_return_must_match_its_annotation(tmp_path, annotation, value, snippet): + m = lower(tmp_path, f""" + @compiled + def f(a: int, b: int) -> {annotation}: + return {value} + """) + assert "f" in m.skipped + assert snippet in m.skipped["f"], m.skipped["f"] + + +@pytest.mark.skipif(not BENCH.exists(), reason="benchmark worktree not present") +def test_fannkuch_lowers(tmp_path): + path = tmp_path / "fannkuch.py" + path.write_text("# typedpython: compiled\n" + (BENCH / "fannkuch.py").read_text()) + m = frontend.lower(path) + assert "fannkuch" in [f.name for f in m.functions], m.skipped.get("fannkuch") + f = function(m, "fannkuch") + assert f.pure and f.may_deopt + assert f.returns == OBJ and f.params == (Param("n", I64),) + assert f.locals["perm1"] == f.locals["count"] == f.locals["perm"] == Type.I64_ARRAY + assert len(nodes(f, NewArray)) == 2 and len(nodes(f, CopyArray)) == 1 + assert nodes(f, Tuple) From 64a77440d082671d36d90917b945c499c0270b5e Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:59:13 +0900 Subject: [PATCH 33/36] Feat: Runtime helpers for local arrays and tuples tp_*_array_new/iota/copy/free and tp_tuple per API.md: negative length is empty, size overflow or malloc failure is MemoryError with a zeroed struct, free is idempotent and never writes back, copy from a parameter array yields a local one, tp_tuple steals nothing. _exit on a local array equals _free. Red: shim build failed (helpers undeclared). Green: 264/264 (default and portable). Mutation (negative n allocates |n|): 22 fail. A standalone ASan+UBSan program over the helpers: clean. Refs #41 --- .../python/typedpython/runtime/tp_runtime.h | 142 +++++++++++ .../src/test/python/runtime_shim/tp_shim.c | 168 +++++++++++++ .../src/test/python/test_runtime.py | 228 ++++++++++++++++++ 3 files changed, 538 insertions(+) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h index 437462878..876a33eb8 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -737,6 +737,148 @@ static inline int tp_i64_array_exit(tp_i64_array *a) return rc; } +/* ------------------------------------------------------------------------------------------ + * Local arrays (ir.NewArray / CopyArray): owned by the function, no list object behind them. + * Same structs as above with list == NULL and dirty == NULL; the slot helpers work on them + * unchanged. rc 0 ok, -1 MemoryError (the struct is then zeroed, so _free is still safe). + * n < 0 gives an empty array, like CPython's [x] * -1 == []. A size that overflows + * n * sizeof(element) is a MemoryError, not a wrapped small allocation. _free never writes + * anything back, zeroes the struct and is idempotent. _exit on a local array is defined to + * behave exactly like _free (nothing to write back since list == NULL, rc 0), so a stray _exit + * is harmless; local arrays are still released with _free. + * ------------------------------------------------------------------------------------------ */ +static inline void *tp__local_alloc(int64_t n, size_t elem, Py_ssize_t *len) +{ + void *p; + if (n < 0) { + n = 0; + } + if (n > (int64_t)PY_SSIZE_T_MAX || (uint64_t)n > (uint64_t)(SIZE_MAX / elem)) { + PyErr_NoMemory(); + return NULL; + } + p = malloc(n ? (size_t)n * elem : elem); + if (p == NULL) { + PyErr_NoMemory(); + return NULL; + } + *len = (Py_ssize_t)n; + return p; +} + +static inline int tp_i64_array_new(tp_i64_array *a, int64_t n, int64_t fill) +{ + Py_ssize_t len = 0, i; + int64_t *p; + a->list = NULL; a->len = 0; a->data = NULL; a->dirty = NULL; + p = (int64_t *)tp__local_alloc(n, sizeof(int64_t), &len); + if (p == NULL) { + return -1; + } + for (i = 0; i < len; i++) { + p[i] = fill; + } + a->data = p; + a->len = len; + return 0; +} + +static inline int tp_i64_array_iota(tp_i64_array *a, int64_t n) +{ + Py_ssize_t len = 0, i; + int64_t *p; + a->list = NULL; a->len = 0; a->data = NULL; a->dirty = NULL; + p = (int64_t *)tp__local_alloc(n, sizeof(int64_t), &len); + if (p == NULL) { + return -1; + } + for (i = 0; i < len; i++) { + p[i] = (int64_t)i; + } + a->data = p; + a->len = len; + return 0; +} + +static inline int tp_f64_array_new(tp_f64_array *a, int64_t n, double fill) +{ + Py_ssize_t len = 0, i; + double *p; + a->list = NULL; a->len = 0; a->data = NULL; a->dirty = NULL; + p = (double *)tp__local_alloc(n, sizeof(double), &len); + if (p == NULL) { + return -1; + } + for (i = 0; i < len; i++) { + p[i] = fill; + } + a->data = p; + a->len = len; + return 0; +} + +/* dst is overwritten (not freed); src may be a parameter array or a local one; dst is local */ +static inline int tp_i64_array_copy(tp_i64_array *dst, const tp_i64_array *src) +{ + Py_ssize_t len = 0; + int64_t *p = (int64_t *)tp__local_alloc((int64_t)src->len, sizeof(int64_t), &len); + dst->list = NULL; dst->len = 0; dst->data = NULL; dst->dirty = NULL; + if (p == NULL) { + return -1; + } + if (len > 0) { + memcpy(p, src->data, (size_t)len * sizeof(int64_t)); + } + dst->data = p; + dst->len = len; + return 0; +} + +static inline int tp_f64_array_copy(tp_f64_array *dst, const tp_f64_array *src) +{ + Py_ssize_t len = 0; + double *p = (double *)tp__local_alloc((int64_t)src->len, sizeof(double), &len); + dst->list = NULL; dst->len = 0; dst->data = NULL; dst->dirty = NULL; + if (p == NULL) { + return -1; + } + if (len > 0) { + memcpy(p, src->data, (size_t)len * sizeof(double)); + } + dst->data = p; + dst->len = len; + return 0; +} + +static inline void tp_i64_array_free(tp_i64_array *a) +{ + free(a->data); + free(a->dirty); + a->list = NULL; a->len = 0; a->data = NULL; a->dirty = NULL; +} + +static inline void tp_f64_array_free(tp_f64_array *a) +{ + free(a->data); + free(a->dirty); + a->list = NULL; a->len = 0; a->data = NULL; a->dirty = NULL; +} + +/* a new tuple of the n items; the items are borrowed (each gets its own new reference) */ +static inline PyObject *tp_tuple(PyObject *const *items, Py_ssize_t n) +{ + Py_ssize_t i; + PyObject *t = PyTuple_New(n); + if (t == NULL) { + return NULL; + } + for (i = 0; i < n; i++) { + Py_INCREF(items[i]); + PyTuple_SET_ITEM(t, i, items[i]); + } + return t; +} + /* aliasing guard: 1 if any two of the n objects are the same object */ static inline int tp_any_same(PyObject *const *objs, Py_ssize_t n) { diff --git a/python-multiplatform-ksp/src/test/python/runtime_shim/tp_shim.c b/python-multiplatform-ksp/src/test/python/runtime_shim/tp_shim.c index 84d053365..2b6b07221 100644 --- a/python-multiplatform-ksp/src/test/python/runtime_shim/tp_shim.c +++ b/python-multiplatform-ksp/src/test/python/runtime_shim/tp_shim.c @@ -288,6 +288,171 @@ static PyObject *s_array_exit_zeroed(PyObject *self, PyObject *arg) return PyLong_FromLong(rc); } + +/* ---- local arrays ---- + * local_run(kind, init, script) -> (rc, error, results) + * kind: "f64" | "i64"; init: ("new", n, fill) | ("iota", n) [i64 only] | ("param", pylist, nocopy) + * ("copy_local", n, fill) / ("copy_param", pylist) build a source, then copy it into the array under test. + * script ops: ("get", i) | ("set", i, v) | ("len",) | ("flags",) -> (list_is_null, dirty_is_null, data_is_null) + * | ("free",) | ("exit",) -> exit rc | ("copy",) -> (list, contents of an independent copy; + * the copy is then set to [i]=v by ("copyset", i, v) which returns (src contents, copy contents)) + * Struct memory is poisoned with 0xAB before every init, so a field left unset shows up. */ +#define LOCAL_RUN(SUF, STRUCT, CTYPE, FROMC, PARSEFMT, PARSET) \ + static int loc_##SUF##_contents(const STRUCT *a, PyObject **out) \ + { \ + PyObject *l = PyList_New(a->len); \ + if (!l) return -1; \ + for (Py_ssize_t i = 0; i < a->len; i++) { \ + PyObject *v = FROMC(a->data[i]); \ + if (!v) { Py_DECREF(l); return -1; } \ + PyList_SET_ITEM(l, i, v); \ + } \ + *out = l; \ + return 0; \ + } \ + static PyObject *s_##SUF##_local_run(PyObject *self, PyObject *args) \ + { \ + PyObject *init, *script; \ + if (!PyArg_ParseTuple(args, "OO", &init, &script)) return NULL; \ + STRUCT a, src; \ + memset(&a, 0xAB, sizeof a); \ + memset(&src, 0xAB, sizeof src); \ + const char *ik = PyUnicode_AsUTF8(PyTuple_GET_ITEM(init, 0)); \ + int rc = 0; \ + int have_src = 0; \ + if (!strcmp(ik, "new")) { \ + long long n = PyLong_AsLongLong(PyTuple_GET_ITEM(init, 1)); \ + PARSET f; \ + if (!PyArg_Parse(PyTuple_GET_ITEM(init, 2), PARSEFMT, &f)) return NULL; \ + rc = tp_##SUF##_array_new(&a, (int64_t)n, (CTYPE)f); \ + } else if (!strcmp(ik, "iota")) { \ + long long n = PyLong_AsLongLong(PyTuple_GET_ITEM(init, 1)); \ + rc = TP_IOTA_##SUF(&a, (int64_t)n); \ + } else { \ + /* copy_local / copy_param: build the source, then copy it */ \ + if (!strcmp(ik, "copy_param")) { \ + int erc = tp_##SUF##_array_enter(PyTuple_GET_ITEM(init, 1), &src); \ + if (erc != 0) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_AssertionError, "enter deopt"); return NULL; } \ + } else { \ + long long n = PyLong_AsLongLong(PyTuple_GET_ITEM(init, 1)); \ + PARSET f; \ + if (!PyArg_Parse(PyTuple_GET_ITEM(init, 2), PARSEFMT, &f)) return NULL; \ + if (tp_##SUF##_array_new(&src, (int64_t)n, (CTYPE)f) != 0) { return NULL; } \ + for (Py_ssize_t i = 0; i < src.len; i++) src.data[i] = (CTYPE)(f + (PARSET)i); \ + } \ + have_src = 1; \ + rc = tp_##SUF##_array_copy(&a, &src); \ + } \ + if (rc != 0) { \ + if (rc != -1 || !PyErr_Occurred()) { \ + PyErr_SetString(PyExc_AssertionError, "local init: rc not 0/-1 or no error"); \ + return NULL; \ + } \ + PyObject *err = PyErr_GetRaisedException(); \ + int zeroed = a.list == NULL && a.len == 0 && a.data == NULL && a.dirty == NULL; \ + if (have_src) tp_##SUF##_array_free(&src); \ + return Py_BuildValue("(iN[N])", rc, err, PyBool_FromLong(zeroed)); \ + } \ + if (PyErr_Occurred()) { PyErr_SetString(PyExc_AssertionError, "error set with rc 0"); return NULL; } \ + PyObject *results = PyList_New(0); \ + Py_ssize_t ns = PyList_GET_SIZE(script); \ + for (Py_ssize_t k = 0; k < ns; k++) { \ + PyObject *op = PyList_GET_ITEM(script, k); \ + const char *kind = PyUnicode_AsUTF8(PyTuple_GET_ITEM(op, 0)); \ + PyObject *item = NULL; \ + if (!strcmp(kind, "get") || !strcmp(kind, "set")) { \ + long long i = PyLong_AsLongLong(PyTuple_GET_ITEM(op, 1)); \ + int store = kind[0] == 's'; \ + Py_ssize_t slot = -7; \ + int r = tp_##SUF##_array_slot(&a, (int64_t)i, store, &slot); \ + if (r < 0) item = PyErr_GetRaisedException(); \ + else if (!store) item = FROMC(a.data[slot]); \ + else { \ + PARSET v; \ + if (!PyArg_Parse(PyTuple_GET_ITEM(op, 2), PARSEFMT, &v)) goto fail; \ + a.data[slot] = (CTYPE)v; \ + item = Py_NewRef(Py_None); \ + } \ + } else if (!strcmp(kind, "len")) { \ + item = PyLong_FromSsize_t(a.len); \ + } else if (!strcmp(kind, "contents")) { \ + if (loc_##SUF##_contents(&a, &item) < 0) goto fail; \ + } else if (!strcmp(kind, "srccontents")) { \ + if (!have_src || loc_##SUF##_contents(&src, &item) < 0) goto fail; \ + } else if (!strcmp(kind, "flags")) { \ + item = Py_BuildValue("(iiin)", a.list == NULL, a.dirty == NULL, a.data == NULL, a.len); \ + } else if (!strcmp(kind, "srcflags")) { \ + item = Py_BuildValue("(iiin)", src.list == NULL, src.dirty == NULL, src.data == NULL, src.len); \ + } else if (!strcmp(kind, "free")) { \ + tp_##SUF##_array_free(&a); \ + item = Py_NewRef(Py_None); \ + } else if (!strcmp(kind, "exit")) { \ + item = PyLong_FromLong(tp_##SUF##_array_exit(&a)); \ + } else if (!strcmp(kind, "srcset")) { \ + long long i = PyLong_AsLongLong(PyTuple_GET_ITEM(op, 1)); \ + PARSET v; \ + if (!have_src || !PyArg_Parse(PyTuple_GET_ITEM(op, 2), PARSEFMT, &v)) goto fail; \ + src.data[i] = (CTYPE)v; \ + item = Py_NewRef(Py_None); \ + } else { \ + PyErr_SetString(PyExc_ValueError, "bad op"); \ + goto fail; \ + } \ + if (!item || PyList_Append(results, item) < 0) { Py_XDECREF(item); goto fail; } \ + Py_DECREF(item); \ + } \ + tp_##SUF##_array_free(&a); \ + if (have_src) tp_##SUF##_array_free(&src); \ + return Py_BuildValue("(iON)", 0, Py_None, results); \ + fail: \ + tp_##SUF##_array_free(&a); \ + if (have_src) tp_##SUF##_array_free(&src); \ + Py_DECREF(results); \ + return NULL; \ + } +static int TP_IOTA_f64(tp_f64_array *a, int64_t n) { (void)a; (void)n; PyErr_SetString(PyExc_ValueError, "no f64 iota"); return -2; } +static int TP_IOTA_i64(tp_i64_array *a, int64_t n) { return tp_i64_array_iota(a, n); } +LOCAL_RUN(f64, tp_f64_array, double, PyFloat_FromDouble, "d", double) +LOCAL_RUN(i64, tp_i64_array, int64_t, PyLong_FromLongLong, "L", long long) + +/* free on a zeroed struct, twice */ +static PyObject *s_local_free_zeroed(PyObject *self, PyObject *arg) +{ + if (PyLong_AsLong(arg) == 0) { + tp_f64_array a; memset(&a, 0, sizeof a); + tp_f64_array_free(&a); tp_f64_array_free(&a); + } else { + tp_i64_array a; memset(&a, 0, sizeof a); + tp_i64_array_free(&a); tp_i64_array_free(&a); + } + Py_RETURN_NONE; +} + +/* copy from a zeroed (never-initialised) source struct: must give an empty local array */ +static PyObject *s_local_copy_of_zeroed(PyObject *self, PyObject *arg) +{ + Py_ssize_t len; + if (PyLong_AsLong(arg) == 0) { + tp_f64_array s0, d; memset(&s0, 0, sizeof s0); memset(&d, 0xAB, sizeof d); + if (tp_f64_array_copy(&d, &s0) != 0) return NULL; + len = d.len; tp_f64_array_free(&d); + } else { + tp_i64_array s0, d; memset(&s0, 0, sizeof s0); memset(&d, 0xAB, sizeof d); + if (tp_i64_array_copy(&d, &s0) != 0) return NULL; + len = d.len; tp_i64_array_free(&d); + } + return PyLong_FromSsize_t(len); +} + +static PyObject *s_tuple(PyObject *self, PyObject *seq) +{ + PyObject *fast = PySequence_Fast(seq, "sequence expected"); + if (!fast) return NULL; + PyObject *t = tp_tuple(PySequence_Fast_ITEMS(fast), PySequence_Fast_GET_SIZE(fast)); + Py_DECREF(fast); + return t; +} + static PyObject *s_any_same(PyObject *self, PyObject *seq) { PyObject *fast = PySequence_Fast(seq, "sequence expected"); @@ -428,6 +593,9 @@ static PyMethodDef methods[] = { M("unbox_bool", s_unbox_bool, METH_O), M("f64_array_run", s_f64_array_run, METH_VARARGS), M("i64_array_run", s_i64_array_run, METH_VARARGS), M("array_exit_zeroed", s_array_exit_zeroed, METH_O), + M("f64_local_run", s_f64_local_run, METH_VARARGS), M("i64_local_run", s_i64_local_run, METH_VARARGS), + M("local_free_zeroed", s_local_free_zeroed, METH_O), M("tuple_", s_tuple, METH_O), + M("local_copy_of_zeroed", s_local_copy_of_zeroed, METH_O), M("global_", s_global, METH_VARARGS), M("getattr_", s_getattr, METH_VARARGS), M("call", s_call, METH_VARARGS), M("call_many", s_call_many, METH_VARARGS), M("getattr_many", s_getattr_many, METH_VARARGS), M("binop_many", s_binop_many, METH_VARARGS), diff --git a/python-multiplatform-ksp/src/test/python/test_runtime.py b/python-multiplatform-ksp/src/test/python/test_runtime.py index d77303dd9..c213475a9 100644 --- a/python-multiplatform-ksp/src/test/python/test_runtime.py +++ b/python-multiplatform-ksp/src/test/python/test_runtime.py @@ -674,6 +674,234 @@ def test_large_arrays_round_trip(rt): assert all(lst[i] == (-float(i) if i % 997 == 0 else float(i)) for i in range(n)) +def bits(x): + return struct.pack(" Date: Sat, 3 Oct 2026 13:59:13 +0900 Subject: [PATCH 34/36] Feat: C back end for local arrays and tuples Local array locals are runtime structs owned by the function: Assign builds into a zeroed temporary and frees the old array on success; every exit (ok, error, deopt) frees them; no write-back or dirty bits. Stores into local arrays are not effects. Tuple builds from owned temporaries via tp_tuple. Red: 4 failed + 11 errors. Green: 58/58 incl. ASan on the new functions. Mutations: skipping frees on the deopt or error path each fail their leak test (64 MB leaked vs a 4 MB bound). Refs #41 --- .../src/main/python/typedpython/cgen.py | 155 +++++-- .../src/test/python/test_cgen.py | 431 +++++++++++++++++- 2 files changed, 555 insertions(+), 31 deletions(-) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py index 4adf5153e..f843bcac9 100644 --- a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -35,6 +35,22 @@ The generated code never decrements a reference except through `tp_release` (API.md); the only increment is `Py_NewRef` when an OBJ local is read or an OBJ parameter is adopted. +Local arrays (ir.NewArray / CopyArray, `Function` docstring "Local arrays"). A local of an array type +is a `tp_i64_array` / `tp_f64_array` struct held by value in the impl, zero-initialised at entry +(`{0}`), with `list == NULL` and `dirty == NULL`. It is owned by the function: + 1. `Assign(local, NewArray | CopyArray)` builds the new array in a zeroed temporary struct + (`tp_*_array_new` / `_iota` / `_copy`), and only when that succeeded frees the old array + (`tp_*_array_free`) and moves the temporary into the slot — so a failed allocation leaves the + old binding intact, as `a = [0] * n` does, and `a = a[:]` copies before it frees; + 2. every exit of the impl — return, error, deopt — frees every local array and every array + temporary in the same exit block that releases the OBJ locals (a deopt of a pure function + frees them before the wrapper redoes the call interpreted); + 3. local arrays never go through `tp_*_array_enter/exit` (there is no list to write back to) and + a store into one sets no dirty bit. +`Index` / `StoreIndex` / `Len` use the same slot helper and `proven` handling as array params. +`Tuple` evaluates each element into an owned temporary, builds the tuple with `tp_tuple` (which +steals nothing), then releases the temporaries. + Generated arithmetic is in separate C statements and the file sets `#pragma STDC FP_CONTRACT OFF` (cbuild also passes -ffp-contract=off): a fused multiply-add would round differently from CPython. """ @@ -52,6 +68,7 @@ class CGenError(Exception): _CT = {Type.I64: "int64_t", Type.F64: "double", Type.BOOL: "int", Type.OBJ: "PyObject *"} _ARRAY_CT = {Type.F64_ARRAY: "tp_f64_array", Type.I64_ARRAY: "tp_i64_array"} +_CT_ARRAY_REV = {"tp_f64_array": Type.F64_ARRAY, "tp_i64_array": Type.I64_ARRAY} _ARRAY_ELEM = {Type.F64_ARRAY: Type.F64, Type.I64_ARRAY: Type.I64} _ARRAY_PFX = {Type.F64_ARRAY: "tp_f64_array", Type.I64_ARRAY: "tp_i64_array"} _UNBOX = {Type.I64: "tp_unbox_i64", Type.F64: "tp_unbox_f64", Type.BOOL: "tp_unbox_bool"} @@ -163,12 +180,14 @@ def _walk_stmts(body): def _sub_exprs(e: ir.Expr): - for name in ("left", "right", "operand", "index", "obj", "callee"): + for name in ("left", "right", "operand", "index", "obj", "callee", "length", "fill"): v = getattr(e, name, None) if isinstance(v, ir.Expr): yield v for v in getattr(e, "args", ()) or (): yield v + for v in getattr(e, "elements", ()) or (): + yield v def _walk_exprs(e: ir.Expr): @@ -312,7 +331,9 @@ def _check_function(f: ir.Function, functions: dict[str, ir.Function]) -> None: raise CGenError(f"{f.name}: may_deopt is False but a node can deopt " f"({type(deopting[0]).__name__})") effects = [e for e in _all_exprs(f) if _is_effect(e)] - stores = [s for s in _walk_stmts(f.body) if isinstance(s, ir.StoreIndex)] + # a store into a local array is not an effect (ir.py "Local arrays"); into a parameter it is + stores = [s for s in _walk_stmts(f.body) + if isinstance(s, ir.StoreIndex) and f.locals.get(s.array) not in ir.ARRAYS] if f.pure and (effects or stores): what = type(effects[0]).__name__ if effects else "StoreIndex" raise CGenError(f"{f.name}: marked pure but contains an effect ({what})") @@ -415,6 +436,7 @@ def __init__(self, mg: _ModGen, f: ir.Function): self.depth = 1 self.temps: list[tuple[str, str]] = [] # (C type, name) self.arrays = {p.name: p for p in f.params if isinstance(p, ir.ArrayParam)} + self.local_arrays: dict[str, Type] = {} # name -> F64_ARRAY / I64_ARRAY self.types: dict[str, Type] = {} for p in f.params: if not isinstance(p, ir.ArrayParam): @@ -424,6 +446,11 @@ def __init__(self, mg: _ModGen, f: ir.Function): for name, t in f.locals.items(): if name in self.arrays or (name in self.types and self.types[name] is not t): raise CGenError(f"{f.name}: local {name} conflicts with a parameter") + if t in ir.ARRAYS: + if name in self.types: + raise CGenError(f"{f.name}: local array {name} conflicts with a parameter") + self.local_arrays[name] = t + continue if t not in _CT: raise CGenError(f"{f.name}: local {name} has type {t}") self.types[name] = t @@ -484,7 +511,12 @@ def _own_release(self, *ts: str) -> None: def _own_release_all(self) -> list[str]: objs = [_ident("l", n) for n, t in self.types.items() if t is Type.OBJ] objs += [n for ct, n in self.temps if ct == "PyObject *"] - return [f" tp_release(&{n});" for n in objs] + out = [f" tp_release(&{n});" for n in objs] + # local arrays and array temporaries: freed on every exit (ok, error, deopt); idempotent + arrs = [(_ident("l", n), t) for n, t in self.local_arrays.items()] + arrs += [(n, _CT_ARRAY_REV[ct]) for ct, n in self.temps if ct in _CT_ARRAY_REV] + out += [f" {_ARRAY_PFX[t]}_free(&{n});" for n, t in arrs] + return out # expressions: return a C expression naming the value (a literal, a scalar local, or a temp); # OBJ results are always owned temporaries. @@ -801,37 +833,99 @@ def ev_CallObject(self, e: ir.CallObject) -> str: self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") return t - def _array(self, name: str) -> ir.ArrayParam: + def _array(self, name: str) -> tuple[Type, str]: + """(array type, C pointer expression) of an array parameter (already a pointer) or a local + array (a struct held by value, so its address).""" a = self.arrays.get(name) - if a is None: - raise CGenError(f"{self.f.name}: {name} is not an array parameter") - return a + if a is not None: + return a.type, _ident("l", name) + t = self.local_arrays.get(name) + if t is None: + raise CGenError(f"{self.f.name}: {name} is not an array parameter or local array") + return t, f"(&{_ident('l', name)})" def ev_Len(self, e: ir.Len) -> str: - self._array(e.array) + _, ptr = self._array(e.array) t = self.tmp("int64_t") - self.emit(f"{t} = (int64_t){_ident('l', e.array)}->len;") + self.emit(f"{t} = (int64_t){ptr}->len;") return t - def _slot(self, a: ir.ArrayParam, index: ir.Expr, proven: bool, store: int) -> str: + def _slot(self, at: Type, ptr: str, index: ir.Expr, proven: bool, store: int) -> str: if index.type is not Type.I64: raise CGenError(f"{self.f.name}: index of type {index.type}") i = self.ev(index) if proven: return f"(Py_ssize_t)({i})" slot = self.tmp("Py_ssize_t") - self.check(f"{_ARRAY_PFX[a.type]}_slot({_ident('l', a.name)}, {i}, {store}, &{slot})") + self.check(f"{_ARRAY_PFX[at]}_slot({ptr}, {i}, {store}, &{slot})") return slot def ev_Index(self, e: ir.Index) -> str: - a = self._array(e.array) - if e.type is not _ARRAY_ELEM[a.type]: - raise CGenError(f"{self.f.name}: Index of {a.type} typed {e.type}") - slot = self._slot(a, e.index, e.proven, 0) + at, ptr = self._array(e.array) + if e.type is not _ARRAY_ELEM[at]: + raise CGenError(f"{self.f.name}: Index of {at} typed {e.type}") + slot = self._slot(at, ptr, e.index, e.proven, 0) t = self.tmp(_CT[e.type]) - self.emit(f"{t} = {_ident('l', a.name)}->data[{slot}];") + self.emit(f"{t} = {ptr}->data[{slot}];") return t + def ev_NewArray(self, e: ir.NewArray) -> str: + raise CGenError(f"{self.f.name}: NewArray is legal only as the value of an Assign to a " + "local array (ir.py)") + + def ev_CopyArray(self, e: ir.CopyArray) -> str: + raise CGenError(f"{self.f.name}: CopyArray is legal only as the value of an Assign to a " + "local array (ir.py)") + + def ev_Tuple(self, e: ir.Tuple) -> str: + if e.type is not Type.OBJ or any(x.type is not Type.OBJ for x in e.elements): + raise CGenError(f"{self.f.name}: Tuple elements must be OBJ (Box scalars)") + items = [self.ev(x) for x in e.elements] # owned temporaries, evaluated in order + t = self.tmp("PyObject *") + if items: + self.emit(f"{{ PyObject *tp_av[{len(items)}] = {{{', '.join(items)}}};") + self.emit(f" {t} = tp_tuple(tp_av, {len(items)}); }}") + else: + self.emit(f"{t} = tp_tuple(NULL, 0);") + self._own_release(*items) # tp_tuple stole nothing + self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def _assign_array(self, s: ir.Assign, at: Type) -> None: + """`local = NewArray | CopyArray`: build into a zeroed temporary, then (only on success) + free the old array and move the temporary in.""" + v = s.value + pfx = _ARRAY_PFX[at] + if v.type is not at: + raise CGenError(f"{self.f.name}: {s.target} is {at}, assigned {v.type}") + dst = _ident("l", s.target) + new = self.tmp(_ARRAY_CT[at]) + if isinstance(v, ir.NewArray): + if v.length.type is not Type.I64: + raise CGenError(f"{self.f.name}: NewArray length of type {v.length.type}") + if v.iota: + if at is not Type.I64_ARRAY or v.fill is not None: + raise CGenError(f"{self.f.name}: iota NewArray must be I64_ARRAY with no fill") + n = self.ev(v.length) + self.check(f"{pfx}_iota(&{new}, {n})") + else: + if v.fill is None or v.fill.type is not _ARRAY_ELEM[at]: + raise CGenError(f"{self.f.name}: NewArray fill must be a " + f"{_ARRAY_ELEM[at]} (got {None if v.fill is None else v.fill.type})") + fill = self.ev(v.fill) # `[fill] * n`: fill first, then n + n = self.ev(v.length) + self.check(f"{pfx}_new(&{new}, {n}, {fill})") + elif isinstance(v, ir.CopyArray): + st, sptr = self._array(v.src) + if st is not at: + raise CGenError(f"{self.f.name}: CopyArray of {st} into {at}") + self.check(f"{pfx}_copy(&{new}, {sptr})") + else: + raise CGenError(f"{self.f.name}: local array {s.target} may only be assigned a " + f"NewArray or CopyArray, not {type(v).__name__}") + self.emit(f"{pfx}_free(&{dst});") + self.emit(f"{dst} = {new}; {new} = ({_ARRAY_CT[at]}){{0}};") + # statements def stmts(self, body) -> None: @@ -842,6 +936,9 @@ def stmts(self, body) -> None: m(s) def st_Assign(self, s: ir.Assign) -> None: + if s.target in self.local_arrays: + self._assign_array(s, self.local_arrays[s.target]) + return t = self.types.get(s.target) if t is None: raise CGenError(f"{self.f.name}: assignment to unknown local {s.target}") @@ -854,16 +951,17 @@ def st_Assign(self, s: ir.Assign) -> None: self.emit(f"{_ident('l', s.target)} = {v};") def st_StoreIndex(self, s: ir.StoreIndex) -> None: - a = self._array(s.array) - if not a.stored: - raise CGenError(f"{self.f.name}: store into {a.name}, declared stored=False") - if s.value.type is not _ARRAY_ELEM[a.type]: - raise CGenError(f"{self.f.name}: store of {s.value.type} into {a.type}") + at, ptr = self._array(s.array) + local = s.array in self.local_arrays + if not local and not self.arrays[s.array].stored: + raise CGenError(f"{self.f.name}: store into {s.array}, declared stored=False") + if s.value.type is not _ARRAY_ELEM[at]: + raise CGenError(f"{self.f.name}: store of {s.value.type} into {at}") v = self.ev(s.value) # CPython: value, then container, then index - slot = self._slot(a, s.index, s.proven, 1) - arr = _ident("l", a.name) - self.emit(f"{arr}->data[{slot}] = {v};") - self.emit(f"{arr}->dirty[{slot}] = 1;") + slot = self._slot(at, ptr, s.index, s.proven, 1) + self.emit(f"{ptr}->data[{slot}] = {v};") + if not local: # a local array has no list to write back to + self.emit(f"{ptr}->dirty[{slot}] = 1;") def st_ExprStmt(self, s: ir.ExprStmt) -> None: v = self.ev(s.value) @@ -983,6 +1081,8 @@ def generate(self) -> str: " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);", " PyObject *tp_dict = PyModule_GetDict(tp_module); /* borrowed: owned by the module */"] params = {p.name for p in f.params} | {g.name for g in f.entry_globals} + for name, t in self.local_arrays.items(): + out.append(f" {_ARRAY_CT[t]} {_ident('l', name)} = {{0}};") for name, t in self.types.items(): if name in params and t is not Type.OBJ: continue @@ -990,11 +1090,12 @@ def generate(self) -> str: ct = _CT[t] out.append(f" {ct}{'' if ct.endswith('*') else ' '}{_ident('l', name)} = {init};") for ct, n in self.temps: - init = "NULL" if ct == "PyObject *" else "0" + init = "NULL" if ct == "PyObject *" else "{0}" if ct in _CT_ARRAY_REV else "0" sep = "" if ct.endswith("*") else " " out.append(f" {ct}{sep}{n} = {init};") out.append(" (void)tp_st; (void)tp_dict; (void)tp_s;" - + "".join(f" (void){_ident('l', n)};" for n in self.types if n not in params)) + + "".join(f" (void){_ident('l', n)};" for n in self.types if n not in params) + + "".join(f" (void){_ident('l', n)};" for n in self.local_arrays)) out += prologue out += self.lines out.append("tp_exit:") diff --git a/python-multiplatform-ksp/src/test/python/test_cgen.py b/python-multiplatform-ksp/src/test/python/test_cgen.py index da069327b..eea61452b 100644 --- a/python-multiplatform-ksp/src/test/python/test_cgen.py +++ b/python-multiplatform-ksp/src/test/python/test_cgen.py @@ -31,6 +31,7 @@ ForRange, Function, If, Index, Len, Local, Module, Param, Return, StoreIndex, Type, Global, GetAttr, CallObject, Truth, CompareObj, ObjToFloat, UnaryOp, UnaryOpKind, Unbox, While, Break, Continue, ToFloat, MathCall, MathFunc, + NewArray, CopyArray, Tuple, ) PACKAGE = Path(__file__).resolve().parents[2] / "main" / "python" / "typedpython" @@ -262,6 +263,68 @@ def scaled(x: float) -> float: def twice_scaled(x: float) -> float: return scaled(x) * 2.0 + + # --- local arrays and tuple returns (fannkuch, M2) --- + def la_build(n: int) -> tuple: + a = list(range(n)) + b = [0] * n + s = 0 + for i in range(n // 2): + t = a[i] + a[i] = a[n - 1 - i] + a[n - 1 - i] = t + c = a[:] + for i in range(len(c)): + b[i] = c[i] * 3 + a[i] + s = s + b[i] + first = -1 + last = -1 + if len(a) > 0: + first = a[0] + last = a[-1] + return (len(a), len(b), first, last, s) + + def la_f64(n: int) -> tuple: + x = [0.5] * n + y = x[:] + total = 0.0 + for i in range(len(x)): + x[i] = x[i] * float(i) + 1.0 + y[i] = x[i] / 2.0 + total = total + y[i] + return (len(y), total) + + def la_overflow(n: int, k: int) -> tuple: + a = list(range(n)) + b = a[:] + acc = 1 + for i in range(len(b)): + acc = acc + k + b[i] = i + return (acc, len(a)) + + def la_index(n: int, i: int) -> int: + a = list(range(n)) + return a[i] + + def la_store(n: int, i: int, v: int) -> int: + a = [0] * n + a[i] = v + return len(a) + + def la_reassign(n: int) -> tuple: + a = list(range(n)) + a = [7] * (n + 1) + a = a[:] + return (len(a), a[0], a[-1]) + + def la_copy_param(xs: list[int]) -> tuple: + c = xs[:] + c[0] = 99 + return (xs[0], c[0], len(c)) + + def la_unit() -> tuple: + return () ''') SOURCE = "import math\n" + SOURCE @@ -1031,6 +1094,314 @@ def __eq__(self, other): assert mod.selfeq(1.0) is True +# --- local arrays and tuples (issue #41, contract 513f10f2) ----------------------------------------- + +I64A, F64A = Type.I64_ARRAY, Type.F64_ARRAY +LOCAL_ARRAY_HELPERS = ("tp_i64_array_new", "tp_i64_array_iota", "tp_i64_array_copy", + "tp_i64_array_free", "tp_f64_array_new", "tp_f64_array_copy", + "tp_f64_array_free", "tp_tuple") + + +def need_local_array_runtime(): + need_runtime() + header = runtime_dir() / "tp_runtime.h" + text = header.read_text() + missing = [h for h in LOCAL_ARRAY_HELPERS if not re.search(rf"\b{h}\s*\(", text)] + if missing: + pytest.skip(f"{header} does not define the local-array helpers yet (the tp-runtime agent " + f"is still writing them): {missing}") + + +def f_la_build(): + n, i, t = L("n", I64), L("i", I64), L("t", I64) + s, first, last = L("s", I64), L("first", I64), L("last", I64) + mirror = binop(BinOpKind.SUB, binop(BinOpKind.SUB, n, c(1)), i) + return fn("la_build", [Param("n", I64)], OBJ, + {"a": I64A, "b": I64A, "c": I64A, "i": I64, "t": I64, "s": I64, "first": I64, "last": I64}, + [Assign("a", NewArray(I64A, n, None, True)), + Assign("b", NewArray(I64A, n, c(0))), + Assign("s", c(0)), + ForRange("i", c(0), binop(BinOpKind.FLOORDIV, n, c(2)), c(1), ( + Assign("t", Index(I64, "a", i)), + StoreIndex("a", i, Index(I64, "a", mirror)), + StoreIndex("a", mirror, t), + )), + Assign("c", CopyArray(I64A, "a")), + ForRange("i", c(0), Len(I64, "c"), c(1), ( + StoreIndex("b", i, binop(BinOpKind.ADD, binop(BinOpKind.MUL, Index(I64, "c", i), c(3)), + Index(I64, "a", i))), + Assign("s", binop(BinOpKind.ADD, s, Index(I64, "b", i))), + )), + Assign("first", c(-1)), Assign("last", c(-1)), + If(Compare(BOOL, CompareKind.GT, Len(I64, "a"), c(0)), ( + Assign("first", Index(I64, "a", c(0))), + Assign("last", Index(I64, "a", c(-1))), + )), + Return(Tuple(OBJ, (Box(OBJ, Len(I64, "a")), Box(OBJ, Len(I64, "b")), Box(OBJ, first), + Box(OBJ, last), Box(OBJ, s))))], + pure=True, may_deopt=True) + + +def f_la_f64(): + n, i, total = L("n", I64), L("i", I64), L("total", F64) + return fn("la_f64", [Param("n", I64)], OBJ, + {"x": F64A, "y": F64A, "total": F64, "i": I64}, + [Assign("x", NewArray(F64A, n, c(0.5))), + Assign("y", CopyArray(F64A, "x")), + Assign("total", c(0.0)), + ForRange("i", c(0), Len(I64, "x"), c(1), ( + StoreIndex("x", i, binop(BinOpKind.ADD, binop(BinOpKind.MUL, Index(F64, "x", i), + ToFloat(F64, i)), c(1.0))), + StoreIndex("y", i, binop(BinOpKind.TRUEDIV, Index(F64, "x", i), c(2.0))), + Assign("total", binop(BinOpKind.ADD, total, Index(F64, "y", i))), + )), + Return(Tuple(OBJ, (Box(OBJ, Len(I64, "y")), Box(OBJ, total))))], + pure=True, may_deopt=False) + + +def f_la_overflow(): + i, acc = L("i", I64), L("acc", I64) + return fn("la_overflow", [Param("n", I64), Param("k", I64)], OBJ, + {"a": I64A, "b": I64A, "acc": I64, "i": I64}, + [Assign("a", NewArray(I64A, L("n", I64), None, True)), + Assign("b", CopyArray(I64A, "a")), + Assign("acc", c(1)), + ForRange("i", c(0), Len(I64, "b"), c(1), ( + Assign("acc", binop(BinOpKind.ADD, acc, L("k", I64))), + StoreIndex("b", i, i), + )), + Return(Tuple(OBJ, (Box(OBJ, acc), Box(OBJ, Len(I64, "a")))))], + pure=True, may_deopt=True) + + +def f_la_index(): + return fn("la_index", [Param("n", I64), Param("i", I64)], I64, {"a": I64A}, + [Assign("a", NewArray(I64A, L("n", I64), None, True)), + Return(Index(I64, "a", L("i", I64)))], + pure=True, may_deopt=False) + + +def f_la_store(): + return fn("la_store", [Param("n", I64), Param("i", I64), Param("v", I64)], I64, {"a": I64A}, + [Assign("a", NewArray(I64A, L("n", I64), c(0))), + StoreIndex("a", L("i", I64), L("v", I64)), + Return(Len(I64, "a"))], + pure=True, may_deopt=False) + + +def f_la_reassign(): + n = L("n", I64) + return fn("la_reassign", [Param("n", I64)], OBJ, {"a": I64A}, + [Assign("a", NewArray(I64A, n, None, True)), + Assign("a", NewArray(I64A, binop(BinOpKind.ADD, n, c(1)), c(7))), + Assign("a", CopyArray(I64A, "a")), + Return(Tuple(OBJ, (Box(OBJ, Len(I64, "a")), Box(OBJ, Index(I64, "a", c(0))), + Box(OBJ, Index(I64, "a", c(-1))))))], + pure=True, may_deopt=True) + + +def f_la_copy_param(): + return fn("la_copy_param", [ArrayParam("xs", I64A, stored=False)], OBJ, {"c": I64A}, + [Assign("c", CopyArray(I64A, "xs")), + StoreIndex("c", c(0), c(99)), + Return(Tuple(OBJ, (Box(OBJ, Index(I64, "xs", c(0))), Box(OBJ, Index(I64, "c", c(0))), + Box(OBJ, Len(I64, "c")))))], + pure=True, may_deopt=False) + + +def f_la_unit(): + return fn("la_unit", [], OBJ, {}, [Return(Tuple(OBJ, ()))], pure=True, may_deopt=False) + + +LOCALARR = (f_la_build, f_la_f64, f_la_overflow, f_la_index, f_la_store, f_la_reassign, + f_la_copy_param, f_la_unit) + + +@pytest.fixture(scope="module") +def la(): + need_local_array_runtime() + return build_module("tp_cgen_la", functions=LOCALARR) + + +def test_gen_local_array_scan_calls_only_runtime_helpers(): + text = gen(LOCALARR, name="tp_gen_la") + body = re.sub(r"/\*.*?\*/", "", text, flags=re.S) + body = re.sub(r'"(?:\\.|[^"\\])*"', '""', body) + called = set(re.findall(r"\b([A-Za-z_][A-Za-z0-9_]*)\s*\(", body)) + helpers = api_helpers() + for h in LOCAL_ARRAY_HELPERS: + assert h in helpers, f"{h} is not in runtime/API.md" + assert h in called, f"the local-array functions never call {h}" + stray = sorted(n for n in called if n.startswith("tp_") and n not in helpers + and not re.search(rf"^static [^\n]*\b{n}\s*\(", body, flags=re.M)) + assert stray == [] + + +def test_gen_local_arrays_never_go_through_the_write_back_exit(): + text = gen((f_la_build,), name="tp_gen_la2") + impl = text[text.index("static int tp_impl_la_build(PyObject *tp_module, int64_t l_n"):] + impl = impl[:impl.index("static PyObject *tp_wrap_la_build")] + assert "_array_exit" not in impl and "_array_enter" not in impl + assert impl.count("tp_i64_array_free(") >= 3 # a, b, c are freed at the exit block + assert ".dirty" not in impl and "->dirty" not in impl # stores into a local set no dirty bit + + +def test_gen_rejects_a_new_array_that_is_not_an_assign_value(): + from typedpython import cgen + + bad = fn("bad", [Param("n", I64)], I64, {"a": I64A}, + [ExprStmt(NewArray(I64A, L("n", I64), c(0))), Return(c(0))], pure=True, may_deopt=False) + with pytest.raises(cgen.CGenError, match="NewArray"): + cgen.generate(Module("tp_bad3", (bad,)), source_file("tp_bad3")) + + +def test_gen_rejects_assigning_an_array_of_the_wrong_element_type(): + from typedpython import cgen + + bad = fn("bad", [Param("n", I64)], I64, {"a": F64A}, + [Assign("a", NewArray(I64A, L("n", I64), c(0))), Return(c(0))], pure=True, may_deopt=False) + with pytest.raises(cgen.CGenError): + cgen.generate(Module("tp_bad4", (bad,)), source_file("tp_bad4")) + + +def test_gen_a_store_into_a_param_array_still_makes_a_pure_function_an_error(): + from typedpython import cgen + + bad = fn("bad", [ArrayParam("xs", I64A, stored=True)], NONE, {}, + [StoreIndex("xs", c(0), c(1))], pure=True, may_deopt=False) + with pytest.raises(cgen.CGenError, match="pure"): + cgen.generate(Module("tp_bad5", (bad,)), source_file("tp_bad5")) + + +def test_gen_a_store_into_a_local_array_is_not_an_effect(): + text = gen((f_la_store,), name="tp_gen_la3") + assert "tp_impl_la_store" in text + + +def test_local_arrays_equal_cpython_for_several_n(la): + for n in (0, 1, 2, 3, 7, 10, 64, -1, -5): + got, want = la.la_build(n), interp(la, "la_build")(n) + assert type(got) is tuple and got == want, (n, got, want) + assert deopts(la) == 0 + + +def test_f64_local_arrays_equal_cpython_bit_for_bit(la): + for n in (0, 1, 5, 100, -3): + got, want = la.la_f64(n), interp(la, "la_f64")(n) + assert repr(got) == repr(want), (n, got, want) + + +def test_a_tuple_with_no_elements_is_the_empty_tuple(la): + assert la.la_unit() == () and type(la.la_unit()) is tuple + + +def test_local_array_index_errors_equal_cpython(la): + for fname, args in (("la_index", (5, 5)), ("la_index", (5, -6)), ("la_index", (0, 0)), + ("la_index", (5, 4)), ("la_index", (5, -5)), + ("la_store", (3, 3, 1)), ("la_store", (3, -4, 1)), ("la_store", (0, 0, 1)), + ("la_store", (3, -1, 9)), ("la_store", (3, 2, 9))): + try: + want = ("ok", interp(la, fname)(*args)) + except Exception as e: + want = (type(e), str(e)) + try: + got = ("ok", getattr(la, fname)(*args)) + except Exception as e: + got = (type(e), str(e)) + assert got == want, (fname, args, got, want) + # and the two messages are really the list ones, not something of our own + with pytest.raises(IndexError, match=r"^list index out of range$"): + la.la_index(2, 2) + with pytest.raises(IndexError, match=r"^list assignment index out of range$"): + la.la_store(2, 2, 0) + + +def test_reassigning_a_local_array_replaces_it_and_copy_of_itself_is_safe(la): + for n in (0, 1, 5, 40): + assert la.la_reassign(n) == interp(la, "la_reassign")(n) + for bad in (-1,): # a == [7]*0: a[0] raises in both + with pytest.raises(IndexError) as g: + la.la_reassign(bad) + with pytest.raises(IndexError) as w: + interp(la, "la_reassign")(bad) + assert str(g.value) == str(w.value) + + +def test_copy_of_an_array_param_leaves_the_list_untouched(la): + xs = [5, 6, 7] + assert la.la_copy_param(xs) == interp(la, "la_copy_param")([5, 6, 7]) == (5, 99, 3) + assert xs == [5, 6, 7] + + +def test_overflow_deopt_with_local_arrays_returns_the_big_int_result(la): + before = deopts(la) + got, want = la.la_overflow(50, 2 ** 62), interp(la, "la_overflow")(50, 2 ** 62) + assert got == want and got[0] == 1 + 50 * 2 ** 62 and got[0] > 2 ** 63 + assert deopts(la) == before + 1 + before = deopts(la) + assert la.la_overflow(10, 2) == interp(la, "la_overflow")(10, 2) and deopts(la) == before + + +def heap_in_use() -> int: + """Bytes the C allocator has handed out right now (macOS: aggregated over all malloc zones).""" + import ctypes + + if sys.platform != "darwin": + pytest.skip("heap-in-use probe is written for macOS (malloc_zone_statistics) only") + + class Stats(ctypes.Structure): + _fields_ = [("blocks_in_use", ctypes.c_uint), ("size_in_use", ctypes.c_size_t), + ("max_size_in_use", ctypes.c_size_t), ("size_allocated", ctypes.c_size_t)] + + libc = ctypes.CDLL(None) + stats = Stats() + libc.malloc_zone_statistics(None, ctypes.byref(stats)) + return stats.size_in_use + + +def leaked_bytes(call, rounds=40) -> int: + import gc + + call() # warm caches and the allocator + gc.collect() + before = heap_in_use() + for _ in range(rounds): + call() + gc.collect() + return heap_in_use() - before + + +# Each call below allocates two 800 KB arrays (n = 100000); a leak of one array per call is 32 MB +# over 40 rounds, far above the allocator noise this bound allows. (LeakSanitizer does not exist on +# macOS, so the leak check is this heap-in-use delta; ASan below checks the memory errors.) +LEAK_LIMIT = 4 * 1024 * 1024 + + +def test_deopt_in_a_pure_function_frees_the_local_arrays(la): + before = deopts(la) + assert la.la_overflow(100_000, 2 ** 62)[1] == 100_000 # deopted: the sum is a big int + assert deopts(la) == before + 1 + assert leaked_bytes(lambda: la.la_overflow(100_000, 2 ** 62)) < LEAK_LIMIT + + +def test_ok_path_frees_the_local_arrays(la): + assert leaked_bytes(lambda: la.la_overflow(100_000, 1)) < LEAK_LIMIT + assert deopts(la) >= 0 + + +def test_error_path_frees_the_local_arrays(la): + def boom(): + try: + la.la_index(100_000, 10 ** 9) + except IndexError: + pass + assert leaked_bytes(boom) < LEAK_LIMIT + + +def test_reassigning_in_a_loop_frees_the_old_arrays(la): + assert leaked_bytes(lambda: la.la_reassign(100_000)) < LEAK_LIMIT + + # --- sanitizers ----------------------------------------------------------------------------------- ASAN_SCRIPT = textwrap.dedent('''\ @@ -1077,7 +1448,9 @@ def __eq__(self, other): ''') -def test_build_with_address_and_undefined_sanitizers(): +def asan_run(name: str, functions, script_template: str) -> tuple[str, Path]: + """Build `functions` with ASan+UBSan, run `script_template` in a sanitized launcher; return the + combined output and the extension path (skips when the platform cannot do it).""" from typedpython import cbuild need_runtime() @@ -1089,10 +1462,9 @@ def test_build_with_address_and_undefined_sanitizers(): asan_rt = Path(probe.stdout.strip()) if probe.returncode != 0 or not asan_rt.is_file(): pytest.skip(f"host compiler {cc!r} has no ASan runtime ({probe.stdout.strip()!r})") - name = "tp_cgen_asan" src = source_file(name) from typedpython import cgen - c_source = cgen.generate(Module(name=name, functions=tuple(f() for f in ALL)), src) + c_source = cgen.generate(Module(name=name, functions=tuple(f() for f in functions)), src) flags = list(cbuild.DEFAULT_FLAGS) + ["-O1", "-g", "-fno-omit-frame-pointer", "-fsanitize=address,undefined", "-fno-sanitize-recover=undefined"] @@ -1117,17 +1489,68 @@ def test_build_with_address_and_undefined_sanitizers(): ASAN_OPTIONS="detect_leaks=0:abort_on_error=0:halt_on_error=1", UBSAN_OPTIONS="halt_on_error=1:print_stacktrace=1") env.pop("DYLD_INSERT_LIBRARIES", None) - script = ASAN_SCRIPT.format(pkg=str(PACKAGE.parent), name=name, so=str(so)) + script = script_template.format(pkg=str(PACKAGE.parent), name=name, so=str(so)) run = subprocess.run([str(launcher), "-c", script], capture_output=True, text=True, env=env) out = run.stdout + run.stderr if "violates platform policy" in out or "Interceptors are not working" in out: pytest.skip(f"ASan runtime could not be loaded: {out[-2000:]}") assert run.returncode == 0 and "ASAN-RUN-OK" in run.stdout, out[-6000:] + return out, so + + +def assert_instrumented(so: Path, out: str) -> None: linked = subprocess.run(["otool", "-L", str(so)], capture_output=True, text=True).stdout assert "libclang_rt.asan" in linked, linked # the extension really is instrumented assert "runtime error" not in out and "AddressSanitizer" not in out, out[-6000:] +def test_build_with_address_and_undefined_sanitizers(): + out, so = asan_run("tp_cgen_asan", ALL, ASAN_SCRIPT) + assert_instrumented(so, out) + + +ASAN_LA_SCRIPT = textwrap.dedent('''\ + import sys + sys.path.insert(0, {pkg!r}) + from typedpython import cbuild + m = cbuild.load({name!r}, {so!r}) + I = m.__typedpython_interpreted__ + for n in (0, 1, 2, 3, 7, 64, -4): + assert m.la_build(n) == I["la_build"](n), n + assert repr(m.la_f64(n)) == repr(I["la_f64"](n)), n + for n in (0, 1, 9): + assert m.la_reassign(n) == I["la_reassign"](n), n + for args in ((5, 5), (5, -6), (0, 0), (5, 4)): + for name in ("la_index",): + try: + want = I[name](*args) + except IndexError as e: + want = str(e) + try: + got = getattr(m, name)(*args) + except IndexError as e: + got = str(e) + assert got == want, (name, args, got, want) + try: + m.la_store(3, 3, 1) + except IndexError: + pass + assert m.la_unit() == () + xs = [5, 6, 7] + assert m.la_copy_param(xs) == (5, 99, 3) and xs == [5, 6, 7] + for _ in range(100): + assert m.la_overflow(1000, 2 ** 62) == I["la_overflow"](1000, 2 ** 62) + assert m.la_overflow(1000, 1) == I["la_overflow"](1000, 1) + print("ASAN-RUN-OK", m.__typedpython_deopts__) + ''') + + +def test_local_arrays_under_address_and_undefined_sanitizers(): + need_local_array_runtime() + out, so = asan_run("tp_cgen_asan_la", LOCALARR, ASAN_LA_SCRIPT) + assert_instrumented(so, out) + + # --- building against the embedded CPython ------------------------------------------------------ EMBEDDED_314 = (WORKTREE.parents[1] / "python-multiplatform" / "build" / "python-standalone" From 82f6d7cecc90f2c0e6543151207666b87851f7db Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:09:05 +0900 Subject: [PATCH 35/36] CI: Plant the ASan self-check fault in the redzone and show the report head The first CI run's self-check failed: a read 56 bytes past an 8-byte block can land outside its redzone, so ASan reported something other than heap-buffer-overflow, and only the report's tail was printed. Read the first byte past the end (always in the redzone) and print the head of the output on failure. Refs #41 --- python-multiplatform-ksp/tools/sanitizers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python-multiplatform-ksp/tools/sanitizers.py b/python-multiplatform-ksp/tools/sanitizers.py index 124edd361..389fd2df7 100644 --- a/python-multiplatform-ksp/tools/sanitizers.py +++ b/python-multiplatform-ksp/tools/sanitizers.py @@ -149,7 +149,8 @@ def cmd_selfcheck() -> int: failures = [] cases = [ ("clean call is silent", "m.oob_read(0); m.overflow(0)", None, 0), - ("ASan reports heap overflow", "m.oob_read(64)", "AddressSanitizer: heap-buffer-overflow", None), + ("ASan reports heap overflow", "m.oob_read(8)", # first byte past the end: always in the redzone + "AddressSanitizer: heap-buffer-overflow", None), ("UBSan reports signed overflow", "m.overflow(1)", "runtime error: signed integer overflow", None), ] for name, call, needle, want_rc in cases: @@ -166,7 +167,7 @@ def cmd_selfcheck() -> int: ok = needle in out and run.returncode != 0 and "RETURNED-WITHOUT-REPORT" not in out print(f"{'ok ' if ok else 'FAIL'} {name} (rc={run.returncode})") if not ok: - failures.append(f"{name}:\n{out[-1500:]}") + failures.append(f"{name}:\n--- head ---\n{out[:2500]}\n--- tail ---\n{out[-800:]}") if failures: print("\n".join(failures), file=sys.stderr) print("SELF-CHECK FAILED: the sanitizer setup does not detect a known fault", file=sys.stderr) From 9a165366d8c35ab71bd254fbe67cead84c74ff91 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:13:43 +0900 Subject: [PATCH 36/36] CI: Let the ASan self-check case reach ASan The second CI run showed the planted read is caught, but by UBSan's object-size check, which halts first. Build the fault-planting extension with -fno-sanitize=object-size so that case proves ASan; the signed-overflow case already proves UBSan. Refs #41 --- python-multiplatform-ksp/tools/sanitizers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python-multiplatform-ksp/tools/sanitizers.py b/python-multiplatform-ksp/tools/sanitizers.py index 389fd2df7..c94018804 100644 --- a/python-multiplatform-ksp/tools/sanitizers.py +++ b/python-multiplatform-ksp/tools/sanitizers.py @@ -136,7 +136,11 @@ def cmd_selfcheck() -> int: src = work / "tp_faulty.c" src.write_text(FAULTY_C) so = work / ("tp_faulty" + sysconfig.get_config_var("EXT_SUFFIX")) - build = cc() + SAN_FLAGS + ["-fPIC", "-shared", f"-I{sysconfig.get_paths()['include']}", str(src), + # UBSan's object-size check would stop the planted out-of-bounds read before ASan sees it (seen + # on the first CI run). Disable it for this fault-planting build only, so the ASan case proves + # ASan itself; UBSan is proven by the signed-overflow case. + build = cc() + SAN_FLAGS + ["-fno-sanitize=object-size", "-fPIC", "-shared", + f"-I{sysconfig.get_paths()['include']}", str(src), "-o", str(so)] if sys.platform == "darwin": build += ["-undefined", "dynamic_lookup"]