diff --git a/.github/workflows/typedpython.yml b/.github/workflows/typedpython.yml new file mode 100644 index 000000000..762b12fe2 --- /dev/null +++ b/.github/workflows/typedpython.yml @@ -0,0 +1,77 @@ +name: TypedPython CI + +# The TypedPython gate and compiler are Python (python-multiplatform-ksp/src/{main,test}/python). +# Gradle does not run them; this workflow does. +on: + push: + branches: [ "develop", "main" ] + paths: [ "python-multiplatform-ksp/**", ".github/workflows/typedpython.yml" ] + pull_request: + branches: [ "develop", "main" ] + paths: [ "python-multiplatform-ksp/**", ".github/workflows/typedpython.yml" ] + +jobs: + test: + name: TypedPython tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + defaults: + run: + working-directory: python-multiplatform-ksp + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + + - name: Create the environment + # pyrefly is pinned exactly: the gate reads its versioned type-report format. + run: | + uv venv .venv --python 3.13 + uv pip install --python .venv/bin/python "pyrefly==1.3.2" pytest + + - name: Behaviour tests + run: .venv/bin/python -m pytest -q -m "not measurement" + + - name: Measurements (recorded, sanity bounds only) + # A shared runner is not an idle machine (AGENTS.md §8): read ratios, not absolute times. + run: .venv/bin/python -m pytest -q -s -m measurement + + sanitizers: + # SPEC N-8: ASan/UBSan runs are the safety net outside the proof. Every extension TypedPython + # builds in these tests is compiled with -fsanitize=address,undefined (TP_SANITIZE=1, see + # conftest.py) and the interpreter runs with the ASan runtime preloaded; any report fails the run. + # Linux only: macOS refuses to load the ASan runtime into the framework Python (the cgen + # sanitizer test has a launcher-based macOS path of its own). + name: TypedPython tests under ASan + UBSan (ubuntu-latest) + runs-on: ubuntu-latest + defaults: + run: + working-directory: python-multiplatform-ksp + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + + - name: Create the environment + run: | + uv venv .venv --python 3.13 + uv pip install --python .venv/bin/python "pyrefly==1.3.2" pytest + + - name: Relax ASLR entropy (ASan shadow mapping on newer kernels) + run: sudo sysctl -w vm.mmap_rnd_bits=28 || true + + - name: Self-check, the sanitizers must catch planted faults + # Builds a deliberately faulty extension (heap out-of-bounds read, signed overflow) and + # REQUIRES both reports. If this step passes, the setup below can detect faults. + run: .venv/bin/python tools/sanitizers.py selfcheck + + - name: cgen, runtime and pipeline tests, sanitized + # detect_leaks=0: CPython itself leaks at exit. LD_PRELOAD is applied to this one command + # only (tools/sanitizers.py env prints it); the gate's pyrefly runs without it. + run: | + env $(.venv/bin/python tools/sanitizers.py env) \ + .venv/bin/python -m pytest -q -m "not measurement" \ + src/test/python/test_cgen.py src/test/python/test_runtime.py src/test/python/test_pipeline.py diff --git a/docs/INTENT.md b/docs/INTENT.md index e3e9084ba..052864626 100644 --- a/docs/INTENT.md +++ b/docs/INTENT.md @@ -26,6 +26,23 @@ CPython interpreter inside a Kotlin Multiplatform application so that: written (`Greeter('Kotlin').greet(2)` in `sample/`). 3. **The same code runs on every target** a Kotlin Multiplatform app ships to: desktop JVM, Android, iOS, Android native, and the web (wasm, experimental). +4. **The project's own Python is statically typed by default** (TypedPython). The maintainer's goal: + + > "파이썬을 컴파일 언어 및 타입 안정화, 정적 타입 검사, 컴파일 타임 타입검사를 지원할 수 있도록" + > — *Support Python as a compiled language with type stability, static type checking and + > compile-time type checking*, so that Python stays usable as AI-friendly languages such as Rust + > and TypeScript gain ground. + + > "TypedPython을 별도 레포가 아니라 PythonMultiplatform의 기본 동작이 될거야" + > — *TypedPython is not a separate repository; it will be PythonMultiplatform's default behaviour.* + + The maintainer chose the scope "검사만 기본, 컴파일은 opt-in" (2026-10-02): every user Python source + is **checked** by default; **compiling** it to native code is opt-in. The compiled code still runs on + the embedded CPython, as an extension module (`docs/design/typedpython.md`). + + > The maintainer also asked to *explore* GPU kernels written in Python without a separate language + > ("GPU 관련 최적화까지도 별도 언어 없이 지원할 방안을 모색해보자"). That is an exploration, not yet + > a commitment; it is recorded in the design note only. The CPython version moved on from 3.13 (the build pins 3.14.x today and offers opt-in free-threaded 3.14t builds on desktop only); the intent — embed **CPython itself**, not a re-implementation — is unchanged. @@ -88,7 +105,7 @@ no per-platform Python → Kotlin binding code to maintain. | A replacement for the `pythonx` packages | §2.3 — they are real packages in their own repository | | Runtime-reflection or name-lookup based binding | §2.1 — impossible under Kotlin/Native and native image | | A hand-written wrapper per Kotlin function or class | Generic adaptation plus generated tables; per-declaration wrappers were tried in 2024 and rotted (`docs/archive/pythonx-adapter-design.md`) | -| A Python re-implementation, transpiler, or JVM-hosted Python | It embeds CPython itself, so C extensions keep working | +| A Python re-implementation, transpiler, or JVM-hosted Python | It embeds CPython itself, so C extensions keep working. TypedPython (§1.4) does not change this: checked code is ordinary Python, and compiled code is a CPython extension module, not a translation to another runtime | | A build tool, dependency resolver or packager | That is pypackpack (the work) and toolchain (the Gradle vocabulary) | | A sub-interpreter parallelism model | Rejected: extensions that do not declare multi-interpreter support fail to import; parallelism comes from free-threaded CPython | | A vendored interpreter | The build downloads and verifies CPython per platform | diff --git a/docs/SPEC.md b/docs/SPEC.md index 6ea3e21b3..19ac3529f 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -317,6 +317,50 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. `androidx.compose.runtime.mutableStateOf(x)` callable, and `.value` of the returned `MutableState` readable and writable on its proxy. Implemented by #38 (B-1, U-10); B-8's test now writes its root through `state.value = root` and the KSP-bound stand-in is gone. `Status: implemented`. +- **N-7** TypedPython soundness gate (INTENT §1.4; `docs/design/typedpython.md` §4.2): checking a set of + user Python files reports every Pyrefly type error, the Any-leak kinds (`implicit-any-parameter`, + `unannotated-return`, `unknown-variable-type`, `no-any-return-implicit`, `no-any-return-explicit`, + `explicit-any`) and any user-code expression of type `Any` outside `cast()` (`any-flow`) as errors; + the forbidden list (`eval`/`exec`, computed-name `getattr`/`setattr`/`delattr`, patching an imported + module or a module-level class, rebinding an unannotated name to a different type, and a list/set/dict + display whose inferred element type is a union) as warnings in `checked` mode and errors in + `compiled` mode. Nothing + inside dependencies is reported. `python -m typedpython check` exits 0 / 1 / 2 for clean / errors / + tool failure. `Status: partial` — the gate is asserted by + `python-multiplatform-ksp/src/test/python/` (run with that module's `.venv`); it is **not yet run by the + build** (design §6 step 1b). Rebinding and mixed containers are found by a second Pyrefly pass over a + probed copy (design §6.1), which runs only when a file has a name assigned twice or a container + display; `for` targets and comprehension variables are not probed. +- **N-8** TypedPython compilation to C (INTENT §1.4; design §4.3; #41). `@compiled` (a builtin name, no + import — #42) and a first-line `# typedpython: compiled` mark functions; the compiler lowers them + through the typed IR (`typedpython/ir.py`) to C against the CPython C API and builds an extension with + the platform C compiler. Compiled functions return what CPython returns (IR contract: checked i64 + with deopt to the interpreted function, Python floor division/modulo, exact int/float comparison, + CPython's exception types and messages; tracebacks are the one accepted difference). `Status: planned`. +- **N-9** TypedPython memory safety is proved on the IR before C is generated (#41; maintainer decision + 2026-10-03). Each property, and what happens when it cannot be proved: + + | Property proved on the IR | If not proved | + |---|---| + | No local read before assignment on every path | function not compiled; diagnostic names the local | + | Every node's operand types match; no implicit conversion | not compiled; diagnostic | + | No deopting node in an impure function | not compiled; diagnostic | + | Array access in bounds, or a runtime bounds check kept | the check stays (never an unchecked access) | + | Array parameters only indexed, stored, `len`-ed; no aliasing (entry guard) | not compiled; diagnostic | + | OBJ references follow one ownership rule; no IR form for free, borrowed alias or NULL read | not compiled; diagnostic | + | Integer overflow checked and promoted (deopt); an op marked `proven` must re-prove its interval fits i64 | not compiled; diagnostic `verify/proven-overflow` | + | `Call.redo` only from an impure caller to a pure, may-deopt callee returning F64/BOOL/NONE | not compiled; diagnostic `verify/redo` | + | `entry_globals` only in a closed function (no user code runs); a call of a function with entry globals is a deopt point in its caller unless it is a legal redo or the caller snapshots the same globals | not compiled; diagnostic `verify/entry-global-open` / `verify/deopt-impure` | + + Generated C reaches memory and `PyObject`s only through the runtime helpers. ASan/UBSan CI, + differential tests against the interpreter and fuzzing back the proof. `Status: planned`. +- **N-10** TypedPython incremental compilation (#41; maintainer decision 2026-10-03). The unit is a module. + Its cache key is the source hash, the interface hashes (typed signatures) of the modules it depends + on, the compiler version, the target platform and the build flags; a body-only change leaves the + interface hash unchanged, so dependent modules are not recompiled. C objects are cached per module. + The Gradle task declares exact inputs and outputs so up-to-date checks and the build cache apply. + Tests: changing one function body recompiles that module only; an unchanged rebuild spends ~0 s in + the compile step. `Status: planned`. --- diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index 5ccb4044d..de89dacfe 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -61,7 +61,7 @@ TypedPython 의 목표는 **새 언어를 만들지 않고, 파이썬 문법 그 |---|---|---|---| | 0 | 서드파티 패키지 (의존성) | 없음 — 검사하지 않는다 | 바이트코드 (pypackpack 이 받은 그대로) | | 1 `checked` | **사용자 프로젝트의 파이썬 코드 전부 — 기본값** | 빌드 타임 타입 오류, `Any` 유출 0 (§4.2) | 바이트코드 | -| 1 `compiled` | `@compiled` 또는 모듈 선언 | `checked` + §3.1 금지 목록 | C 확장 모듈 | +| 1 `compiled` | `@compiled` (내장 이름, import 없음) 또는 모듈 선언 `# typedpython: compiled` | `checked` + §3.1 금지 목록 | C 확장 모듈 | | 2 `kernel` | `@kernel` | 객체·힙 할당 없음, 고정 dtype | SPIR-V / MSL / WGSL / PTX | - **사용자 코드는 표시하지 않아도 검사된다.** 타입 오류가 있으면 Kotlin 컴파일 에러처럼 빌드가 @@ -125,22 +125,49 @@ API 안정성을 확인한 뒤로 미룬다. **이 게이트만으로도 독립적인 가치가 있다.** 컴파일러가 없어도 "사용자 코드에는 `Any` 가 없다" 는 보장이 생기고, 그것이 AI 친화적 피드백 루프의 핵심이다. 이후 모든 단계가 이 게이트를 전제로 한다. -### 4.3 백엔드 — 단계별 +### 4.3 백엔드 — 자체 typed IR → C (결정 2026-10-03, #41) -| 단계 | 경로 | 이유 | -|---|---|---| -| 1 | 추론 결과 → Cython pure-Python 모드 애노테이션 → C | 성숙한 백엔드. pypackpack 의 빈 Cython 슬롯(`native`/`mixed` 빌드 레벨)에 그대로 들어간다 | -| 2 | 자체 typed IR → C / LLVM | Cython 의 의미론 제약에서 벗어남. 등급 2 와 IR 을 공유할 여지 | +**Cython 은 쓰지 않는다** (사용자: "Cython 의존 별로야. 2단계로 바로 진행하도록 해"). Cython 경로 +(PR #39)는 닫았고, 거기서 검증된 의미 보존 규칙만 IR 계약으로 옮겼다. -Cython 은 착지용 디딤돌이지 종착지가 아니다. + 소스 ─ Pyrefly 추론 ─▶ frontend ─▶ typed IR ─▶ verify ─▶ cgen ─▶ C ─▶ 플랫폼 C 컴파일러 ─▶ 확장 모듈 + (ir.py) (증명) (CPython C API + tp_runtime.h) + +- **IR 이 계약이다** (`typedpython/ir.py`). 각 노드는 CPython 동작으로 정의된다: 검사되는 i64 산술과 + 오버플로 시 deopt(인터프리터 재실행), 파이썬식 floor 나눗셈·나머지, 정확한 int/float 비교, 리스트식 + 인덱싱, `math` 모듈의 예외. C 런타임 API 는 `typedpython/runtime/API.md` 에 고정했다. +- **왜 C 인가, LLVM 이 아니라.** 모든 타깃에 C 컴파일러가 이미 있다(clang, NDK, Xcode) — 새 의존이 없다. + CPython C API 가 C 이고, 플랫폼 디버거가 그대로 쓰인다. 타입이 확정된 스칼라·배열 코드에서 중요한 + 최적화(인라이닝, 벡터화)는 C 컴파일러가 이미 한다. IR 이 안정 계층이라 LLVM 백엔드는 나중에도 붙일 수 있다. +- **왜 Rust 가 아닌가.** 파이썬의 별칭과 공유 변경은 Rust 소유권에 맞지 않아 생성 코드가 + `Rc`/`unsafe` 로 채워지고, 앱 빌드마다 Rust 툴체인 비용이 붙는다(사용자 확정). +- **모듈 모양.** 확장 모듈은 원래 소스를 자기 네임스페이스에서 먼저 실행하고, 컴파일된 함수만 C 함수로 + 바꾼다. 원래 함수는 `__typedpython_interpreted__` 에 남아 deopt 때 같은 전역·같은 클래스로 실행된다. +- **순서.** 스칼라 int/float/bool 과 컴파일된 함수끼리의 호출 → `list[float]`/`list[int]` 의 네이티브 + 배열 → 고정 레이아웃 클래스. + +### 4.3.1 안전성은 IR 에서 증명한다 (SPEC N-9) + +사용자 결정: "IR 단계에서 안전성을 증명하도록 해". C 를 만들기 전에 `verify.py` 가 IR 에서 증명한다 — +정의 전 사용 없음, 노드 타입 일치, 비순수 함수에 deopt 노드 없음, 배열 접근은 경계 증명 또는 런타임 검사 +유지, 배열 파라미터는 인덱싱·저장·len 만, OBJ 참조는 하나의 소유 규칙. **증명하지 못한 함수는 컴파일하지 +않고** 인터프리터에 남기며 이유를 진단으로 낸다. 생성 C 는 `tp_runtime.h` 헬퍼로만 메모리와 `PyObject` +를 다룬다. 증명 밖의 안전망: ASan/UBSan, 인터프리터와의 차등 테스트, IR 수준 퍼징. + +### 4.3.2 증분 컴파일 (SPEC N-10) + +사용자 결정: "증분 컴파일도 지원하도록 해. 빌드 속도 너무 오래 안걸리게". 단위는 모듈. 키는 소스 해시 + +의존 모듈의 **인터페이스** 해시(타입 시그니처) + 컴파일러 버전 + 플랫폼 + 빌드 플래그. 본문만 바뀌면 +인터페이스 해시가 그대로라 의존 모듈은 다시 컴파일하지 않는다. C 오브젝트는 모듈별로 캐시한다. +Gradle 태스크는 입력·출력을 정확히 선언해 up-to-date 판정과 빌드 캐시가 함께 작동한다. 검토하고 기각한 경로: +- **Cython** — 사용자 결정으로 폐기(위). - **mypyc** — 타입 체커가 mypy 에 묶여 있어 Pyrefly 와 판정이 갈라진다. -- **Nuitka** — 동작 동일성을 우선해 타입 특수화를 하지 않는다. 등급 0 모듈의 전체 컴파일 폴백으로는 유효. -- **LPython** — ASR 설계는 좋지만 업스트림 활동이 LFortran 쪽으로 옮겨간 것으로 보인다. -- **Rust / Kotlin 으로 번역** — 생성기가 이미 아는 사실을 borrow checker 나 다른 런타임에 다시 - 증명해야 하고, 파이썬 의미론(임의 정밀도 정수, 슬라이싱, 예외)을 그쪽에 재구현해야 한다. +- **Nuitka** — 동작 동일성을 우선해 타입 특수화를 하지 않는다. +- **LPython** — 업스트림 활동이 LFortran 쪽으로 옮겨간 것으로 보인다. +- **Rust 생성** — 위. ### 4.4 Kotlin 경계의 직접 호출 @@ -205,7 +232,7 @@ TypedPython 은 그래서 자체 텐서 런타임을 만들지 않고 **torch op |---|---|---|---| | 1 | 건전성 게이트 | Pyrefly 출력을 받아 `Any` 유출을 에러로, 금지 목록 위반을 경고/에러로 내는 도구 | 없음 | | 1b | 빌드 기본 단계로 연결 | PythonMultiplatform Gradle 플러그인이 사용자 파이썬 소스에 게이트를 항상 돌리고, 에러면 빌드 실패 | 1 | -| 2 | Cython 백엔드 | `@compiled` 모듈 → C 확장, 데스크톱 타깃에서 끝까지 | pypackpack Cython 슬롯 인터페이스 결정 | +| 2 | typed IR → C 백엔드 (#41) | 프론트엔드·검증기·C 생성·런타임, 데스크톱에서 끝까지 | 없음 (pypackpack 슬롯은 3 의 의존) | | 3 | Kotlin 직접 호출 | 스텁 타입 기반 C ABI 호출 생성 | 스텁 배선 완료 | | 4 | 커널 하나 | 융합 softmax 를 Vulkan · Metal 양쪽에서 upstream 과 수치 일치 | torchnative 착지 승인 | | 5 | 커널 컴파일러 일반화 | SIMT 서브셋 전체, 이후 타일 | 4 | @@ -213,10 +240,35 @@ TypedPython 은 그래서 자체 텐서 런타임을 만들지 않고 **torch op 각 단계는 AGENTS.md §5·§14 의 TDD 와 측정 테스트 규정을 따른다. 특히 3 은 경계 비용 감소를, 4 는 upstream 대비 수치 오차를 기준선으로 박아 둔다. +### 6.1 단계 1 의 현재 상태 (2026-10-03, #16) + +- **위치:** `python-multiplatform-ksp/src/main/python/typedpython/`, 테스트는 + `python-multiplatform-ksp/src/test/python/`. 사용자 지시("ksp랑 통합")에 따라 KSP 모듈 안에 둔다. + KSP 프로세서는 Kotlin 심볼만 보고 파이썬 파일 변경으로는 다시 돌지 않으므로, **실행**은 1b 에서 + Gradle 태스크가 맡는다 — KSP 모듈은 소스와 배포(jar 리소스)의 자리다. +- **구현된 규칙:** SPEC N-7. Pyrefly 의 `Any` 유출 kind 6개를 에러로 올리고, Pyrefly 의 + per-expression 타입 리포트(`--report-pysa`)로 사용자 코드 안의 `Any` 식을 찾는다(`any-flow`). + `cast()` 인자, 버려지는 식 문장의 값, 함수 시그니처 자체는 제외한다. +- **재할당·혼합 컨테이너 (#22):** Pyrefly 1.3.2 는 둘 다 보고하지 않고, 타입 리포트에 대입 대상의 + 타입이 없다. 대신 이름을 *읽는* 위치의 타입은 기록한다. 그래서 대입문 바로 뒤에 같은 줄로 + `; x` 를 붙인 사본을 만들어(줄 번호 불변) 한 번 더 돌리고, 그 위치의 타입으로 판정한다. 두 번 + 이상 대입되는 이름이나 컨테이너 디스플레이가 있는 파일에서만 돈다. `for` 대상과 컴프리헨션 + 변수는 아직 탐침하지 않는다. +- **측정:** 1800줄 합성 모듈에서 plain `pyrefly check` 160–235 ms, 게이트 1.3–1.9 s — 두 번 잰 비율이 + 모두 약 8배다. 탐침 패스를 별도 실행으로 붙이면 약 15배(232 ms → 3.6 s, load 41)였다 — 합성 모듈의 + `total += ...` 가 탐침 대상이라 두 번째 실행이 돌았기 때문이다. 지금은 탐침 대상이 있으면 + **탐침 사본(과 같은 검사의 나머지 파일)에 대해 Pyrefly 를 한 번만** 돌리고, 에러·`any-flow`· + 타입을 모두 그 한 번에서 읽는다(에러 열은 삽입 길이만큼 왼쪽으로 되돌린다). 같은 합성 모듈에서 + 825 ms → 6.2 s, 약 7.6배(load 30 — 유효한 측정이 아니다. load 4 초과). 단독 측정이 아니다(두 번째는 load average 18.6). 절대값보다 비율을 기준으로 본다. 차이의 + 대부분은 타입 리포트가 typeshed 사본까지 디스크에 쓰는 비용으로 보인다(추정, 미분해). 1b 에서 + 빌드마다 돌리기 전에 줄일 대상이다. +- **고정:** Pyrefly `==1.3.2`. 타입 리포트는 Pyrefly 가 버전을 매기는 포맷이지 공개 API 가 아니다. + ## 7. 위험 - **Pyrefly 의 외부 API 안정성.** 라이브러리 임베드는 확인 전까지 미룬다 (§4.1). -- **Cython pure 모드의 표현 한계.** 게이트가 통과시킨 코드를 Cython 이 거부하는 경우가 나올 수 있다. +- **IR 이 표현하지 못하는 코드.** 게이트를 통과한 코드라도 IR 밖의 구문이 있으면 그 함수는 컴파일되지 + 않고 인터프리터에 남는다(`Module.skipped`). 그 목록이 IR 확장의 요구사항이 된다. 그 목록이 2단계 IR 의 요구사항이 된다. - **등급 2 컴파일러의 규모.** 가장 큰 작업이다. 커널 하나를 끝까지 하는 것(§6 의 4)을 첫 검증 단위로 삼아 범위를 제한한다. @@ -230,9 +282,16 @@ upstream 대비 수치 오차를 기준선으로 박아 둔다. ## 8. 미결 사항 (사용자 결정) -1. `compiled` 의 표기 — 모듈 선언 방식, 데코레이터 이름. -2. 고정폭 정수의 오버플로 의미. -3. pypackpack Cython 백엔드 슬롯의 인터페이스를 누가 정하는가. +1. ~~`compiled` 의 표기~~ — **결정됨 (2026-10-03, 메인테이너):** `compiled` 는 **내장 이름**이다. 패키지 + 아래에 두지 않고(`@typedpython.compiled` 아님) import 없이 `@compiled` 로 쓴다. 런타임은 임베디드 + 인터프리터 시작 시 `builtins.compiled` 를 설치하고(#42), 일반 CPython 에서는 `typedpython` wheel 의 + `typedpython_builtins.pth` 가 `builtins` 에 없을 때만 항등 함수를 설치한다(런타임이 넣은 것을 덮어쓰지 + 않는다). 타입 검사기에는 `__builtins__.pyi` 로 알린다 — Pyrefly 1.3.2 는 검색 경로의 그 파일을 추가 + 내장으로 읽는다(실측). 모듈 단위 표기는 `# typedpython: compiled` 주석을 유지한다: `# pyright: strict` 와 같은 + 도구 pragma 형태이고, 실행 시 이름 조회가 필요 없어 일반 CPython 에서 부작용이 없으며, 데코레이터 없이 + 모듈 전체를 지정할 수 있다. +2. ~~고정폭 정수의 오버플로 의미~~ — **결정됨:** `int` 는 i64 로 컴파일하고 오버플로 시 인터프리터 재실행으로 큰 정수로 승격(의미 보존). +3. ~~pypackpack Cython 슬롯 인터페이스 소유~~ — **결정됨:** pypackpack 이 정한다. TypedPython 은 생성한 C 소스, 빌드 플래그, C 를 낸 모듈 목록을 넘긴다. 확장이 원본 소스를 품으므로 별도 `.py` 폴백은 없다(Cython 폐기 후 갱신). 4. GPU 착지를 torchnative 커스텀 op 로 할 것인가 (다른 레포 변경). 5. ~~TypedPython 을 별도 레포로 분리할 것인가~~ — **결정됨 (2026-10-02):** PythonMultiplatform 의 기본 동작. 검사는 기본, 컴파일은 opt-in. diff --git a/python-multiplatform-ksp/.gitignore b/python-multiplatform-ksp/.gitignore new file mode 100644 index 000000000..4dc423b60 --- /dev/null +++ b/python-multiplatform-ksp/.gitignore @@ -0,0 +1,5 @@ +.venv/ +__pycache__/ +.pytest_cache/ +*.egg-info/ +.pytest_tmp/ diff --git a/python-multiplatform-ksp/pyproject.toml b/python-multiplatform-ksp/pyproject.toml new file mode 100644 index 000000000..460631868 --- /dev/null +++ b/python-multiplatform-ksp/pyproject.toml @@ -0,0 +1,33 @@ +# TypedPython soundness gate (docs/design/typedpython.md §4.2). +# The Kotlin processor in this module is built by Gradle; this file only describes the Python +# package that lives next to it in src/main/python. +[project] +name = "typedpython" +version = "0.1.0" +requires-python = ">=3.13" +# Pinned exactly: the gate reads Pyrefly's per-expression type report (--report-pysa), whose +# format is versioned by Pyrefly, not by a public API. +dependencies = ["pyrefly==1.3.2"] + +[project.scripts] +typedpython = "typedpython.__main__:cli" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/main/python/typedpython"] + +# `compiled` is a builtin name (design §2.3): this .pth installs it into `builtins` on plain CPython. +[tool.hatch.build.targets.wheel.force-include] +"src/main/python/typedpython_builtins.pth" = "typedpython_builtins.pth" + +[project.optional-dependencies] +test = ["pytest"] + +[tool.pytest.ini_options] +testpaths = ["src/test/python"] +pythonpath = ["src/main/python"] +addopts = "--basetemp=.pytest_tmp" +markers = ["measurement: records cost; asserts only sanity bounds"] diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__init__.py b/python-multiplatform-ksp/src/main/python/typedpython/__init__.py new file mode 100644 index 000000000..507809dab --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/__init__.py @@ -0,0 +1,4 @@ +"""TypedPython — static checking for the Python code of a PythonMultiplatform project. + +See docs/design/typedpython.md. This package is the soundness gate (§4.2). +""" diff --git a/python-multiplatform-ksp/src/main/python/typedpython/__main__.py b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py new file mode 100644 index 000000000..6534afebf --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/__main__.py @@ -0,0 +1,155 @@ +"""`python -m typedpython check [--mode checked|compiled] [--format text|json] PATHS...` + +Exit status: 0 no errors (warnings allowed), 1 errors, 2 the gate itself could not run. +""" +import argparse +import json +import sys +from pathlib import Path + +from typedpython import gate +from typedpython.paths import python_files +from typedpython.pyrefly import PyreflyError + +EXIT_CLEAN, EXIT_ERRORS, EXIT_FAILURE = 0, 1, 2 + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(prog="typedpython") + commands = parser.add_subparsers(dest="command", required=True) + check = commands.add_parser("check", help="type-check user Python code") + check.add_argument("paths", nargs="+", type=Path, help="files, or directories to check recursively") + check.add_argument("--mode", choices=["checked", "compiled"], default="checked") + check.add_argument("--format", choices=["text", "json"], default="text") + check.add_argument("--search-path", action="append", type=Path, default=[], + help="directory to resolve imports from (repeatable)") + demo = commands.add_parser( + "demo", help="compile a module, run its main() interpreted and compiled, and compare", + ) + demo.add_argument("module", type=Path) + demo.add_argument("argv", nargs="*", help="sys.argv[1:] for the module's main()") + demo.add_argument("--out", type=Path, default=Path("build/typedpython"), + help="where the extension is built (default: build/typedpython)") + build = commands.add_parser("build", help="compile modules incrementally (cached per module)") + build.add_argument("paths", nargs="+", type=Path, help="files, or directories to build recursively") + build.add_argument("--cache", type=Path, default=Path("build/typedpython-cache"), + help="incremental cache directory (default: build/typedpython-cache)") + build.add_argument("--out", type=Path, default=None, + help="copy the built extensions here (default: leave them in the cache)") + args = parser.parse_args(argv) + + if args.command == "build": + return _build(args.paths, args.cache, args.out) + if args.command == "demo": + return _demo(args.module, args.argv, args.out) + + try: + found = gate.check(python_files(args.paths), mode=args.mode, search_paths=args.search_path) + except PyreflyError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_FAILURE + + if args.format == "json": + print(json.dumps([d.to_json() for d in found], indent=2)) + else: + for d in found: + print(f"{d.severity.upper()} {d.path}:{d.line}:{d.column}: {d.message} [{d.rule}]") + + return EXIT_ERRORS if any(d.severity == "error" for d in found) else EXIT_CLEAN + + +def _build(paths: list[Path], cache: Path, out: Path | None) -> int: + import os + import shutil + + from typedpython import pipeline + + sources = python_files(paths) + if not sources: + print("typedpython: no Python files to build", file=sys.stderr) + return EXIT_FAILURE + dirs = [p.resolve() for p in paths if p.is_dir()] or [p.resolve().parent for p in paths] + root = Path(os.path.commonpath([str(d) for d in dirs])) + try: + result = pipeline.compile_project(sources, root, cache) + except pipeline.CompileError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_ERRORS + except PyreflyError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_FAILURE + for m in result.modules: + why = f" ({', '.join(m.reasons)})" if m.reasons else "" + what = m.extension.name if m.extension else "no extension (interpreted)" + print(f"{m.status:7} {m.name}: {what}{why}") + for name, reason in sorted(m.skipped.items()): + print(f" interpreted: {name} — {reason}") + if out is not None and m.extension is not None: + out.mkdir(parents=True, exist_ok=True) + shutil.copy2(m.extension, out / m.extension.name) + return EXIT_CLEAN + + +def _demo(module: Path, module_argv: list[str], out: Path) -> int: + import contextlib + import importlib.util + import io + import time + + from typedpython import pipeline + + try: + result = pipeline.compile_module(module, out / module.stem) + except pipeline.CompileError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_ERRORS + except PyreflyError as e: + print(f"typedpython: {e}", file=sys.stderr) + return EXIT_FAILURE + + def interpreted() -> object: + spec = importlib.util.spec_from_file_location(module.stem + "_interpreted", module) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + def run(load: object) -> tuple[str, float]: + mod = load() # type: ignore[operator] + saved = sys.argv + sys.argv = [str(module), *module_argv] + captured = io.StringIO() + try: + with contextlib.redirect_stdout(captured): + start = time.perf_counter() + mod.main() + elapsed = time.perf_counter() - start + finally: + sys.argv = saved + return captured.getvalue(), elapsed + + compiled_names = [f.name for f in result.module.functions] + print(f"compiled functions: {', '.join(compiled_names) or '(none)'}") + for name, reason in sorted(result.skipped.items()): + print(f" interpreted: {name} — {reason}") + out_i, t_i = run(interpreted) + out_c, t_c = run(lambda: pipeline.load(result)) + print(f"interpreted: {t_i * 1000:.1f} ms compiled: {t_c * 1000:.1f} ms " + f"speedup {t_i / t_c:.1f}x") + if out_i != out_c: + print("output differs:") + print(f" interpreted: {out_i!r}") + print(f" compiled: {out_c!r}") + return EXIT_ERRORS + print("identical output:") + print(out_c, end="") + return EXIT_CLEAN + + +def cli() -> None: + """Console-script entry point (`typedpython check ...`).""" + sys.exit(main(sys.argv[1:])) + + +if __name__ == "__main__": + cli() diff --git a/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi b/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi new file mode 100644 index 000000000..f773368b9 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/builtin_stubs/__builtins__.pyi @@ -0,0 +1,3 @@ +# TypedPython's builtin names (docs/design/typedpython.md §2.3). A `__builtins__.pyi` on the type +# checker's search path declares names the interpreter provides without an import. +def compiled[T](f: T, /) -> T: ... diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py b/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py new file mode 100644 index 000000000..3872a4bbe --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/cbuild.py @@ -0,0 +1,84 @@ +"""Build a generated C file (cgen) into a CPython extension with the platform C compiler. + +No setuptools, no Cython: one compiler invocation, `-shared -fPIC`, on macOS +`-undefined dynamic_lookup` (the extension resolves the C API from the process that loads it). +The headers and the extension suffix default to the host interpreter's (`sysconfig`), and can be +pointed at the CPython the app embeds (`python_include`, `ext_suffix`). +""" +from __future__ import annotations + +import importlib.machinery +import importlib.util +import shlex +import subprocess +import sys +import sysconfig +from pathlib import Path + +RUNTIME_DIR = Path(__file__).resolve().parent / "runtime" + +# -ffp-contract=off: a fused multiply-add rounds once where CPython rounds twice. +DEFAULT_FLAGS: tuple[str, ...] = ( + "-O2", + "-ffp-contract=off", + "-fno-fast-math", + "-Werror=implicit-function-declaration", +) + + +class CBuildError(RuntimeError): + """The C compiler rejected the generated source; the message carries its stderr.""" + + +def _python_include(python_include: Path | None) -> Path: + if python_include is None: + return Path(sysconfig.get_paths()["include"]) + p = Path(python_include) + if (p / "Python.h").is_file(): + return p + found = sorted(d for d in p.glob("python3*") if (d / "Python.h").is_file()) + if len(found) == 1: + return found[0] + raise CBuildError(f"no Python.h in {p} (nor in exactly one python3.*/ below it: {found})") + + +def compiler() -> list[str]: + return shlex.split(sysconfig.get_config_var("CC") or "cc") + + +def build(c_source: str, module_name: str, out_dir: Path, runtime_dir: Path | None = None, + flags: list[str] | tuple[str, ...] = DEFAULT_FLAGS, python_include: Path | None = None, + ext_suffix: str | None = None) -> Path: + """Compile `c_source` to `/`; return it.""" + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + stem = module_name.rpartition(".")[2] + suffix = ext_suffix if ext_suffix is not None else sysconfig.get_config_var("EXT_SUFFIX") + if not suffix: + raise CBuildError("no extension suffix (sysconfig EXT_SUFFIX is empty); pass ext_suffix") + runtime = Path(runtime_dir) if runtime_dir is not None else RUNTIME_DIR + if not (runtime / "tp_runtime.h").is_file(): + raise CBuildError(f"tp_runtime.h not found in {runtime}") + include = _python_include(python_include) + c_file = out_dir / f"{stem}.c" + c_file.write_text(c_source, encoding="utf-8") + so = out_dir / f"{stem}{suffix}" + cmd = compiler() + list(flags) + ["-fPIC", "-shared", f"-I{include}", f"-I{runtime}"] + if sys.platform == "darwin": + cmd += ["-undefined", "dynamic_lookup"] + cmd += [str(c_file), "-o", str(so)] + run = subprocess.run(cmd, capture_output=True, text=True) + if run.returncode != 0: + raise CBuildError(f"C compiler failed ({run.returncode}): {shlex.join(cmd)}\n{run.stderr}") + return so + + +def load(module_name: str, path: Path): + """Import the built extension at `path` as `module_name` (not registered in sys.modules).""" + loader = importlib.machinery.ExtensionFileLoader(module_name, str(path)) + spec = importlib.util.spec_from_file_location(module_name, str(path), loader=loader) + if spec is None: + raise ImportError(f"cannot make a spec for {path}") + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module diff --git a/python-multiplatform-ksp/src/main/python/typedpython/cgen.py b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py new file mode 100644 index 000000000..f843bcac9 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/cgen.py @@ -0,0 +1,1332 @@ +"""TypedPython C back end: `ir.Module` -> one C source file (a CPython extension, PEP 489). + +The contract is `ir.py`; this file implements each node exactly as its docstring says, and reaches +anything with Python semantics only through the runtime helpers of `runtime/API.md` +(`tp_runtime.h`). What the generated file contains: + +Module shape (ir.py "Module shape"). Multi-phase init. The exec slot runs the module's ORIGINAL +source (read at build time and embedded as a C string literal, compiled with its own path as the +filename) in the extension module's own dict, so every name, class and global is the interpreted +one. It then stores each compiled function's interpreted original in `__typedpython_interpreted__` +(a dict), sets `__typedpython_deopts__ = 0`, and replaces the global `f` with a C function +(METH_FASTCALL | METH_KEYWORDS) bound to the module. + +Per function `f`: + `static int tp_impl_f(PyObject *tp_module, , *tp_out)` returns 0 ok, 1 deopt, + -1 error. Scalars are passed by value, OBJ params borrowed (the impl takes its own reference), + array params as `tp_*_array *`. + `tp_wrap_f` is the Python-visible function: wrong arity or any keyword -> the interpreted + function (CPython raises its own TypeError); entry guards (exact scalar types through + tp_unbox_*, the array aliasing guard tp_any_same, then tp_*_array_enter); the impl; then + tp_*_array_exit on every path (ok, error, deopt) before returning or redoing. A deopt (failed + guard, or the impl's 1 in a pure function) increments `__typedpython_deopts__` and calls + `__typedpython_interpreted__[f]` with the original arguments. + +Ownership rule (ir.py "Safety"), implemented by the `_own_*` emit helpers below and nowhere else: + 1. every OBJ local — including each OBJ parameter, which the impl copies with a new reference at + entry — owns one strong reference, or is NULL; + 2. every OBJ-typed expression evaluates into a fresh temporary that owns a new reference; + temporaries are NULL at every statement boundary; + 3. a temporary is consumed exactly once: moved into a local (the local's old reference is + released), moved into `*tp_out` by Return, or released (`tp_release`) right after the helper + that borrowed it returns; + 4. every exit of the impl — return, error, deopt — goes through one exit block that releases + every OBJ local and temporary. + The generated code never decrements a reference except through `tp_release` (API.md); the only + increment is `Py_NewRef` when an OBJ local is read or an OBJ parameter is adopted. + +Local arrays (ir.NewArray / CopyArray, `Function` docstring "Local arrays"). A local of an array type +is a `tp_i64_array` / `tp_f64_array` struct held by value in the impl, zero-initialised at entry +(`{0}`), with `list == NULL` and `dirty == NULL`. It is owned by the function: + 1. `Assign(local, NewArray | CopyArray)` builds the new array in a zeroed temporary struct + (`tp_*_array_new` / `_iota` / `_copy`), and only when that succeeded frees the old array + (`tp_*_array_free`) and moves the temporary into the slot — so a failed allocation leaves the + old binding intact, as `a = [0] * n` does, and `a = a[:]` copies before it frees; + 2. every exit of the impl — return, error, deopt — frees every local array and every array + temporary in the same exit block that releases the OBJ locals (a deopt of a pure function + frees them before the wrapper redoes the call interpreted); + 3. local arrays never go through `tp_*_array_enter/exit` (there is no list to write back to) and + a store into one sets no dirty bit. +`Index` / `StoreIndex` / `Len` use the same slot helper and `proven` handling as array params. +`Tuple` evaluates each element into an owned temporary, builds the tuple with `tp_tuple` (which +steals nothing), then releases the temporaries. + +Generated arithmetic is in separate C statements and the file sets `#pragma STDC FP_CONTRACT OFF` +(cbuild also passes -ffp-contract=off): a fused multiply-add would round differently from CPython. +""" +from __future__ import annotations + +from pathlib import Path + +from typedpython import ir +from typedpython.ir import Type + + +class CGenError(Exception): + """The IR cannot be lowered as specified (a contract violation by the producer of the IR).""" + + +_CT = {Type.I64: "int64_t", Type.F64: "double", Type.BOOL: "int", Type.OBJ: "PyObject *"} +_ARRAY_CT = {Type.F64_ARRAY: "tp_f64_array", Type.I64_ARRAY: "tp_i64_array"} +_CT_ARRAY_REV = {"tp_f64_array": Type.F64_ARRAY, "tp_i64_array": Type.I64_ARRAY} +_ARRAY_ELEM = {Type.F64_ARRAY: Type.F64, Type.I64_ARRAY: Type.I64} +_ARRAY_PFX = {Type.F64_ARRAY: "tp_f64_array", Type.I64_ARRAY: "tp_i64_array"} +_UNBOX = {Type.I64: "tp_unbox_i64", Type.F64: "tp_unbox_f64", Type.BOOL: "tp_unbox_bool"} +_BOX = {Type.I64: "tp_box_i64", Type.F64: "tp_box_f64", Type.BOOL: "tp_box_bool"} +_PYCMP = {ir.CompareKind.EQ: "Py_EQ", ir.CompareKind.NE: "Py_NE", ir.CompareKind.LT: "Py_LT", + ir.CompareKind.LE: "Py_LE", ir.CompareKind.GT: "Py_GT", ir.CompareKind.GE: "Py_GE"} +_I64_OP = {ir.BinOpKind.ADD: "tp_add_i64", ir.BinOpKind.SUB: "tp_sub_i64", + ir.BinOpKind.MUL: "tp_mul_i64", ir.BinOpKind.FLOORDIV: "tp_floordiv_i64", + ir.BinOpKind.MOD: "tp_mod_i64", ir.BinOpKind.TRUEDIV: "tp_truediv_i64"} +_F64_OP = {ir.BinOpKind.TRUEDIV: "tp_truediv_f64", ir.BinOpKind.FLOORDIV: "tp_floordiv_f64", + ir.BinOpKind.MOD: "tp_mod_f64"} +_F64_INLINE = {ir.BinOpKind.ADD: "+", ir.BinOpKind.SUB: "-", ir.BinOpKind.MUL: "*"} +_OBJ_OP = {k: f"TP_BINOP_{k.name}" for k in ir.BinOpKind} +_MATH = {ir.MathFunc.SQRT: 1, ir.MathFunc.EXP: 1, ir.MathFunc.LOG: 1, ir.MathFunc.SIN: 1, + ir.MathFunc.COS: 1, ir.MathFunc.TAN: 1, ir.MathFunc.FABS: 1, ir.MathFunc.ATAN2: 2, + ir.MathFunc.HYPOT: 2} +# I64 operations whose helper can return 1 (API.md): MOD never overflows (MIN % -1 == 0). +_I64_DEOPT = {ir.BinOpKind.ADD, ir.BinOpKind.SUB, ir.BinOpKind.MUL, ir.BinOpKind.FLOORDIV, + ir.BinOpKind.TRUEDIV} +_INT64_MIN, _INT64_MAX = -(2 ** 63), 2 ** 63 - 1 + + +# --- C literals and identifiers ------------------------------------------------------------------ + +def c_string(data: str | bytes) -> str: + """A C string literal for `data` (UTF-8). Octal escapes only (hex escapes are greedy), `?` + escaped against trigraphs; split after each newline so long sources stay readable.""" + raw = data.encode("utf-8") if isinstance(data, str) else data + out, pieces = [], [] + for b in raw: + ch = chr(b) + if ch in '\\"?': + out.append("\\" + ch) + elif 0x20 <= b < 0x7F: + out.append(ch) + else: + out.append("\\%03o" % b) + if b == 0x0A: + pieces.append('"' + "".join(out) + '"') + out = [] + if out or not pieces: + pieces.append('"' + "".join(out) + '"') + return "\n ".join(pieces) + + +def _ident(prefix: str, name: str) -> str: + """A C identifier for a Python name: ASCII names keep their spelling, others are hex-encoded + under a distinct prefix so the two forms cannot collide.""" + if name.isascii() and name.isidentifier(): + return f"{prefix}_{name}" + return f"{prefix}x_{name.encode('utf-8').hex()}" + + +def _i64_lit(v: int) -> str: + if isinstance(v, bool) or not isinstance(v, int) or not (_INT64_MIN <= v <= _INT64_MAX): + raise CGenError(f"I64 constant out of range or not an int: {v!r}") + if v == _INT64_MIN: + return "(-INT64_C(9223372036854775807) - 1)" + return f"INT64_C({v})" + + +def _f64_lit(v: float) -> str: + v = float(v) + if v != v: + return "(-Py_NAN)" if _sign_bit(v) else "Py_NAN" + if v in (float("inf"), float("-inf")): + return "Py_HUGE_VAL" if v > 0 else "(-Py_HUGE_VAL)" + return f"({v.hex()})" + + +def _sign_bit(v: float) -> bool: + import struct + return struct.pack(">d", v)[0] >> 7 == 1 + + +# --- IR walks ------------------------------------------------------------------------------------ + +def _exprs_of_stmt(s: ir.Stmt): + if isinstance(s, ir.Assign): + yield s.value + elif isinstance(s, ir.StoreIndex): + yield s.value + yield s.index + elif isinstance(s, ir.ExprStmt): + yield s.value + elif isinstance(s, ir.If): + yield s.cond + elif isinstance(s, ir.While): + yield s.cond + elif isinstance(s, ir.ForRange): + yield s.start + yield s.stop + elif isinstance(s, ir.Return) and s.value is not None: + yield s.value + + +def _children(s: ir.Stmt): + if isinstance(s, ir.If): + return s.then + s.orelse + if isinstance(s, (ir.While, ir.ForRange)): + return s.body + return () + + +def _walk_stmts(body): + for s in body: + yield s + yield from _walk_stmts(_children(s)) + + +def _sub_exprs(e: ir.Expr): + for name in ("left", "right", "operand", "index", "obj", "callee", "length", "fill"): + v = getattr(e, name, None) + if isinstance(v, ir.Expr): + yield v + for v in getattr(e, "args", ()) or (): + yield v + for v in getattr(e, "elements", ()) or (): + yield v + + +def _walk_exprs(e: ir.Expr): + yield e + for c in _sub_exprs(e): + yield from _walk_exprs(c) + + +def _all_exprs(f: ir.Function): + for s in _walk_stmts(f.body): + for e in _exprs_of_stmt(s): + yield from _walk_exprs(e) + + +def _can_deopt(e: ir.Expr, functions: dict[str, ir.Function]) -> bool: + if isinstance(e, ir.BinOp) and e.left.type is Type.I64 and e.op in _I64_DEOPT: + return not e.proven # a proven op cannot deopt (overflow = SystemError) + if isinstance(e, ir.UnaryOp) and e.op is ir.UnaryOpKind.NEG and e.operand.type is Type.I64: + return not e.proven + if isinstance(e, ir.Unbox): + return True + if isinstance(e, ir.Call): + if e.redo: + return False # the callee's deopt is absorbed by its interpreted redo + callee = functions.get(e.function) + # a callee's entry-global guard runs at the call (`_FnGen.ev_Call`) and can fail there + return callee is not None and (callee.may_deopt or bool(callee.entry_globals)) + return False + + +def _is_effect(e: ir.Expr) -> bool: + if isinstance(e, (ir.GetAttr, ir.CallObject, ir.Truth, ir.CompareObj, ir.ObjToFloat)): + return True + return isinstance(e, ir.BinOp) and e.type is Type.OBJ + + +# --- module -------------------------------------------------------------------------------------- + +class _ModGen: + def __init__(self, module: ir.Module): + self.module = module + self.functions = {f.name: f for f in module.functions} + if len(self.functions) != len(module.functions): + raise CGenError("duplicate compiled function names") + self.names: list[str] = [] + self.kwtuples: list[tuple[str, ...]] = [] + + def name_index(self, s: str) -> int: + if s not in self.names: + self.names.append(s) + return self.names.index(s) + + def kw_index(self, kw: tuple[str, ...]) -> int: + for k in kw: + self.name_index(k) + if kw not in self.kwtuples: + self.kwtuples.append(kw) + return self.kwtuples.index(kw) + + +def generate(module: ir.Module, source_path: Path) -> str: + """One C file implementing `module`; `source_path` is the original Python source, embedded.""" + source_path = Path(source_path) + source = source_path.read_text(encoding="utf-8") + mg = _ModGen(module) + for f in module.functions: + _check_function(f, mg.functions) + for f in module.functions: + mg.name_index(f.name) + bodies = [_FnGen(mg, f).generate() for f in module.functions] + init_name = module.name.rpartition(".")[2] + if not (init_name.isascii() and init_name.isidentifier()): + raise CGenError(f"module name {module.name!r} cannot name a PyInit_ function") + + out: list[str] = [] + w = out.append + w(f"/* Generated by typedpython.cgen from {source_path.name} — do not edit. */") + w("#define PY_SSIZE_T_CLEAN") + w("#include ") + w("#include ") + w('#include "tp_runtime.h"') + w("") + w("#if PY_VERSION_HEX < 0x030D0000") + w('#error "typedpython extensions need CPython 3.13 or newer (PyDict_GetItemRef)"') + w("#endif") + w("#pragma STDC FP_CONTRACT OFF") + w("") + w(f"static const char tp_source[] =\n {c_string(source)};") + w(f"static const char tp_source_path[] = {c_string(str(source_path))};") + w("") + n_names, n_kw = len(mg.names), len(mg.kwtuples) + w("/* Module state: interned names (globals, attributes, keywords, function names) and the") + w(" kwnames tuples of keyword calls. Owned by the module; released in tp_clear. */") + w("typedef struct {") + w(f" PyObject *names[{n_names + 1}];") + w(f" PyObject *kwnames[{n_kw + 1}];") + w("} tp_state;") + w("") + w("static const char *const tp_name_strings[] = {") + for s in mg.names: + w(f" {c_string(s)},") + w(" NULL") + w("};") + w("") + for f in module.functions: + w(_impl_signature(f) + ";") + w("") + w(_TEMPLATE_REDO) + for b in bodies: + w(b) + # method table + w("static PyMethodDef tp_methods[] = {") + for f in module.functions: + w(f" {{{c_string(f.name)}, (PyCFunction)(void (*)(void)){_ident('tp_wrap', f.name)}, " + f"METH_FASTCALL | METH_KEYWORDS, NULL}},") + w(" {NULL, NULL, 0, NULL}") + w("};") + w("") + w(_exec_function(mg)) + w(_TEMPLATE_STATE) + w("static PyModuleDef_Slot tp_slots[] = {") + w(" {Py_mod_exec, (void *)tp_exec},") + w(" {0, NULL}") + w("};") + w("") + w("static struct PyModuleDef tp_moduledef = {") + w(f" PyModuleDef_HEAD_INIT, {c_string(module.name)}, NULL, sizeof(tp_state), NULL, tp_slots,") + w(" tp_traverse, tp_clear, tp_free") + w("};") + w("") + w(f"PyMODINIT_FUNC PyInit_{init_name}(void) {{ return PyModuleDef_Init(&tp_moduledef); }}") + return "\n".join(out) + "\n" + + +def _check_function(f: ir.Function, functions: dict[str, ir.Function]) -> None: + deopting = [e for e in _all_exprs(f) if _can_deopt(e, functions)] + if deopting and not f.pure: + raise CGenError(f"{f.name}: a node that can deopt ({type(deopting[0]).__name__}) in an " + "impure function (ir.py: only a pure function may deopt after entry)") + if deopting and not f.may_deopt: + raise CGenError(f"{f.name}: may_deopt is False but a node can deopt " + f"({type(deopting[0]).__name__})") + effects = [e for e in _all_exprs(f) if _is_effect(e)] + # a store into a local array is not an effect (ir.py "Local arrays"); into a parameter it is + stores = [s for s in _walk_stmts(f.body) + if isinstance(s, ir.StoreIndex) and f.locals.get(s.array) not in ir.ARRAYS] + if f.pure and (effects or stores): + what = type(effects[0]).__name__ if effects else "StoreIndex" + raise CGenError(f"{f.name}: marked pure but contains an effect ({what})") + for e in _all_exprs(f): + if isinstance(e, (ir.BinOp, ir.UnaryOp)) and e.proven: + checked = (e.op in (ir.BinOpKind.ADD, ir.BinOpKind.SUB, ir.BinOpKind.MUL) + if isinstance(e, ir.BinOp) else e.op is ir.UnaryOpKind.NEG) + operand = e.left if isinstance(e, ir.BinOp) else e.operand + if not checked or operand.type is not Type.I64: + raise CGenError(f"{f.name}: proven is set on {e.op.name} of {operand.type} " + "(ir.py: I64 ADD/SUB/MUL and NEG only)") + if isinstance(e, ir.Call): + callee = functions.get(e.function) + if callee is None: + raise CGenError(f"{f.name}: Call of {e.function!r}, which is not compiled here") + if any(isinstance(p, ir.ArrayParam) for p in callee.params): + raise CGenError(f"{f.name}: Call of {e.function!r}, which takes array params") + if e.redo: + if callee.returns not in (Type.F64, Type.BOOL, Type.NONE): + raise CGenError(f"{f.name}: redo Call of {e.function!r}, which returns " + f"{callee.returns} (ir.py: F64, BOOL or NONE only)") + if not callee.pure: + raise CGenError(f"{f.name}: redo Call of impure {e.function!r} " + "(redoing it could repeat an effect)") + elif (callee.may_deopt or callee.entry_globals) and not f.pure: + raise CGenError(f"{f.name}: Call of deopting {e.function!r} from an impure caller " + "without redo") + if f.entry_globals: + _check_entry_globals(f, functions) + for p in f.params: + if isinstance(p, ir.ArrayParam): + if p.type not in ir.ARRAYS: + raise CGenError(f"{f.name}: array param {p.name} has type {p.type}") + if p.stored and f.pure: + raise CGenError(f"{f.name}: array param {p.name} is stored into in a pure function") + elif p.type not in _CT: + raise CGenError(f"{f.name}: param {p.name} has type {p.type}") + + +def _is_closed(f: ir.Function, functions: dict[str, ir.Function], seen=None) -> bool: + """ir.Function.entry_globals: runs no user code — no effect node, Calls only to closed + functions — so nothing can rebind a module global while it runs.""" + seen = set() if seen is None else seen + if f.name in seen: + return True + seen.add(f.name) + for e in _all_exprs(f): + if _is_effect(e): + return False + if isinstance(e, ir.Call): + callee = functions.get(e.function) + if callee is None or not _is_closed(callee, functions, seen): + return False + return True + + +def _check_entry_globals(f: ir.Function, functions: dict[str, ir.Function]) -> None: + names = [g.name for g in f.entry_globals] + if len(set(names)) != len(names): + raise CGenError(f"{f.name}: entry_globals lists a name twice") + taken = {p.name for p in f.params} | set(f.locals) + for g in f.entry_globals: + if not isinstance(g, ir.Param) or g.type not in ir.SCALARS: + raise CGenError(f"{f.name}: entry global {g.name} must be a scalar Param, not {g.type}") + if g.name in taken: + raise CGenError(f"{f.name}: entry global {g.name} is also a parameter or local") + for s in _walk_stmts(f.body): + if isinstance(s, ir.Assign) and s.target in names: + raise CGenError(f"{f.name}: assignment to entry global {s.target}") + if isinstance(s, ir.ForRange) and s.var in names: + raise CGenError(f"{f.name}: entry global {s.var} used as a range variable") + if not _is_closed(f, functions): + raise CGenError(f"{f.name}: entry_globals in a function that is not closed (it can run " + "user code, which could rebind the global between entry and use)") + + +def _impl_signature(f: ir.Function) -> str: + params = ["PyObject *tp_module"] + for p in f.params: + if isinstance(p, ir.ArrayParam): + params.append(f"{_ARRAY_CT[p.type]} *{_ident('l', p.name)}") + elif p.type is Type.OBJ: + params.append(f"PyObject *{_ident('p', p.name)}") + else: + params.append(f"{_CT[p.type]} {_ident('l', p.name)}") + for g in f.entry_globals: # read and unboxed by the caller (wrapper or Call) + params.append(f"{_CT[g.type]} {_ident('l', g.name)}") + if f.returns is not Type.NONE: + ct = _CT[f.returns] + params.append(f"{ct}{'' if ct.endswith('*') else ' '}*tp_out") + return f"static int {_ident('tp_impl', f.name)}({', '.join(params)})" + + +# --- function bodies ----------------------------------------------------------------------------- + +class _FnGen: + def __init__(self, mg: _ModGen, f: ir.Function): + self.mg, self.f = mg, f + self.lines: list[str] = [] + self.depth = 1 + self.temps: list[tuple[str, str]] = [] # (C type, name) + self.arrays = {p.name: p for p in f.params if isinstance(p, ir.ArrayParam)} + self.local_arrays: dict[str, Type] = {} # name -> F64_ARRAY / I64_ARRAY + self.types: dict[str, Type] = {} + for p in f.params: + if not isinstance(p, ir.ArrayParam): + self.types[p.name] = p.type + for g in f.entry_globals: # the body reads each as Local(name) + self.types[g.name] = g.type + for name, t in f.locals.items(): + if name in self.arrays or (name in self.types and self.types[name] is not t): + raise CGenError(f"{f.name}: local {name} conflicts with a parameter") + if t in ir.ARRAYS: + if name in self.types: + raise CGenError(f"{f.name}: local array {name} conflicts with a parameter") + self.local_arrays[name] = t + continue + if t not in _CT: + raise CGenError(f"{f.name}: local {name} has type {t}") + self.types[name] = t + + # emit primitives + + def emit(self, line: str) -> None: + self.lines.append(" " * self.depth + line) + + def tmp(self, ctype: str) -> str: + name = f"tp_t{len(self.temps)}" + self.temps.append((ctype, name)) + return name + + def check(self, call: str) -> None: + """A helper with the 0 / 1 deopt / -1 error convention.""" + self.emit(f"tp_s = {call};") + self.emit("if (tp_s != 0) { tp_rc = tp_s; goto tp_exit; }") + + def check_proven(self, call: str) -> None: + """A checked I64 helper on a `proven` op (ir.BinOp.proven): it can only return 0 or 1, and + 1 means the front end's and the verifier's proof was wrong — an internal error, never a + deopt (a proven op may sit after an effect).""" + self.emit(f"tp_s = {call};") + self.emit("if (tp_s != 0) { PyErr_SetString(PyExc_SystemError, " + "\"typedpython: proven operation overflowed\"); tp_rc = -1; goto tp_exit; }") + + def fail(self) -> None: + self.emit("{ tp_rc = -1; goto tp_exit; }") + + # ownership helpers — the only places that create or drop references (see module docstring) + + def _own_new(self, call: str) -> str: + """A fresh temporary owning the new reference `call` returns (NULL -> error exit).""" + t = self.tmp("PyObject *") + self.emit(f"{t} = {call};") + self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def _own_copy(self, local: str) -> str: + """A fresh temporary owning a new reference to what an OBJ local holds.""" + t = self.tmp("PyObject *") + self.emit(f"{t} = Py_NewRef({local});") + return t + + def _own_move(self, dst: str, t: str) -> None: + """Move temporary `t` into local `dst`, releasing the reference `dst` held.""" + self.emit(f"{{ PyObject *tp_old = {dst}; {dst} = {t}; {t} = NULL; tp_release(&tp_old); }}") + + def _own_move_out(self, t: str) -> None: + """Move temporary `t` into the caller's `*tp_out` (Return of OBJ).""" + self.emit(f"*tp_out = {t}; {t} = NULL;") + + def _own_release(self, *ts: str) -> None: + for t in ts: + self.emit(f"tp_release(&{t});") + + def _own_release_all(self) -> list[str]: + objs = [_ident("l", n) for n, t in self.types.items() if t is Type.OBJ] + objs += [n for ct, n in self.temps if ct == "PyObject *"] + out = [f" tp_release(&{n});" for n in objs] + # local arrays and array temporaries: freed on every exit (ok, error, deopt); idempotent + arrs = [(_ident("l", n), t) for n, t in self.local_arrays.items()] + arrs += [(n, _CT_ARRAY_REV[ct]) for ct, n in self.temps if ct in _CT_ARRAY_REV] + out += [f" {_ARRAY_PFX[t]}_free(&{n});" for n, t in arrs] + return out + + # expressions: return a C expression naming the value (a literal, a scalar local, or a temp); + # OBJ results are always owned temporaries. + + def ev(self, e: ir.Expr) -> str: + m = getattr(self, "ev_" + type(e).__name__, None) + if m is None: + raise CGenError(f"{self.f.name}: no lowering for {type(e).__name__}") + return m(e) + + def ev_Const(self, e: ir.Const) -> str: + if e.type is Type.BOOL: + return "1" if e.value else "0" + if e.type is Type.I64: + return _i64_lit(e.value) + if e.type is Type.F64: + return _f64_lit(e.value) + if e.type is Type.OBJ: + if e.value is None: + return self._own_copy("Py_None") + if isinstance(e.value, bool): + return self._own_copy("Py_True" if e.value else "Py_False") + raise CGenError(f"{self.f.name}: OBJ constant {e.value!r} (use Box of a scalar Const)") + raise CGenError(f"{self.f.name}: constant of type {e.type}") + + def ev_Local(self, e: ir.Local) -> str: + t = self.types.get(e.name) + if t is None: + raise CGenError(f"{self.f.name}: unknown local {e.name}") + if t is not e.type: + raise CGenError(f"{self.f.name}: local {e.name} is {t}, read as {e.type}") + if t is Type.OBJ: + return self._own_copy(_ident("l", e.name)) + return _ident("l", e.name) + + def ev_BinOp(self, e: ir.BinOp) -> str: + lt, rt = e.left.type, e.right.type + if lt is not rt: + raise CGenError(f"{self.f.name}: BinOp operands {lt} and {rt}") + a, b = self.ev(e.left), self.ev(e.right) + if lt is Type.I64: + t = self.tmp("double" if e.op is ir.BinOpKind.TRUEDIV else "int64_t") + if e.proven: + self.check_proven(f"{_I64_OP[e.op]}({a}, {b}, &{t})") + else: + self.check(f"{_I64_OP[e.op]}({a}, {b}, &{t})") + return t + if lt is Type.F64: + t = self.tmp("double") + if e.op in _F64_INLINE: + self.emit(f"{t} = {a} {_F64_INLINE[e.op]} {b};") + else: + self.check(f"{_F64_OP[e.op]}({a}, {b}, &{t})") + return t + if lt is Type.OBJ: + t = self.tmp("PyObject *") + self.emit(f"{t} = tp_binop_obj({a}, {b}, {_OBJ_OP[e.op]});") + self._own_release(a, b) + self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + return t + raise CGenError(f"{self.f.name}: BinOp on {lt}") + + def ev_UnaryOp(self, e: ir.UnaryOp) -> str: + ot = e.operand.type + v = self.ev(e.operand) + if e.op is ir.UnaryOpKind.NOT: + if ot is not Type.BOOL: + raise CGenError(f"{self.f.name}: NOT of {ot}") + t = self.tmp("int") + self.emit(f"{t} = !({v});") + return t + if ot is Type.I64 and e.op is ir.UnaryOpKind.NEG: + t = self.tmp("int64_t") + if e.proven: + self.check_proven(f"tp_neg_i64({v}, &{t})") + else: + self.check(f"tp_neg_i64({v}, &{t})") + return t + if ot is Type.F64 and e.op is ir.UnaryOpKind.NEG: + t = self.tmp("double") + self.emit(f"{t} = -({v});") + return t + if ot in (Type.I64, Type.F64) and e.op is ir.UnaryOpKind.POS: + t = self.tmp(_CT[ot]) + self.emit(f"{t} = {v};") + return t + raise CGenError(f"{self.f.name}: {e.op} of {ot}") + + def ev_Compare(self, e: ir.Compare) -> str: + lt, rt = e.left.type, e.right.type + a, b = self.ev(e.left), self.ev(e.right) + t = self.tmp("int") + if lt is rt and lt in ir.SCALARS: + self.emit(f"{t} = ({a}) {e.op.value} ({b});") + elif (lt, rt) == (Type.I64, Type.F64): + self.emit(f"{t} = tp_cmp_i64_f64({a}, {b}, {_PYCMP[e.op]});") + elif (lt, rt) == (Type.F64, Type.I64): + self.emit(f"{t} = tp_cmp_f64_i64({a}, {b}, {_PYCMP[e.op]});") + else: + raise CGenError(f"{self.f.name}: Compare of {lt} and {rt} (OBJ uses CompareObj)") + return t + + def ev_CompareObj(self, e: ir.CompareObj) -> str: + if e.left.type is not Type.OBJ or e.right.type is not Type.OBJ: + raise CGenError(f"{self.f.name}: CompareObj operands must be OBJ (Box scalars)") + a, b = self.ev(e.left), self.ev(e.right) + t = self.tmp("int") + self.emit(f"{t} = tp_compare_bool({a}, {b}, {_PYCMP[e.op]});") + self._own_release(a, b) + self.emit(f"if ({t} < 0) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def _short_circuit(self, e, take_right_if: str) -> str: + if e.left.type is not Type.BOOL or e.right.type is not Type.BOOL: + raise CGenError(f"{self.f.name}: And/Or on non-BOOL operands") + t = self.tmp("int") + a = self.ev(e.left) + self.emit(f"{t} = {a};") + self.emit(f"if ({take_right_if}{t}) {{") + self.depth += 1 + b = self.ev(e.right) + self.emit(f"{t} = {b};") + self.depth -= 1 + self.emit("}") + return t + + def ev_And(self, e: ir.And) -> str: + return self._short_circuit(e, "") + + def ev_Or(self, e: ir.Or) -> str: + return self._short_circuit(e, "!") + + def ev_Truth(self, e: ir.Truth) -> str: + if e.operand.type is not Type.OBJ: + raise CGenError(f"{self.f.name}: Truth of {e.operand.type}") + o = self.ev(e.operand) + t = self.tmp("int") + self.emit(f"{t} = tp_truth({o});") + self._own_release(o) + self.emit(f"if ({t} < 0) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def ev_ToFloat(self, e: ir.ToFloat) -> str: + if e.operand.type is not Type.I64: + raise CGenError(f"{self.f.name}: ToFloat of {e.operand.type}") + v = self.ev(e.operand) + t = self.tmp("double") + self.emit(f"{t} = tp_i64_to_f64({v});") + return t + + def ev_ObjToFloat(self, e: ir.ObjToFloat) -> str: + if e.operand.type is not Type.OBJ: + raise CGenError(f"{self.f.name}: ObjToFloat of {e.operand.type}") + o = self.ev(e.operand) + t = self.tmp("double") + self.emit(f"tp_s = tp_obj_to_f64({o}, &{t});") + self._own_release(o) + self.emit("if (tp_s != 0) { tp_rc = -1; goto tp_exit; }") + return t + + def ev_Box(self, e: ir.Box) -> str: + ot = e.operand.type + if ot not in _BOX: + raise CGenError(f"{self.f.name}: Box of {ot}") + v = self.ev(e.operand) + return self._own_new(f"{_BOX[ot]}({v})") + + def ev_Unbox(self, e: ir.Unbox) -> str: + if e.operand.type is not Type.OBJ or e.type not in _UNBOX: + raise CGenError(f"{self.f.name}: Unbox {e.operand.type} -> {e.type}") + o = self.ev(e.operand) + t = self.tmp(_CT[e.type]) + self.emit(f"tp_s = {_UNBOX[e.type]}({o}, &{t});") + self._own_release(o) + self.emit("if (tp_s != 0) { tp_rc = tp_s; goto tp_exit; }") + return t + + def ev_MathCall(self, e: ir.MathCall) -> str: + if len(e.args) != _MATH[e.func] or any(a.type is not Type.F64 for a in e.args): + raise CGenError(f"{self.f.name}: math.{e.func.value} arguments") + args = [self.ev(a) for a in e.args] + t = self.tmp("double") + self.check(f"tp_math_{e.func.value}({', '.join(args)}, &{t})") + return t + + def ev_Call(self, e: ir.Call) -> str | None: + callee = self.mg.functions[e.function] + if len(e.args) != len(callee.params): + raise CGenError(f"{self.f.name}: Call of {e.function} with {len(e.args)} args") + args = [] + owned = [] + for p, a in zip(callee.params, e.args): + if a.type is not p.type: + raise CGenError(f"{self.f.name}: Call {e.function} arg {p.name}: {a.type} for {p.type}") + v = self.ev(a) + args.append(v) + if a.type is Type.OBJ: + owned.append(v) + values = list(args) # the evaluated arguments, for a redo + t = None + self.emit("tp_s = 0;") + # The callee's entry globals (ir.Function.entry_globals) are its entry guards: read them + # here, at the callee's entry. A mismatch is the callee's deopt (tp_s = 1), handled below + # exactly as a deopt inside the callee. + for g in callee.entry_globals: + args.append(self._read_entry_global(g, "tp_s = 1;")) + if callee.returns is not Type.NONE: + t = self.tmp(_CT[callee.returns]) + args.append(f"&{t}") + self.emit("if (tp_s == 0) {") + self.depth += 1 + # C-to-C recursion has no Python frame: guard the C stack as CPython guards its own. + self.emit('if (Py_EnterRecursiveCall(" in compiled code")) { tp_rc = -1; goto tp_exit; }') + self.emit(f"tp_s = {_ident('tp_impl', e.function)}(tp_module{''.join(', ' + x for x in args)});") + self.emit("Py_LeaveRecursiveCall();") + self.depth -= 1 + self.emit("}") + if e.redo: + self._redo_call(callee, e.args, values, t) + self._own_release(*owned) + self.emit("if (tp_s != 0) { tp_rc = tp_s; goto tp_exit; }") + return t + + def _read_entry_global(self, g: ir.Param, on_mismatch: str, on_error: str = "tp_rc = -1; goto tp_exit;", + guard: str = "tp_s == 0") -> str: + """Emit the entry read of module global `g` (tp_global, exact-type unbox, release) guarded + by `tp_s == 0`; on a wrong type or an unbound name run `on_mismatch` (a deopt: the + interpreted function then sees the same binding and raises CPython's own NameError where + CPython would). Returns the C scalar holding the value.""" + k = self.mg.name_index(g.name) + gv, go = self.tmp(_CT[g.type]), self.tmp("PyObject *") + self.emit(f"if ({guard}) {{") + self.emit(f" {go} = tp_global(tp_dict, tp_st->names[{k}]);") + self.emit(f" if ({go} == NULL) {{") + self.emit(f" if (!PyErr_ExceptionMatches(PyExc_NameError)) {{ {on_error} }}") + self.emit(f" PyErr_Clear(); {on_mismatch}") + self.emit(" } else {") + self.emit(f" tp_s = {_UNBOX[g.type]}({go}, &{gv});") + self.emit(f" tp_release(&{go});") + self.emit(f" if (tp_s < 0) {{ {on_error} }}") + self.emit(f" if (tp_s != 0) {{ {on_mismatch} }}") + self.emit(" }") + self.emit("}") + return gv + + def _redo_call(self, callee: ir.Function, arg_exprs, values: list[str], t: str | None) -> None: + """ir.Call.redo: on the callee's deopt (tp_s == 1) run `__typedpython_interpreted__[callee]` + on the already-evaluated arguments (boxed), count the deopt, convert the result back, and + continue in the caller. The callee is pure, so the redo is unobservable.""" + k = self.mg.name_index(callee.name) + self.emit("if (tp_s == 1) {") + self.depth += 1 + boxes = [] + av = [] + for a, v in zip(arg_exprs, values): + if a.type is Type.OBJ: + av.append(v) # still owned by the caller; released after + else: + b = self._own_new(f"{_BOX[a.type]}({v})") + boxes.append(b) + av.append(b) + rr = self.tmp("PyObject *") + if av: + self.emit(f"{{ PyObject *tp_av[{len(av)}] = {{{', '.join(av)}}};") + self.emit(f" {rr} = tp_cg_redo(tp_module, {k}, tp_av, {len(av)}, NULL); }}") + else: + self.emit(f"{rr} = tp_cg_redo(tp_module, {k}, NULL, 0, NULL);") + self._own_release(*boxes) + self.emit(f"if ({rr} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + rt = callee.returns + if rt is Type.NONE: + self.emit(f"tp_s = ({rr} == Py_None) ? 0 : 1;") + what = "None" + else: + self.emit(f"tp_s = {_UNBOX[rt]}({rr}, &{t});") + what = {Type.F64: "a float", Type.BOOL: "True or False"}[rt] + self._own_release(rr) + msg = c_string(f"typedpython: the interpreted redo of {callee.name} did not return {what}") + self.emit(f"if (tp_s != 0) {{ PyErr_SetString(PyExc_SystemError, {msg}); tp_rc = -1; goto tp_exit; }}") + self.depth -= 1 + self.emit("}") + + def ev_Global(self, e: ir.Global) -> str: + k = self.mg.name_index(e.name) + return self._own_new(f"tp_global(tp_dict, tp_st->names[{k}])") + + def ev_GetAttr(self, e: ir.GetAttr) -> str: + if e.obj.type is not Type.OBJ: + raise CGenError(f"{self.f.name}: GetAttr on {e.obj.type}") + o = self.ev(e.obj) + k = self.mg.name_index(e.name) + t = self.tmp("PyObject *") + self.emit(f"{t} = tp_getattr({o}, tp_st->names[{k}]);") + self._own_release(o) + self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def ev_CallObject(self, e: ir.CallObject) -> str: + if e.callee.type is not Type.OBJ or any(a.type is not Type.OBJ for a in e.args): + raise CGenError(f"{self.f.name}: CallObject operands must be OBJ") + if len(e.kwnames) > len(e.args) or len(set(e.kwnames)) != len(e.kwnames): + raise CGenError(f"{self.f.name}: CallObject kwnames {e.kwnames}") + callee = self.ev(e.callee) # CPython evaluates the callee first + args = [self.ev(a) for a in e.args] + npos = len(e.args) - len(e.kwnames) + kw = f"tp_st->kwnames[{self.mg.kw_index(tuple(e.kwnames))}]" if e.kwnames else "NULL" + t = self.tmp("PyObject *") + if args: + self.emit(f"{{ PyObject *tp_av[{len(args)}] = {{{', '.join(args)}}};") + self.emit(f" {t} = tp_call({callee}, tp_av, {npos}, {kw}); }}") + else: + self.emit(f"{t} = tp_call({callee}, NULL, 0, NULL);") + self._own_release(callee, *args) + self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def _array(self, name: str) -> tuple[Type, str]: + """(array type, C pointer expression) of an array parameter (already a pointer) or a local + array (a struct held by value, so its address).""" + a = self.arrays.get(name) + if a is not None: + return a.type, _ident("l", name) + t = self.local_arrays.get(name) + if t is None: + raise CGenError(f"{self.f.name}: {name} is not an array parameter or local array") + return t, f"(&{_ident('l', name)})" + + def ev_Len(self, e: ir.Len) -> str: + _, ptr = self._array(e.array) + t = self.tmp("int64_t") + self.emit(f"{t} = (int64_t){ptr}->len;") + return t + + def _slot(self, at: Type, ptr: str, index: ir.Expr, proven: bool, store: int) -> str: + if index.type is not Type.I64: + raise CGenError(f"{self.f.name}: index of type {index.type}") + i = self.ev(index) + if proven: + return f"(Py_ssize_t)({i})" + slot = self.tmp("Py_ssize_t") + self.check(f"{_ARRAY_PFX[at]}_slot({ptr}, {i}, {store}, &{slot})") + return slot + + def ev_Index(self, e: ir.Index) -> str: + at, ptr = self._array(e.array) + if e.type is not _ARRAY_ELEM[at]: + raise CGenError(f"{self.f.name}: Index of {at} typed {e.type}") + slot = self._slot(at, ptr, e.index, e.proven, 0) + t = self.tmp(_CT[e.type]) + self.emit(f"{t} = {ptr}->data[{slot}];") + return t + + def ev_NewArray(self, e: ir.NewArray) -> str: + raise CGenError(f"{self.f.name}: NewArray is legal only as the value of an Assign to a " + "local array (ir.py)") + + def ev_CopyArray(self, e: ir.CopyArray) -> str: + raise CGenError(f"{self.f.name}: CopyArray is legal only as the value of an Assign to a " + "local array (ir.py)") + + def ev_Tuple(self, e: ir.Tuple) -> str: + if e.type is not Type.OBJ or any(x.type is not Type.OBJ for x in e.elements): + raise CGenError(f"{self.f.name}: Tuple elements must be OBJ (Box scalars)") + items = [self.ev(x) for x in e.elements] # owned temporaries, evaluated in order + t = self.tmp("PyObject *") + if items: + self.emit(f"{{ PyObject *tp_av[{len(items)}] = {{{', '.join(items)}}};") + self.emit(f" {t} = tp_tuple(tp_av, {len(items)}); }}") + else: + self.emit(f"{t} = tp_tuple(NULL, 0);") + self._own_release(*items) # tp_tuple stole nothing + self.emit(f"if ({t} == NULL) {{ tp_rc = -1; goto tp_exit; }}") + return t + + def _assign_array(self, s: ir.Assign, at: Type) -> None: + """`local = NewArray | CopyArray`: build into a zeroed temporary, then (only on success) + free the old array and move the temporary in.""" + v = s.value + pfx = _ARRAY_PFX[at] + if v.type is not at: + raise CGenError(f"{self.f.name}: {s.target} is {at}, assigned {v.type}") + dst = _ident("l", s.target) + new = self.tmp(_ARRAY_CT[at]) + if isinstance(v, ir.NewArray): + if v.length.type is not Type.I64: + raise CGenError(f"{self.f.name}: NewArray length of type {v.length.type}") + if v.iota: + if at is not Type.I64_ARRAY or v.fill is not None: + raise CGenError(f"{self.f.name}: iota NewArray must be I64_ARRAY with no fill") + n = self.ev(v.length) + self.check(f"{pfx}_iota(&{new}, {n})") + else: + if v.fill is None or v.fill.type is not _ARRAY_ELEM[at]: + raise CGenError(f"{self.f.name}: NewArray fill must be a " + f"{_ARRAY_ELEM[at]} (got {None if v.fill is None else v.fill.type})") + fill = self.ev(v.fill) # `[fill] * n`: fill first, then n + n = self.ev(v.length) + self.check(f"{pfx}_new(&{new}, {n}, {fill})") + elif isinstance(v, ir.CopyArray): + st, sptr = self._array(v.src) + if st is not at: + raise CGenError(f"{self.f.name}: CopyArray of {st} into {at}") + self.check(f"{pfx}_copy(&{new}, {sptr})") + else: + raise CGenError(f"{self.f.name}: local array {s.target} may only be assigned a " + f"NewArray or CopyArray, not {type(v).__name__}") + self.emit(f"{pfx}_free(&{dst});") + self.emit(f"{dst} = {new}; {new} = ({_ARRAY_CT[at]}){{0}};") + + # statements + + def stmts(self, body) -> None: + for s in body: + m = getattr(self, "st_" + type(s).__name__, None) + if m is None: + raise CGenError(f"{self.f.name}: no lowering for {type(s).__name__}") + m(s) + + def st_Assign(self, s: ir.Assign) -> None: + if s.target in self.local_arrays: + self._assign_array(s, self.local_arrays[s.target]) + return + t = self.types.get(s.target) + if t is None: + raise CGenError(f"{self.f.name}: assignment to unknown local {s.target}") + if s.value.type is not t: + raise CGenError(f"{self.f.name}: {s.target} is {t}, assigned {s.value.type}") + v = self.ev(s.value) + if t is Type.OBJ: + self._own_move(_ident("l", s.target), v) + else: + self.emit(f"{_ident('l', s.target)} = {v};") + + def st_StoreIndex(self, s: ir.StoreIndex) -> None: + at, ptr = self._array(s.array) + local = s.array in self.local_arrays + if not local and not self.arrays[s.array].stored: + raise CGenError(f"{self.f.name}: store into {s.array}, declared stored=False") + if s.value.type is not _ARRAY_ELEM[at]: + raise CGenError(f"{self.f.name}: store of {s.value.type} into {at}") + v = self.ev(s.value) # CPython: value, then container, then index + slot = self._slot(at, ptr, s.index, s.proven, 1) + self.emit(f"{ptr}->data[{slot}] = {v};") + if not local: # a local array has no list to write back to + self.emit(f"{ptr}->dirty[{slot}] = 1;") + + def st_ExprStmt(self, s: ir.ExprStmt) -> None: + v = self.ev(s.value) + if v is None: + return + if s.value.type is Type.OBJ: + self._own_release(v) + else: + self.emit(f"(void){v};") + + def _cond(self, e: ir.Expr) -> str: + if e.type is not Type.BOOL: + raise CGenError(f"{self.f.name}: condition of type {e.type} (use Truth/CompareObj)") + return self.ev(e) + + def st_If(self, s: ir.If) -> None: + c = self._cond(s.cond) + self.emit(f"if ({c}) {{") + self.depth += 1 + self.stmts(s.then) + self.depth -= 1 + if s.orelse: + self.emit("} else {") + self.depth += 1 + self.stmts(s.orelse) + self.depth -= 1 + self.emit("}") + + def st_While(self, s: ir.While) -> None: + self.emit("for (;;) {") + self.depth += 1 + c = self._cond(s.cond) + self.emit(f"if (!({c})) break;") + self.stmts(s.body) + self.depth -= 1 + self.emit("}") + + def st_ForRange(self, s: ir.ForRange) -> None: + if self.types.get(s.var) is not Type.I64: + raise CGenError(f"{self.f.name}: range variable {s.var} must be an I64 local") + if not isinstance(s.step, ir.Const) or s.step.type is not Type.I64 or s.step.value == 0: + raise CGenError(f"{self.f.name}: range step must be a non-zero I64 Const") + if s.start.type is not Type.I64 or s.stop.type is not Type.I64: + raise CGenError(f"{self.f.name}: range bounds must be I64") + step = s.step.value + _i64_lit(step) + ts, te = self.tmp("int64_t"), self.tmp("int64_t") + a = self.ev(s.start) + self.emit(f"{ts} = {a};") # range() evaluates its arguments once + b = self.ev(s.stop) + self.emit(f"{te} = {b};") + tn, tk = self.tmp("uint64_t"), self.tmp("uint64_t") + mag = f"UINT64_C({abs(step)})" + var = _ident("l", s.var) + # Iteration count computed in unsigned arithmetic, as range's length: no overflow for any + # start/stop/step, and the variable stays between start and stop. + if step > 0: + self.emit(f"if ({ts} < {te}) {{") + self.emit(f" {tn} = ((uint64_t){te} - (uint64_t){ts} - 1u) / {mag} + 1u;") + sign = "+" + else: + self.emit(f"if ({ts} > {te}) {{") + self.emit(f" {tn} = ((uint64_t){ts} - (uint64_t){te} - 1u) / {mag} + 1u;") + sign = "-" + self.depth += 1 + self.emit(f"for ({tk} = 0; {tk} < {tn}; {tk}++) {{") + self.depth += 1 + self.emit(f"{var} = (int64_t)((uint64_t){ts} {sign} {tk} * {mag});") + self.stmts(s.body) + self.depth -= 1 + self.emit("}") + self.depth -= 1 + self.emit("}") + + def st_Return(self, s: ir.Return) -> None: + rt = self.f.returns + if rt is Type.NONE: + if s.value is not None and not (isinstance(s.value, ir.Const) and s.value.value is None): + raise CGenError(f"{self.f.name}: returns None but Return has a value") + self.emit("tp_rc = 0; goto tp_exit;") + return + if s.value is None or s.value.type is not rt: + raise CGenError(f"{self.f.name}: Return type mismatch (function returns {rt})") + v = self.ev(s.value) + if rt is Type.OBJ: + self._own_move_out(v) + else: + self.emit(f"*tp_out = {v};") + self.emit("tp_rc = 0; goto tp_exit;") + + def st_Break(self, s) -> None: + self.emit("break;") + + def st_Continue(self, s) -> None: + self.emit("continue;") + + # whole function + + def generate(self) -> str: + f = self.f + prologue = [] + for p in f.params: + if not isinstance(p, ir.ArrayParam) and p.type is Type.OBJ: + # adopt the borrowed argument: the local owns its own reference (rule 1) + prologue.append(f" {_ident('l', p.name)} = Py_NewRef({_ident('p', p.name)});") + self.stmts(f.body) + if f.returns is Type.NONE: + self.emit("tp_rc = 0;") + else: + self.emit(f'PyErr_SetString(PyExc_SystemError, {c_string("compiled function " + f.name + " ended without a return")});') + self.emit("tp_rc = -1;") + self.emit("goto tp_exit;") + + out = [_impl_signature(f), "{", + " int tp_rc = 0;", + " int tp_s = 0;", + " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);", + " PyObject *tp_dict = PyModule_GetDict(tp_module); /* borrowed: owned by the module */"] + params = {p.name for p in f.params} | {g.name for g in f.entry_globals} + for name, t in self.local_arrays.items(): + out.append(f" {_ARRAY_CT[t]} {_ident('l', name)} = {{0}};") + for name, t in self.types.items(): + if name in params and t is not Type.OBJ: + continue + init = "NULL" if t is Type.OBJ else "0" + ct = _CT[t] + out.append(f" {ct}{'' if ct.endswith('*') else ' '}{_ident('l', name)} = {init};") + for ct, n in self.temps: + init = "NULL" if ct == "PyObject *" else "{0}" if ct in _CT_ARRAY_REV else "0" + sep = "" if ct.endswith("*") else " " + out.append(f" {ct}{sep}{n} = {init};") + out.append(" (void)tp_st; (void)tp_dict; (void)tp_s;" + + "".join(f" (void){_ident('l', n)};" for n in self.types if n not in params) + + "".join(f" (void){_ident('l', n)};" for n in self.local_arrays)) + out += prologue + out += self.lines + out.append("tp_exit:") + out += self._own_release_all() + out.append(" return tp_rc;") + out.append("}") + out.append("") + out.append(self.wrapper()) + return "\n".join(out) + + def wrapper(self) -> str: + f = self.f + name_k = self.mg.name_index(f.name) + n = len(f.params) + L = [f"static PyObject *{_ident('tp_wrap', f.name)}(PyObject *tp_module, PyObject *const *tp_args, " + "Py_ssize_t tp_nargs, PyObject *tp_kwnames)", "{", + " int tp_s = 0, tp_x = 0;"] + if f.returns not in (Type.NONE,): + init = "NULL" if f.returns is Type.OBJ else "0" + L.append(f" {_CT[f.returns]} tp_r = {init};") + args = [] + arrays = [] + for i, p in enumerate(f.params): + if isinstance(p, ir.ArrayParam): + L.append(f" {_ARRAY_CT[p.type]} tp_a{i} = {{0}};") + args.append(f"&tp_a{i}") + arrays.append((i, p)) + elif p.type is Type.OBJ: + args.append(f"tp_args[{i}]") # borrowed for the call's duration + else: + L.append(f" {_CT[p.type]} tp_a{i} = 0;") + args.append(f"tp_a{i}") + if f.returns is not Type.NONE: + args.append("&tp_r") + exits = [f" if ({_ARRAY_PFX[p.type]}_exit(&tp_a{i}) < 0) tp_x = -1;" for i, p in arrays] + L.append(" (void)tp_x;") + # arity / keywords -> the interpreted function raises CPython's own TypeError + L.append(f" if (tp_nargs != {n} || (tp_kwnames != NULL && PyTuple_GET_SIZE(tp_kwnames) != 0)) goto tp_deopt;") + # scalar guards: exact type (and i64 range) + for i, p in enumerate(f.params): + if not isinstance(p, ir.ArrayParam) and p.type in _UNBOX: + L.append(f" tp_s = {_UNBOX[p.type]}(tp_args[{i}], &tp_a{i});") + L.append(" if (tp_s < 0) goto tp_error;") + L.append(" if (tp_s != 0) goto tp_deopt;") + # aliasing guard, then copy-in + if len(arrays) > 1: + objs = ", ".join(f"tp_args[{i}]" for i, _ in arrays) + L.append(f" {{ PyObject *const tp_objs[{len(arrays)}] = {{{objs}}};") + L.append(f" if (tp_any_same(tp_objs, {len(arrays)})) goto tp_deopt; }}") + for i, p in arrays: + L.append(f" tp_s = {_ARRAY_PFX[p.type]}_enter(tp_args[{i}], &tp_a{i});") + L.append(" if (tp_s < 0) goto tp_error;") + L.append(" if (tp_s != 0) goto tp_deopt;") + gnames = [] + if f.entry_globals: + L.insert(3, " PyObject *tp_dict = PyModule_GetDict(tp_module); /* borrowed */") + L.insert(3, " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);") + for j, g in enumerate(f.entry_globals): + # entry read of a module global: exact type or deopt (still before any effect) + k = self.mg.name_index(g.name) + L.append(f" {_CT[g.type]} tp_g{j} = 0;") + L.append(f" {{ PyObject *tp_go = tp_global(tp_dict, tp_st->names[{k}]);") + L.append(" if (tp_go == NULL) {") + L.append(" if (!PyErr_ExceptionMatches(PyExc_NameError)) goto tp_error;") + L.append(" PyErr_Clear(); goto tp_deopt; }") + L.append(f" tp_s = {_UNBOX[g.type]}(tp_go, &tp_g{j}); Py_DECREF(tp_go); }}") + L.append(" if (tp_s < 0) goto tp_error;") + L.append(" if (tp_s != 0) goto tp_deopt;") + gnames.append(f"tp_g{j}") + if gnames: + at = len(args) - (1 if f.returns is not Type.NONE else 0) + args[at:at] = gnames + L.append(f" tp_s = {_ident('tp_impl', f.name)}(tp_module{''.join(', ' + a for a in args)});") + if f.pure: + L.append(" if (tp_s == 1) goto tp_deopt;") + else: + # cgen proved there is no deopting node in an impure function; never redo after effects + L.append(" if (tp_s == 1) { PyErr_SetString(PyExc_SystemError, " + "\"typedpython: deopt after an effect\"); goto tp_error; }") + L.append(" if (tp_s < 0) goto tp_error;") + # ok: write back, then box + L += exits + if f.returns is Type.OBJ: + L.append(" if (tp_x < 0) { tp_release(&tp_r); return NULL; }") + L.append(" return tp_r;") + else: + L.append(" if (tp_x < 0) return NULL;") + if f.returns is Type.NONE: + L.append(" return Py_NewRef(Py_None);") + else: + L.append(f" return {_BOX[f.returns]}(tp_r);") + L.append("tp_error:") + if arrays: + # keep CPython's exception; write-back still runs (partial updates stay visible) + L.append(" { PyObject *tp_exc = PyErr_GetRaisedException();") + L += [" " + e for e in exits] + L.append(" if (tp_x < 0) { PyObject *tp_exc2 = PyErr_GetRaisedException();") + L.append(" PyException_SetContext(tp_exc2, tp_exc); /* steals tp_exc */") + L.append(" PyErr_SetRaisedException(tp_exc2); }") + L.append(" else PyErr_SetRaisedException(tp_exc); }") + L.append(" return NULL;") + L.append("tp_deopt:") + L += exits + L.append(" if (tp_x < 0) return NULL;") + L.append(f" return tp_cg_redo(tp_module, {name_k}, tp_args, tp_nargs, tp_kwnames);") + L.append("}") + L.append("") + return "\n".join(L) + + +# --- fixed parts of the module ------------------------------------------------------------------- + +_TEMPLATE_REDO = r''' +/* Deopt: count it, then call __typedpython_interpreted__[name] with the original arguments. */ +static PyObject *tp_cg_redo(PyObject *tp_module, int tp_name, PyObject *const *tp_args, + Py_ssize_t tp_nargs, PyObject *tp_kwnames) +{ + tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module); + PyObject *tp_dict = PyModule_GetDict(tp_module); /* borrowed: owned by the module */ + PyObject *tp_count = NULL, *tp_one = NULL, *tp_next = NULL, *tp_table = NULL, *tp_f = NULL; + PyObject *tp_result = NULL; + int tp_k; + tp_k = PyDict_GetItemStringRef(tp_dict, "__typedpython_deopts__", &tp_count); + if (tp_k == 0) PyErr_SetString(PyExc_RuntimeError, "typedpython: __typedpython_deopts__ is missing"); + if (tp_k <= 0) goto tp_done; + tp_one = PyLong_FromLong(1); + if (tp_one == NULL) goto tp_done; + tp_next = PyNumber_Add(tp_count, tp_one); + if (tp_next == NULL || PyDict_SetItemString(tp_dict, "__typedpython_deopts__", tp_next) < 0) goto tp_done; + tp_k = PyDict_GetItemStringRef(tp_dict, "__typedpython_interpreted__", &tp_table); + if (tp_k == 0) PyErr_SetString(PyExc_RuntimeError, "typedpython: __typedpython_interpreted__ is missing"); + if (tp_k <= 0) goto tp_done; + if (!PyDict_Check(tp_table)) { + PyErr_SetString(PyExc_TypeError, "typedpython: __typedpython_interpreted__ is not a dict"); + goto tp_done; + } + tp_k = PyDict_GetItemRef(tp_table, tp_st->names[tp_name], &tp_f); + if (tp_k == 0) PyErr_Format(PyExc_RuntimeError, "typedpython: no interpreted function %R", tp_st->names[tp_name]); + if (tp_k <= 0) goto tp_done; + tp_result = PyObject_Vectorcall(tp_f, tp_args, (size_t)tp_nargs, tp_kwnames); +tp_done: + tp_release(&tp_count); tp_release(&tp_one); tp_release(&tp_next); + tp_release(&tp_table); tp_release(&tp_f); + return tp_result; +} +''' + +_TEMPLATE_STATE = r''' +static int tp_traverse(PyObject *tp_module, visitproc visit, void *arg) +{ + tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module); + size_t i; + if (tp_st == NULL) return 0; + for (i = 0; i < sizeof(tp_st->names) / sizeof(tp_st->names[0]); i++) Py_VISIT(tp_st->names[i]); + for (i = 0; i < sizeof(tp_st->kwnames) / sizeof(tp_st->kwnames[0]); i++) Py_VISIT(tp_st->kwnames[i]); + return 0; +} + +static int tp_clear(PyObject *tp_module) +{ + tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module); + size_t i; + if (tp_st == NULL) return 0; + for (i = 0; i < sizeof(tp_st->names) / sizeof(tp_st->names[0]); i++) tp_release(&tp_st->names[i]); + for (i = 0; i < sizeof(tp_st->kwnames) / sizeof(tp_st->kwnames[0]); i++) tp_release(&tp_st->kwnames[i]); + return 0; +} + +static void tp_free(void *tp_module) { (void)tp_clear((PyObject *)tp_module); } +''' + + +def _exec_function(mg: _ModGen) -> str: + L = [ + "/* Exec slot: run the original source in this module's dict, then swap in the C functions. */", + "static int tp_exec(PyObject *tp_module)", + "{", + " tp_state *tp_st = (tp_state *)PyModule_GetState(tp_module);", + " PyObject *tp_dict = PyModule_GetDict(tp_module); /* borrowed: owned by the module */", + " PyObject *tp_code = NULL, *tp_res = NULL, *tp_table = NULL, *tp_zero = NULL;", + " PyObject *tp_modname = NULL, *tp_f = NULL, *tp_cf = NULL;", + " int tp_rc = -1, tp_k;", + " size_t i;", + " for (i = 0; tp_name_strings[i] != NULL; i++) {", + " tp_st->names[i] = PyUnicode_InternFromString(tp_name_strings[i]);", + " if (tp_st->names[i] == NULL) goto tp_done;", + " }", + ] + for k, kw in enumerate(mg.kwtuples): + items = ", ".join(f"tp_st->names[{mg.name_index(x)}]" for x in kw) + L.append(f" tp_st->kwnames[{k}] = PyTuple_Pack({len(kw)}, {items});") + L.append(f" if (tp_st->kwnames[{k}] == NULL) goto tp_done;") + L += [ + " tp_k = PyDict_ContainsString(tp_dict, \"__builtins__\");", + " if (tp_k < 0) goto tp_done;", + " if (tp_k == 0) {", + " PyObject *tp_b = PyImport_ImportModule(\"builtins\");", + " if (tp_b == NULL) goto tp_done;", + " tp_k = PyDict_SetItemString(tp_dict, \"__builtins__\", PyModule_GetDict(tp_b));", + " tp_release(&tp_b);", + " if (tp_k < 0) goto tp_done;", + " }", + " tp_code = Py_CompileString(tp_source, tp_source_path, Py_file_input);", + " if (tp_code == NULL) goto tp_done;", + " tp_res = PyEval_EvalCode(tp_code, tp_dict, tp_dict);", + " if (tp_res == NULL) goto tp_done;", + " tp_table = PyDict_New();", + " if (tp_table == NULL || PyDict_SetItemString(tp_dict, \"__typedpython_interpreted__\", tp_table) < 0) goto tp_done;", + " tp_zero = PyLong_FromLong(0);", + " if (tp_zero == NULL || PyDict_SetItemString(tp_dict, \"__typedpython_deopts__\", tp_zero) < 0) goto tp_done;", + " tp_modname = PyModule_GetNameObject(tp_module);", + " if (tp_modname == NULL) goto tp_done;", + ] + for j, f in enumerate(mg.module.functions): + k = mg.name_index(f.name) + L += [ + f" tp_k = PyDict_GetItemRef(tp_dict, tp_st->names[{k}], &tp_f);", + f" if (tp_k == 0) PyErr_Format(PyExc_ImportError, \"typedpython: the module source does not define %R\", tp_st->names[{k}]);", + " if (tp_k <= 0) goto tp_done;", + f" if (PyDict_SetItem(tp_table, tp_st->names[{k}], tp_f) < 0) goto tp_done;", + f" tp_cf = PyCFunction_NewEx(&tp_methods[{j}], tp_module, tp_modname);", + f" if (tp_cf == NULL || PyDict_SetItem(tp_dict, tp_st->names[{k}], tp_cf) < 0) goto tp_done;", + " tp_release(&tp_f); tp_release(&tp_cf);", + ] + L += [ + " tp_rc = 0;", + "tp_done:", + " tp_release(&tp_code); tp_release(&tp_res); tp_release(&tp_table); tp_release(&tp_zero);", + " tp_release(&tp_modname); tp_release(&tp_f); tp_release(&tp_cf);", + " return tp_rc;", + "}", + "", + ] + return "\n".join(L) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/diagnostic.py b/python-multiplatform-ksp/src/main/python/typedpython/diagnostic.py new file mode 100644 index 000000000..2ec322edc --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/diagnostic.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass, asdict +from typing import Literal + +Severity = Literal["error", "warning"] + + +@dataclass(frozen=True, order=True) +class Diagnostic: + path: str + line: int + column: int + rule: str + severity: Severity + message: str + + def to_json(self) -> dict[str, object]: + return asdict(self) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/forbidden.py b/python-multiplatform-ksp/src/main/python/typedpython/forbidden.py new file mode 100644 index 000000000..cccc1651f --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/forbidden.py @@ -0,0 +1,110 @@ +"""The forbidden list (docs/design/typedpython.md §3.1), found by walking the AST. + +These are the dynamic features that make a static type unreliable at compile time. The gate +reports them as warnings for `checked` code and as errors for `compiled` code; this module only +finds them. + +Rebinding a name to a different type and containers that mix element types need inferred types +for assignment targets; they live in typedpython.rebinding. +""" +import ast +from dataclasses import dataclass + +DYNAMIC_ATTRIBUTE_CALLS = {"getattr", "setattr", "delattr"} +# Index of the attribute-name argument in each of the calls above. +NAME_ARGUMENT = 1 + + +@dataclass(frozen=True) +class Finding: + rule: str + line: int + column: int + message: str + + +def scan(source: str, path: str) -> list[Finding]: + tree = ast.parse(source, filename=path) + patchable = _module_level_modules_and_classes(tree) + findings: list[Finding] = [] + + for node in ast.walk(tree): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + findings += _check_call(node, node.func.id, patchable) + elif isinstance(node, (ast.Assign, ast.AugAssign, ast.AnnAssign, ast.Delete)): + for target in _targets(node): + if _is_patch_target(target, patchable): + assert isinstance(target, ast.Attribute) and isinstance(target.value, ast.Name) + findings.append(_finding( + "forbidden/monkeypatch", target, + f"`{target.value.id}.{target.attr}` changes a module or class from outside it", + )) + + return sorted(findings, key=lambda f: (f.line, f.column, f.rule)) + + +def _check_call(call: ast.Call, name: str, patchable: set[str]) -> list[Finding]: + if name in ("eval", "exec"): + return [_finding(f"forbidden/{name}", call, f"`{name}` runs code the checker cannot see")] + if name not in DYNAMIC_ATTRIBUTE_CALLS or len(call.args) <= NAME_ARGUMENT: + return [] + + attribute = call.args[NAME_ARGUMENT] + if not (isinstance(attribute, ast.Constant) and isinstance(attribute.value, str)): + return [_finding( + "forbidden/dynamic-attr", call, + f"`{name}` with a computed attribute name hides the attribute from the checker", + )] + + # A literal name is static, but setattr/delattr on a module or class is still a patch. + receiver = call.args[0] + if name != "getattr" and isinstance(receiver, ast.Name) and receiver.id in patchable: + return [_finding( + "forbidden/monkeypatch", call, + f"`{name}` on `{receiver.id}` changes a module or class from outside it", + )] + return [] + + +def _module_level_modules_and_classes(tree: ast.Module) -> set[str]: + """Names bound at module scope to an imported module/name or to a class definition.""" + names: set[str] = set() + for node in tree.body: + if isinstance(node, ast.Import): + names |= {(alias.asname or alias.name).split(".")[0] for alias in node.names} + elif isinstance(node, ast.ImportFrom): + names |= {alias.asname or alias.name for alias in node.names if alias.name != "*"} + elif isinstance(node, ast.ClassDef): + names.add(node.name) + return names + + +def _targets(node: ast.stmt) -> list[ast.expr]: + if isinstance(node, ast.Assign): + return [t for target in node.targets for t in _flatten(target)] + if isinstance(node, ast.Delete): + return [t for target in node.targets for t in _flatten(target)] + if isinstance(node, (ast.AugAssign, ast.AnnAssign)): + return [node.target] + return [] + + +def _flatten(target: ast.expr) -> list[ast.expr]: + if isinstance(target, (ast.Tuple, ast.List)): + return [t for element in target.elts for t in _flatten(element)] + if isinstance(target, ast.Starred): + return _flatten(target.value) + return [target] + + +def _is_patch_target(target: ast.expr, patchable: set[str]) -> bool: + return ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id in patchable + ) + + +def _finding(rule: str, node: ast.expr, message: str) -> Finding: + # ast columns are 0-based; Pyrefly's (and every editor's) are 1-based. + return Finding(rule, node.lineno, node.col_offset + 1, message) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/frontend.py b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py new file mode 100644 index 000000000..0d841f704 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/frontend.py @@ -0,0 +1,1453 @@ +"""Front end: Python AST + Pyrefly-inferred types -> the typed IR (`ir.py`, #41, SPEC N-8). + +`lower(path)` reads one module and returns an `ir.Module`: + +- Which functions: module-level `def`s decorated with the builtin `@compiled` (a bare name, no + import — #42), or every module-level `def` when the first non-empty line is + `# typedpython: compiled`. +- A function is lowered only when **all** of it is expressible in the IR; otherwise it is left out + and `Module.skipped[name]` says why, with the line. There is no partial lowering. +- Types: parameters and returns from their annotations (`int`/`float`/`bool`, `list[float]` and + `list[int]` parameters as native arrays, anything else an opaque object); locals from their + annotation, a `for ... in range()` target, or else the Pyrefly-inferred type of their + assignments (read back through the same probe the gate uses, `rebinding.probe`). + +Purity and deopt (ir.Function). Each function is lowered first as if pure; if the result is impure +(it stores into an array, calls an object, reads an attribute, or calls an impure function), it is +lowered again in impure mode, where nothing may deopt: an int whose i64 arithmetic is not proved +free of overflow is a CPython int object (OBJ, `PyNumber_*`), a call to a compiled function that +may deopt goes through the module global (`CallObject(Global)`), and only `for ... in range()` +targets whose every binding is such a loop stay I64. The proof is a small interval analysis: +constants, range targets (between their bounds), `len()` and the i64 range of a guarded parameter. +It is recorded in the IR: an I64 ADD/SUB/MUL/NEG it proves cannot overflow is `proven=True` (the +verifier re-proves it; a proven op does not deopt). Division and modulo of ints cannot be recorded, +so they count as deopting. An impure caller of a pure callee that may deopt and returns F64, BOOL +or NONE emits `Call(redo=True)`: only the callee is redone on a deopt. + +Entry globals (ir.Function.entry_globals). A module-level name assigned once at top level, of type +float/int/bool (annotated, or literals and arithmetic of such names), never rebound or deleted in +the module, and read in a *closed* function (one that runs no user code, decided after lowering) +is read once at entry: a `Param` of the function and `Local(name)` at every read. Elsewhere it is a +`Global` (OBJ). +""" +from __future__ import annotations + +import ast +import dataclasses +import tempfile +from collections.abc import Iterable +from dataclasses import dataclass, field +from pathlib import Path + +from typedpython import gate, ir, pyrefly, rebinding +from typedpython.ir import Type + +MARKER = "# typedpython: compiled" + +I64_MIN, I64_MAX = -2**63, 2**63 - 1 +TOP = (I64_MIN, I64_MAX) +EXACT_INT_FLOAT = 2**53 # |n| <= this: int/int true division is exact-rounded in C (ir.BinOp) + +Interval = tuple[int, int] | None # None: no value at all (not yet known to be reached) + +TYPE_NAME = { + Type.I64: "int", Type.F64: "float", Type.BOOL: "bool", Type.NONE: "None", Type.OBJ: "object", + Type.F64_ARRAY: "list[float]", Type.I64_ARRAY: "list[int]", +} +SCALAR_KIND = {"int": Type.I64, "float": Type.F64, "bool": Type.BOOL} +ARRAY_KIND = {"list[float]": Type.F64_ARRAY, "list[int]": Type.I64_ARRAY} + +BINOP = { + ast.Add: ir.BinOpKind.ADD, ast.Sub: ir.BinOpKind.SUB, ast.Mult: ir.BinOpKind.MUL, + ast.Div: ir.BinOpKind.TRUEDIV, ast.FloorDiv: ir.BinOpKind.FLOORDIV, ast.Mod: ir.BinOpKind.MOD, +} +COMPARE = { + ast.Eq: ir.CompareKind.EQ, ast.NotEq: ir.CompareKind.NE, ast.Lt: ir.CompareKind.LT, + ast.LtE: ir.CompareKind.LE, ast.Gt: ir.CompareKind.GT, ast.GtE: ir.CompareKind.GE, +} +MATH_ARITY = { + ir.MathFunc.SQRT: 1, ir.MathFunc.EXP: 1, ir.MathFunc.LOG: 1, ir.MathFunc.SIN: 1, + ir.MathFunc.COS: 1, ir.MathFunc.TAN: 1, ir.MathFunc.FABS: 1, ir.MathFunc.ATAN2: 2, + ir.MathFunc.HYPOT: 2, +} + +UNSUPPORTED = { + "Try": "try statement", "TryStar": "try statement", "With": "with statement", + "AsyncWith": "async with statement", "Match": "match statement", "Raise": "raise statement", + "Assert": "assert statement", "Global": "global declaration", + "Nonlocal": "nonlocal declaration", "FunctionDef": "nested def", + "AsyncFunctionDef": "nested async def", "ClassDef": "nested class", + "Import": "import inside a function", "ImportFrom": "import inside a function", + "Delete": "del statement", "AsyncFor": "async for", "TypeAlias": "type alias", + "Lambda": "lambda", "ListComp": "list comprehension", "SetComp": "set comprehension", + "DictComp": "dict comprehension", "GeneratorExp": "generator expression", + "List": "list display", "Tuple": "tuple display", "Dict": "dict display", + "Set": "set display", "Yield": "yield (generator)", "YieldFrom": "yield from (generator)", + "Await": "await", "IfExp": "conditional expression", "JoinedStr": "f-string", + "NamedExpr": "assignment expression", "Starred": "starred expression", "Slice": "slicing", +} + + +class Skip(Exception): + """This function cannot be expressed in the IR; it stays interpreted.""" + + def __init__(self, line: int, message: str) -> None: + super().__init__(f"line {line}: {message}") + self.reason = f"line {line}: {message}" + + +def _unsupported(node: ast.AST) -> Skip: + what = UNSUPPORTED.get(type(node).__name__, f"`{type(node).__name__}`") + return Skip(getattr(node, "lineno", 0), f"{what} is not supported") + + +def _src(node: ast.AST) -> str: + return ast.unparse(node) + + +# --- the module ---------------------------------------------------------------------------------- + +@dataclass +class _ModuleInfo: + bindings: dict[str, list[tuple[int, str]]] # module-level name -> [(line, how)] + stdlib_math: bool + scalar_globals: dict[str, Type] = field(default_factory=dict) # entry-global candidates + + def bound(self, name: str) -> bool: + return name in self.bindings + + +@dataclass +class _Signature: + node: ast.FunctionDef + params: tuple[ir.Param | ir.ArrayParam, ...] + returns: Type + # `-> tuple[a, b]`: the element kinds (a scalar type, else OBJ); None when not a tuple annotation. + tuple_returns: tuple[Type, ...] | None = None + tuple_ellipsis: bool = False # `tuple[int, ...]`: a variable length + return_annotation: str = "" + + +def lower(path: Path) -> ir.Module: + path = Path(path).resolve() + source = path.read_text() + tree = ast.parse(source, filename=str(path)) + info = _module_info(tree) + + skipped: dict[str, str] = {} + signatures: dict[str, _Signature] = {} + for node, reason in _targets(tree, source, info): + if reason is not None: + skipped[node.name] = reason + continue + assert isinstance(node, ast.FunctionDef) + try: + signatures[node.name] = _signature(node) + except Skip as e: + skipped[node.name] = e.reason + + for name, cycle in _call_cycles(signatures).items(): + skipped[name] = (f"line {signatures[name].node.lineno}: recursion " + f"({' -> '.join(cycle)}) is not compiled: C has no recursion limit " + "where CPython raises RecursionError") + del signatures[name] + + kinds = _inferred_kinds(path, source, tree, [s.node for s in signatures.values()]) + + # Optimistic start: every candidate lowered, pure, not deopting; then iterate to a fixpoint + # (purity is the greatest fixpoint through calls, may_deopt the least). + env: dict[str, ir.Function] = { + name: ir.Function(name=name, params=s.params, returns=s.returns, locals={}, body=(), pure=True, + may_deopt=False, source_line=s.node.lineno) + for name, s in signatures.items() + } + results: dict[str, ir.Function | str] = {} + for _ in range(25): + new = {name: _lower_function(s, env, info, kinds) for name, s in signatures.items()} + if new == results: + break + results = new + env = {n: r for n, r in results.items() if isinstance(r, ir.Function)} + else: + results = {n: f"line {s.node.lineno}: purity/deopt analysis did not converge" + for n, s in signatures.items()} + + functions = [] + for name, result in results.items(): + if isinstance(result, ir.Function): + functions.append(result) + else: + skipped[name] = result + functions.sort(key=lambda f: f.source_line) + return ir.Module(path.stem, tuple(functions), skipped) + + +def _call_cycles(signatures: dict[str, _Signature]) -> dict[str, list[str]]: + """Candidates on a call cycle through calls by name to other candidates (syntactic, so a + call that would end up as CallObject is counted too: conservative).""" + calls = { + name: {n.func.id for n in ast.walk(s.node) if isinstance(n, ast.Call) + and isinstance(n.func, ast.Name) and n.func.id in signatures} + for name, s in signatures.items() + } + cycles: dict[str, list[str]] = {} + for start in signatures: + stack = [(start, [start])] + seen: set[str] = set() + while stack and start not in cycles: + name, path = stack.pop() + for callee in sorted(calls[name]): + if callee == start: + cycles[start] = path + [start] + break + if callee not in seen: + seen.add(callee) + stack.append((callee, path + [callee])) + return cycles + + +def _module_info(tree: ast.Module) -> _ModuleInfo: + bindings: dict[str, list[tuple[int, str]]] = {} + + def add(name: str, line: int, how: str) -> None: + bindings.setdefault(name, []).append((line, how)) + + for stmt in rebinding._own_statements(tree): + if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + add(stmt.name, stmt.lineno, "def") + elif isinstance(stmt, ast.Import): + for alias in stmt.names: + stdlib = alias.name == "math" and alias.asname is None + add(alias.asname or alias.name.split(".")[0], stmt.lineno, + "import math" if stdlib else "import") + elif isinstance(stmt, ast.ImportFrom): + for alias in stmt.names: + add(alias.asname or alias.name, stmt.lineno, "import") + else: + for target in _binding_targets(stmt): + for n in ast.walk(target): + if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store): + add(n.id, stmt.lineno, "assign") + for node in ast.walk(tree): + if isinstance(node, ast.Global): + for name in node.names: + add(name, node.lineno, "global") + + math = bindings.get("math", []) + return _ModuleInfo(bindings, len(math) == 1 and math[0][1] == "import math", + _scalar_globals(tree, bindings)) + + +def _scalar_globals(tree: ast.Module, bindings: dict[str, list[tuple[int, str]]]) -> dict[str, Type]: + """Module-level names that are assigned exactly once, at top level, with a float/int/bool + value, and are never rebound (`global`, `del`, `:=`, a loop target ...) anywhere in the module.""" + spoiled = {n.id for n in ast.walk(tree) if isinstance(n, ast.Name) + and isinstance(n.ctx, ast.Del)} + spoiled |= {n.target.id for n in ast.walk(tree) + if isinstance(n, ast.NamedExpr) and isinstance(n.target, ast.Name)} + found: dict[str, Type] = {} + for stmt in tree.body: + if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name) \ + and stmt.value is not None: + name, annotated = stmt.target.id, SCALAR_KIND.get(_annotation_kind(stmt.annotation)) + t = annotated + elif isinstance(stmt, ast.Assign) and len(stmt.targets) == 1 \ + and isinstance(stmt.targets[0], ast.Name): + name, t = stmt.targets[0].id, _literal_type(stmt.value, found) + else: + continue + if t is not None and len(bindings.get(name, [])) == 1 and name not in spoiled: + found[name] = t + return found + + +def _literal_type(node: ast.expr, known: dict[str, Type]) -> Type | None: + """The type of an expression made only of literals, arithmetic and `known` globals.""" + if isinstance(node, ast.Constant): + v = node.value + if isinstance(v, bool): + return Type.BOOL + if isinstance(v, int): + return Type.I64 if I64_MIN <= v <= I64_MAX else None + return Type.F64 if isinstance(v, float) else None + if isinstance(node, ast.Name): + return known.get(node.id) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.USub, ast.UAdd)): + t = _literal_type(node.operand, known) + return t if t in (Type.I64, Type.F64) else None + if isinstance(node, ast.BinOp) and type(node.op) in BINOP: + a, b = _literal_type(node.left, known), _literal_type(node.right, known) + if a not in (Type.I64, Type.F64) or b not in (Type.I64, Type.F64): + return None + if a == b == Type.I64 and not isinstance(node.op, ast.Div): + return Type.I64 + return Type.F64 + return None + + +def _binding_targets(stmt: ast.stmt) -> list[ast.expr]: + if isinstance(stmt, ast.Assign): + return list(stmt.targets) + if isinstance(stmt, (ast.AugAssign, ast.AnnAssign, ast.For, ast.AsyncFor)): + return [stmt.target] + if isinstance(stmt, (ast.With, ast.AsyncWith)): + return [i.optional_vars for i in stmt.items if i.optional_vars is not None] + return [] + + +def _targets( + tree: ast.Module, source: str, info: _ModuleInfo, +) -> list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]]: + first = next((line.strip() for line in source.splitlines() if line.strip()), "") + whole_module = first == MARKER + found: list[tuple[ast.FunctionDef | ast.AsyncFunctionDef, str | None]] = [] + for node in tree.body: + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + marked = [d for d in node.decorator_list if isinstance(d, ast.Name) and d.id == "compiled"] + if not (whole_module or marked): + continue + reason = None + if marked and info.bound("compiled"): + line = info.bindings["compiled"][0][0] + reason = (f"line {node.lineno}: `compiled` is bound in this module (line {line}), " + "so `@compiled` is not the builtin") + elif other := [d for d in node.decorator_list if d not in marked]: + reason = (f"line {other[0].lineno}: decorator `@{_src(other[0])}` is not supported " + "(only `@compiled`)") + elif isinstance(node, ast.AsyncFunctionDef): + reason = f"line {node.lineno}: async function is not supported" + elif len(info.bindings.get(node.name, [])) > 1: + lines = ", ".join(str(line) for line, _ in info.bindings[node.name]) + reason = f"line {node.lineno}: `{node.name}` is bound more than once (lines {lines})" + found.append((node, reason)) + return found + + +def _signature(fn: ast.FunctionDef) -> _Signature: + args = fn.args + line = fn.lineno + if args.posonlyargs: + raise Skip(line, "positional-only parameters are not supported") + if args.kwonlyargs: + raise Skip(line, "keyword-only parameters are not supported") + if args.vararg is not None: + raise Skip(line, f"`*{args.vararg.arg}` is not supported") + if args.kwarg is not None: + raise Skip(line, f"`**{args.kwarg.arg}` is not supported") + if args.defaults: + raise Skip(line, "default parameter values are not supported") + params: list[ir.Param | ir.ArrayParam] = [] + for a in args.args: + if a.annotation is None: + raise Skip(a.lineno, f"parameter `{a.arg}` has no annotation") + kind = _annotation_kind(a.annotation) + if kind in ARRAY_KIND: + params.append(ir.ArrayParam(a.arg, ARRAY_KIND[kind], False)) # `stored` set later + else: + params.append(ir.Param(a.arg, SCALAR_KIND.get(kind, Type.OBJ))) + if fn.returns is None: + raise Skip(line, "no return annotation") + if isinstance(fn.returns, ast.Constant) and fn.returns.value is None: + returns = Type.NONE + else: + returns = SCALAR_KIND.get(_annotation_kind(fn.returns), Type.OBJ) + elements, ellipsis = _tuple_annotation(fn.returns) + return _Signature(fn, tuple(params), returns, elements, ellipsis, _src(fn.returns)) + + +def _tuple_annotation(annotation: ast.expr) -> tuple[tuple[Type, ...] | None, bool]: + """`tuple[int, float]` -> ((I64, F64), False); `tuple[int, ...]` -> (None, True).""" + if not (isinstance(annotation, ast.Subscript) and isinstance(annotation.value, ast.Name) + and annotation.value.id == "tuple"): + return None, False + inner = annotation.slice + items = list(inner.elts) if isinstance(inner, ast.Tuple) else [inner] + if any(isinstance(i, ast.Constant) and i.value is Ellipsis for i in items): + return None, True + return tuple(SCALAR_KIND.get(_annotation_kind(i), Type.OBJ) for i in items), False + + +def _annotation_kind(annotation: ast.expr) -> str: + if isinstance(annotation, ast.Name) and annotation.id in SCALAR_KIND: + return annotation.id + return _src(annotation).replace(" ", "") + + +# --- local types from Pyrefly -------------------------------------------------------------------- + +def _inferred_kinds( + path: Path, source: str, tree: ast.Module, functions: list[ast.FunctionDef], +) -> dict[str, dict[str, list[tuple[int, str]]]]: + """function -> local -> [(line, base type)] for every unannotated assignment (Pyrefly).""" + scopes = rebinding._scopes(tree) + wanted_scopes = {scopes.index(fn) for fn in functions} + wanted = [a for a in rebinding.assignments(tree) if a.scope in wanted_scopes] + if not wanted: + return {} + probed, probes = rebinding.probe(source, wanted) + with tempfile.TemporaryDirectory(prefix="typedpython-frontend-") as work: + copies = gate._mirror(Path(work), {str(path): probed}) + [copy] = copies + roots = pyrefly.import_roots([str(path)]) + report = pyrefly.run([Path(copy)], [Path(work) / "0", Path(copy).parent, *roots]) + types = report.expression_types.get(copy, {}) + + found: dict[str, dict[str, list[tuple[int, str]]]] = {} + for p in probes: + a = p.assignment + if a is None: + continue + spelled = types.get((p.line, p.column), "Unknown") + function = scopes[a.scope] + assert isinstance(function, ast.FunctionDef) + found.setdefault(function.name, {}).setdefault(p.name, []).append( + (a.line, rebinding._base(spelled))) + return found + + +# --- one function -------------------------------------------------------------------------------- + +def _lower_function( + sig: _Signature, env: dict[str, ir.Function], info: _ModuleInfo, + kinds: dict[str, dict[str, list[tuple[int, str]]]], +) -> ir.Function | str: + inferred = kinds.get(sig.node.name, {}) + pure_error: Skip | None = None + try: + fn = _lower_in_mode(sig, env, info, inferred, impure=False) + if fn.pure: + return fn + except Skip as e: + pure_error = e + try: + return _lower_in_mode(sig, env, info, inferred, impure=True) + except Skip as e: + return (pure_error or e).reason + + +def _lower_in_mode(sig, env, info, inferred, impure: bool) -> ir.Function: + fn = _lower_with_bounds(sig, env, info, inferred, impure, info.scalar_globals) + if fn.entry_globals and not _closed(fn, env): + # A global read once at entry is only the same as a read at each use if nothing between + # can run user code: here something can, so the reads stay `Global`. + fn = _lower_with_bounds(sig, env, info, inferred, impure, {}) + return fn + + +def _lower_with_bounds(sig, env, info, inferred, impure: bool, + candidates: dict[str, Type]) -> ir.Function: + # Range-target intervals: least fixpoint by iteration, widened to TOP if it does not settle. + bounds: dict[str, Interval] = {} + for _ in range(8): + lowerer = _Lowerer(sig, env, info, inferred, impure, bounds, candidates) + fn = lowerer.run() + joined = {**bounds} + for var, iv in lowerer.found_bounds.items(): + joined[var] = _join(joined.get(var), iv) + if joined == bounds: + return fn + bounds = joined + widened = {var: TOP for var in bounds} + return _Lowerer(sig, env, info, inferred, impure, widened, candidates).run() + + +def _join(a: Interval, b: Interval) -> Interval: + if a is None: + return b + if b is None: + return a + return (min(a[0], b[0]), max(a[1], b[1])) + + +@dataclass +class _Binding: + line: int + how: str # "range" (a for-over-range target), "annotation" (no value), "assign", "for" + + +class _Lowerer: + def __init__(self, sig: _Signature, env: dict[str, ir.Function], info: _ModuleInfo, + inferred: dict[str, list[tuple[int, str]]], impure: bool, + bounds: dict[str, Interval], candidates: dict[str, Type]) -> None: + self.sig = sig + self.candidates = candidates + self.entry_used: dict[str, Type] = {} + self.fn = sig.node + self.env = env + self.info = info + self.inferred = inferred + self.impure_mode = impure + self.bounds = bounds + self.found_bounds: dict[str, Interval] = {} + + self.params = {p.name: p for p in sig.params} + self.arrays = {p.name: p.type for p in sig.params if isinstance(p, ir.ArrayParam)} + self.bindings: dict[str, list[_Binding]] = {} + self.annotations: dict[str, list[tuple[int, str]]] = {} + self._collect_bindings() + # Local arrays (ir.NewArray / CopyArray): name -> array type; also in `self.arrays`. + self.local_arrays = self._find_local_arrays() + self.arrays.update(self.local_arrays) + self.range_vars = { + n for n, bs in self.bindings.items() + if n not in self.params and any(b.how == "range" for b in bs) + and all(b.how in ("range", "annotation") for b in bs) + } + self.all_names = {n.id for n in ast.walk(self.fn) if isinstance(n, ast.Name)} \ + | set(self.params) + + self.types: dict[str, Type] = {} + self.temps: dict[str, Type] = {} + self.pre: list[ir.Stmt] = [] + self.hoist_ok = True + self.impure = False # an effect outside the locals + self.deopts = False # a node that can deopt + self.stored: set[str] = set() + self.current: ast.stmt | None = None # the statement being lowered (for messages) + + # --- entry --- + + def run(self) -> ir.Function: + body = self.block(self.fn.body) + if self.sig.returns != Type.NONE and _falls_through(body): + last = self.fn.body[-1] + raise Skip(getattr(last, "end_lineno", None) or last.lineno, + f"can reach the end without returning a {TYPE_NAME[self.sig.returns]} " + "(CPython would return None)") + locals_ = {} + for name in self.bindings: + if name not in self.params: + locals_[name] = self.local_type(name, self.bindings[name][0].line) + locals_.update(self.temps) + params = tuple( + ir.ArrayParam(p.name, p.type, p.name in self.stored) if isinstance(p, ir.ArrayParam) + else p for p in self.sig.params + ) + pure = not self.impure + assert pure or not self.deopts or not self.impure_mode, \ + "impure mode emitted a deopting node" + return ir.Function( + name=self.fn.name, params=params, returns=self.sig.returns, locals=locals_, body=body, + pure=pure, may_deopt=self.deopts, source_line=self.fn.lineno, + entry_globals=tuple(ir.Param(n, t) for n, t in self.entry_used.items())) + + # --- names and types --- + + def _collect_bindings(self) -> None: + def add(name: str, line: int, how: str) -> None: + self.bindings.setdefault(name, []).append(_Binding(line, how)) + + for stmt in rebinding._own_statements(self.fn): + if stmt is self.fn: + continue + if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name): + add(stmt.target.id, stmt.lineno, "assign" if stmt.value is not None else "annotation") + self.annotations.setdefault(stmt.target.id, []).append( + (stmt.lineno, _annotation_kind(stmt.annotation))) + elif isinstance(stmt, (ast.For, ast.AsyncFor)): + how = "range" if _is_range_call(stmt.iter) else "for" + for n in ast.walk(stmt.target): + if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store): + add(n.id, stmt.lineno, how) + else: + for target in _binding_targets(stmt): + for n in ast.walk(target): + if isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store): + add(n.id, stmt.lineno, "assign") + + def _find_local_arrays(self) -> dict[str, Type]: + """Locals created by `list(range(n))`, `[v] * n` or `src[:]` (src an array). Every binding + of such a name must be one of those creations: a local array is never rebound to anything + else, so it has one element type for its whole life.""" + creations: dict[str, list[tuple[ast.stmt, str, ast.expr]]] = {} + for stmt in rebinding._own_statements(self.fn): + if isinstance(stmt, ast.Assign) and len(stmt.targets) == 1 \ + and isinstance(stmt.targets[0], ast.Name): + name, value = stmt.targets[0].id, stmt.value + elif isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name) \ + and stmt.value is not None: + name, value = stmt.target.id, stmt.value + else: + continue + kind = _creation_kind(value) + if kind is not None and name not in self.params: + creations.setdefault(name, []).append((stmt, kind, value)) + known = set(self.arrays) | {n for n, cs in creations.items() + if any(k != "copy" for _, k, _ in cs)} + while True: + more = {n for n, cs in creations.items() if n not in known and any( + k == "copy" and v.value.id in known for _, k, v in cs)} # type: ignore[attr-defined] + if not more: + break + known |= more + found: dict[str, Type] = {} + for name, cs in creations.items(): + if name not in known: + continue + cs = [c for c in cs if c[1] != "copy" or c[2].value.id in known] # type: ignore[attr-defined] + lines = {c[0].lineno for c in cs} + other = [b for b in self.bindings[name] if b.how != "annotation" and b.line not in lines] + if other: + raise Skip(other[0].line, f"local `{name}` is a native array (created at line " + f"{cs[0][0].lineno}) but is also bound by something " + "that does not create an array") + found[name] = self._array_type(name, cs, found) + return found + + def _array_type(self, name: str, cs, found: dict[str, Type]) -> Type: + line = cs[0][0].lineno + kinds: list[tuple[int, str]] = list(self.annotations.get(name, [])) + if name not in self.annotations: + kinds += self.inferred.get(name, []) + distinct = sorted({k for _, k in kinds}) + if len(distinct) > 1: + raise Skip(line, f"local `{name}` is given values of types " + + " and ".join(f"`{k}`" for k in distinct)) + if distinct: + if distinct[0] not in ARRAY_KIND: + raise Skip(line, f"local `{name}: {distinct[0]}`: a native array's element type " + "must be exactly int or float") + return ARRAY_KIND[distinct[0]] + for _, kind, value in cs: # unknown to Pyrefly: a copy takes its source's type + if kind == "copy": + src = value.value.id + if src in self.arrays: + return self.arrays[src] + if src in found: + return found[src] + raise Skip(line, f"no type is known for local `{name}`") + + def is_local(self, name: str) -> bool: + return name in self.params or name in self.bindings or name in self.temps + + def local_type(self, name: str, line: int) -> Type: + if name in self.temps: + return self.temps[name] + if name in self.params: + return self.params[name].type + if name in self.local_arrays: + return self.local_arrays[name] + if name in self.types: + return self.types[name] + kinds: list[tuple[int, str]] = list(self.annotations.get(name, [])) + kinds += [(b.line, "int") for b in self.bindings.get(name, []) if b.how == "range"] + if name not in self.annotations: + kinds += self.inferred.get(name, []) + distinct = sorted({k for _, k in kinds}) + if not distinct: + raise Skip(self.bindings[name][0].line, f"no type is known for local `{name}`") + if len(distinct) > 1: + where = next(line for line, k in kinds if k != kinds[0][1]) + raise Skip(where, f"local `{name}` is given values of types " + + " and ".join(f"`{k}`" for k in distinct)) + kind = distinct[0] + if kind in ARRAY_KIND or kind.startswith(("list[float]", "list[int]")): + raise Skip(kinds[0][0], f"local `{name}: {kind}`: a list local is a native array only " + "when created by `list(range(n))`, `[v] * n` or `src[:]`") + if kind == "int": + t = Type.OBJ if self.impure_mode and name not in self.range_vars else Type.I64 + else: + t = SCALAR_KIND.get(kind, Type.OBJ) + self.types[name] = t + return t + + def temp(self, value: ir.Expr) -> ir.Local: + n = len(self.temps) + while f"_tp{n}" in self.all_names or f"_tp{n}" in self.temps: + n += 1 + name = f"_tp{n}" + self.temps[name] = value.type + self.pre.append(ir.Assign(name, value)) + return ir.Local(value.type, name) + + # --- statements --- + + def block(self, stmts: Iterable[ast.stmt]) -> tuple[ir.Stmt, ...]: + out: list[ir.Stmt] = [] + for s in stmts: + out += self.stmt(s) + return tuple(out) + + def start(self) -> None: + self.pre = [] + self.hoist_ok = True + + def take(self) -> list[ir.Stmt]: + pre, self.pre = self.pre, [] + return pre + + def stmt(self, node: ast.stmt) -> list[ir.Stmt]: + self.start() + self.current = node + line = node.lineno + if isinstance(node, ast.Pass): + return [] + if isinstance(node, ast.Break): + return [ir.Break()] + if isinstance(node, ast.Continue): + return [ir.Continue()] + if isinstance(node, ast.Expr): + if isinstance(node.value, ast.Constant): + return [] # a docstring or a bare literal: no effect + value = self.sub(node.value) + return self.take() + [ir.ExprStmt(value)] + if isinstance(node, ast.Assign): + if len(node.targets) != 1: + raise Skip(line, "chained assignment `a = b = ...` is not supported") + return self.assign(node.targets[0], node.value, node) + if isinstance(node, ast.AnnAssign): + if not isinstance(node.target, ast.Name): + raise Skip(line, f"annotated assignment to `{_src(node.target)}` is not supported") + if node.value is None: + return [] + return self.assign(node.target, node.value, node) + if isinstance(node, ast.AugAssign): + return self.aug_assign(node) + if isinstance(node, ast.If): + cond = self.condition(node.test) + pre = self.take() + return pre + [ir.If(cond, self.block(node.body), self.block(node.orelse))] + if isinstance(node, ast.While): + if node.orelse: + raise Skip(line, "`while ... else` is not supported") + cond = self.condition(node.test) + pre = self.take() + body = self.block(node.body) + if not pre: + return [ir.While(cond, body)] + # The condition needs statements (a chain's temporary): run them on every iteration. + exit_ = ir.If(cond, (), (ir.Break(),)) # cond stays directly a condition + return [ir.While(ir.Const(Type.BOOL, True), (*pre, exit_, *body))] + if isinstance(node, ast.For): + return self.for_range(node) + if isinstance(node, ast.Return): + return self.return_(node) + raise _unsupported(node) + + def assign(self, target: ast.expr, value_node: ast.expr, node: ast.stmt) -> list[ir.Stmt]: + line = node.lineno + if isinstance(target, ast.Name): + name = target.id + if name in self.local_arrays: + return self.new_array(name, value_node, node) + if name in self.arrays: + raise Skip(line, f"array parameter `{name}` is rebound") + value = self.value(value_node) + t = self.local_type(name, line) + return self.take() + [ir.Assign(name, self.coerce(value, t, value_node, + f"local `{name}`"))] + if isinstance(target, ast.Subscript): + array, elem = self.array_of(target) + # CPython evaluates the value first, then the index. + value = self.value(value_node) + value = self.coerce(value, elem, value_node, f"`{array}: {TYPE_NAME[self.arrays[array]]}`") + if not _simple(target.slice) and not _trivial(value): + value = self.temp(value) + index = self.index(target.slice) + self.store_effect(array) + return self.take() + [ir.StoreIndex(array, index, value)] + if isinstance(target, ast.Attribute): + raise Skip(line, f"assignment to attribute `{_src(target)}` is not supported") + if isinstance(target, (ast.Tuple, ast.List)): + raise Skip(line, "tuple unpacking is not supported") + raise _unsupported(target) + + def array_word(self, name: str) -> str: + return "local array" if name in self.local_arrays else "array parameter" + + def store_effect(self, array: str) -> None: + """A store into a parameter list is an effect (write-back); into a local array it is not.""" + if array not in self.local_arrays: + self.stored.add(array) + self.impure = True + + def new_array(self, name: str, value_node: ast.expr, node: ast.stmt) -> list[ir.Stmt]: + line = node.lineno + t = self.local_arrays[name] + elem = Type.F64 if t == Type.F64_ARRAY else Type.I64 + what = f"local array `{name}: {TYPE_NAME[t]}`" + kind = _creation_kind(value_node) + if kind == "iota": + call = value_node.args[0] # type: ignore[attr-defined] + if not (self.builtin("list") and self.builtin("range")): + raise Skip(line, f"`{_src(value_node)}` with `list` or `range` rebound") + if elem != Type.I64: + raise Skip(line, f"`{_src(value_node)}` is a list of int, but {what} is " + f"{TYPE_NAME[t]}") + length = self.range_bound(call.args[0]) + return self.take() + [ir.Assign(name, ir.NewArray(t, length, None, True))] + if kind == "fill": + assert isinstance(value_node, ast.BinOp) and isinstance(value_node.left, ast.List) + fill = self.coerce(self.value(value_node.left.elts[0]), elem, + value_node.left.elts[0], what) + if not _trivial(fill): + fill = self.temp(fill) # `[v] * n` evaluates v before n + length = self.value(value_node.right) + if length.type != Type.I64: + why = (" (it is a CPython int here: its arithmetic is not proved free of i64 " + "overflow in an impure function)" if length.type == Type.OBJ + and self.impure_mode else "") + raise Skip(line, f"array length `{_src(value_node.right)}` is " + f"{TYPE_NAME[length.type]}, not a native int{why}") + return self.take() + [ir.Assign(name, ir.NewArray(t, length, fill))] + if kind == "copy": + assert isinstance(value_node, ast.Subscript) and isinstance(value_node.value, ast.Name) + src = value_node.value.id + if self.arrays[src] != t: + raise Skip(line, f"`{_src(value_node)}` is {TYPE_NAME[self.arrays[src]]}, but " + f"{what} is {TYPE_NAME[t]}") + return [ir.Assign(name, ir.CopyArray(t, src))] + raise Skip(line, f"{what} is rebound to `{_src(value_node)}`, which does not create an array") + + def aug_assign(self, node: ast.AugAssign) -> list[ir.Stmt]: + line = node.lineno + op = self.binop_kind(node.op, node) + target = node.target + if isinstance(target, ast.Name): + name = target.id + if name in self.arrays: + raise Skip(line, f"{self.array_word(name)} `{name}` is rebound") + if not self.is_local(name): + raise _unsupported(node) + t = self.local_type(name, line) + current = ir.Local(t, name) + result = self.arith(op, current, self.sub(node.value), node) + return self.take() + [ir.Assign(name, self.coerce(result, t, node, f"local `{name}`"))] + if isinstance(target, ast.Subscript): + array, elem = self.array_of(target) + index = self.index(target.slice) + if not isinstance(index, (ir.Local, ir.Const)): + index = self.temp(index) # `a[i] += e` evaluates `i` once + load = ir.Index(elem, array, index) + self.hoist_ok = False + result = self.arith(op, load, self.sub(node.value), node) + result = self.coerce(result, elem, node, f"`{array}: {TYPE_NAME[self.arrays[array]]}`") + self.store_effect(array) + return self.take() + [ir.StoreIndex(array, index, result)] + if isinstance(target, ast.Attribute): + raise Skip(line, f"assignment to attribute `{_src(target)}` is not supported") + raise _unsupported(target) + + def for_range(self, node: ast.For) -> list[ir.Stmt]: + line = node.lineno + if node.orelse: + raise Skip(line, "`for ... else` is not supported") + if not isinstance(node.target, ast.Name): + raise Skip(line, f"loop target `{_src(node.target)}` is not supported (only a name)") + if isinstance(node.iter, ast.Name) and node.iter.id in self.arrays: + raise self.array_misuse(node.iter.id, node.iter) + if not (_is_range_call(node.iter) and self.builtin("range")): + raise Skip(line, f"`for` over `{_src(node.iter)}` is not supported (only range())") + call = node.iter + assert isinstance(call, ast.Call) + var = node.target.id + if var in self.params: + raise Skip(line, f"loop variable `{var}` is a parameter") + for inner in ast.walk(ast.Module(body=node.body, type_ignores=[])): + for target in _binding_targets(inner) if isinstance(inner, ast.stmt) else []: + if any(isinstance(n, ast.Name) and isinstance(n.ctx, ast.Store) and n.id == var + for n in ast.walk(target)): + raise Skip(inner.lineno, f"loop variable `{var}` is assigned in the loop body") + if self.local_type(var, line) != Type.I64: + other = next(b.line for b in self.bindings[var] if b.how != "range") + raise Skip(line, f"loop variable `{var}` is also assigned at line {other}; in an " + "impure function that makes it a CPython int, not a range index") + args = call.args + if len(args) == 3: + step = _int_literal(args[2]) + if step is None or step == 0: + raise Skip(line, f"range step `{_src(args[2])}` must be a non-zero int literal") + else: + step = 1 + if len(args) == 1: + start: ir.Expr = ir.Const(Type.I64, 0) + stop = self.range_bound(args[0]) + else: + start = self.range_bound(args[0]) + stop = self.range_bound(args[1]) + pre = self.take() + + s, e = self.interval(start), self.interval(stop) + if s is not None and e is not None: + if step > 0: + iv = (s[0], max(s[0], e[1] - 1)) + else: + iv = (min(s[1], e[0] + 1), s[1]) + self.found_bounds[var] = _join(self.found_bounds.get(var), iv) + + body = self.block(node.body) + return pre + [ir.ForRange(var, start, stop, ir.Const(Type.I64, step), body)] + + def range_bound(self, node: ast.expr) -> ir.Expr: + e = self.value(node) + if e.type != Type.I64: + why = (" (it is a CPython int here: its arithmetic is not proved free of i64 overflow " + "in an impure function)" if e.type == Type.OBJ and self.impure_mode else "") + raise Skip(node.lineno, f"range bound `{_src(node)}` is not a native int{why}") + return e + + def return_(self, node: ast.Return) -> list[ir.Stmt]: + returns = self.sig.returns + if node.value is None or (isinstance(node.value, ast.Constant) and node.value.value is None): + if returns != Type.NONE: + raise Skip(node.lineno, f"returns None from a function declared to return " + f"{TYPE_NAME[returns]}") + return [ir.Return(None)] + if returns == Type.NONE: + raise Skip(node.lineno, "returns a value from a function declared to return None") + if isinstance(node.value, ast.Tuple): + return self.return_tuple(node.value) + value = self.value(node.value) + value = self.coerce(value, returns, node.value, "the return value") + return self.take() + [ir.Return(value)] + + def return_tuple(self, node: ast.Tuple) -> list[ir.Stmt]: + sig = self.sig + if any(isinstance(e, ast.Starred) for e in node.elts): + raise Skip(node.lineno, f"`return {_src(node)}` with a starred element is not supported") + if sig.tuple_returns is None: + why = ("a variable-length `tuple[..., ...]`" if sig.tuple_ellipsis + else f"`{sig.return_annotation}`") + raise Skip(node.lineno, f"returns `{_src(node)}`, but the declared return type is " + f"{why}, not a fixed `tuple[...]`") + kinds = sig.tuple_returns + if len(kinds) != len(node.elts): + raise Skip(node.lineno, f"returns {len(node.elts)} values, but the return type " + f"`{sig.return_annotation}` declares {len(kinds)}") + elements = [] + for kind, e_node in zip(kinds, node.elts): + e = self.value(e_node) + if e.type in ir.SCALARS and kind in ir.SCALARS and e.type != kind: + raise Skip(e_node.lineno, f"`{_src(e_node)}` is {TYPE_NAME[e.type]}, but the " + f"return type `{sig.return_annotation}` declares " + f"{TYPE_NAME[kind]} there") + elements.append(self.boxed(e, e_node)) + return self.take() + [ir.Return(ir.Tuple(Type.OBJ, tuple(elements)))] + + # --- expressions --- + + def sub(self, node: ast.expr, cond: int = 0) -> ir.Expr: + """Lower a subexpression in evaluation order; after a non-trivial one, nothing later in + the statement may be hoisted in front of it.""" + e = self.expr(node, cond) + if not _trivial(e): + self.hoist_ok = False + return e + + def value(self, node: ast.expr) -> ir.Expr: + e = self.sub(node) + if e.type == Type.NONE: + raise Skip(node.lineno, f"uses the result of `{_src(node)}`, which returns None") + return e + + def condition(self, node: ast.expr) -> ir.Expr: + return self.truth(self.sub(node, cond=2), node) + + def truth(self, e: ir.Expr, node: ast.expr) -> ir.Expr: + if e.type == Type.BOOL: + return e + if e.type == Type.I64: + return ir.Compare(Type.BOOL, ir.CompareKind.NE, e, ir.Const(Type.I64, 0)) + if e.type == Type.F64: + return ir.Compare(Type.BOOL, ir.CompareKind.NE, e, ir.Const(Type.F64, 0.0)) + if e.type == Type.OBJ: + self.impure = True + return ir.Truth(Type.BOOL, e) + raise Skip(node.lineno, f"truth value of `{_src(node)}` ({TYPE_NAME[e.type]})") + + def expr(self, node: ast.expr, cond: int = 0) -> ir.Expr: + """`cond`: 2 when `node` is itself an If/While condition, 1 when it is an operand of an + And/Or that is the condition, 0 elsewhere (ir.CompareObj is allowed only at 2 and 1).""" + line = node.lineno + if isinstance(node, ast.Constant): + v = node.value + if isinstance(v, bool): + return ir.Const(Type.BOOL, v) + if isinstance(v, int): + if not I64_MIN <= v <= I64_MAX: + raise Skip(line, f"int literal {v} is outside i64") + return ir.Const(Type.I64, v) + if isinstance(v, float): + return ir.Const(Type.F64, v) + raise Skip(line, f"{type(v).__name__} constant `{_src(node)}` is not supported") + if isinstance(node, ast.Name): + return self.name(node) + if isinstance(node, ast.BinOp): + op = self.binop_kind(node.op, node) + left = self.sub(node.left) + right = self.sub(node.right) + return self.arith(op, left, right, node) + if isinstance(node, ast.UnaryOp): + return self.unary(node) + if isinstance(node, ast.Compare): + return self.compare(node, cond) + if isinstance(node, ast.BoolOp): + return self.boolop(node, cond) + if isinstance(node, ast.Call): + return self.call(node) + if isinstance(node, ast.Attribute): + if isinstance(node.value, ast.Name) and node.value.id in self.arrays: + raise self.array_misuse(node.value.id, node) + obj = self.boxed(self.sub(node.value), node.value) + self.impure = True + return ir.GetAttr(Type.OBJ, obj, node.attr) + if isinstance(node, ast.Subscript): + if isinstance(node.value, ast.Name) and node.value.id in self.arrays: + if isinstance(node.slice, ast.Slice): + raise Skip(line, f"`{_src(node)}` slices {self.array_word(node.value.id)} " + f"`{node.value.id}`: only `y = {node.value.id}[:]` as the whole " + "right-hand side of an assignment is a native copy") + array, elem = self.array_of(node) + return ir.Index(elem, array, self.index(node.slice)) + raise Skip(line, f"subscript `{_src(node)}` of an object is not supported") + raise _unsupported(node) + + def name(self, node: ast.Name) -> ir.Expr: + name = node.id + if name in self.arrays: + raise self.array_misuse(name, node) + if self.is_local(name): + if name not in self.params and name not in self.temps and all( + b.how == "annotation" for b in self.bindings[name]): + raise Skip(node.lineno, f"`{name}` is read but never assigned") + return ir.Local(self.local_type(name, node.lineno), name) + if name in self.candidates: + self.entry_used.setdefault(name, self.candidates[name]) + return ir.Local(self.candidates[name], name) + return ir.Global(Type.OBJ, name) + + def array_misuse(self, name: str, node: ast.AST) -> Skip: + where = self.current if isinstance(self.current, ( + ast.Assign, ast.AnnAssign, ast.AugAssign, ast.Return, ast.Expr)) else node + if name in self.local_arrays: + return Skip(node.lineno, f"local array `{name}` is used other than by indexing, len() " + f"or as a `[:]` source (in `{_src(where)}`)") + return Skip(node.lineno, f"array parameter `{name}` is used other than by indexing or " + f"len() (in `{_src(where)}`)") + + def array_of(self, node: ast.Subscript) -> tuple[str, Type]: + if not (isinstance(node.value, ast.Name) and node.value.id in self.arrays): + raise Skip(node.lineno, f"subscript `{_src(node)}` of an object is not supported") + name = node.value.id + return name, Type.F64 if self.arrays[name] == Type.F64_ARRAY else Type.I64 + + def index(self, node: ast.expr) -> ir.Expr: + if isinstance(node, ast.Slice): + raise _unsupported(node) + e = self.value(node) + if e.type != Type.I64: + why = (" (it is a CPython int here: its arithmetic is not proved free of i64 overflow " + "in an impure function)" if e.type == Type.OBJ and self.impure_mode else "") + raise Skip(node.lineno, f"index `{_src(node)}` is {TYPE_NAME[e.type]}, not a native " + f"int{why}") + return e + + def binop_kind(self, op: ast.operator, node: ast.AST) -> ir.BinOpKind: + kind = BINOP.get(type(op)) + if kind is None: + raise Skip(node.lineno, f"operator `{_src(node)}` is not supported") + return kind + + def arith(self, op: ir.BinOpKind, left: ir.Expr, right: ir.Expr, node: ast.AST) -> ir.Expr: + lt, rt = left.type, right.type + if Type.BOOL in (lt, rt) or Type.NONE in (lt, rt): + raise Skip(node.lineno, f"arithmetic on {TYPE_NAME[Type.BOOL if Type.BOOL in (lt, rt) else Type.NONE]} " + f"in `{_src(node)}` is not supported") + if Type.OBJ in (lt, rt): + return self.obj_binop(op, left, right) + if lt == Type.I64 and rt == Type.I64: + result = Type.F64 if op == ir.BinOpKind.TRUEDIV else Type.I64 + e = ir.BinOp(result, op, left, right) + if self.recordable(e) and self.proved(e): + return dataclasses.replace(e, proven=True) + if self.impure_mode: + return self.obj_binop(op, left, right) + self.deopts = True + return e + if lt == Type.I64: + left = ir.ToFloat(Type.F64, left) + if rt == Type.I64: + right = ir.ToFloat(Type.F64, right) + return ir.BinOp(Type.F64, op, left, right) + + def obj_binop(self, op: ir.BinOpKind, left: ir.Expr, right: ir.Expr) -> ir.Expr: + self.impure = True + return ir.BinOp(Type.OBJ, op, _box(left), _box(right)) + + def unary(self, node: ast.UnaryOp) -> ir.Expr: + if isinstance(node.op, ast.Not): + operand = self.truth(self.sub(node.operand), node.operand) + return ir.UnaryOp(Type.BOOL, ir.UnaryOpKind.NOT, operand) + if not isinstance(node.op, (ast.USub, ast.UAdd)): + raise Skip(node.lineno, f"operator `{_src(node)}` is not supported") + operand = self.sub(node.operand) + kind = ir.UnaryOpKind.NEG if isinstance(node.op, ast.USub) else ir.UnaryOpKind.POS + if operand.type == Type.F64: + return ir.UnaryOp(Type.F64, kind, operand) + if operand.type == Type.I64: + e = ir.UnaryOp(Type.I64, kind, operand) + if kind == ir.UnaryOpKind.POS: + return e + if self.proved(e): + return dataclasses.replace(e, proven=True) + if self.impure_mode: # -x of a CPython int is 0 - x + return self.obj_binop(ir.BinOpKind.SUB, ir.Const(Type.I64, 0), operand) + self.deopts = True + return e + raise Skip(node.lineno, f"unary `{_src(node)}` on {TYPE_NAME[operand.type]} is not " + "supported") + + def compare(self, node: ast.Compare, cond: int) -> ir.Expr: + # A chain is an And of its links: as a condition (2) that And is the condition; as an + # operand of the condition's And/Or (1) it would be nested, so only a single link there. + obj_ok = cond == 2 or (cond == 1 and len(node.ops) == 1) + left = self.sub(node.left) + links: list[ir.Expr] = [] + last = len(node.ops) - 1 + for i, (op, comparator) in enumerate(zip(node.ops, node.comparators)): + if i > 0: + self.hoist_ok = False # later links run only if the earlier ones held + hoist_ok = self.hoist_ok + right = self.expr(comparator) + if right.type == Type.NONE: + raise Skip(comparator.lineno, f"uses the result of `{_src(comparator)}`, which " + "returns None") + if i < last and not isinstance(right, (ir.Local, ir.Const)): + # The middle operand of a chain is evaluated once: bind it to a temporary. + if hoist_ok: + right = self.temp(right) + elif not _trivial(right): + raise Skip(comparator.lineno, f"chained comparison: `{_src(comparator)}` " + "cannot be evaluated once here") + if not _trivial(right): + self.hoist_ok = False + links.append(self.link(op, left, right, obj_ok, node)) + left = right + result = links[0] + for link in links[1:]: + result = ir.And(Type.BOOL, result, link) + return result + + def link(self, op: ast.cmpop, left: ir.Expr, right: ir.Expr, obj_ok: bool, + node: ast.Compare) -> ir.Expr: + kind = COMPARE.get(type(op)) + if kind is None: + raise Skip(node.lineno, f"comparison `{_src(node)}` is not supported " + "(only == != < <= > >=)") + lt, rt = left.type, right.type + if Type.OBJ in (lt, rt): + if not obj_ok: + raise Skip(node.lineno, f"comparison `{_src(node)}` with an object is not " + "directly a condition (its result need not be a bool)") + self.impure = True + return ir.CompareObj(Type.BOOL, kind, _box(left), _box(right)) + if (lt == Type.BOOL) != (rt == Type.BOOL): + raise Skip(node.lineno, f"comparison `{_src(node)}` of a bool with a number is not " + "supported") + return ir.Compare(Type.BOOL, kind, left, right) + + def boolop(self, node: ast.BoolOp, cond: int) -> ir.Expr: + word = "and" if isinstance(node.op, ast.And) else "or" + operands = [] + for i, v in enumerate(node.values): + if i > 0: + self.hoist_ok = False # evaluated only if the earlier operands did not decide + e = self.sub(v, 1 if cond == 2 else 0) + if cond == 2: + e = self.truth(e, v) + elif e.type != Type.BOOL: + raise Skip(node.lineno, f"`{word}` on a non-bool operand `{_src(v)}` outside a " + "condition (it returns the operand)") + operands.append(e) + result = operands[0] + for e in operands[1:]: + result = (ir.And if word == "and" else ir.Or)(Type.BOOL, result, e) + return result + + def builtin(self, name: str) -> bool: + return not self.info.bound(name) and not self.is_local(name) + + def call(self, node: ast.Call) -> ir.Expr: + line = node.lineno + if any(isinstance(a, ast.Starred) for a in node.args) or any( + k.arg is None for k in node.keywords): + raise Skip(line, f"`*`/`**` arguments in `{_src(node)}` are not supported") + func = node.func + plain = not node.keywords + + if isinstance(func, ast.Name): + name = func.id + if name in self.arrays: + raise self.array_misuse(name, node) + if plain and len(node.args) == 1 and name == "len" and self.builtin("len") \ + and isinstance(node.args[0], ast.Name) and node.args[0].id in self.arrays: + return ir.Len(Type.I64, node.args[0].id) + if plain and len(node.args) == 1 and name == "float" and self.builtin("float"): + arg = self.value(node.args[0]) + if arg.type == Type.I64: + return ir.ToFloat(Type.F64, arg) + if arg.type == Type.F64: + return arg # float(x) of an exact float is x itself + self.impure = True + return ir.ObjToFloat(Type.F64, _box(arg)) + if not self.is_local(name) and name in self.env: + direct = self.direct_call(name, node) + if direct is not None: + return direct + + if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name) \ + and func.value.id == "math" and self.info.stdlib_math \ + and not self.is_local("math") and plain: + math = self.math_call(func.attr, node) + if math is not None: + return math + + if isinstance(func, ast.Name) and func.id in self.arrays: + raise self.array_misuse(func.id, node) + callee = self.boxed(self.sub(func), func) + self.hoist_ok = False + args = [self.boxed(self.value(a), a) for a in node.args] + args += [self.boxed(self.value(k.value), k.value) for k in node.keywords] + self.impure = True + return ir.CallObject(Type.OBJ, callee, tuple(args), + tuple(k.arg for k in node.keywords if k.arg is not None)) + + def direct_call(self, name: str, node: ast.Call) -> ir.Expr | None: + """`Call` of another compiled function, else `CallObject` of its module global; None when + the generic path should lower the call (decided before any argument is lowered).""" + callee = self.env[name] + if node.keywords or len(node.args) != len(callee.params): + return None # CPython's own TypeError, through the interpreted path + if any(isinstance(p, ir.ArrayParam) for p in callee.params): + return None + self.hoist_ok = False + args = [self.value(a) for a in node.args] + redo = callee.may_deopt and self.impure_mode and callee.pure \ + and callee.returns in (Type.F64, Type.BOOL, Type.NONE) + # else a deopt could not be redone (an I64 result of a redo could be a big int) + direct = redo or not (callee.may_deopt and self.impure_mode) + for p, e in zip(callee.params, args): + if p.type != Type.OBJ and e.type != p.type: + direct = False # the callee's guard would deopt; the global keeps CPython's result + if not direct: + self.impure = True + return ir.CallObject(Type.OBJ, ir.Global(Type.OBJ, name), + tuple(self.boxed(e, a) for e, a in zip(args, node.args))) + if not callee.pure: + self.impure = True + if callee.may_deopt and not redo: + self.deopts = True + typed = tuple(self.boxed(e, a) if p.type == Type.OBJ else e + for p, e, a in zip(callee.params, args, node.args)) + return ir.Call(callee.returns, name, typed, redo=redo) + + def math_call(self, attr: str, node: ast.Call) -> ir.Expr | None: + try: + func = ir.MathFunc(attr) + except ValueError: + return None + if len(node.args) != MATH_ARITY[func]: + return None + self.hoist_ok = False + args = [self.value(a) for a in node.args] + if all(e.type in (Type.I64, Type.F64) for e in args): + return ir.MathCall(Type.F64, func, tuple( + ir.ToFloat(Type.F64, e) if e.type == Type.I64 else e for e in args)) + # math's own conversion of an object (or a bool): call `math.` as an object. + self.impure = True + callee = ir.GetAttr(Type.OBJ, ir.Global(Type.OBJ, "math"), attr) + return ir.CallObject(Type.OBJ, callee, tuple(self.boxed(e, a) + for e, a in zip(args, node.args))) + + def boxed(self, e: ir.Expr, node: ast.AST) -> ir.Expr: + if e.type == Type.NONE: + raise Skip(node.lineno, f"uses the result of `{_src(node)}`, which returns None") + return _box(e) + + def coerce(self, value: ir.Expr, target: Type, node: ast.AST, what: str) -> ir.Expr: + if value.type == target: + return value + if target == Type.OBJ and value.type in ir.SCALARS: + return ir.Box(Type.OBJ, value) + extra = "" + if value.type == Type.OBJ: + extra = " (CPython would keep the object as it is; there is no conversion to make)" + elif value.type == Type.I64 and target == Type.F64: + extra = " (CPython keeps an int there, it does not convert it to float)" + raise Skip(getattr(node, "lineno", 0), + f"`{_src(node)}` is {TYPE_NAME[value.type]}, but {what} is " + f"{TYPE_NAME[target]}{extra}") + + # --- intervals --- + + def interval(self, e: ir.Expr) -> Interval: + if isinstance(e, ir.Const) and e.type == Type.I64: + return (e.value, e.value) # type: ignore[return-value] + if isinstance(e, ir.Local) and e.type == Type.I64 and e.name in self.range_vars: + return self.bounds.get(e.name) + if isinstance(e, ir.Len): + return (0, I64_MAX) + if isinstance(e, (ir.BinOp, ir.UnaryOp)) and e.type == Type.I64 and self.proved(e): + return self._result_interval(e) + return TOP + + @staticmethod + def recordable(e: ir.BinOp) -> bool: + """`proven` exists for ADD/SUB/MUL only; the verifier cannot re-prove the others.""" + return e.op in (ir.BinOpKind.ADD, ir.BinOpKind.SUB, ir.BinOpKind.MUL) + + def proved(self, e: ir.BinOp | ir.UnaryOp) -> bool: + """The i64 operation provably does not deopt (no overflow, exact division).""" + if isinstance(e, ir.UnaryOp): + if e.op == ir.UnaryOpKind.POS: + return True + a = self.interval(e.operand) + return a is None or a[0] > I64_MIN + a, b = self.interval(e.left), self.interval(e.right) + if a is None or b is None: + return True # no value reaches here (yet) + if e.op == ir.BinOpKind.TRUEDIV: + return all(-EXACT_INT_FLOAT <= v <= EXACT_INT_FLOAT for v in (*a, *b)) + if e.op == ir.BinOpKind.MOD: + return True # the runtime computes -2**63 % -1 == 0 (ir.BinOp); only a raise remains + if e.op == ir.BinOpKind.FLOORDIV: + return not (a[0] == I64_MIN and b[0] <= -1 <= b[1]) + r = self._result_interval(e) + return r is None or (r[0] >= I64_MIN and r[1] <= I64_MAX) + + def _result_interval(self, e: ir.BinOp | ir.UnaryOp) -> Interval: + if isinstance(e, ir.UnaryOp): + a = self.interval(e.operand) + if a is None: + return None + return a if e.op == ir.UnaryOpKind.POS else (-a[1], -a[0]) + a, b = self.interval(e.left), self.interval(e.right) + if a is None or b is None: + return None + if e.op == ir.BinOpKind.ADD: + return (a[0] + b[0], a[1] + b[1]) + if e.op == ir.BinOpKind.SUB: + return (a[0] - b[1], a[1] - b[0]) + if e.op == ir.BinOpKind.MUL: + products = [x * y for x in a for y in b] + return (min(products), max(products)) + if e.op == ir.BinOpKind.FLOORDIV: + m = max(abs(a[0]), abs(a[1])) + return (max(-m, I64_MIN), min(m, I64_MAX)) + if e.op == ir.BinOpKind.MOD: + d = max(abs(b[0]), abs(b[1]), 1) + return (-(d - 1), d - 1) + return TOP + + +# --- helpers ------------------------------------------------------------------------------------- + +def _walk(x): + if isinstance(x, (tuple, list)): + for y in x: + yield from _walk(y) + elif dataclasses.is_dataclass(x) and not isinstance(x, type): + yield x + for f in dataclasses.fields(x): + yield from _walk(getattr(x, f.name)) + + +def _closed(fn: ir.Function, env: dict[str, ir.Function], seen: frozenset[str] = frozenset()) -> bool: + """`fn` runs no user code: no GetAttr, CallObject, Truth, CompareObj, ObjToFloat or OBJ + BinOp, and calls only closed functions.""" + for n in _walk(fn.body): + if isinstance(n, (ir.GetAttr, ir.CallObject, ir.Truth, ir.CompareObj, ir.ObjToFloat)): + return False + if isinstance(n, ir.BinOp) and n.type == Type.OBJ: + return False + if isinstance(n, ir.Call): + callee = env.get(n.function) + if callee is None or n.function in seen or not _closed(callee, env, seen | {fn.name}): + return False + return True + + +def _box(e: ir.Expr) -> ir.Expr: + return ir.Box(Type.OBJ, e) if e.type in ir.SCALARS else e + + +def _trivial(e: ir.Expr) -> bool: + """No effect, cannot raise, cannot deopt: may be evaluated in any order, or twice.""" + if isinstance(e, (ir.Const, ir.Local, ir.Len)): + return True + if isinstance(e, ir.ToFloat): + return _trivial(e.operand) + if isinstance(e, ir.UnaryOp): + return (e.op == ir.UnaryOpKind.NOT or e.type == Type.F64) and _trivial(e.operand) + if isinstance(e, ir.BinOp): + return e.type == Type.F64 and e.op in (ir.BinOpKind.ADD, ir.BinOpKind.SUB, + ir.BinOpKind.MUL) \ + and _trivial(e.left) and _trivial(e.right) + if isinstance(e, (ir.Compare, ir.And, ir.Or)): + return _trivial(e.left) and _trivial(e.right) + return False + + +def _creation_kind(node: ast.expr) -> str | None: + """How a statement's value creates a native array: "iota" `list(range(n))`, "fill" `[v] * n`, + "copy" `name[:]` (whether `name` is an array is decided by the caller), else None.""" + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "list" \ + and not node.keywords and len(node.args) == 1 and _is_range_call(node.args[0]) \ + and len(node.args[0].args) == 1: + return "iota" + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Mult) \ + and isinstance(node.left, ast.List) and len(node.left.elts) == 1 \ + and not isinstance(node.left.elts[0], ast.Starred): + return "fill" + if isinstance(node, ast.Subscript) and isinstance(node.value, ast.Name) \ + and isinstance(node.slice, ast.Slice) and node.slice.lower is None \ + and node.slice.upper is None and node.slice.step is None: + return "copy" + return None + + +def _simple(node: ast.expr) -> bool: + return isinstance(node, (ast.Name, ast.Constant)) + + +def _is_range_call(node: ast.expr) -> bool: + return isinstance(node, ast.Call) and isinstance(node.func, ast.Name) \ + and node.func.id == "range" and not node.keywords and 1 <= len(node.args) <= 3 \ + and not any(isinstance(a, ast.Starred) for a in node.args) + + +def _int_literal(node: ast.expr) -> int | None: + if isinstance(node, ast.Constant) and type(node.value) is int: + return node.value + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub) \ + and isinstance(node.operand, ast.Constant) and type(node.operand.value) is int: + return -node.operand.value + return None + + +def _falls_through(body: tuple[ir.Stmt, ...]) -> bool: + if not body: + return True + last = body[-1] + if isinstance(last, ir.Return): + return False + if isinstance(last, ir.If): + return _falls_through(last.then) or _falls_through(last.orelse) + if isinstance(last, ir.While): + forever = isinstance(last.cond, ir.Const) and last.cond.value is True + return not forever or _breaks(last.body) + return True + + +def _breaks(body: tuple[ir.Stmt, ...]) -> bool: + """A `break` of this loop (not of a nested one).""" + for s in body: + if isinstance(s, ir.Break): + return True + if isinstance(s, ir.If) and (_breaks(s.then) or _breaks(s.orelse)): + return True + return False + + +__all__ = ["lower", "MARKER"] diff --git a/python-multiplatform-ksp/src/main/python/typedpython/gate.py b/python-multiplatform-ksp/src/main/python/typedpython/gate.py new file mode 100644 index 000000000..1f34e2303 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/gate.py @@ -0,0 +1,258 @@ +"""The soundness gate (docs/design/typedpython.md §4.2). + +A type checker may fall back to `Any` when it is unsure; a compiler cannot. The gate closes that +gap for the files it is given: + + Pyrefly errors, Any-leak kinds included -> error + an expression of type Any in user code -> error (`any-flow`), unless inside cast() + the §3.1 forbidden list -> warning (`checked`) / error (`compiled`) + +Only the given files are user code. Their dependencies are type-checked by Pyrefly only as far +as their stubs or sources say; nothing inside them is reported. +""" +import ast +import shutil +import tempfile +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from typedpython import forbidden, pyrefly, rebinding +from typedpython.diagnostic import Diagnostic, Severity + +Mode = Literal["checked", "compiled"] + +Position = tuple[int, int] + + +@dataclass(frozen=True) +class _Run: + """One user file as Pyrefly sees it: the text it was given and where that text differs.""" + + original: str + text: str # the source Pyrefly reads: the original, or the probed copy of it + probes: list[rebinding.Probe] + remap: "_Remap" + + +class _Remap: + """Maps positions in probed text back to the original source. + + A probe inserts `; name` on the line of its assignment, so a column to the right of an + insertion is `len("; name")` further right than in the original; line numbers never change. + """ + + def __init__(self, probes: Sequence[rebinding.Probe]) -> None: + self._inserted: dict[int, list[tuple[int, int]]] = {} + for p in probes: + self._inserted.setdefault(p.line, []).append((p.column - 2, len(p.name) + 2)) + + def to_original(self, line: int, column: int) -> Position | None: + """The original position, or None when it lies inside an inserted probe load.""" + shift = 0 + for start, length in self._inserted.get(line, ()): + if column >= start + length: + shift += length + elif column >= start: + return None + return line, column - shift + + +def check( + paths: Sequence[Path], + mode: Mode = "checked", + search_paths: Sequence[Path] = (), +) -> list[Diagnostic]: + runs = _plan([str(Path(p).resolve()) for p in paths]) + forbidden_severity: Severity = "error" if mode == "compiled" else "warning" + diagnostics: list[Diagnostic] = [] + + # One Pyrefly run for everything. When any file has something to probe, it runs on a mirrored + # package tree holding the probed copies (and the other files unchanged); otherwise on the + # files themselves. + with tempfile.TemporaryDirectory(prefix="typedpython-probe-") as work: + probing = any(r.probes for r in runs.values()) + if probing: + copies = _mirror(Path(work), {o: r.text for o, r in runs.items()}) + roots = pyrefly.import_roots(list(runs)) + mirror = _Mirror(copies, [((Path(work) / str(i)).resolve(), r) + for i, r in enumerate(roots)]) + report = pyrefly.run( + [Path(c) for c in copies], + [*[Path(work) / str(i) for i in range(len(roots))], + *sorted({Path(c).parent for c in copies}), + *search_paths, *roots], + ) + else: + mirror = _Mirror({o: o for o in runs}, []) + report = pyrefly.run([Path(p) for p in runs], search_paths) + original_of = mirror.copies + run_of = {c: runs[o] for c, o in original_of.items()} + + reported: set[tuple[str, int, int]] = set() + for e in report.errors: + run = run_of.get(e.path) + if run is None: + diagnostics.append(Diagnostic( + mirror.original(e.path), e.line, e.column, f"pyrefly/{e.kind}", "error", e.message, + )) + continue + position = run.remap.to_original(e.line, e.column) + if position is None: + # An error inside an inserted `; name` means the probe itself is wrong. + diagnostics.append(Diagnostic( + run.original, e.line, e.column, "internal/probe-error", "error", + f"pyrefly/{e.kind} inside a probe load (`{e.message}`); this is a gate bug", + )) + continue + diagnostics.append(Diagnostic( + run.original, *position, f"pyrefly/{e.kind}", "error", e.message, + )) + reported.add((run.original, *position)) + + for copy, run in run_of.items(): + path = run.original + try: + tree = ast.parse(run.text, filename=path) + except SyntaxError: + continue # Pyrefly has already reported the parse error + + # Judged on the probed text: the probe loads are expression statements, so they are + # discarded values and never become any-flow findings. + cast_spans = _cast_argument_spans(tree) + discarded = _discarded_values(tree) + for expr in report.any_expressions: + if expr.path != copy: + continue + start = run.remap.to_original(expr.line, expr.column) + if start is None or (path, *start) in reported: + continue + if _is_signature(expr.type): + continue # a callable whose signature mentions Any; its result is judged on its own + if ((expr.line, expr.column), (expr.stop_line, expr.stop_column)) in discarded: + continue # an expression statement's value goes nowhere + if any(_inside((expr.line, expr.column), span) for span in cast_spans): + continue + diagnostics.append(Diagnostic( + path, *start, "any-flow", "error", + f"this expression has type `{expr.type}`; give it a type, or cast() it " + "where it leaves untyped code", + )) + reported.add((path, *start)) + + for finding in forbidden.scan(Path(path).read_text(), path): + diagnostics.append(Diagnostic( + path, finding.line, finding.column, finding.rule, forbidden_severity, + finding.message, + )) + + types = report.expression_types.get(copy, {}) + for f in rebinding.judge(run.probes, types): + diagnostics.append(Diagnostic(path, f.line, f.column, f.rule, forbidden_severity, + f.message)) + + return sorted(diagnostics) + + +def _plan(paths: Sequence[str]) -> dict[str, _Run]: + """Each file's text for Pyrefly: probed when it has rebinding/mixed-container candidates.""" + runs: dict[str, _Run] = {} + for path in paths: + source = Path(path).read_text() + probed, probes = source, [] + try: + tree = ast.parse(source, filename=path) + except SyntaxError: + tree = None + if tree is not None and (wanted := rebinding.candidates(rebinding.assignments(tree))): + probed, probes = rebinding.probe(source, wanted) + runs[path] = _Run(path, probed, probes, _Remap(probes)) + return runs + + +@dataclass(frozen=True) +class _Mirror: + copies: dict[str, str] # copy -> original + bases: list[tuple[Path, Path]] # (mirrored root, original root) + + def original(self, path: str) -> str: + """The original path of a file in the mirrored tree (a dependency's, for instance).""" + for base, root in self.bases: + try: + return str(root / Path(path).relative_to(base)) + except ValueError: + continue + return path + + +def _mirror(work: Path, probed: dict[str, str]) -> dict[str, str]: + """Copy each file's package tree under `work`, with the probed text in place of the file. + + The copy keeps the file's path relative to its import root, so relative imports and imports + of sibling modules inside the package resolve to the copied tree. Returns copy -> original. + """ + roots = pyrefly.import_roots(list(probed)) + copies: dict[str, str] = {} + copied: set[Path] = set() + for original, text in probed.items(): + root = pyrefly.import_roots([original])[0] + base = work / str(roots.index(root)) + relative = Path(original).relative_to(root) + if len(relative.parts) > 1: + package = root / relative.parts[0] + if package not in copied: + shutil.copytree( + package, base / relative.parts[0], dirs_exist_ok=True, + ignore=shutil.ignore_patterns("__pycache__", ".*"), + ) + copied.add(package) + target = base / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + copies[str(target.resolve())] = original + return copies + + +def _cast_argument_spans(tree: ast.Module) -> list[tuple[Position, Position]]: + """Source spans of the value argument of every `cast(T, value)` / `typing.cast(T, value)`.""" + spans = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call) or len(node.args) != 2: + continue + func = node.func + is_cast = (isinstance(func, ast.Name) and func.id == "cast") or ( + isinstance(func, ast.Attribute) and func.attr == "cast" + and isinstance(func.value, ast.Name) and func.value.id == "typing" + ) + value = node.args[1] + if is_cast and value.end_lineno is not None and value.end_col_offset is not None: + spans.append(( + (value.lineno, value.col_offset + 1), + (value.end_lineno, value.end_col_offset + 1), + )) + return spans + + +def _discarded_values(tree: ast.Module) -> set[tuple[Position, Position]]: + """Spans of values computed and thrown away by an expression statement (`eval(s)` alone).""" + spans = set() + for node in ast.walk(tree): + if isinstance(node, ast.Expr): + value = node.value + if value.end_lineno is not None and value.end_col_offset is not None: + spans.add(( + (value.lineno, value.col_offset + 1), + (value.end_lineno, value.end_col_offset + 1), + )) + return spans + + +def _is_signature(spelled: str) -> bool: + # Pyrefly spells callables as `(x: int) -> str` and overload sets as `Overload[...]`. + return "->" in spelled or spelled.startswith("Overload[") + + +def _inside(position: Position, span: tuple[Position, Position]) -> bool: + start, stop = span + return start <= position <= stop diff --git a/python-multiplatform-ksp/src/main/python/typedpython/incremental.py b/python-multiplatform-ksp/src/main/python/typedpython/incremental.py new file mode 100644 index 000000000..fad342e05 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/incremental.py @@ -0,0 +1,330 @@ +"""Incremental compilation (SPEC N-10, #41): a per-module build cache. + +The unit is a module. Its cache key covers the source, the *interface* hashes of the project modules +it imports, the compiler version, the target platform and the build flags. A body-only change in a +dependency leaves that dependency's interface hash unchanged, so dependents stay cached. + +The front end, the C back end and the C compiler are passed in as callables, so this file depends on +none of them (only on `ir`, the contract). + +Key encoding (v1): sha256 of the compact, key-sorted JSON of + {"v": 1, "source": sha256-hex(source bytes), + "deps": [[, ], ...] sorted, + "compiler_version": str, "platform": str, "flags": [str, ...] in the given order} +Interface hash (v1): sha256 of the compact, key-sorted JSON of + {"v": 1, "functions": [[name, [[param name, type value, "array" | "scalar"], ...], return type value], ...] + sorted by function name, "skipped": sorted skipped names} +Only signatures enter it: no bodies, locals, purity flags, line numbers or skip reasons. + +Cache layout: /entries//{meta.json, } and /index.json (the last +key components per module, used only to explain a rebuild). An entry is written to a temporary +directory inside cache_dir and renamed into place, so a crash never leaves a half entry under a key. +An entry whose metadata is unreadable or whose artefact is missing or has a different sha256 is a +miss and is rebuilt. +""" +from __future__ import annotations + +import ast +import hashlib +import json +import os +import shutil +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable + +from typedpython import ir +from typedpython.pyrefly import import_roots + +_V = 1 + + +def _sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _canon(obj) -> bytes: + return json.dumps(obj, sort_keys=True, separators=(",", ":")).encode() + + +def interface_hash(module: ir.Module) -> str: + """sha256 over the compiled functions' signatures and the skipped names — nothing else.""" + functions = [] + for f in sorted(module.functions, key=lambda f: f.name): + params = [[p.name, p.type.value, "array" if isinstance(p, ir.ArrayParam) else "scalar"] + for p in f.params] + functions.append([f.name, params, f.returns.value]) + return _sha(_canon({"v": _V, "functions": functions, "skipped": sorted(module.skipped)})) + + +# --- dependencies -------------------------------------------------------------------------------- + +def _abs(p: Path) -> Path: + return Path(os.path.abspath(p)) + + +def _within(p: Path, root: Path) -> bool: + try: + p.relative_to(root) + return True + except ValueError: + return False + + +def _find(base: Path, parts: list[str]) -> list[Path]: + """Files executed by importing dotted `parts` under `base`: each package's __init__ on the way, + then the module itself (module file or package __init__). Stops at the first missing part.""" + found: list[Path] = [] + cur = base + for part in parts: + pkg = cur / part / "__init__.py" + mod = cur / f"{part}.py" + if pkg.is_file(): + found.append(pkg) + cur = cur / part + elif mod.is_file(): + found.append(mod) + break + else: + break + return found + + +def dependencies(path: Path, project_root: Path) -> list[Path]: + """Project-local modules `path` imports (stdlib and third-party are ignored), in first-seen order.""" + path, project_root = _abs(Path(path)), _abs(Path(project_root)) + try: + tree = ast.parse(path.read_bytes(), filename=str(path)) + except (SyntaxError, ValueError, OSError): + return [] + roots = [_abs(r) for r in import_roots([str(path)])] + if project_root not in roots: + roots.append(project_root) + + out: list[Path] = [] + + def add(found: list[Path]) -> None: + for p in found: + p = _abs(p) + if p != path and _within(p, project_root) and p not in out: + out.append(p) + + def resolve(parts: list[str], bases: list[Path]) -> list[Path]: + best: list[Path] = [] + for b in bases: + got = _find(b, parts) + if len(got) > len(best): + best = got + return best + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + add(resolve(alias.name.split("."), roots)) + elif isinstance(node, ast.ImportFrom): + parts = node.module.split(".") if node.module else [] + if node.level: + base = path.parent + for _ in range(node.level - 1): + base = base.parent + bases = [base] + if not parts: + add([base / "__init__.py"] if (base / "__init__.py").is_file() else []) + else: + bases = roots + got = resolve(parts, bases) if parts else [] + add(got) + # `from pkg import mod`: the imported name may itself be a submodule + for alias in node.names: + if alias.name == "*": + continue + for b in bases: + if parts: + sub_base = b + for part in parts: + sub_base = sub_base / part + else: + sub_base = b + add(_find(sub_base, [alias.name]) if sub_base.is_dir() else []) + return out + + +# --- report -------------------------------------------------------------------------------------- + +@dataclass +class ModuleReport: + path: Path + name: str + status: str # "rebuilt" | "reused" + reasons: list[str] = field(default_factory=list) + artifact: Path | None = None + interface: str = "" + key: str = "" + + +@dataclass +class BuildReport: + modules: list[ModuleReport] = field(default_factory=list) + + @property + def rebuilt(self) -> list[ModuleReport]: + return [m for m in self.modules if m.status == "rebuilt"] + + @property + def reused(self) -> list[ModuleReport]: + return [m for m in self.modules if m.status == "reused"] + + +# --- cache --------------------------------------------------------------------------------------- + +class BuildCache: + def __init__(self, cache_dir: Path): + self.dir = Path(cache_dir) + self.entries = self.dir / "entries" + self.entries.mkdir(parents=True, exist_ok=True) + + # entries + + def _load(self, key: str) -> dict | None: + """The entry's metadata plus its artefact path, or None if it is absent or damaged.""" + d = self.entries / key + try: + meta = json.loads((d / "meta.json").read_text()) + art = d / meta["artifact"] + if meta["key"] != key or _sha(art.read_bytes()) != meta["artifact_sha256"]: + return None + meta["path"] = art + return meta + except (OSError, ValueError, KeyError, TypeError): + return None + + def _store(self, key: str, name: str, interface: str, c_source: str, + compile_c: Callable[[str, str, Path], Path]) -> Path: + tmp = self.dir / f"tmp-{uuid.uuid4().hex}" + tmp.mkdir() + try: + art = Path(compile_c(c_source, name, tmp)) + if art.parent != tmp: # the artefact must live in the entry + shutil.copy2(art, tmp / art.name) + art = tmp / art.name + meta = {"key": key, "name": name, "interface": interface, "artifact": art.name, + "artifact_sha256": _sha(art.read_bytes())} + (tmp / "meta.json").write_text(json.dumps(meta)) + final = self.entries / key + if final.exists(): + shutil.rmtree(final, ignore_errors=True) # a damaged entry under this key + try: + os.rename(tmp, final) + except OSError: + if self._load(key) is None: # lost a race to a valid entry? else fail + raise + shutil.rmtree(tmp, ignore_errors=True) + return final / art.name + except BaseException: + shutil.rmtree(tmp, ignore_errors=True) + raise + + # index (explains rebuilds only; never decides a hit) + + def _read_index(self) -> dict: + try: + data = json.loads((self.dir / "index.json").read_text()) + return data if isinstance(data, dict) else {} + except (OSError, ValueError): + return {} + + def _write_index(self, index: dict) -> None: + tmp = self.dir / f"index-{uuid.uuid4().hex}.tmp" + tmp.write_text(json.dumps(index, sort_keys=True)) + os.replace(tmp, self.dir / "index.json") + + # build + + def build_project(self, sources: list[Path], project_root: Path, *, + lower: Callable[[Path], ir.Module], + generate: Callable[[ir.Module, Path], str], + compile_c: Callable[[str, str, Path], Path], + compiler_version: str, platform: str, flags: list[str]) -> BuildReport: + root = _abs(Path(project_root)) + paths = [_abs(Path(s)) for s in sources] + known = set(paths) + deps = {p: dependencies(p, root) for p in paths} + order = _topological(paths, deps) + rel = lambda p: p.relative_to(root).as_posix() + + index = self._read_index() + interfaces: dict[Path, str] = {} + by_path: dict[Path, ModuleReport] = {} + + for p in order: + source_hash = _sha(p.read_bytes()) + dep_hashes: dict[str, str] = {} + for d in deps[p]: + if d in interfaces: + dep_hashes[rel(d)] = interfaces[d] + else: # not compiled here, or in a cycle: its whole source is its interface + dep_hashes[rel(d)] = "src:" + _sha(d.read_bytes()) + comps = {"source": source_hash, "deps": dep_hashes, "compiler_version": compiler_version, + "platform": platform, "flags": list(flags)} + key = _sha(_canon({"v": _V, "source": source_hash, "deps": sorted(dep_hashes.items()), + "compiler_version": compiler_version, "platform": platform, + "flags": list(flags)})) + name = _module_name(p, root) + hit = self._load(key) + if hit is not None: + interfaces[p] = hit["interface"] + by_path[p] = ModuleReport(p, name, "reused", [], hit["path"], hit["interface"], key) + else: + reasons = _why(index.get(rel(p)), comps, (self.entries / key).exists()) + module = lower(p) + c_source = generate(module, p) + iface = interface_hash(module) + art = self._store(key, name, iface, c_source, compile_c) + interfaces[p] = iface + by_path[p] = ModuleReport(p, name, "rebuilt", reasons, art, iface, key) + index[rel(p)] = comps + + self._write_index(index) + return BuildReport([by_path[p] for p in paths]) + + +def _module_name(p: Path, root: Path) -> str: + parts = list(p.relative_to(root).with_suffix("").parts) + if parts[-1] == "__init__" and len(parts) > 1: + parts.pop() + return ".".join(parts) + + +def _why(prev: dict | None, now: dict, entry_dir_exists: bool) -> list[str]: + if not prev: + return ["new"] + reasons: list[str] = [] + if prev.get("source") != now["source"]: + reasons.append("source") + old_deps, new_deps = prev.get("deps", {}), now["deps"] + for d in sorted(set(old_deps) | set(new_deps)): + if old_deps.get(d) != new_deps.get(d): + reasons.append(f"dependency interface: {d}") + for k in ("compiler_version", "platform", "flags"): + if prev.get(k) != now[k]: + reasons.append(k) + return reasons or ["corrupt" if entry_dir_exists else "missing"] + + +def _topological(paths: list[Path], deps: dict[Path, list[Path]]) -> list[Path]: + """Dependencies first, input order otherwise; a cycle is broken where it is first re-entered.""" + known, order, state = set(paths), [], {} + + def visit(p: Path) -> None: + if state.get(p): + return + state[p] = 1 + for d in deps[p]: + if d in known: + visit(d) + order.append(p) + + for p in paths: + visit(p) + return order diff --git a/python-multiplatform-ksp/src/main/python/typedpython/ir.py b/python-multiplatform-ksp/src/main/python/typedpython/ir.py new file mode 100644 index 000000000..78fae7b96 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/ir.py @@ -0,0 +1,522 @@ +"""TypedPython typed IR (design §4.3 stage 2, #41). + +The IR sits between the front end (AST + Pyrefly types → IR, `frontend.py`) and the back end +(IR → C, `cgen.py`). Both sides are written against this file; it is the contract. + +The governing rule is INTENT §1.4: **compiled code computes what CPython computes.** The IR +encodes that rule, so a back end that implements each node as specified below cannot change a +result: + +- Every node says what it does in CPython terms. Where a C operation differs from CPython (integer + overflow, floor division and modulo of negatives, true division of large ints, int/float + comparison, list indexing), the node is defined by the CPython behaviour and the back end must + implement that behaviour, not the C one. +- When the C path cannot produce CPython's result, the IR says **deopt**: the compiled call gives + up and the whole call is redone by the interpreted function. A deopt is allowed only where + redoing is unobservable: before any effect (`Function.guards`, evaluated at entry), or anywhere + inside a `pure` function. The front end guarantees that an impure function contains no node + that can deopt (`Function.may_deopt` is False for impure functions); the back end may assert it. +- Exceptions are CPython's exceptions, with CPython's messages (`ZeroDivisionError("division by + zero")`, `IndexError("list index out of range")`, `ValueError("math domain error")`, ...). + The traceback differs (there is no Python frame for compiled code); that is the one accepted + difference, as for every C extension. + +Safety (`verify.py`, maintainer decision 2026-10-03: "IR 단계에서 안전성을 증명하도록 해"). Before any +C is generated, the verifier proves on the IR that the C will be memory-safe and keep the rules +above; **a function it cannot prove is not compiled** — it stays interpreted and the reason is +reported as a diagnostic. What it proves: + +- definite assignment: no local is read before it is assigned on every path; +- types: every node's operands have the types the node requires; no implicit conversion; +- deopt placement: no node that can deopt in an impure function (`may_deopt` consistent); +- arrays: every `Index`/`StoreIndex` is either proved in bounds (and marked `proven`) or keeps its + runtime check; array params are used only by `Index`/`StoreIndex`/`Len`; +- ownership: OBJ values are created only by nodes that return a new reference (Box, Global, + GetAttr, CallObject, OBJ BinOp) and the back end releases them by one rule (every OBJ local owns its reference; assigning + releases the old one; every exit releases all); the IR has no node that frees, aliases a + borrowed reference, or reads a possibly-NULL object, so use-after-free and NULL dereference have + no IR form. + +The generated C touches memory and `PyObject`s only through the runtime helpers (`tp_runtime.h`), +which are tested separately; AddressSanitizer/UBSan runs, differential tests against the +interpreter, and fuzzing are the safety net outside the proof. + +Module shape (back end). The extension module executes the module's original source in its own +namespace first, so every name, class and global is the interpreted one and lives in one +namespace. Then each compiled function `f` replaces the global `f` with a C function, and the +original stays reachable as `__typedpython_interpreted__[f]` for deopt. Because there is one +namespace, interpreted code that calls `f` gets the compiled `f`, and a deopted call sees the same +globals and classes as compiled code. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum + + +# --- types --------------------------------------------------------------------------------------- + +class Type(Enum): + """The C representation of a value. Python semantics are attached per type below.""" + + I64 = "i64" # a Python int known to fit in [-2**63, 2**63-1]; arithmetic is checked + F64 = "f64" # a Python float (IEEE double, exactly as CPython) + BOOL = "bool" # a Python bool + NONE = "none" # the value None (only as a return type) + OBJ = "obj" # any Python object, held as a strong reference (PyObject*) + F64_ARRAY = "f64[]" # a list[float] parameter (ArrayParam) or a local array (NewArray/CopyArray) + I64_ARRAY = "i64[]" # a list[int] parameter (ArrayParam) or a local array (NewArray/CopyArray) + + +SCALARS = (Type.I64, Type.F64, Type.BOOL) +ARRAYS = (Type.F64_ARRAY, Type.I64_ARRAY) + + +# --- expressions --------------------------------------------------------------------------------- + +@dataclass(frozen=True) +class Expr: + type: Type + + +@dataclass(frozen=True) +class Const(Expr): + """A literal. `value` is a Python int/float/bool/None matching `type`.""" + + value: int | float | bool | None + + +@dataclass(frozen=True) +class Local(Expr): + """Reading a local variable or parameter (always definitely assigned: the front end checks).""" + + name: str + + +class BinOpKind(Enum): + ADD = "+" + SUB = "-" + MUL = "*" + TRUEDIV = "/" + FLOORDIV = "//" + MOD = "%" + + +@dataclass(frozen=True) +class BinOp(Expr): + """Arithmetic with CPython's result. Both operands have the same scalar type (the front end + inserts `ToFloat` for mixed int/float, as CPython converts the int). + + I64 (result I64, except TRUEDIV → F64): + ADD/SUB/MUL — checked; overflow → deopt. + FLOORDIV/MOD — Python floor semantics (result sign follows the divisor for MOD; the runtime + handles -2**63 % -1 == 0 without C's undefined behaviour); divisor 0 + → ZeroDivisionError with the running interpreter's message (3.13: "integer division or + modulo by zero" / "integer modulo by zero"; 3.14: "division by zero" for every case — the + runtime selects by version and its differential tests pin both); -2**63 // -1 overflows → + deopt. Every ZeroDivisionError/math-domain message below follows the same rule. + TRUEDIV — CPython rounds int/int correctly; when both |operands| <= 2**53 the C double + division is exact-rounded too, otherwise → deopt. Divisor 0 → ZeroDivisionError("division + by zero"). + F64: + ADD/SUB/MUL — IEEE, as CPython. + TRUEDIV — divisor 0.0 (either sign) → ZeroDivisionError("float division by zero"). + FLOORDIV/MOD — CPython's float floor division and modulo (sign of divisor, `-0.0` cases), + divisor 0.0 → ZeroDivisionError("float floor division by zero") / ("float modulo by + zero"). + OBJ: CPython's generic operation on objects (PyNumber_*), never deopts. + """ + + op: BinOpKind + left: Expr + right: Expr + # I64 ADD/SUB/MUL only: set by the front end when an interval proof shows the result cannot + # leave the i64 range; the verifier re-proves it (never trusts it) and a proven op cannot + # deopt, so it is allowed in an impure function. The back end still uses the checked helper + # and treats an overflow there as an internal error (SystemError), never as a deopt. + proven: bool = False + + +class UnaryOpKind(Enum): + NEG = "-" + POS = "+" + NOT = "not" + + +@dataclass(frozen=True) +class UnaryOp(Expr): + """NEG of I64 is checked (-(-2**63) → deopt). NOT takes a BOOL and gives a BOOL.""" + + op: UnaryOpKind + operand: Expr + proven: bool = False # NEG of I64: as BinOp.proven + + +class CompareKind(Enum): + EQ = "==" + NE = "!=" + LT = "<" + LE = "<=" + GT = ">" + GE = ">=" + + +@dataclass(frozen=True) +class Compare(Expr): + """Result BOOL. Chains (`a < b < c`) are lowered by the front end into `And` with each middle + operand evaluated once (via a temporary). Mixed I64/F64 compares **exactly**, as CPython does — + not by converting the int to double (2**53+1 != float(2**53)). NaN compares as in IEEE.""" + + op: CompareKind + left: Expr + right: Expr + + +@dataclass(frozen=True) +class And(Expr): + """`a and b` on BOOL operands only (result BOOL), short-circuit. The front end does not lower + `and`/`or` on non-bool operands (they return an operand, not a bool).""" + + left: Expr + right: Expr + + +@dataclass(frozen=True) +class Or(Expr): + left: Expr + right: Expr + + +@dataclass(frozen=True) +class ToFloat(Expr): + """`float(x)` of an I64 (round-to-nearest-even, as CPython's int→float), or the implicit + int→float conversion of mixed arithmetic. Type F64.""" + + operand: Expr + + +@dataclass(frozen=True) +class Box(Expr): + """A scalar or array element converted to a Python object (type OBJ): I64 → int, F64 → float, + BOOL → True/False. Used when a typed value flows into OBJ code.""" + + operand: Expr + + +@dataclass(frozen=True) +class Unbox(Expr): + """An OBJ converted to a scalar. The runtime checks the exact type (`tp_unbox_*`) and the i64 + range; anything else → deopt. Because it can deopt, it is legal only in a pure function (the + verifier enforces this); impure code converts objects with `ObjToFloat` instead.""" + + operand: Expr + + +class MathFunc(Enum): + SQRT = "sqrt" + EXP = "exp" + LOG = "log" + SIN = "sin" + COS = "cos" + TAN = "tan" + ATAN2 = "atan2" + FABS = "fabs" + HYPOT = "hypot" + + +@dataclass(frozen=True) +class MathCall(Expr): + """`math.(x...)` on F64, type F64 (ATAN2 and HYPOT take exactly two arguments; other arities + stay interpreted), with the stdlib `math` module's errors: + a domain error → ValueError("math domain error"), a range overflow → + OverflowError("math range error") (CPython's math_1/math_2 rules on errno/inf/nan).""" + + func: MathFunc + args: tuple[Expr, ...] + + +@dataclass(frozen=True) +class Call(Expr): + """A direct call of another compiled function in the same module (C to C). If the callee may + deopt, the caller must be pure (the deopt propagates and the outermost compiled call is + redone); otherwise the front end calls it as an object (`CallObject(Global(name), ...)`). + Recursion (any call cycle) is rejected by the verifier: compiled C has no recursion limit where + CPython raises RecursionError.""" + + function: str + args: tuple[Expr, ...] + # Set when an impure caller calls a may_deopt (therefore pure) callee: on a deopt inside the + # callee, only the callee is redone by its interpreted function — unobservable, because the + # callee is pure — and the caller continues. Allowed only for callees returning F64, BOOL or + # NONE: an F64-typed IR expression is a Python float in CPython too (ints reach floats only + # through explicit ToFloat or float arithmetic), so the redone result is exactly a float; an + # I64 result could be a big int and is not allowed here. + redo: bool = False + + +@dataclass(frozen=True) +class Global(Expr): + """Reading a module-level name (then builtins) as a new reference, looked up **at the time of the read** in the + module namespace, exactly as CPython's LOAD_GLOBAL: rebinding the global later is seen. + Unbound → NameError("name 'x' is not defined"). Type OBJ. This is how compiled code reaches + functions and classes imported from Kotlin packages through the binder (they are ordinary + module globals). A read alone has no effect, so it does not make a function impure.""" + + name: str + + +@dataclass(frozen=True) +class GetAttr(Expr): + """`obj.name` on an OBJ (PyObject_GetAttr — descriptors, properties and binder proxies' attribute + hooks run exactly as in CPython). Type OBJ. Makes the function impure: a property may have + effects, so the call can never be redone.""" + + obj: Expr + name: str + + +@dataclass(frozen=True) +class CallObject(Expr): + """Calling an OBJ callable — a binder Kotlin function or proxy method, any Python callable — + with positional `args` and keyword arguments `kwnames` (the last len(kwnames) args are the + keyword values, as in vectorcall). Arguments are OBJ (scalars arrive through Box). Type OBJ. + Makes the function impure. This is the general path that keeps Kotlin interop available to + compiled code: whatever the binder accepts from interpreted code, it accepts from here, and + whatever it returns (a proxy, a str, a list) flows on as an opaque OBJ.""" + + callee: Expr + args: tuple[Expr, ...] + kwnames: tuple[str, ...] = () + + +@dataclass(frozen=True) +class Truth(Expr): + """`bool(obj)` as a condition (PyObject_IsTrue; may raise as CPython does). Type BOOL.""" + + operand: Expr + + +@dataclass(frozen=True) +class CompareObj(Expr): + """A comparison of two OBJ operands (a scalar side arrives through `Box`) **directly as a + condition**: an If/While cond, or an And/Or operand when that And/Or is itself the condition. + Semantics of `if a == b:` in CPython: the rich comparison runs, then the truth value of its + result — **without** PyObject_RichCompareBool's identity shortcut (a NaN float compared with + itself is False in CPython's `if x == x`). Type BOOL. + Anywhere the comparison's own result object would be kept (`x = a < b`, `r = c and a < b`, + `not (a < b)`) it is not allowed, because that object need not be a bool.""" + + op: CompareKind + left: Expr + right: Expr + + +@dataclass(frozen=True) +class ObjToFloat(Expr): + """`float(obj)` (PyNumber_Float: __float__/__index__ run as in CPython), result as F64. Raises as + CPython raises; never deopts, so it is allowed in impure functions. Type F64.""" + + operand: Expr + + +@dataclass(frozen=True) +class NewArray(Expr): + """A fresh **local** array (type F64_ARRAY / I64_ARRAY), owned by the function: + `[fill] * length` (`iota=False`), or `list(range(length))` (`iota=True`, I64 only, elements + 0..length-1). A negative length gives an empty array, as in CPython. Allocation failure → + MemoryError. Only legal as the value of an `Assign` to a local of the same array type.""" + + length: Expr # I64 + fill: Expr | None = None # element-typed; None when iota + iota: bool = False + + +@dataclass(frozen=True) +class CopyArray(Expr): + """`src[:]` of a local or parameter array: a fresh local array with the same elements. + Only legal as the value of an `Assign` to a local of the same array type.""" + + src: str + + +@dataclass(frozen=True) +class Tuple(Expr): + """`(a, b, ...)` of scalars/OBJs — a new tuple (type OBJ). Scalar elements arrive through Box.""" + + elements: tuple[Expr, ...] + + +@dataclass(frozen=True) +class Len(Expr): + """`len(array)` of an array parameter. Type I64.""" + + array: str + + +@dataclass(frozen=True) +class Index(Expr): + """`array[i]` of an array parameter, with list semantics: a negative index counts from the + end; out of range → IndexError("list index out of range"). Element type F64 or I64.""" + + array: str + index: Expr + # Set only by `verify` when it has proved 0 <= index < len for the non-negative case (e.g. a + # `ForRange` index over range(len(array))); then the back end may omit the bounds check. + # Never set by the front end. Unproved accesses keep the full check. + proven: bool = False + + +# --- statements ---------------------------------------------------------------------------------- + +@dataclass(frozen=True) +class Stmt: + pass + + +@dataclass(frozen=True) +class Assign(Stmt): + target: str + value: Expr + + +@dataclass(frozen=True) +class StoreIndex(Stmt): + """`array[i] = value` with list semantics (negative index; out of range → + IndexError("list assignment index out of range")). Marks the element dirty for write-back.""" + + array: str + index: Expr + value: Expr + proven: bool = False # as in Index + + +@dataclass(frozen=True) +class ExprStmt(Stmt): + value: Expr + + +@dataclass(frozen=True) +class If(Stmt): + cond: Expr # BOOL + then: tuple[Stmt, ...] + orelse: tuple[Stmt, ...] = () + + +@dataclass(frozen=True) +class While(Stmt): + cond: Expr # BOOL + body: tuple[Stmt, ...] + + +@dataclass(frozen=True) +class ForRange(Stmt): + """`for var in range(start, stop, step)`: `step` is a non-zero I64 Const; `start`/`stop` are + evaluated once before the loop, as range() does. `var` is I64 and is not assigned in the body + (the front end checks), so it never overflows: it stays between start and stop. After the loop + `var` keeps its last value, and if the loop ran zero times it keeps its previous binding — the + front end rejects reading it after the loop unless it was definitely assigned before.""" + + var: str + start: Expr + stop: Expr + step: Const + body: tuple[Stmt, ...] + + +@dataclass(frozen=True) +class Return(Stmt): + value: Expr | None = None + + +@dataclass(frozen=True) +class Break(Stmt): + pass + + +@dataclass(frozen=True) +class Continue(Stmt): + pass + + +# --- functions and modules ----------------------------------------------------------------------- + +@dataclass(frozen=True) +class Param: + name: str + type: Type + + +@dataclass(frozen=True) +class ArrayParam: + """How a `list[float]`/`list[int]` parameter becomes a native array, keeping list semantics: + + At entry (guards, before any effect): the argument is exactly a `list`, every element is exactly + `float` (F64_ARRAY) or exactly `int` within i64 (I64_ARRAY), and no two array parameters are + the same list object (aliasing would make separate copies diverge). Then the elements are copied + into a C array. The function body only indexes, stores into and takes `len` of it (no calls that + could observe the list, no appends: the front end checks), so the list and the array cannot + diverge observably. On **every** exit — return, exception, or deopt — dirty elements are + written back as new float/int objects before control leaves the function, so CPython's + partial-update-then-raise behaviour is preserved, and a deopt redo starts from the same list + state the interpreted function would see... which is only correct because a function with array + stores is impure and therefore can never deopt after entry (`Function.may_deopt` is False). + """ + + name: str + type: Type # F64_ARRAY or I64_ARRAY + stored: bool # whether the body stores into it (write-back needed) + + +@dataclass(frozen=True) +class Function: + """One compiled function. `params` lists every positional parameter in order (scalar Param, + ArrayParam, or Param of type OBJ for anything else). Keyword-only, *args, **kwargs, defaults + and decorators other than `compiled` are not compiled in this stage (the function stays + interpreted; `frontend` records why). + + Local arrays (fannkuch, M2): a local of an array type holds a native array the function owns, + created only by NewArray/CopyArray. It never escapes — used only by Index/StoreIndex/Len and + as a CopyArray source; never returned, passed, boxed or captured — so stores into it are not + effects (a function whose only stores are into local arrays can be pure), and no list object + ever exists for it. Reassigning the local frees the old array; every exit frees all of them. + Its length never changes after creation, so a bounds proof against Len(local) holds. + + `pure`: no effect outside its own locals — no stores into anything but locals, no CallObject, + no GetAttr (a property can have effects), no Truth/CompareObj/ObjToFloat/OBJ BinOp (they run + user methods), only `Call`s to pure functions. + + Kotlin interop (maintainer: compiled code must still talk to Kotlin freely): a function that + calls into the binder is impure, so its int arithmetic cannot use the deopting I64 path — + unbounded ints there are OBJ (CPython's own int objects, PyNumber_*), while floats (F64 never + deopts), bounded loop indices, and the loops around the calls are still native. Only a pure + function may contain deopting nodes after entry. + + Entry guards are generated by the back end from the params: exact runtime type per scalar/array + param (`type(x) is float`, not isinstance — CPython would compute with the int or the subclass + it was given), i64 range for I64, the ArrayParam checks. A failing guard → deopt (always safe: + nothing has happened yet). Wrong arity or keyword use → deopt too, so CPython raises its own + TypeError. A function returning a value must return on every path (no falling off the end). + """ + + name: str + params: tuple[Param | ArrayParam, ...] + returns: Type + locals: dict[str, Type] # non-parameter locals only (temporaries included) + body: tuple[Stmt, ...] + pure: bool + may_deopt: bool + source_line: int + # Module globals of a scalar type read ONCE at entry, as part of the guards: each must be + # exactly that type (else deopt, before any effect), and the body reads it as `Local(name)`. + # Legal only for a *closed* function — one that runs no user code (no GetAttr, CallObject, + # Truth, CompareObj, ObjToFloat, OBJ BinOp; Calls only to closed functions) — so nothing can + # rebind the global between entry and use, and reading it once equals reading it at each use. + entry_globals: tuple[Param, ...] = () + + +@dataclass(frozen=True) +class Module: + name: str + functions: tuple[Function, ...] + # Functions the front end left interpreted, with the reason (for `demo` and the coverage table). + skipped: dict[str, str] = field(default_factory=dict) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/paths.py b/python-multiplatform-ksp/src/main/python/typedpython/paths.py new file mode 100644 index 000000000..34f342238 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/paths.py @@ -0,0 +1,21 @@ +"""Turning the paths a caller gives into the Python files to check.""" +from collections.abc import Sequence +from pathlib import Path + +SKIPPED_DIRS = {"__pycache__", "build"} + + +def python_files(paths: Sequence[Path]) -> list[Path]: + """Files as given; directories expanded to their `.py` files, skipping hidden and cache dirs.""" + found: list[Path] = [] + for path in paths: + path = Path(path) + if not path.is_dir(): + found.append(path) + continue + for file in sorted(path.rglob("*.py")): + parts = file.relative_to(path).parts[:-1] + if any(p.startswith(".") or p in SKIPPED_DIRS for p in parts): + continue + found.append(file) + return found diff --git a/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py new file mode 100644 index 000000000..db3e2ffe1 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/pipeline.py @@ -0,0 +1,176 @@ +"""Source → gate → frontend → verify → cgen → cbuild (design §4.3, SPEC N-8/N-9, #41). + +`compile_module` is the one entry point the CLI, the demo and the build slot use: + +1. The gate in `compiled` mode: a module with type errors is not compiled at all. +2. `frontend.lower`: functions outside the IR are left interpreted (`skipped`, with reasons). +3. `verify.verify`: functions the verifier cannot prove are left interpreted too (SPEC N-9 — + unsafe code is never generated silently). +4. If nothing is left to compile, no C is produced (`extension is None`): the module ships as + ordinary Python. +5. `cgen.generate` + `cbuild.build`, against the host's CPython or a target's headers. +""" +import hashlib +import json +import sysconfig +from dataclasses import dataclass, field +from pathlib import Path +from types import ModuleType + +from typedpython import cbuild, cgen, frontend, gate, incremental, ir, verify + + +class CompileError(RuntimeError): + """The module cannot be compiled at all (it fails the gate).""" + + +@dataclass +class Result: + source: Path + module: ir.Module # what was compiled (verified functions only) + extension: Path | None # None: nothing to compile, ship the .py as is + skipped: dict[str, str] = field(default_factory=dict) # function -> why it stays interpreted + diagnostics: list[verify.Diagnostic] = field(default_factory=list) + + +def _gate_or_raise(path: Path) -> None: + errors = [d for d in gate.check([path], mode="compiled") if d.severity == "error"] + if errors: + listing = "\n".join(f"{d.path}:{d.line}:{d.column}: {d.message} [{d.rule}]" for d in errors) + raise CompileError(f"{path.name} does not pass the gate in compiled mode:\n{listing}") + + +def compile_module( + path: Path, + out_dir: Path, + *, + python_include: Path | None = None, + ext_suffix: str | None = None, + runtime_dir: Path | None = None, +) -> Result: + path = Path(path).resolve() + _gate_or_raise(path) + + lowered = frontend.lower(path) + proved, diagnostics = verify.verify(lowered) + skipped = dict(proved.skipped) + if not proved.functions: + return Result(path, proved, None, skipped, diagnostics) + + c_source = cgen.generate(proved, path) + extension = cbuild.build( + c_source, path.stem, Path(out_dir), + runtime_dir=runtime_dir, python_include=python_include, ext_suffix=ext_suffix, + ) + return Result(path, proved, extension, skipped, diagnostics) + + +def load(result: Result) -> ModuleType: + """Import the built extension (or, with nothing compiled, the source) from its path.""" + if result.extension is None: + import importlib.util + spec = importlib.util.spec_from_file_location(result.source.stem, result.source) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + return cbuild.load(result.source.stem, result.extension) + + +# --- incremental project build (SPEC N-10) ---------------------------------------------------------- + +def _tree_hash(*dirs: Path) -> str: + h = hashlib.sha256() + for d in dirs: + for f in sorted(p for p in Path(d).rglob("*") if p.is_file() and "__pycache__" not in p.parts): + h.update(f.relative_to(d).as_posix().encode() + b"\0" + f.read_bytes() + b"\0") + return h.hexdigest() + + +_PACKAGE = Path(__file__).resolve().parent + +# What the compiler is: every file of this package (front end, verifier, C back end, runtime header, +# builtin stubs). Any change to any of them can change the C produced for an unchanged source, so it +# invalidates the whole cache. Content-based, not a release number: it needs no manual bump and is +# right for a working tree; the cost is a rebuild after any edit to the compiler, which is correct. +COMPILER_VERSION = _tree_hash(_PACKAGE) + +_NO_EXTENSION = ".noext" # cache artefact of a module with nothing compiled + + +@dataclass +class ModuleResult: + path: Path + name: str + extension: Path | None # None: nothing compiled, ship the .py as is + status: str # "rebuilt" | "reused" + reasons: list[str] = field(default_factory=list) # why it was rebuilt (empty if reused) + skipped: dict[str, str] = field(default_factory=dict) # function -> why it stays interpreted + interface: str = "" + + +@dataclass +class ProjectResult: + modules: list[ModuleResult] + + @property + def rebuilt(self) -> list[ModuleResult]: + return [m for m in self.modules if m.status == "rebuilt"] + + +def compile_project( + sources: list[Path], + project_root: Path, + cache_dir: Path, + *, + python_include: Path | None = None, + ext_suffix: str | None = None, + runtime_dir: Path | None = None, +) -> ProjectResult: + sources = [Path(s).resolve() for s in sources] + cache_dir = Path(cache_dir) + verified: dict[Path, ir.Module] = {} + + def lower(path: Path) -> ir.Module: + _gate_or_raise(path) + proved, _ = verify.verify(frontend.lower(path)) + verified[path] = proved # the VERIFIED module: its signatures are what gets compiled + return proved + + def generate(module: ir.Module, path: Path) -> str: + return cgen.generate(module, path) if module.functions else "" + + def compile_c(c_source: str, name: str, out_dir: Path) -> Path: + if not c_source: + marker = Path(out_dir) / (name.rpartition(".")[2] + _NO_EXTENSION) + marker.write_text("nothing compiled\n") + return marker + return cbuild.build(c_source, name, out_dir, runtime_dir=runtime_dir, flags=cbuild.DEFAULT_FLAGS, + python_include=python_include, ext_suffix=ext_suffix) + + version = COMPILER_VERSION + if runtime_dir is not None: + version += ":" + _tree_hash(Path(runtime_dir)) + suffix = ext_suffix if ext_suffix is not None else sysconfig.get_config_var("EXT_SUFFIX") + include = cbuild._python_include(python_include) + flags = [*cbuild.DEFAULT_FLAGS, f"-I{include}"] + + cache = incremental.BuildCache(cache_dir) + report = cache.build_project( + sources, project_root, lower=lower, generate=generate, compile_c=compile_c, + compiler_version=version, platform=f"{sysconfig.get_platform()} {suffix}", flags=flags) + + side = cache_dir / "skipped" + side.mkdir(exist_ok=True) + out: list[ModuleResult] = [] + for m in report.modules: + record = side / f"{m.key}.json" + if m.status == "rebuilt": + record.write_text(json.dumps(dict(verified[m.path].skipped))) + try: + skipped = json.loads(record.read_text()) + except (OSError, ValueError): + skipped = {} + ext = None if m.artifact is None or m.artifact.suffix == _NO_EXTENSION else m.artifact + out.append(ModuleResult(m.path, m.name, ext, m.status, list(m.reasons), skipped, m.interface)) + return ProjectResult(out) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py b/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py new file mode 100644 index 000000000..016c7b6bb --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/pyrefly.py @@ -0,0 +1,181 @@ +"""Running Pyrefly and reading what it reports (docs/design/typedpython.md §4.1). + +Pyrefly is used as a CLI, not embedded. Two outputs are read: + +- its JSON error list, with the Any-leak error kinds raised to `error` by a generated config; +- its per-expression type report (`--report-pysa`), which says which expressions in user code + have type `Any`. That report is a Pyrefly-versioned format, which is why pyproject.toml pins + Pyrefly exactly. +""" +import json +import os +import re +import subprocess +import sys +import tempfile +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + +# Pyrefly error kinds that mean "an Any got in". Pyrefly leaves several of them off by default. +ANY_LEAK_KINDS = ( + "implicit-any-parameter", + "unannotated-return", + "unknown-variable-type", + "no-any-return-implicit", + "no-any-return-explicit", + "explicit-any", +) + +# How Pyrefly spells an Any in the type report: `typing.Any` for an explicit one and `Unknown` +# for an implicit one, alone or inside a larger type such as `list[typing.Any]`. +ANY_IN_TYPE = re.compile(r"(? (line, column) -> type, for every expression the type report records. + expression_types: dict[str, dict[tuple[int, int], str]] + + +# Directory holding `__builtins__.pyi`: names the interpreter provides without an import (`compiled`). +# Pyrefly 1.3.2 reads that file from any search-path entry as extra builtins. +BUILTIN_STUBS = Path(__file__).parent / "builtin_stubs" + + +def command() -> list[str]: + """The Pyrefly command line: `TYPEDPYTHON_PYREFLY`, or the copy installed next to this Python.""" + override = os.environ.get("TYPEDPYTHON_PYREFLY") + return [override] if override else [sys.executable, "-m", "pyrefly"] + + +def run(paths: Sequence[Path], search_paths: Sequence[Path] = ()) -> Report: + files = [str(Path(p).resolve()) for p in paths] + with tempfile.TemporaryDirectory(prefix="typedpython-") as work: + config = Path(work) / "pyrefly.toml" + config.write_text(_config([*search_paths, *import_roots(files), BUILTIN_STUBS])) + report_dir = Path(work) / "pysa" + argv = [ + *command(), "check", *files, + "--config", str(config), + "--output-format", "json", + "--summary=none", + "--report-pysa", str(report_dir), + "--report-pysa-format", "json", + ] + try: + result = subprocess.run(argv, capture_output=True, text=True, check=False) + except OSError as e: + raise PyreflyError(f"cannot run {argv[0]}: {e}") from e + + # Exit code 1 only means "type errors found"; anything else is a failure of the tool. + if result.returncode not in (0, 1): + raise PyreflyError(f"pyrefly exited {result.returncode}: {result.stderr.strip()}") + try: + raw_errors = json.loads(result.stdout)["errors"] + except (json.JSONDecodeError, KeyError) as e: + raise PyreflyError(f"unreadable pyrefly output: {result.stdout[:200]!r}") from e + + errors = [ + Error(_resolve(e["path"]), e["line"], e["column"], e["name"], e["description"]) + for e in raw_errors + ] + types = _expression_types(report_dir, set(files)) + return Report(errors, _any_expressions(types), _types_by_start(types)) + + +def import_roots(files: Sequence[str]) -> list[Path]: + """The directory each file is imported from: above its outermost package, else its own. + + The config lives in a temporary directory, and Pyrefly would otherwise take that as the import + root, so a package's own imports (relative or absolute) would not resolve. + """ + roots: list[Path] = [] + for file in files: + root = Path(file).parent + while (root / "__init__.py").exists(): + root = root.parent + if root not in roots: + roots.append(root) + return roots + + +def _config(search_paths: Sequence[Path]) -> str: + lines = [] + if search_paths: + quoted = ", ".join(json.dumps(str(Path(p).resolve())) for p in search_paths) + lines.append(f"search-path = [{quoted}]") + lines.append("[errors]") + lines += [f'{kind} = "error"' for kind in ANY_LEAK_KINDS] + return "\n".join(lines) + "\n" + + +def _expression_types(report_dir: Path, files: set[str]) -> list[AnyExpression]: + """Every typed expression the report records in the given files (typeshed and deps skipped).""" + try: + index = json.loads((report_dir / "pyrefly.pysa.json").read_text()) + except (OSError, json.JSONDecodeError) as e: + raise PyreflyError(f"pyrefly wrote no readable type report: {e}") from e + + found: list[AnyExpression] = [] + for module in index["modules"].values(): + source = module.get("source_path", {}).get("FileSystem") + if source is None or _resolve(source) not in files: + continue # typeshed and dependencies: not user code + table_file = report_dir / "type_of_expressions" / module["info_filename"] + if not table_file.exists(): + continue + table = json.loads(table_file.read_text()) + for function in table["functions"].values(): + types = function["type_table"] + for location, type_index in function["locations"].items(): + found.append(_expression(_resolve(source), location, types[type_index]["string"])) + return found + + +def _any_expressions(expressions: list[AnyExpression]) -> list[AnyExpression]: + return [e for e in expressions if ANY_IN_TYPE.search(e.type)] + + +def _types_by_start(expressions: list[AnyExpression]) -> dict[str, dict[tuple[int, int], str]]: + table: dict[str, dict[tuple[int, int], str]] = {} + for e in expressions: + table.setdefault(e.path, {})[(e.line, e.column)] = e.type + return table + + +def _expression(path: str, location: str, spelled: str) -> AnyExpression: + # "6:9-6:15" -> start line/column, stop line/column (1-based, as in Pyrefly's errors). + start, stop = location.split("-") + line, column = map(int, start.split(":")) + stop_line, stop_column = map(int, stop.split(":")) + return AnyExpression(path, line, column, stop_line, stop_column, spelled) + + +def _resolve(path: str) -> str: + return str(Path(path).resolve()) diff --git a/python-multiplatform-ksp/src/main/python/typedpython/rebinding.py b/python-multiplatform-ksp/src/main/python/typedpython/rebinding.py new file mode 100644 index 000000000..8f26cef8e --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/rebinding.py @@ -0,0 +1,258 @@ +"""Rebinding a name to a different type, and mixed-element containers (design §3.1). + +Pyrefly 1.3.2 reports neither, and its per-expression type report has no entries for assignment +targets. It does record every *load* of a name, with the name's type at that point. So the gate +writes a probed copy of the file in which each assignment of interest is followed, on the same +line, by a bare load of each target (`x = 1` becomes `x = 1; x`), runs Pyrefly on the copy, and +reads the target types back at the probe positions. Line numbers do not change, and every column +before the insertion point stays where it was. +""" +import ast +import re +from collections.abc import Iterable +from dataclasses import dataclass, field + +# Container types whose element type should be one type, not a union. Tuples are excluded: a +# fixed-length tuple is typed per slot, so `tuple[int, str]` is not "mixed". +MIXABLE = re.compile(r"^(list|set|frozenset|dict)\[(.*)\]$", re.DOTALL) +LITERAL = re.compile(r"^Literal\[(.*)\]$", re.DOTALL) + + +@dataclass(frozen=True) +class Assignment: + """One assignment to a plain name, in the original source.""" + + scope: int + name: str + line: int + column: int + end_line: int + end_byte: int # ast end_col_offset of the whole statement (UTF-8 bytes) + augmented: bool + display: bool # the value is a list/set/dict display with more than one element + + +@dataclass(frozen=True) +class Probe: + name: str + line: int + column: int + assignment: Assignment | None = field(default=None, compare=False) + + +@dataclass(frozen=True) +class Finding: + rule: str + line: int + column: int + message: str + + +def assignments(tree: ast.Module) -> list[Assignment]: + """Assignments to unannotated names, scope by scope (module, class bodies, functions).""" + found: list[Assignment] = [] + for scope_id, scope in enumerate(_scopes(tree)): + declared = _declared_names(scope) + for stmt in _own_statements(scope): + found += [a for a in _assignments_in(stmt, scope_id) if a.name not in declared] + return found + + +def candidates(found: Iterable[Assignment]) -> list[Assignment]: + """The assignments worth probing: names assigned more than once, and container displays.""" + found = list(found) + counts: dict[tuple[int, str], int] = {} + for a in found: + counts[(a.scope, a.name)] = counts.get((a.scope, a.name), 0) + 1 + return [a for a in found if counts[(a.scope, a.name)] > 1 or a.display] + + +def probe(source: str, wanted: Iterable[Assignment] | None = None) -> tuple[str, list[Probe]]: + """Insert a load of each target after its statement; `wanted=None` probes every assignment.""" + if wanted is None: + wanted = assignments(ast.parse(source)) + lines = source.splitlines(keepends=True) + # Insert right-to-left within a line so earlier offsets stay valid. + by_position: dict[tuple[int, int], list[Assignment]] = {} + for a in wanted: + by_position.setdefault((a.end_line, a.end_byte), []).append(a) + + probes: list[Probe] = [] + for (line_no, end_byte) in sorted(by_position, reverse=True): + encoded = lines[line_no - 1].encode() + head, tail = encoded[:end_byte].decode(), encoded[end_byte:].decode() + inserted = "" + added: list[Probe] = [] + for a in by_position[(line_no, end_byte)]: + inserted += "; " + added.append(Probe(a.name, line_no, len(head) + len(inserted) + 1, a)) + inserted += a.name + lines[line_no - 1] = head + inserted + tail + # Probes already placed further right on this line moved by what was just inserted. + probes = [ + Probe(q.name, q.line, q.column + len(inserted), q.assignment) + if q.line == line_no and q.column > len(head) else q + for q in probes + ] + probes += added + + return "".join(lines), probes + + +def judge(probes: list[Probe], types: dict[tuple[int, int], str]) -> list[Finding]: + """Turn probed types into findings.""" + findings: list[Finding] = [] + history: dict[tuple[int, str], tuple[str, Assignment]] = {} + + for p in sorted(probes, key=lambda p: (p.line, p.column)): + a = p.assignment + spelled = types.get((p.line, p.column)) + if a is None or spelled is None or _is_unknown(spelled): + continue + + if a.display and (mixed := _mixed_element(spelled)): + findings.append(Finding( + "forbidden/mixed-container", a.line, a.column, + f"`{a.name}` is `{spelled}`: its elements mix `{mixed}`; annotate the union " + "on purpose or keep one element type", + )) + + key = (a.scope, a.name) + current = _base(spelled) + if key in history and history[key][0] != current: + before = history[key][0] + findings.append(Finding( + "forbidden/rebinding", a.line, a.column, + f"`{a.name}` was `{before}` and is rebound to `{current}`; annotate it " + f"(`{a.name}: {before} | {current}`) or use another name", + )) + history[key] = (current, a) + + return findings + + +# --- AST helpers ------------------------------------------------------------------------------- + +Scope = ast.Module | ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef + + +def _scopes(tree: ast.Module) -> list[Scope]: + return [tree] + [ + n for n in ast.walk(tree) + if isinstance(n, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)) + ] + + +def _own_statements(scope: Scope) -> Iterable[ast.stmt]: + """Statements of this scope, descending into blocks but not into nested scopes.""" + stack = list(reversed(scope.body)) + while stack: + stmt = stack.pop() + yield stmt + if isinstance(stmt, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for name in ("body", "orelse", "finalbody"): + stack.extend(reversed(getattr(stmt, name, []))) + for handler in getattr(stmt, "handlers", []): + stack.extend(reversed(handler.body)) + for case in getattr(stmt, "cases", []): + stack.extend(reversed(case.body)) + + +def _declared_names(scope: Scope) -> set[str]: + """Names whose type is declared (annotated, parameters) or owned elsewhere (global).""" + names: set[str] = set() + if isinstance(scope, (ast.FunctionDef, ast.AsyncFunctionDef)): + args = scope.args + for arg in [*args.posonlyargs, *args.args, *args.kwonlyargs]: + names.add(arg.arg) + names |= {a.arg for a in (args.vararg, args.kwarg) if a is not None} + for stmt in _own_statements(scope): + if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name): + names.add(stmt.target.id) + elif isinstance(stmt, (ast.Global, ast.Nonlocal)): + names |= set(stmt.names) + return names + + +def _assignments_in(stmt: ast.stmt, scope_id: int) -> list[Assignment]: + if isinstance(stmt, ast.Assign): + targets = [t for target in stmt.targets for t in _names(target)] + display = len(stmt.targets) == 1 and isinstance(stmt.targets[0], ast.Name) \ + and _is_display(stmt.value) + augmented = False + elif isinstance(stmt, ast.AugAssign) and isinstance(stmt.target, ast.Name): + targets, display, augmented = [stmt.target], False, True + else: + return [] + assert stmt.end_lineno is not None and stmt.end_col_offset is not None + return [ + Assignment(scope_id, t.id, t.lineno, t.col_offset + 1, + stmt.end_lineno, stmt.end_col_offset, augmented, display) + for t in targets + ] + + +def _names(target: ast.expr) -> list[ast.Name]: + if isinstance(target, ast.Name): + return [target] + if isinstance(target, (ast.Tuple, ast.List)): + return [n for element in target.elts for n in _names(element)] + if isinstance(target, ast.Starred): + return _names(target.value) + return [] + + +def _is_display(value: ast.expr) -> bool: + if isinstance(value, (ast.List, ast.Set)): + return len(value.elts) > 1 + if isinstance(value, ast.Dict): + return len(value.keys) > 1 + return False + + +# --- type spelling ----------------------------------------------------------------------------- + +def _is_unknown(spelled: str) -> bool: + return spelled in ("Unknown", "typing.Any", "Any") + + +def _base(spelled: str) -> str: + """`Literal[1]` -> `int`, `Literal['a']` -> `str`, `Literal[True]` -> `bool`.""" + m = LITERAL.match(spelled) + if not m: + return spelled + value = m.group(1).strip() + if value in ("True", "False"): + return "bool" + if value[:1] in ("'", '"'): + return "str" + if value[:1] in ("b",) and value[1:2] in ("'", '"'): + return "bytes" + return "int" + + +def _mixed_element(spelled: str) -> str | None: + m = MIXABLE.match(spelled) + if not m: + return None + for argument in _split_top_level(m.group(2)): + if len(_split_top_level(argument, "|")) > 1: + return argument.strip() + return None + + +def _split_top_level(text: str, separator: str = ",") -> list[str]: + parts, depth, current = [], 0, "" + for ch in text: + if ch in "[(": + depth += 1 + elif ch in "])": + depth -= 1 + if ch == separator and depth == 0: + parts.append(current) + current = "" + else: + current += ch + parts.append(current) + return parts diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md new file mode 100644 index 000000000..6bfed5ddd --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/API.md @@ -0,0 +1,86 @@ +# tp_runtime.h — the C runtime API generated code may call + +Generated C (cgen.py) reaches memory and `PyObject`s **only** through these helpers (SPEC N-9). +Header-only, `static inline`, C99 + CPython C API, no other dependencies. Every helper implements +the CPython behaviour named in `ir.py`. Return convention unless stated: `0` ok, `1` deopt (no +Python error set, nothing to undo), `-1` Python error set (CPython's exception type and message — **of the running interpreter**: 3.14 changed several ZeroDivisionError and math-domain messages, and the runtime selects them by PY_VERSION_HEX). + +## Integers (int64_t) + int tp_add_i64(int64_t a, int64_t b, int64_t *out); /* 0 / 1 on overflow */ + int tp_sub_i64(int64_t a, int64_t b, int64_t *out); + int tp_mul_i64(int64_t a, int64_t b, int64_t *out); + int tp_neg_i64(int64_t a, int64_t *out); /* 1 for INT64_MIN */ + int tp_floordiv_i64(int64_t a, int64_t b, int64_t *out); /* -1 ZeroDivisionError; 1 for MIN // -1 */ + int tp_mod_i64(int64_t a, int64_t b, int64_t *out); /* -1 ZeroDivisionError; sign of b */ + int tp_truediv_i64(int64_t a, int64_t b, double *out); /* -1 ZeroDivisionError; 1 if |a| or |b| > 2**53 */ + double tp_i64_to_f64(int64_t a); /* as float(int): round half to even */ + +## Floats (double) + int tp_truediv_f64(double a, double b, double *out); /* 0 / -1 ZeroDivisionError("float division by zero") */ + int tp_floordiv_f64(double a, double b, double *out); /* CPython float_floor_div semantics */ + int tp_mod_f64(double a, double b, double *out); /* CPython float_rem semantics */ + +## Comparisons + int tp_cmp_i64_f64(int64_t a, double b, int op); /* exact, op is Py_LT..Py_GE; returns 0/1 */ + int tp_cmp_f64_i64(double a, int64_t b, int op); + +## math (stdlib math module semantics: math_1 / math_2 errno and inf/nan rules) + int tp_math_sqrt(double x, double *out); /* -1 ValueError("math domain error") */ + int tp_math_exp(double x, double *out); /* -1 OverflowError("math range error") */ + int tp_math_log(double x, double *out); + int tp_math_sin(double x, double *out); + int tp_math_cos(double x, double *out); + int tp_math_tan(double x, double *out); + int tp_math_fabs(double x, double *out); + int tp_math_atan2(double y, double x, double *out); + int tp_math_hypot(double x, double y, double *out); + +## Boxing (new references; NULL with MemoryError set on failure) + PyObject *tp_box_i64(int64_t v); + PyObject *tp_box_f64(double v); + PyObject *tp_box_bool(int v); + /* exact-type unboxing for guards: 0 ok, 1 wrong exact type or out of i64 range (deopt) */ + int tp_unbox_i64(PyObject *o, int64_t *out); /* type(o) is int and fits */ + int tp_unbox_f64(PyObject *o, double *out); /* type(o) is float */ + int tp_unbox_bool(PyObject *o, int *out); /* o is Py_True / Py_False */ + +## Native arrays for list[float] / list[int] parameters (ir.ArrayParam) + typedef struct { PyObject *list; Py_ssize_t len; double *data; unsigned char *dirty; } tp_f64_array; + typedef struct { PyObject *list; Py_ssize_t len; int64_t *data; unsigned char *dirty; } tp_i64_array; + + int tp_f64_array_enter(PyObject *obj, tp_f64_array *a); /* 1 unless exactly list of exactly float; -1 MemoryError */ + int tp_i64_array_enter(PyObject *obj, tp_i64_array *a); /* 1 unless exactly list of exactly int in i64 range */ + /* resolve a Python index (negative counts from the end) to a slot; -1 IndexError with + "list index out of range" (load) or "list assignment index out of range" (store) */ + int tp_f64_array_slot(const tp_f64_array *a, int64_t i, int store, Py_ssize_t *slot); + int tp_i64_array_slot(const tp_i64_array *a, int64_t i, int store, Py_ssize_t *slot); + /* write dirty elements back as new float/int objects and free the buffers. Always frees, even + when it fails (then -1 with MemoryError). Safe to call on a zeroed struct. */ + int tp_f64_array_exit(tp_f64_array *a); + int tp_i64_array_exit(tp_i64_array *a); + /* aliasing guard: 1 if any two of the n objects are the same object */ + int tp_any_same(PyObject *const *objs, Py_ssize_t n); + +## Opaque objects (Kotlin interop and any Python object; ir.Global/GetAttr/CallObject/Truth/CompareObj/ObjToFloat) +All return new references or NULL with CPython's exception set; none deopts. + PyObject *tp_global(PyObject *module_dict, PyObject *name); /* globals, then builtins; NameError("name 'x' is not defined") */ + PyObject *tp_getattr(PyObject *obj, PyObject *name); /* PyObject_GetAttr */ + PyObject *tp_call(PyObject *callee, PyObject *const *args, size_t nargs, PyObject *kwnames); /* PyObject_Vectorcall: nargs = positional count; keyword values follow at args[nargs..] */ + /* tp_binop_obj op codes: TP_BINOP_ADD..TP_BINOP_MOD = 0..5 in ir.BinOpKind declaration order */ + int tp_truth(PyObject *obj); /* 0/1, -1 error (PyObject_IsTrue) */ + int tp_compare_bool(PyObject *a, PyObject *b, int op); /* 0/1, -1 error: PyObject_RichCompare then PyObject_IsTrue (no identity shortcut, as `if a == b`) */ + int tp_obj_to_f64(PyObject *obj, double *out); /* 0 / -1 (PyNumber_Float, then exact double) */ + PyObject *tp_binop_obj(PyObject *a, PyObject *b, int op); /* PyNumber_Add/... for ir.BinOpKind on OBJ */ + void tp_release(PyObject **slot); /* Py_CLEAR: the only way generated code drops a reference */ + +## Local arrays (ir.NewArray / CopyArray; owned by the function, no list object) +Same structs as above with `list == NULL` and `dirty == NULL`. 0 ok, -1 MemoryError. + int tp_i64_array_new(tp_i64_array *a, int64_t n, int64_t fill); /* n < 0 → empty */ + int tp_i64_array_iota(tp_i64_array *a, int64_t n); /* 0..n-1; n < 0 → empty */ + int tp_f64_array_new(tp_f64_array *a, int64_t n, double fill); + int tp_i64_array_copy(tp_i64_array *dst, const tp_i64_array *src); + int tp_f64_array_copy(tp_f64_array *dst, const tp_f64_array *src); + void tp_i64_array_free(tp_i64_array *a); /* no write-back; zeroes; idempotent */ + void tp_f64_array_free(tp_f64_array *a); + /* tp_*_array_slot works unchanged on local arrays (same IndexError messages as list). */ + PyObject *tp_tuple(PyObject *const *items, Py_ssize_t n); /* new tuple; steals nothing */ diff --git a/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h new file mode 100644 index 000000000..876a33eb8 --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/runtime/tp_runtime.h @@ -0,0 +1,1011 @@ +/* tp_runtime.h -- the C runtime API generated TypedPython code may call (SPEC N-8, API.md). + * + * Header-only, `static inline`, C99 + the CPython C API (3.12+; exception messages follow CPython + * 3.13 and 3.14, which differ -- see TP_MSG_* below). Every helper implements the CPython + * behaviour named in ir.py; the differential tests in src/test/python/test_runtime.py compare each + * helper with CPython itself, bit for bit and message for message. + * + * Return convention unless stated: 0 ok, 1 deopt (no Python error set, nothing to undo), + * -1 Python error set. + */ +#ifndef TP_RUNTIME_H +#define TP_RUNTIME_H + +#include +#include +#include +#include +#include +#include + +#if PY_VERSION_HEX < 0x030C0000 +#error "tp_runtime.h needs CPython 3.12 or newer (PyErr_GetRaisedException)" +#endif + +/* ------------------------------------------------------------------------------------------ + * Exception messages that CPython changed between 3.13 and 3.14. + * 3.13: "integer division or modulo by zero" (int //), "integer modulo by zero" (int %), + * "float division by zero" / "float floor division by zero" / "float modulo by zero", + * "math domain error" for every math-domain ValueError. + * 3.14: "division by zero" for all of them; math_1 domain errors name the argument + * ("expected a nonnegative input, got -1.0", "expected a finite input, got inf", + * "expected a positive input, got 0.0"). + * ------------------------------------------------------------------------------------------ */ +#if PY_VERSION_HEX >= 0x030E0000 +#define TP_MSG_INT_FLOORDIV_ZERO "division by zero" +#define TP_MSG_INT_MOD_ZERO "division by zero" +#define TP_MSG_FLOAT_DIV_ZERO "division by zero" +#define TP_MSG_FLOAT_FLOORDIV_ZERO "division by zero" +#define TP_MSG_FLOAT_MOD_ZERO "division by zero" +#else +#define TP_MSG_INT_FLOORDIV_ZERO "integer division or modulo by zero" +#define TP_MSG_INT_MOD_ZERO "integer modulo by zero" +#define TP_MSG_FLOAT_DIV_ZERO "float division by zero" +#define TP_MSG_FLOAT_FLOORDIV_ZERO "float floor division by zero" +#define TP_MSG_FLOAT_MOD_ZERO "float modulo by zero" +#endif +#define TP_MSG_INT_TRUEDIV_ZERO "division by zero" /* long_true_divide: same in 3.13 and 3.14 */ + +#define TP_MATH_MSG_NONNEG "expected a nonnegative input, got %s" +#define TP_MATH_MSG_FINITE "expected a finite input, got %s" +#define TP_MATH_MSG_POSITIVE "expected a positive input, got %s" + +/* ir.BinOpKind order, for tp_binop_obj */ +#define TP_BINOP_ADD 0 +#define TP_BINOP_SUB 1 +#define TP_BINOP_MUL 2 +#define TP_BINOP_TRUEDIV 3 +#define TP_BINOP_FLOORDIV 4 +#define TP_BINOP_MOD 5 + +/* ------------------------------------------------------------------------------------------ + * Integers (int64_t) + * ------------------------------------------------------------------------------------------ */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(TP_RUNTIME_FORCE_PORTABLE_OVERFLOW) +#define TP_RUNTIME_PORTABLE_OVERFLOW 0 +#else +#define TP_RUNTIME_PORTABLE_OVERFLOW 1 +#endif + +static inline int tp_add_i64(int64_t a, int64_t b, int64_t *out) +{ +#if !TP_RUNTIME_PORTABLE_OVERFLOW + return __builtin_add_overflow(a, b, out) ? 1 : 0; +#else + if ((b > 0 && a > INT64_MAX - b) || (b < 0 && a < INT64_MIN - b)) { + return 1; + } + *out = a + b; + return 0; +#endif +} + +static inline int tp_sub_i64(int64_t a, int64_t b, int64_t *out) +{ +#if !TP_RUNTIME_PORTABLE_OVERFLOW + return __builtin_sub_overflow(a, b, out) ? 1 : 0; +#else + if ((b < 0 && a > INT64_MAX + b) || (b > 0 && a < INT64_MIN + b)) { + return 1; + } + *out = a - b; + return 0; +#endif +} + +static inline int tp_mul_i64(int64_t a, int64_t b, int64_t *out) +{ +#if !TP_RUNTIME_PORTABLE_OVERFLOW + return __builtin_mul_overflow(a, b, out) ? 1 : 0; +#else + if (a == 0 || b == 0) { + *out = 0; + return 0; + } + if (a == -1) { + if (b == INT64_MIN) return 1; + *out = -b; + return 0; + } + if (b == -1) { + if (a == INT64_MIN) return 1; + *out = -a; + return 0; + } + if (a > 0) { + if (b > 0 ? a > INT64_MAX / b : b < INT64_MIN / a) return 1; + } + else { + if (b > 0 ? a < INT64_MIN / b : a < INT64_MAX / b) return 1; + } + *out = a * b; + return 0; +#endif +} + +static inline int tp_neg_i64(int64_t a, int64_t *out) +{ + if (a == INT64_MIN) { + return 1; + } + *out = -a; + return 0; +} + +/* Python floor division: the quotient rounds toward negative infinity. */ +static inline int tp_floordiv_i64(int64_t a, int64_t b, int64_t *out) +{ + if (b == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, TP_MSG_INT_FLOORDIV_ZERO); + return -1; + } + if (b == -1) { + if (a == INT64_MIN) { + return 1; /* 2**63 does not fit */ + } + *out = -a; + return 0; + } + int64_t q = a / b; + int64_t r = a % b; + if (r != 0 && ((r < 0) != (b < 0))) { + q -= 1; + } + *out = q; + return 0; +} + +/* Python modulo: the result has the sign of the divisor. */ +static inline int tp_mod_i64(int64_t a, int64_t b, int64_t *out) +{ + if (b == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, TP_MSG_INT_MOD_ZERO); + return -1; + } + if (b == -1) { + *out = 0; /* also INT64_MIN % -1, which traps in C */ + return 0; + } + int64_t r = a % b; + if (r != 0 && ((r < 0) != (b < 0))) { + r += b; + } + *out = r; + return 0; +} + +/* CPython rounds int/int correctly; the C double division is the correctly rounded quotient when + * both operands are exactly representable, i.e. |x| <= 2**53. Otherwise: deopt. A zero divisor is + * an error whatever the dividend. */ +static inline int tp_truediv_i64(int64_t a, int64_t b, double *out) +{ + const int64_t lim = (int64_t)1 << 53; + if (b == 0) { + PyErr_SetString(PyExc_ZeroDivisionError, TP_MSG_INT_TRUEDIV_ZERO); + return -1; + } + if (a > lim || a < -lim || b > lim || b < -lim) { + return 1; + } + *out = (double)a / (double)b; + return 0; +} + +/* float(int): the C conversion rounds to nearest, ties to even in the default rounding mode. */ +static inline double tp_i64_to_f64(int64_t a) +{ + return (double)a; +} + +/* ------------------------------------------------------------------------------------------ + * Floats (double) + * ------------------------------------------------------------------------------------------ */ +static inline int tp_truediv_f64(double a, double b, double *out) +{ + if (b == 0.0) { + PyErr_SetString(PyExc_ZeroDivisionError, TP_MSG_FLOAT_DIV_ZERO); + return -1; + } + *out = a / b; + return 0; +} + +/* floatobject.c: _float_div_mod */ +static inline void tp__float_div_mod(double vx, double wx, double *floordiv, double *mod) +{ + double div; + *mod = fmod(vx, wx); + div = (vx - *mod) / wx; + if (*mod) { + if ((wx < 0) != (*mod < 0)) { + *mod += wx; + div -= 1.0; + } + } + else { + *mod = copysign(0.0, wx); + } + if (div) { + *floordiv = floor(div); + if (div - *floordiv > 0.5) { + *floordiv += 1.0; + } + } + else { + *floordiv = copysign(0.0, vx / wx); + } +} + +static inline int tp_floordiv_f64(double a, double b, double *out) +{ + double mod, floordiv; + if (b == 0.0) { + PyErr_SetString(PyExc_ZeroDivisionError, TP_MSG_FLOAT_FLOORDIV_ZERO); + return -1; + } + tp__float_div_mod(a, b, &floordiv, &mod); + *out = floordiv; + return 0; +} + +/* floatobject.c: float_rem */ +static inline int tp_mod_f64(double a, double b, double *out) +{ + double mod; + if (b == 0.0) { + PyErr_SetString(PyExc_ZeroDivisionError, TP_MSG_FLOAT_MOD_ZERO); + return -1; + } + mod = fmod(a, b); + if (mod) { + if ((b < 0) != (mod < 0)) { + mod += b; + } + } + else { + mod = copysign(0.0, b); + } + *out = mod; + return 0; +} + +/* ------------------------------------------------------------------------------------------ + * Comparisons: exact int/float comparison (float_richcompare), never via (double)int. + * ------------------------------------------------------------------------------------------ */ +static inline int tp__apply_cmp(int c, int op) +{ + switch (op) { + case Py_LT: return c < 0; + case Py_LE: return c <= 0; + case Py_EQ: return c == 0; + case Py_NE: return c != 0; + case Py_GT: return c > 0; + case Py_GE: return c >= 0; + } + return 0; +} + +/* three-way compare of an int64 with a non-NaN double: -1, 0, 1 */ +static inline int tp__cmp3_i64_f64(int64_t a, double b) +{ + if (b >= 9223372036854775808.0) return -1; /* b >= 2**63 > any int64 (also +inf) */ + if (b < -9223372036854775808.0) return 1; /* b < -2**63 <= any int64 (also -inf) */ + int64_t bi = (int64_t)b; /* truncation toward zero, b in range */ + if (a < bi) return -1; + if (a > bi) return 1; + double frac = b - (double)bi; /* exact: bi is b with the fraction dropped */ + if (frac > 0.0) return -1; /* b = a + frac */ + if (frac < 0.0) return 1; + return 0; +} + +static inline int tp_cmp_i64_f64(int64_t a, double b, int op) +{ + if (b != b) { + return op == Py_NE; + } + return tp__apply_cmp(tp__cmp3_i64_f64(a, b), op); +} + +static inline int tp_cmp_f64_i64(double a, int64_t b, int op) +{ + if (a != a) { + return op == Py_NE; + } + return tp__apply_cmp(-tp__cmp3_i64_f64(b, a), op); +} + +/* ------------------------------------------------------------------------------------------ + * math (mathmodule.c: math_1 / math_2 / is_error rules) + * ------------------------------------------------------------------------------------------ */ +static inline void tp__math_domain_error(double x, const char *msg314) +{ +#if PY_VERSION_HEX >= 0x030E0000 + if (msg314) { + char *buf = PyOS_double_to_string(x, 'r', 0, Py_DTSF_ADD_DOT_0, NULL); + if (buf) { + PyErr_Format(PyExc_ValueError, msg314, buf); + PyMem_Free(buf); + } + return; + } +#else + (void)x; + (void)msg314; +#endif + PyErr_SetString(PyExc_ValueError, "math domain error"); +} + +/* mathmodule.c is_error: 1 if an exception was set, 0 if the errno is to be ignored */ +static inline int tp__math_is_error(double x) +{ + int result = 1; + if (errno == EDOM) { + PyErr_SetString(PyExc_ValueError, "math domain error"); + } + else if (errno == ERANGE) { + if (fabs(x) < 1.5) { + result = 0; + } + else { + PyErr_SetString(PyExc_OverflowError, "math range error"); + } + } + else { + PyErr_SetFromErrno(PyExc_ValueError); + } + return result; +} + +/* math_1: x is the argument, r the libm result computed with errno cleared just before */ +static inline int tp__math_1_finish(double x, double r, int can_overflow, const char *msg314, + double *out) +{ + if (isnan(r) && !isnan(x)) { + tp__math_domain_error(x, msg314); + return -1; + } + if (isinf(r) && isfinite(x)) { + if (can_overflow) { + PyErr_SetString(PyExc_OverflowError, "math range error"); + } + else { + tp__math_domain_error(x, msg314); + } + return -1; + } + if (isfinite(r) && errno && tp__math_is_error(r)) { + return -1; + } + *out = r; + return 0; +} + +/* mathmodule.c m_log */ +static inline double tp__m_log(double x) +{ + if (isfinite(x)) { + if (x > 0.0) return log(x); + errno = EDOM; + if (x == 0.0) return -HUGE_VAL; + return NAN; + } + else if (isnan(x)) { + return x; + } + else if (x > 0.0) { + return x; + } + errno = EDOM; + return NAN; +} + +#define TP__DEF_MATH1(NAME, CALL, CAN_OVERFLOW, MSG) \ + static inline int tp_math_##NAME(double x, double *out) \ + { \ + double r; \ + errno = 0; \ + r = CALL; \ + return tp__math_1_finish(x, r, CAN_OVERFLOW, MSG, out); \ + } +TP__DEF_MATH1(sqrt, sqrt(x), 0, TP_MATH_MSG_NONNEG) +TP__DEF_MATH1(exp, exp(x), 1, NULL) +TP__DEF_MATH1(log, tp__m_log(x), 0, TP_MATH_MSG_POSITIVE) +TP__DEF_MATH1(sin, sin(x), 0, TP_MATH_MSG_FINITE) +TP__DEF_MATH1(cos, cos(x), 0, TP_MATH_MSG_FINITE) +TP__DEF_MATH1(tan, tan(x), 0, TP_MATH_MSG_FINITE) +TP__DEF_MATH1(fabs, fabs(x), 0, NULL) +#undef TP__DEF_MATH1 + +/* mathmodule.c m_atan2: the special values are defined here, the rest is libm */ +static inline double tp__m_atan2(double y, double x) +{ + const double pi = 3.14159265358979323846; + if (isnan(x) || isnan(y)) return NAN; + if (isinf(y)) { + if (isinf(x)) { + if (copysign(1., x) == 1.) return copysign(0.25 * pi, y); + return copysign(0.75 * pi, y); + } + return copysign(0.5 * pi, y); + } + if (isinf(x) || y == 0.) { + if (copysign(1., x) == 1.) return copysign(0., y); + return copysign(pi, y); + } + return atan2(y, x); +} + +/* math_2 rules for atan2 */ +static inline int tp_math_atan2(double y, double x, double *out) +{ + double r; + errno = 0; + r = tp__m_atan2(y, x); + if (isnan(r)) { + errno = (!isnan(x) && !isnan(y)) ? EDOM : 0; + } + else if (isinf(r)) { + errno = (isfinite(x) && isfinite(y)) ? ERANGE : 0; + } + if (errno && tp__math_is_error(r)) { + return -1; + } + *out = r; + return 0; +} + +#define TP__DBL_MIN 2.2250738585072014e-308 /* DBL_MIN, without */ + +/* mathmodule.c vector_norm (3.13 and 3.14 are identical): the correctly rounded hypot */ +typedef struct { double hi; double lo; } tp__dl; + +static inline tp__dl tp__dl_fast_sum(double a, double b) +{ + double x = a + b; + double y = (a - x) + b; + tp__dl r = {x, y}; + return r; +} + +static inline tp__dl tp__dl_mul(double x, double y) +{ + double z = x * y; + double zz = fma(x, y, -z); + tp__dl r = {z, zz}; + return r; +} + +static inline double tp__vector_norm(Py_ssize_t n, double *vec, double max, int found_nan) +{ + double x, h, scale, csum = 1.0, frac1 = 0.0, frac2 = 0.0; + tp__dl pr, sm; + int max_e; + Py_ssize_t i; + + if (isinf(max)) return max; + if (found_nan) return NAN; + if (max == 0.0 || n <= 1) return max; + frexp(max, &max_e); + if (max_e < -1023) { + for (i = 0; i < n; i++) { + vec[i] /= TP__DBL_MIN; + } + return TP__DBL_MIN * tp__vector_norm(n, vec, max / TP__DBL_MIN, found_nan); + } + scale = ldexp(1.0, -max_e); + for (i = 0; i < n; i++) { + x = vec[i]; + x *= scale; + pr = tp__dl_mul(x, x); + sm = tp__dl_fast_sum(csum, pr.hi); + csum = sm.hi; + frac1 += pr.lo; + frac2 += sm.lo; + } + h = sqrt(csum - 1.0 + (frac1 + frac2)); + pr = tp__dl_mul(-h, h); + sm = tp__dl_fast_sum(csum, pr.hi); + csum = sm.hi; + frac1 += pr.lo; + frac2 += sm.lo; + x = csum - 1.0 + (frac1 + frac2); + h += x / (2.0 * h); + return h / scale; +} + +static inline int tp_math_hypot(double x, double y, double *out) +{ + double vec[2]; + double max = 0.0; + int found_nan; + vec[0] = fabs(x); + vec[1] = fabs(y); + found_nan = isnan(vec[0]) | isnan(vec[1]); + if (vec[0] > max) max = vec[0]; + if (vec[1] > max) max = vec[1]; + *out = tp__vector_norm(2, vec, max, found_nan); + return 0; +} + +/* ------------------------------------------------------------------------------------------ + * Boxing + * ------------------------------------------------------------------------------------------ */ +static inline PyObject *tp_box_i64(int64_t v) { return PyLong_FromLongLong((long long)v); } +static inline PyObject *tp_box_f64(double v) { return PyFloat_FromDouble(v); } +static inline PyObject *tp_box_bool(int v) { return PyBool_FromLong(v ? 1 : 0); } + +static inline int tp_unbox_i64(PyObject *o, int64_t *out) +{ + int overflow = 0; + long long v; + if (!PyLong_CheckExact(o)) { + return 1; + } + v = PyLong_AsLongLongAndOverflow(o, &overflow); + if (overflow) { + return 1; + } + if (v == -1 && PyErr_Occurred()) { + PyErr_Clear(); /* cannot happen for an exact int; deopt rather than leak an error */ + return 1; + } + *out = (int64_t)v; + return 0; +} + +static inline int tp_unbox_f64(PyObject *o, double *out) +{ + if (!PyFloat_CheckExact(o)) { + return 1; + } + *out = PyFloat_AS_DOUBLE(o); + return 0; +} + +static inline int tp_unbox_bool(PyObject *o, int *out) +{ + if (o == Py_True) { + *out = 1; + return 0; + } + if (o == Py_False) { + *out = 0; + return 0; + } + return 1; +} + +/* ------------------------------------------------------------------------------------------ + * Native arrays for list[float] / list[int] parameters (ir.ArrayParam) + * + * enter: rc 0 -> the struct holds the copy; rc 1 or -1 -> the struct is zeroed (so exit is a safe + * no-op) and nothing was changed. `list` is borrowed: the caller keeps the argument alive. + * exit: writes the dirty slots back as new objects (PyList_SetItem steals the new reference and, on + * failure, drops it), always frees, and zeroes the struct. On a write-back failure it stops, + * frees, and returns -1 with that error set (MemoryError, or IndexError if the list shrank). + * ------------------------------------------------------------------------------------------ */ +typedef struct { PyObject *list; Py_ssize_t len; double *data; unsigned char *dirty; } tp_f64_array; +typedef struct { PyObject *list; Py_ssize_t len; int64_t *data; unsigned char *dirty; } tp_i64_array; + +static inline int tp_f64_array_enter(PyObject *obj, tp_f64_array *a) +{ + Py_ssize_t n, i; + a->list = NULL; + a->len = 0; + a->data = NULL; + a->dirty = NULL; + if (!PyList_CheckExact(obj)) { + return 1; + } + n = PyList_GET_SIZE(obj); + for (i = 0; i < n; i++) { + if (!PyFloat_CheckExact(PyList_GET_ITEM(obj, i))) { + return 1; + } + } + a->data = (double *)malloc(((size_t)n ? (size_t)n : 1) * sizeof(double)); + a->dirty = (unsigned char *)calloc((size_t)n ? (size_t)n : 1, 1); + if (a->data == NULL || a->dirty == NULL) { + free(a->data); + free(a->dirty); + a->data = NULL; + a->dirty = NULL; + PyErr_NoMemory(); + return -1; + } + for (i = 0; i < n; i++) { + a->data[i] = PyFloat_AS_DOUBLE(PyList_GET_ITEM(obj, i)); + } + a->list = obj; + a->len = n; + return 0; +} + +static inline int tp_i64_array_enter(PyObject *obj, tp_i64_array *a) +{ + Py_ssize_t n, i; + a->list = NULL; + a->len = 0; + a->data = NULL; + a->dirty = NULL; + if (!PyList_CheckExact(obj)) { + return 1; + } + n = PyList_GET_SIZE(obj); + a->data = (int64_t *)malloc(((size_t)n ? (size_t)n : 1) * sizeof(int64_t)); + a->dirty = (unsigned char *)calloc((size_t)n ? (size_t)n : 1, 1); + if (a->data == NULL || a->dirty == NULL) { + free(a->data); + free(a->dirty); + a->data = NULL; + a->dirty = NULL; + PyErr_NoMemory(); + return -1; + } + for (i = 0; i < n; i++) { + if (tp_unbox_i64(PyList_GET_ITEM(obj, i), &a->data[i]) != 0) { + free(a->data); + free(a->dirty); + a->data = NULL; + a->dirty = NULL; + return 1; + } + } + a->list = obj; + a->len = n; + return 0; +} + +/* list indexing: a negative index counts from the end; out of range -> IndexError */ +static inline int tp__array_slot(Py_ssize_t len, int64_t i, int store, Py_ssize_t *slot) +{ + if (i < 0) { + if (i < -(int64_t)len) goto out_of_range; + i += (int64_t)len; + } + else if (i >= (int64_t)len) { + goto out_of_range; + } + *slot = (Py_ssize_t)i; + return 0; +out_of_range: + PyErr_SetString(PyExc_IndexError, + store ? "list assignment index out of range" : "list index out of range"); + return -1; +} + +static inline int tp_f64_array_slot(const tp_f64_array *a, int64_t i, int store, Py_ssize_t *slot) +{ + return tp__array_slot(a->len, i, store, slot); +} + +static inline int tp_i64_array_slot(const tp_i64_array *a, int64_t i, int store, Py_ssize_t *slot) +{ + return tp__array_slot(a->len, i, store, slot); +} + +static inline int tp_f64_array_exit(tp_f64_array *a) +{ + int rc = 0; + if (a->data != NULL && a->dirty != NULL && a->list != NULL) { + Py_ssize_t i; + for (i = 0; i < a->len; i++) { + PyObject *v; + if (!a->dirty[i]) { + continue; + } + v = PyFloat_FromDouble(a->data[i]); + if (v == NULL || PyList_SetItem(a->list, i, v) < 0) { + rc = -1; /* SetItem consumed v on failure; the error is set */ + break; + } + } + } + free(a->data); + free(a->dirty); + a->list = NULL; + a->len = 0; + a->data = NULL; + a->dirty = NULL; + return rc; +} + +static inline int tp_i64_array_exit(tp_i64_array *a) +{ + int rc = 0; + if (a->data != NULL && a->dirty != NULL && a->list != NULL) { + Py_ssize_t i; + for (i = 0; i < a->len; i++) { + PyObject *v; + if (!a->dirty[i]) { + continue; + } + v = PyLong_FromLongLong((long long)a->data[i]); + if (v == NULL || PyList_SetItem(a->list, i, v) < 0) { + rc = -1; + break; + } + } + } + free(a->data); + free(a->dirty); + a->list = NULL; + a->len = 0; + a->data = NULL; + a->dirty = NULL; + return rc; +} + +/* ------------------------------------------------------------------------------------------ + * Local arrays (ir.NewArray / CopyArray): owned by the function, no list object behind them. + * Same structs as above with list == NULL and dirty == NULL; the slot helpers work on them + * unchanged. rc 0 ok, -1 MemoryError (the struct is then zeroed, so _free is still safe). + * n < 0 gives an empty array, like CPython's [x] * -1 == []. A size that overflows + * n * sizeof(element) is a MemoryError, not a wrapped small allocation. _free never writes + * anything back, zeroes the struct and is idempotent. _exit on a local array is defined to + * behave exactly like _free (nothing to write back since list == NULL, rc 0), so a stray _exit + * is harmless; local arrays are still released with _free. + * ------------------------------------------------------------------------------------------ */ +static inline void *tp__local_alloc(int64_t n, size_t elem, Py_ssize_t *len) +{ + void *p; + if (n < 0) { + n = 0; + } + if (n > (int64_t)PY_SSIZE_T_MAX || (uint64_t)n > (uint64_t)(SIZE_MAX / elem)) { + PyErr_NoMemory(); + return NULL; + } + p = malloc(n ? (size_t)n * elem : elem); + if (p == NULL) { + PyErr_NoMemory(); + return NULL; + } + *len = (Py_ssize_t)n; + return p; +} + +static inline int tp_i64_array_new(tp_i64_array *a, int64_t n, int64_t fill) +{ + Py_ssize_t len = 0, i; + int64_t *p; + a->list = NULL; a->len = 0; a->data = NULL; a->dirty = NULL; + p = (int64_t *)tp__local_alloc(n, sizeof(int64_t), &len); + if (p == NULL) { + return -1; + } + for (i = 0; i < len; i++) { + p[i] = fill; + } + a->data = p; + a->len = len; + return 0; +} + +static inline int tp_i64_array_iota(tp_i64_array *a, int64_t n) +{ + Py_ssize_t len = 0, i; + int64_t *p; + a->list = NULL; a->len = 0; a->data = NULL; a->dirty = NULL; + p = (int64_t *)tp__local_alloc(n, sizeof(int64_t), &len); + if (p == NULL) { + return -1; + } + for (i = 0; i < len; i++) { + p[i] = (int64_t)i; + } + a->data = p; + a->len = len; + return 0; +} + +static inline int tp_f64_array_new(tp_f64_array *a, int64_t n, double fill) +{ + Py_ssize_t len = 0, i; + double *p; + a->list = NULL; a->len = 0; a->data = NULL; a->dirty = NULL; + p = (double *)tp__local_alloc(n, sizeof(double), &len); + if (p == NULL) { + return -1; + } + for (i = 0; i < len; i++) { + p[i] = fill; + } + a->data = p; + a->len = len; + return 0; +} + +/* dst is overwritten (not freed); src may be a parameter array or a local one; dst is local */ +static inline int tp_i64_array_copy(tp_i64_array *dst, const tp_i64_array *src) +{ + Py_ssize_t len = 0; + int64_t *p = (int64_t *)tp__local_alloc((int64_t)src->len, sizeof(int64_t), &len); + dst->list = NULL; dst->len = 0; dst->data = NULL; dst->dirty = NULL; + if (p == NULL) { + return -1; + } + if (len > 0) { + memcpy(p, src->data, (size_t)len * sizeof(int64_t)); + } + dst->data = p; + dst->len = len; + return 0; +} + +static inline int tp_f64_array_copy(tp_f64_array *dst, const tp_f64_array *src) +{ + Py_ssize_t len = 0; + double *p = (double *)tp__local_alloc((int64_t)src->len, sizeof(double), &len); + dst->list = NULL; dst->len = 0; dst->data = NULL; dst->dirty = NULL; + if (p == NULL) { + return -1; + } + if (len > 0) { + memcpy(p, src->data, (size_t)len * sizeof(double)); + } + dst->data = p; + dst->len = len; + return 0; +} + +static inline void tp_i64_array_free(tp_i64_array *a) +{ + free(a->data); + free(a->dirty); + a->list = NULL; a->len = 0; a->data = NULL; a->dirty = NULL; +} + +static inline void tp_f64_array_free(tp_f64_array *a) +{ + free(a->data); + free(a->dirty); + a->list = NULL; a->len = 0; a->data = NULL; a->dirty = NULL; +} + +/* a new tuple of the n items; the items are borrowed (each gets its own new reference) */ +static inline PyObject *tp_tuple(PyObject *const *items, Py_ssize_t n) +{ + Py_ssize_t i; + PyObject *t = PyTuple_New(n); + if (t == NULL) { + return NULL; + } + for (i = 0; i < n; i++) { + Py_INCREF(items[i]); + PyTuple_SET_ITEM(t, i, items[i]); + } + return t; +} + +/* aliasing guard: 1 if any two of the n objects are the same object */ +static inline int tp_any_same(PyObject *const *objs, Py_ssize_t n) +{ + Py_ssize_t i, j; + for (i = 0; i < n; i++) { + for (j = i + 1; j < n; j++) { + if (objs[i] == objs[j]) { + return 1; + } + } + } + return 0; +} + +/* ------------------------------------------------------------------------------------------ + * Opaque objects (Kotlin interop and any Python object). All return new references or NULL with + * CPython's exception set; none deopts. + * ------------------------------------------------------------------------------------------ */ + +/* LOAD_GLOBAL: module globals, then builtins, looked up now; NameError("name 'x' is not defined") + * with .name set, as ceval's format_exc_check_arg does. `module_dict` is the module's __dict__; + * its "__builtins__" (a module or a dict) is searched next, the interpreter's builtins if absent. */ +static inline PyObject *tp_global(PyObject *module_dict, PyObject *name) +{ + PyObject *v, *builtins; + v = PyDict_GetItemWithError(module_dict, name); + if (v != NULL) { + Py_INCREF(v); + return v; + } + if (PyErr_Occurred()) { + return NULL; + } + builtins = PyDict_GetItemString(module_dict, "__builtins__"); /* borrowed, errors ignored */ + if (builtins != NULL && PyModule_Check(builtins)) { + builtins = PyModule_GetDict(builtins); + } + if (builtins == NULL || !PyDict_Check(builtins)) { + builtins = PyEval_GetBuiltins(); + } + if (builtins != NULL) { + v = PyDict_GetItemWithError(builtins, name); + if (v != NULL) { + Py_INCREF(v); + return v; + } + if (PyErr_Occurred()) { + return NULL; + } + } + { + PyObject *exc; + PyErr_Format(PyExc_NameError, "name '%.200U' is not defined", name); + exc = PyErr_GetRaisedException(); + if (exc != NULL) { + if (PyObject_SetAttrString(exc, "name", name) < 0) { + PyErr_Clear(); + } + PyErr_SetRaisedException(exc); + } + } + return NULL; +} + +static inline PyObject *tp_getattr(PyObject *obj, PyObject *name) +{ + return PyObject_GetAttr(obj, name); +} + +/* vectorcall: the last len(kwnames) of the nargs values are keyword values */ +static inline PyObject *tp_call(PyObject *callee, PyObject *const *args, size_t nargs, + PyObject *kwnames) +{ + return PyObject_Vectorcall(callee, args, nargs, kwnames); +} + +static inline int tp_truth(PyObject *obj) +{ + return PyObject_IsTrue(obj); +} + +/* A comparison in a condition, as CPython's `if a < b:` runs it: the rich comparison, then the truth + * value of its result. NOT PyObject_RichCompareBool: that one short-circuits `a is b` for == and != + * (so a NaN would equal itself), which `if x == x:` does not. */ +static inline int tp_compare_bool(PyObject *a, PyObject *b, int op) +{ + int r; + PyObject *res = PyObject_RichCompare(a, b, op); + if (res == NULL) { + return -1; + } + r = PyObject_IsTrue(res); + Py_DECREF(res); + return r; +} + +/* float(obj): PyNumber_Float, then the double of the result */ +static inline int tp_obj_to_f64(PyObject *obj, double *out) +{ + PyObject *f = PyNumber_Float(obj); + if (f == NULL) { + return -1; + } + *out = PyFloat_AS_DOUBLE(f); + Py_DECREF(f); + return 0; +} + +/* op is an ir.BinOpKind: TP_BINOP_ADD .. TP_BINOP_MOD (declaration order) */ +static inline PyObject *tp_binop_obj(PyObject *a, PyObject *b, int op) +{ + switch (op) { + case TP_BINOP_ADD: return PyNumber_Add(a, b); + case TP_BINOP_SUB: return PyNumber_Subtract(a, b); + case TP_BINOP_MUL: return PyNumber_Multiply(a, b); + case TP_BINOP_TRUEDIV: return PyNumber_TrueDivide(a, b); + case TP_BINOP_FLOORDIV: return PyNumber_FloorDivide(a, b); + case TP_BINOP_MOD: return PyNumber_Remainder(a, b); + } + PyErr_SetString(PyExc_SystemError, "tp_binop_obj: unknown operator"); + return NULL; +} + +/* Py_CLEAR: the only way generated code drops a reference */ +static inline void tp_release(PyObject **slot) +{ + Py_CLEAR(*slot); +} + +#endif /* TP_RUNTIME_H */ diff --git a/python-multiplatform-ksp/src/main/python/typedpython/verify.py b/python-multiplatform-ksp/src/main/python/typedpython/verify.py new file mode 100644 index 000000000..b649fb3ff --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython/verify.py @@ -0,0 +1,1137 @@ +"""TypedPython IR safety verifier (SPEC N-9, #41; maintainer decision 2026-10-03). + +`verify(module)` proves, per function and on the IR alone, the properties listed in the "Safety" +section of `ir.py`. A function it cannot prove is removed from `Module.functions` and moved to +`Module.skipped` with the reason, so it stays interpreted; every reason is also returned as a +`Diagnostic`. A proved function comes back with each `Index`/`StoreIndex` whose bounds were proved +marked `proven=True`; every other access has `proven=False` (an input `proven` is never trusted). + +The verifier reads nothing but `ir` — it is independent of the front end — and it trusts no flag +the front end set (`pure`, `may_deopt`, `ArrayParam.stored`, node `type`s): each is recomputed from +the body and compared. + +Rules (the `rule` of a Diagnostic): + + verify/definite-assignment a local may be read before it is assigned on some path + verify/missing-return a function returning a value can reach the end of its body + verify/type an operand, result, declaration, constant, or call signature is wrong + verify/ownership an OBJ is produced by a node that does not create a new reference, or + crosses into a scalar slot without Unbox (or a scalar into an OBJ + slot without Box) + verify/deopt-impure a node that can deopt appears in an impure function + verify/flag-may-deopt `may_deopt` differs from what the body computes + verify/flag-pure `pure` is set but the body has an effect + verify/array-use an array parameter is used other than by Index/StoreIndex/Len (or as a + CopyArray source), or those nodes name something that is not an array + verify/array-stored `ArrayParam.stored` differs from the body + verify/array-escape a local array is used other than by Index/StoreIndex/Len and as a + CopyArray source (returned, passed, boxed, stored in a tuple, aliased, + compared...), or NewArray/CopyArray stand anywhere but as the value of + an Assign to a local array + verify/unknown-node a node (or enum field) that `ir` does not define + verify/call-unresolved a Call names a function that is not compiled in this module + verify/call-unverified a Call names a function that the verifier rejected + verify/recursion the function is on a cycle of Calls (C stack depth is unbounded; + CPython would raise RecursionError instead) + verify/condition-only a CompareObj outside a condition (If/While cond, or an And/Or + operand that is itself in a condition): its rich-comparison result + need not be a bool, so only its truth value may be used + verify/structure break/continue outside a loop, the ForRange var assigned in its body, + a zero or non-constant step, duplicate names, bad declarations, an + entry global that collides with a parameter/local/other entry global or + is assigned in the body + verify/proven-overflow `proven` is set on an op the verifier cannot prove stays inside i64 (or + on a node `proven` does not apply to, or it is not a bool) + verify/redo `Call.redo` where it is not legal (see below) + verify/entry-global-open a function with entry_globals is not closed + +Proven i64 operations. The verifier never trusts `BinOp.proven` / `UnaryOp.proven`: for every I64 +expression it computes an interval [lo, hi] (exact Python ints) and accepts `proven=True` only on an +I64 ADD/SUB/MUL (BinOp) or NEG (UnaryOp) whose exact result interval lies inside [-2**63, 2**63-1]; +anything else (a different op, a non-I64 node, a result that may leave i64, a non-bool flag) is +rejected, and so is the whole function. A proven op is not a deopting node (an impure function may +contain it; a pure one does not count it toward `may_deopt`); every unproven I64 ADD/SUB/MUL/NEG +still deopts. The interval rules, exactly: + + Const v [v, v] + Local, a ForRange var inside its own loop body: + step > 0 [lo(start), max(lo(start), hi(stop) - 1)] + step < 0 [min(hi(start), lo(stop) + 1), hi(start)] + (the bounds are the intervals of the start/stop expressions, both + evaluated before the loop; then intersected with the i64 range) + Len(array) [0, 2**62] + ADD / SUB / MUL interval arithmetic on the operand intervals (for MUL the min/max of + the four endpoint products); a proven op must fit i64 exactly, an + unproven one is clamped to i64 (a result outside it deopts, so only + in-range values flow on) + NEG / POS [-hi, -lo] / [lo, hi] + everything else the full i64 range: I64 params and entry globals, every local that is + not the ForRange var of an enclosing loop (the verifier does no flow + analysis, so a local assigned a constant is still "any i64"), a loop + var after its loop, Index elements, Unbox, Call results, FLOORDIV, + MOD, TRUEDIV + +Local arrays (NewArray / CopyArray, ir.Function "Local arrays"). A local of an array type holds a +native array the function owns. The verifier proves, never trusting the front end: + + * NewArray/CopyArray are the value of an Assign to a local declared with the same array type, and + nowhere else (`verify/array-escape` elsewhere, `verify/type` on a type mismatch). NewArray: length + I64; fill absent iff `iota` (a bool), `iota` only for I64_ARRAY; a fill has the element type. + CopyArray: the source is a local or parameter array of the same type. + * No escape: a local array is read only by Index/StoreIndex/Len and as a CopyArray source. Any + `Local` naming it (a return value, a Call/CallObject/Box/Tuple operand, another assignment...) is + `verify/array-escape`. It cannot be a loop variable, a parameter, or an entry global. + * Purity: a StoreIndex into a local array is not an effect (and does not set `stored`, which is + for ArrayParams); a function whose only stores are into local arrays can be `pure`. A store into + an array parameter still is. `pure` and `may_deopt` are recomputed as before and compared. + * Definite assignment: the allocation's operands are read before the array is assigned, and every + Index/StoreIndex/Len/CopyArray source of a local array needs it assigned on every path. + * Bounds. (1) The ForRange rule (var over range(c >= 0, Len(a), step > 0)) now also covers a local + array, but only if the loop body never assigns that array (a reassignment may change its + length; the stop is evaluated once). (2) A local array assigned by exactly one Assign statement + in the whole body has a known length interval [lo, hi]: for NewArray the interval of the length + expression clamped at 0 below (a negative length is an empty array), for CopyArray the interval + of the source if the source is itself such a local (otherwise [0, 2**62]). Then Len(a) has that + interval, and an Index/StoreIndex whose index interval lies inside [0, lo - 1] is proven. That + proves constant indices and bounded index expressions; a negative index is never proven (the + `proven` flag means 0 <= index < len). An array assigned twice or more has no known length. + +Tuple. `Tuple(OBJ, elements)`: a tuple of OBJ expressions (a scalar needs Box), result OBJ, a +reference creator in the ownership rule. Building one is not an effect. + +What this cannot prove: anything that needs knowing a value through assignments (accumulators, +`x = 1; x + 1`), relations between variables (`i < n` does not bound `i + n`), bounds that come from +guards or conditions (`if n < 100`), values through array elements, calls or Unbox, the result of +`//` and `%` (even by a constant), a loop var after the loop, or `range` bounds that are themselves +full-range (`range(n)` bounds the var only by i64). A function needing such a proof keeps its +checked, deopting ops (and is then legal only if pure). + +Call.redo. Legal only if the caller is impure, the callee is pure and may_deopt, and the callee +returns F64, BOOL or NONE (`verify/redo` otherwise). A redo call is not a deopting node in the +caller: the callee alone is redone, unobservably, since it is pure. + +Function.entry_globals. Scalar `Param`s (I64/F64/BOOL), read once at entry as part of the guards; +assigned at entry for definite assignment; the body reads them as `Local(name)` of that type and may +not assign them; their names collide with no parameter, local or other entry global. The function +must be *closed*: no GetAttr, CallObject, Truth, CompareObj, ObjToFloat or OBJ BinOp, and Calls only +to closed functions (a fixpoint over the call graph), else `verify/entry-global-open`. Beyond that +rule (conservative, not in ir.py): a Call of a function that has entry globals can fail the callee's +entry guard in the middle of the caller, which is a deopt there unless the caller is itself guarded +the same way (it lists the same name and type as an entry global, and being closed cannot see it +rebound) or the call is a legal `redo`. +""" +from __future__ import annotations + +import dataclasses +from dataclasses import dataclass + +from . import ir +from .ir import ( + And, ArrayParam, Assign, BinOp, BinOpKind, Box, Break, Call, CallObject, Compare, CompareKind, + CompareObj, Const, Continue, CopyArray, ExprStmt, ForRange, Function, GetAttr, Global, If, Index, Len, + Local, MathCall, MathFunc, Module, NewArray, ObjToFloat, Or, Param, Return, StoreIndex, ToFloat, + Truth, Tuple, Type, Unbox, UnaryOp, UnaryOpKind, While, +) + +DA = "verify/definite-assignment" +MISSING_RETURN = "verify/missing-return" +TYPE = "verify/type" +OWNERSHIP = "verify/ownership" +DEOPT = "verify/deopt-impure" +FLAG_DEOPT = "verify/flag-may-deopt" +FLAG_PURE = "verify/flag-pure" +ARRAY_USE = "verify/array-use" +ARRAY_STORED = "verify/array-stored" +ESCAPE = "verify/array-escape" +UNKNOWN = "verify/unknown-node" +UNRESOLVED = "verify/call-unresolved" +UNVERIFIED = "verify/call-unverified" +RECURSION = "verify/recursion" +STRUCTURE = "verify/structure" +CONDITION_ONLY = "verify/condition-only" +PROVEN = "verify/proven-overflow" +REDO = "verify/redo" +ENTRY_OPEN = "verify/entry-global-open" + + +@dataclass(frozen=True) +class Diagnostic: + function: str + rule: str + message: str + source_line: int + + +# Exact classes, not isinstance: a subclass of an ir node is not an ir node (its meaning is not +# the one the back end implements). +_EXPRS = frozenset({Const, Local, BinOp, UnaryOp, Compare, And, Or, ToFloat, Box, Unbox, MathCall, + Call, Global, GetAttr, CallObject, Truth, CompareObj, ObjToFloat, Len, Index, + NewArray, CopyArray, Tuple}) +_STMTS = frozenset({Assign, StoreIndex, ExprStmt, If, While, ForRange, Return, Break, Continue}) +# Nodes whose OBJ result is a new reference (ir.py "ownership": Box, Global, GetAttr, CallObject, +# OBJ BinOp, Tuple), plus reads of an OBJ local (the local owns its reference; the back end increfs or +# borrows under that rule) and Calls of a compiled function returning OBJ (which returns a new +# reference by the same rule). +_OBJ_PRODUCERS = frozenset({Box, Global, GetAttr, CallObject, BinOp, Local, Call, Tuple}) +_VALUE_TYPES = (Type.I64, Type.F64, Type.BOOL, Type.OBJ) # what a local may hold +_RETURN_TYPES = _VALUE_TYPES + (Type.NONE,) +_ELEMENT = {Type.F64_ARRAY: Type.F64, Type.I64_ARRAY: Type.I64} +_MATH_ARITY = {MathFunc.ATAN2: 2, MathFunc.HYPOT: 2} # every other MathFunc: 1 +_I64_DEOPT_OPS = frozenset({BinOpKind.ADD, BinOpKind.SUB, BinOpKind.MUL, BinOpKind.FLOORDIV, + BinOpKind.MOD, BinOpKind.TRUEDIV}) +_I64_MIN, _I64_MAX = -2 ** 63, 2 ** 63 - 1 +_FULL = (_I64_MIN, _I64_MAX) +_LEN_MAX = 2 ** 62 +_PROVABLE_OPS = frozenset({BinOpKind.ADD, BinOpKind.SUB, BinOpKind.MUL}) + +# Definite assignment uses `None` for "unreachable" (bottom: every name counts as assigned). +_State = frozenset[str] | None + + +def _meet(a: _State, b: _State) -> _State: + if a is None: + return b + if b is None: + return a + return a & b + + +class _Loop: + def __init__(self, var: str | None, bounds_array: str | None): + self.var = var # ForRange var (None for While) + self.bounds_array = bounds_array # array whose indices by `var` are in bounds + self.interval = _FULL # the values `var` takes inside the body + self.breaks: list[_State] = [] + + +class _Checker: + """Checks one function against the module's signatures. Collects diagnostics; rebuilds the + body with proven bounds.""" + + def __init__(self, function: Function, signatures: dict[str, Function]): + self.fn = function + self.sigs = signatures + self.diags: list[Diagnostic] = [] + self.can_deopt = False + self.effects: list[str] = [] # reasons the body is impure + self.stored: set[str] = set() + self.callees: set[str] = set() + self.loops: list[_Loop] = [] + self.scalars: dict[str, Type] = {} # name -> type of every scalar/OBJ variable + self.arrays: dict[str, Type] = {} # name -> array type of every ArrayParam + self.larrays: dict[str, Type] = {} # name -> array type of every local array + self.assigns: dict[str, int] = {} # local array -> number of Assign statements to it + self.lens: dict[str, tuple[int, int]] = {} # single-assignment local array -> length + self.entry: dict[str, Type] = {} # name -> type of every entry global + self.opens: list[str] = [] # reasons the function is not closed + + def error(self, rule: str, message: str) -> None: + d = Diagnostic(self.fn.name, rule, message, self.fn.source_line) + if d not in self.diags: + self.diags.append(d) + + # --- declarations ------------------------------------------------------------------------- + + def declarations(self) -> frozenset[str]: + fn = self.fn + params: set[str] = set() + for p in fn.params: + if type(p) is ArrayParam: + if p.type not in ir.ARRAYS: + self.error(TYPE, f"array parameter '{p.name}' has non-array type {p.type}") + elif not isinstance(p.stored, bool): + self.error(TYPE, f"array parameter '{p.name}' has a non-bool `stored`") + else: + self.arrays[p.name] = p.type + elif type(p) is Param: + if p.type not in _VALUE_TYPES: + self.error(TYPE, f"parameter '{p.name}' has type {p.type}; arrays must be " + f"ArrayParam") + else: + self.scalars[p.name] = p.type + else: + self.error(UNKNOWN, f"parameter {p!r} is not an ir Param/ArrayParam") + continue + if p.name in params: + self.error(STRUCTURE, f"parameter '{p.name}' appears twice") + params.add(p.name) + eg = fn.entry_globals + if not isinstance(eg, tuple): + self.error(STRUCTURE, "entry_globals is not a tuple") + eg = () + for p in eg: + if type(p) is ArrayParam: + self.error(TYPE, f"entry global '{p.name}' is an array") + continue + if type(p) is not Param: + self.error(UNKNOWN, f"entry global {p!r} is not an ir Param") + continue + if p.type not in ir.SCALARS: + self.error(TYPE, f"entry global '{p.name}' has type {p.type}; only i64, f64 and " + f"bool globals can be read at entry") + continue + if p.name in params or p.name in self.entry: + self.error(STRUCTURE, f"entry global '{p.name}' collides with a parameter or " + f"another entry global") + continue + self.entry[p.name] = p.type + self.scalars[p.name] = p.type + params |= set(self.entry) + if not isinstance(fn.locals, dict): + self.error(STRUCTURE, "locals is not a dict") + return frozenset(params) + for name, t in fn.locals.items(): + if name in self.entry: + self.error(STRUCTURE, f"local '{name}' collides with an entry global") + elif name in self.arrays: + self.error(ARRAY_USE, f"local '{name}' shadows array parameter '{name}'") + elif t in ir.ARRAYS: + if name in self.scalars: + self.error(TYPE, f"local array '{name}' collides with a scalar parameter") + else: + self.larrays[name] = t + elif t not in _VALUE_TYPES: + self.error(TYPE, f"local '{name}' has type {t}") + elif name in self.scalars and self.scalars[name] is not t: + self.error(TYPE, f"local '{name}' declared {t} but parameter is " + f"{self.scalars[name]}") + else: + self.scalars[name] = t + if fn.returns not in _RETURN_TYPES: + self.error(TYPE, f"return type {fn.returns} is not a value type") + for flag in ("pure", "may_deopt"): + if not isinstance(getattr(fn, flag), bool): + self.error(TYPE, f"`{flag}` is not a bool") + return frozenset(params) + + # --- expressions -------------------------------------------------------------------------- + + def slot(self, e: ir.Expr, want: Type, what: str) -> None: + """`e` flows into a slot of type `want` (already checked as an expression).""" + got = getattr(e, "type", None) + if got is want: + return + if got is Type.OBJ and want in ir.SCALARS: + self.error(OWNERSHIP, f"{what}: OBJ flows into a {want.value} slot without Unbox") + elif want is Type.OBJ and got in ir.SCALARS: + self.error(OWNERSHIP, f"{what}: {got.value} flows into an OBJ slot without Box") + elif got in ir.ARRAYS: + pass # reported as array-use where the array was read + else: + self.error(TYPE, f"{what}: expected {want.value}, got " + f"{got.value if isinstance(got, Type) else got!r}") + + def expr(self, e: object, state: _State, cond: bool = False) -> ir.Expr: + """Check `e`, returning it rebuilt (with proven bounds). `state` is the definitely + assigned set at this point; `cond` says `e` is used only for its truth value (an If/While + condition, or an And/Or operand in one).""" + cls = type(e) + if cls not in _EXPRS: + self.error(UNKNOWN, f"expression node {cls.__name__} is not defined by ir") + return e # type: ignore[return-value] + t = e.type + if not isinstance(t, Type): + self.error(UNKNOWN, f"{cls.__name__} has non-Type type {t!r}") + return e + if t is Type.OBJ and cls not in _OBJ_PRODUCERS: + self.error(OWNERSHIP, f"{cls.__name__} cannot produce an OBJ (only Box, Global, GetAttr, CallObject, OBJ " + f"BinOp, Tuple, OBJ locals and OBJ-returning Calls create " + f"references)") + if cls is CompareObj and not cond: + self.error(CONDITION_ONLY, "CompareObj outside a condition: the rich comparison's " + "result object would be kept, and it need not be a bool") + method = getattr(self, "e_" + cls.__name__) + if cls in (And, Or): + return method(e, state, cond) + return method(e, state) + + def e_Const(self, e: Const, state: _State) -> ir.Expr: + v, t = e.value, e.type + ok = ((t is Type.I64 and type(v) is int and _I64_MIN <= v <= _I64_MAX) + or (t is Type.F64 and type(v) is float) + or (t is Type.BOOL and type(v) is bool) + or (t is Type.NONE and v is None)) + if not ok and t is not Type.OBJ: + self.error(TYPE, f"constant {v!r} does not match type {t.value}") + return e + + def e_Local(self, e: Local, state: _State) -> ir.Expr: + name = e.name + if name in self.arrays: + self.error(ARRAY_USE, f"array parameter '{name}' is used as a value (only Index, " + f"StoreIndex and Len may use it)") + return e + if name in self.larrays: + self.error(ESCAPE, f"local array '{name}' is used as a value (only Index, StoreIndex, " + f"Len and CopyArray may use it)") + return e + declared = self.scalars.get(name) + if declared is None: + self.error(TYPE, f"local '{name}' is not declared") + return e + if declared is not e.type: + self.error(TYPE, f"local '{name}' is {declared.value} but read as {e.type.value}") + if state is not None and name not in state: + self.error(DA, f"local '{name}' may be read before it is assigned") + return e + + def interval(self, e: ir.Expr) -> tuple[int, int]: + """The interval rules in the module docstring: [lo, hi] bounding every value `e` (an I64 + expression, already checked) can take when it completes without deopting.""" + cls = type(e) + if getattr(e, "type", None) is not Type.I64: + return _FULL + if cls is Const: + return (e.value, e.value) if type(e.value) is int else _FULL + if cls is Local: + for loop in reversed(self.loops): + if loop.var == e.name: + return loop.interval + return _FULL + if cls is Len: + return self.lens.get(e.array, (0, _LEN_MAX)) + if cls is BinOp and e.op in _PROVABLE_OPS: + lo, hi = _exact(e.op, self.interval(e.left), self.interval(e.right)) + return (max(lo, _I64_MIN), min(hi, _I64_MAX)) + if cls is UnaryOp and e.op in (UnaryOpKind.NEG, UnaryOpKind.POS): + lo, hi = self.interval(e.operand) + lo, hi = (-hi, -lo) if e.op is UnaryOpKind.NEG else (lo, hi) + return (max(lo, _I64_MIN), min(hi, _I64_MAX)) + return _FULL + + def check_proven(self, shape_ok: bool, interval: tuple[int, int], what: str) -> bool: + """`proven=True` (already known to be a bool) is accepted only if re-proved. Returns + whether the node counts as non-deopting.""" + if not shape_ok: + self.error(PROVEN, f"`proven` is set on {what}, which has no i64 overflow to prove") + elif not (_I64_MIN <= interval[0] and interval[1] <= _I64_MAX): + self.error(PROVEN, f"`proven` is set on {what} but its result interval " + f"[{interval[0]}, {interval[1]}] is not inside i64") + else: + return True + return False + + def e_BinOp(self, e: BinOp, state: _State) -> ir.Expr: + if type(e.op) is not BinOpKind: + self.error(UNKNOWN, f"BinOp op {e.op!r} is not a BinOpKind") + return e + left, right = self.expr(e.left, state), self.expr(e.right, state) + lt, rt = getattr(left, "type", None), getattr(right, "type", None) + proven = False + if type(e.proven) is not bool: + self.error(TYPE, f"BinOp `proven` {e.proven!r} is not a bool") + elif e.proven: + shape = (e.type is Type.I64 and lt is Type.I64 and rt is Type.I64 + and e.op in _PROVABLE_OPS) + exact = _exact(e.op, self.interval(left), self.interval(right)) if shape else _FULL + proven = self.check_proven(shape, exact, f"{e.type.value} {e.op.value}") + if lt not in (Type.I64, Type.F64, Type.OBJ): + self.error(TYPE if lt is not Type.OBJ else OWNERSHIP, + f"BinOp {e.op.value} on {getattr(lt, 'value', lt)} operands") + elif lt is not rt: + self.slot(right, lt, f"right operand of {e.op.value}") + else: + want = Type.F64 if (lt is Type.I64 and e.op is BinOpKind.TRUEDIV) else lt + if e.type is not want: + self.error(TYPE, f"{lt.value} {e.op.value} {rt.value} is {want.value}, node says " + f"{e.type.value}") + if lt is Type.I64 and e.op in _I64_DEOPT_OPS and not proven: + self.can_deopt = True + if lt is Type.OBJ: + self.effects.append(f"OBJ {e.op.value} (runs user methods)") + self.opens.append(f"OBJ {e.op.value} (runs user methods)") + return dataclasses.replace(e, left=left, right=right) + + def e_UnaryOp(self, e: UnaryOp, state: _State) -> ir.Expr: + if type(e.op) is not UnaryOpKind: + self.error(UNKNOWN, f"UnaryOp op {e.op!r} is not a UnaryOpKind") + return e + operand = self.expr(e.operand, state) + ot = getattr(operand, "type", None) + proven = False + if type(e.proven) is not bool: + self.error(TYPE, f"UnaryOp `proven` {e.proven!r} is not a bool") + elif e.proven: + shape = e.type is Type.I64 and ot is Type.I64 and e.op is UnaryOpKind.NEG + lo, hi = self.interval(operand) if shape else _FULL + proven = self.check_proven(shape, (-hi, -lo) if shape else _FULL, + f"unary {e.op.value} of {getattr(ot, 'value', ot)}") + if e.op is UnaryOpKind.NOT: + self.slot(operand, Type.BOOL, "operand of not") + if e.type is not Type.BOOL: + self.error(TYPE, f"not gives bool, node says {e.type.value}") + else: + if ot is Type.OBJ: + self.error(OWNERSHIP, f"unary {e.op.value} on OBJ has no IR form") + elif ot not in (Type.I64, Type.F64): + self.error(TYPE, f"unary {e.op.value} on {getattr(ot, 'value', ot)}") + elif e.type is not ot: + self.error(TYPE, f"unary {e.op.value} of {ot.value} is {ot.value}, node says " + f"{e.type.value}") + elif ot is Type.I64 and e.op is UnaryOpKind.NEG and not proven: + self.can_deopt = True + return dataclasses.replace(e, operand=operand) + + def e_Compare(self, e: Compare, state: _State) -> ir.Expr: + if type(e.op) is not CompareKind: + self.error(UNKNOWN, f"Compare op {e.op!r} is not a CompareKind") + return e + left, right = self.expr(e.left, state), self.expr(e.right, state) + lt, rt = getattr(left, "type", None), getattr(right, "type", None) + numeric = (Type.I64, Type.F64) + if Type.OBJ in (lt, rt): + self.error(OWNERSHIP, "Compare on an OBJ operand (rich comparison returns an object, " + "not a bool)") + elif not ((lt in numeric and rt in numeric) or (lt is Type.BOOL and rt is Type.BOOL)): + self.error(TYPE, f"Compare of {getattr(lt, 'value', lt)} and " + f"{getattr(rt, 'value', rt)}") + if e.type is not Type.BOOL: + self.error(TYPE, f"Compare gives bool, node says {e.type.value}") + return dataclasses.replace(e, left=left, right=right) + + def _bool_pair(self, e: And | Or, state: _State, cond: bool = False) -> ir.Expr: + name = type(e).__name__ + left, right = self.expr(e.left, state, cond), self.expr(e.right, state, cond) + self.slot(left, Type.BOOL, f"left operand of {name}") + self.slot(right, Type.BOOL, f"right operand of {name}") + if e.type is not Type.BOOL: + self.error(TYPE, f"{name} gives bool, node says {e.type.value}") + return dataclasses.replace(e, left=left, right=right) + + e_And = _bool_pair + e_Or = _bool_pair + + def e_ToFloat(self, e: ToFloat, state: _State) -> ir.Expr: + operand = self.expr(e.operand, state) + self.slot(operand, Type.I64, "operand of ToFloat") + if e.type is not Type.F64: + self.error(TYPE, f"ToFloat gives f64, node says {e.type.value}") + return dataclasses.replace(e, operand=operand) + + def e_Box(self, e: Box, state: _State) -> ir.Expr: + operand = self.expr(e.operand, state) + ot = getattr(operand, "type", None) + if ot is Type.OBJ: + self.error(OWNERSHIP, "Box of an OBJ (it is already an object)") + elif ot not in ir.SCALARS and ot not in ir.ARRAYS: + self.error(TYPE, f"Box of {getattr(ot, 'value', ot)}") + if e.type is not Type.OBJ: + self.error(TYPE, f"Box gives obj, node says {e.type.value}") + return dataclasses.replace(e, operand=operand) + + def e_Unbox(self, e: Unbox, state: _State) -> ir.Expr: + operand = self.expr(e.operand, state) + ot = getattr(operand, "type", None) + if ot in ir.SCALARS: + self.error(OWNERSHIP, f"Unbox of a {ot.value} (only an OBJ can be unboxed)") + elif ot is not Type.OBJ and ot not in ir.ARRAYS: + self.error(TYPE, f"Unbox of {getattr(ot, 'value', ot)}") + if e.type not in ir.SCALARS: + self.error(TYPE, f"Unbox gives a scalar, node says {e.type.value}") + elif e.type is Type.I64: + self.can_deopt = True + return dataclasses.replace(e, operand=operand) + + def e_MathCall(self, e: MathCall, state: _State) -> ir.Expr: + if type(e.func) is not MathFunc: + self.error(UNKNOWN, f"MathCall func {e.func!r} is not a MathFunc") + return e + args = self.args(e.args, state, f"math.{e.func.value}") + arity = _MATH_ARITY.get(e.func, 1) + if len(args) != arity: + self.error(TYPE, f"math.{e.func.value} takes {arity} argument(s), got {len(args)}") + for k, a in enumerate(args): + self.slot(a, Type.F64, f"argument {k + 1} of math.{e.func.value}") + if e.type is not Type.F64: + self.error(TYPE, f"math.{e.func.value} gives f64, node says {e.type.value}") + return dataclasses.replace(e, args=args) + + def args(self, args: object, state: _State, what: str) -> tuple[ir.Expr, ...]: + if not isinstance(args, tuple): + self.error(STRUCTURE, f"arguments of {what} are not a tuple") + return () + return tuple(self.expr(a, state) for a in args) + + def e_Call(self, e: Call, state: _State) -> ir.Expr: + args = self.args(e.args, state, f"call of '{e.function}'") + callee = self.sigs.get(e.function) + if callee is None: + self.error(UNRESOLVED, f"call of '{e.function}', which is not a compiled function of " + f"this module") + return dataclasses.replace(e, args=args) + self.callees.add(e.function) + if len(args) != len(callee.params): + self.error(TYPE, f"'{e.function}' takes {len(callee.params)} argument(s), got " + f"{len(args)}") + for k, (a, p) in enumerate(zip(args, callee.params)): + if type(p) is ArrayParam: + self.error(ARRAY_USE, f"'{e.function}' takes array parameter '{p.name}'; an array " + f"cannot be passed by Call (the entry guard and write-back " + f"belong to the Python-level call)") + else: + self.slot(a, p.type, f"argument {k + 1} of '{e.function}'") + if e.type is not callee.returns: + self.error(TYPE, f"'{e.function}' returns {callee.returns.value}, node says " + f"{e.type.value}") + redo = False + if type(e.redo) is not bool: + self.error(TYPE, f"Call `redo` {e.redo!r} is not a bool") + elif e.redo: + if self.fn.pure is True: + self.error(REDO, f"redo call of '{e.function}' in a pure function (a pure caller " + f"propagates the deopt instead)") + elif callee.pure is not True or callee.may_deopt is not True: + self.error(REDO, f"redo call of '{e.function}', which is not a pure may_deopt " + f"function") + elif callee.returns not in (Type.F64, Type.BOOL, Type.NONE): + self.error(REDO, f"redo call of '{e.function}' returning " + f"{getattr(callee.returns, 'value', callee.returns)}: only f64, " + f"bool and none results are exactly reproducible") + else: + redo = True + if callee.may_deopt is True and not redo: + self.can_deopt = True + if not redo and not self.covers(callee): + self.can_deopt = True + if callee.pure is not True: + self.effects.append(f"calls impure '{e.function}'") + return dataclasses.replace(e, args=args) + + def covers(self, callee: Function) -> bool: + """Every entry global of `callee` is also an entry global of this function with the same + type: the callee's entry guard cannot fail mid-caller (this function, being closed, + cannot see the global rebound).""" + eg = callee.entry_globals + if not isinstance(eg, tuple): + return False + return all(type(p) is Param and self.entry.get(p.name) is p.type for p in eg) + + def obj_operand(self, e: object, state: _State, what: str) -> ir.Expr: + x = self.expr(e, state) + self.slot(x, Type.OBJ, what) + return x + + def result(self, e: ir.Expr, want: Type) -> None: + if e.type is not want: + self.error(TYPE, f"{type(e).__name__} gives {want.value}, node says {e.type.value}") + + def e_Global(self, e: Global, state: _State) -> ir.Expr: + if not isinstance(e.name, str): + self.error(TYPE, f"Global name {e.name!r} is not a str") + self.result(e, Type.OBJ) + return e # a read alone has no effect (ir.Global) + + def e_GetAttr(self, e: GetAttr, state: _State) -> ir.Expr: + obj = self.obj_operand(e.obj, state, f"object of .{e.name}") + if not isinstance(e.name, str): + self.error(TYPE, f"GetAttr name {e.name!r} is not a str") + self.result(e, Type.OBJ) + self.effects.append(f"GetAttr .{e.name} (a property may have effects)") + self.opens.append(f"GetAttr .{e.name}") + return dataclasses.replace(e, obj=obj) + + def e_CallObject(self, e: CallObject, state: _State) -> ir.Expr: + callee = self.obj_operand(e.callee, state, "callee of CallObject") + args = self.args(e.args, state, "CallObject") + for k, a in enumerate(args): + self.slot(a, Type.OBJ, f"argument {k + 1} of CallObject") + kw = e.kwnames + if not (isinstance(kw, tuple) and all(isinstance(n, str) for n in kw)): + self.error(TYPE, f"CallObject kwnames {kw!r} is not a tuple of str") + elif len(kw) > len(args) or len(set(kw)) != len(kw): + self.error(TYPE, f"CallObject has {len(kw)} keyword name(s) {kw!r} for {len(args)} " + f"argument(s) (they must name distinct trailing arguments)") + self.result(e, Type.OBJ) + self.effects.append("CallObject") + self.opens.append("CallObject") + return dataclasses.replace(e, callee=callee, args=args) + + def e_Truth(self, e: Truth, state: _State) -> ir.Expr: + operand = self.obj_operand(e.operand, state, "operand of Truth") + self.result(e, Type.BOOL) + self.effects.append("Truth (runs __bool__/__len__)") + self.opens.append("Truth") + return dataclasses.replace(e, operand=operand) + + def e_CompareObj(self, e: CompareObj, state: _State) -> ir.Expr: + if type(e.op) is not CompareKind: + self.error(UNKNOWN, f"CompareObj op {e.op!r} is not a CompareKind") + return e + left, right = self.expr(e.left, state), self.expr(e.right, state) + types = (getattr(left, "type", None), getattr(right, "type", None)) + if Type.OBJ not in types: + self.error(TYPE, "CompareObj without an OBJ operand (use Compare)") + for t in types: + if t is not Type.OBJ and t not in ir.SCALARS and t not in ir.ARRAYS: + self.error(TYPE, f"CompareObj operand of type {getattr(t, 'value', t)}") + self.result(e, Type.BOOL) + self.effects.append("CompareObj (runs rich comparison)") + self.opens.append("CompareObj") + return dataclasses.replace(e, left=left, right=right) + + def e_ObjToFloat(self, e: ObjToFloat, state: _State) -> ir.Expr: + operand = self.obj_operand(e.operand, state, "operand of ObjToFloat") + self.result(e, Type.F64) + self.effects.append("ObjToFloat (runs __float__/__index__)") + self.opens.append("ObjToFloat") + return dataclasses.replace(e, operand=operand) + + def array(self, name: str, node: str, state: _State = None) -> Type | None: + """The array type of `name` (an array parameter or a local array); a local array must be + definitely assigned.""" + t = self.arrays.get(name) + if t is not None: + return t + t = self.larrays.get(name) + if t is None: + self.error(ARRAY_USE, f"{node} of '{name}', which is not an array") + elif state is not None and name not in state: + self.error(DA, f"local array '{name}' may be used ({node}) before it is assigned") + return t + + def e_Len(self, e: Len, state: _State) -> ir.Expr: + self.array(e.array, "len", state) + if e.type is not Type.I64: + self.error(TYPE, f"len gives i64, node says {e.type.value}") + return e + + def index(self, array: str, index: object, state: _State, node: str + ) -> tuple[ir.Expr, Type | None, bool]: + at = self.array(array, node, state) + idx = self.expr(index, state) + self.slot(idx, Type.I64, f"index of {node} '{array}'") + return idx, (_ELEMENT[at] if at else None), self.in_bounds(array, idx) + + def e_Index(self, e: Index, state: _State) -> ir.Expr: + idx, element, proven = self.index(e.array, e.index, state, "index") + if element is not None and e.type is not element and e.type is not Type.OBJ: + self.error(TYPE, f"element of '{e.array}' is {element.value}, node says " + f"{e.type.value}") + return dataclasses.replace(e, index=idx, proven=proven) + + def in_bounds(self, array: str, idx: ir.Expr) -> bool: + """0 <= idx < len(array): idx is the var of an enclosing ForRange over + range(c >= 0, len(array), step > 0) (for a local array, one whose body never assigns it), or + -- for a local array of known length -- idx has an interval inside [0, minimum length - 1]. + The var is never assigned in the body (checked), the stop is evaluated once, and no IR node + changes an array's length.""" + if array in self.lens and getattr(idx, "type", None) is Type.I64: + lo, hi = self.interval(idx) + if 0 <= lo and hi < self.lens[array][0]: + return True + if type(idx) is not Local or not (array in self.arrays or array in self.larrays): + return False + for loop in reversed(self.loops): + if loop.var == idx.name: + return loop.bounds_array == array + return False + + # --- tuples and local arrays -------------------------------------------------------------- + + def e_Tuple(self, e: Tuple, state: _State) -> ir.Expr: + if not isinstance(e.elements, tuple): + self.error(STRUCTURE, "Tuple elements are not a tuple") + return e + elements = tuple(self.obj_operand(x, state, f"element {k + 1} of Tuple") + for k, x in enumerate(e.elements)) + self.result(e, Type.OBJ) + return dataclasses.replace(e, elements=elements) + + def e_NewArray(self, e: NewArray, state: _State) -> ir.Expr: + return self.stray(e, state) + + def e_CopyArray(self, e: CopyArray, state: _State) -> ir.Expr: + return self.stray(e, state) + + def stray(self, e: NewArray | CopyArray, state: _State) -> ir.Expr: + self.error(ESCAPE, f"{type(e).__name__} is only legal as the value of an Assign to a local " + f"array") + return e + + def allocation(self, e: NewArray | CopyArray, target: str, state: _State) -> ir.Expr: + """`e` is the value of `Assign(target, e)`. Checks the operands, and that the node's type + is an array type equal to the target's declared one.""" + name = type(e).__name__ + want = self.larrays.get(target) + if not isinstance(e.type, Type) or e.type not in ir.ARRAYS: + self.error(TYPE, f"{name} gives an array, node says " + f"{getattr(e.type, 'value', e.type)}") + elif want is not None and e.type is not want: + self.error(TYPE, f"{name} gives {e.type.value} but local array '{target}' is " + f"{want.value}") + element = _ELEMENT.get(e.type) + if type(e) is CopyArray: + src = self.arrays.get(e.src) or self.larrays.get(e.src) + if not isinstance(e.src, str) or src is None: + self.error(ARRAY_USE, f"CopyArray of '{e.src}', which is not an array") + else: + self.array(e.src, "copy", state) + if src is not e.type: + self.error(TYPE, f"CopyArray of {src.value} '{e.src}' gives {e.type.value}") + return e + length = self.expr(e.length, state) + self.slot(length, Type.I64, "length of NewArray") + fill = e.fill + if type(e.iota) is not bool: + self.error(TYPE, f"NewArray `iota` {e.iota!r} is not a bool") + elif e.iota: + if e.type is not Type.I64_ARRAY: + self.error(TYPE, "NewArray iota is only for i64 arrays") + if fill is not None: + self.error(TYPE, "NewArray iota takes no fill") + elif fill is None: + self.error(TYPE, "NewArray needs a fill unless it is iota") + if fill is not None: + fill = self.expr(fill, state) + if element is not None: + self.slot(fill, element, "fill of NewArray") + return dataclasses.replace(e, length=length, fill=fill) + + def length_of(self, e: NewArray | CopyArray) -> tuple[int, int]: + """The interval of the length of the array `e` creates (the operands are checked).""" + if type(e) is CopyArray: + return self.lens.get(e.src, (0, _LEN_MAX)) + if getattr(e.length, "type", None) is not Type.I64: + return (0, _LEN_MAX) + lo, hi = self.interval(e.length) + return (min(max(lo, 0), _LEN_MAX), min(max(hi, 0), _LEN_MAX)) + + # --- statements --------------------------------------------------------------------------- + + def block(self, body: object, state: _State) -> tuple[tuple[ir.Stmt, ...], _State]: + if not isinstance(body, tuple): + self.error(STRUCTURE, "a statement block is not a tuple") + return (), state + out = [] + for s in body: + s2, state = self.stmt(s, state) + out.append(s2) + return tuple(out), state + + def stmt(self, s: object, state: _State) -> tuple[ir.Stmt, _State]: + cls = type(s) + if cls not in _STMTS: + self.error(UNKNOWN, f"statement node {cls.__name__} is not defined by ir") + return s, state # type: ignore[return-value] + return getattr(self, "s_" + cls.__name__)(s, state) + + def s_Assign(self, s: Assign, state: _State): + target = s.target + alloc = type(s.value) in (NewArray, CopyArray) + if alloc: + if target in self.larrays: + value = self.allocation(s.value, target, state) + if self.assigns.get(target) == 1: + self.lens[target] = self.length_of(value) + else: + value = self.allocation(s.value, target, state) + self.error(ESCAPE, f"{type(s.value).__name__} is assigned to '{target}', which is " + f"not a local array") + else: + value = self.expr(s.value, state) + if target in self.larrays: + self.error(TYPE, f"local array '{target}' may only be assigned NewArray or " + f"CopyArray") + if target in self.larrays: + pass + elif target in self.arrays: + self.error(ARRAY_USE, f"assignment to array parameter '{target}'") + elif target not in self.scalars: + self.error(TYPE, f"assignment to undeclared local '{target}'") + elif not alloc: + self.slot(value, self.scalars[target], f"assignment to '{target}'") + if getattr(value, "type", None) is Type.NONE: + self.error(TYPE, f"None assigned to '{target}'") + if target in self.entry: + self.error(STRUCTURE, f"entry global '{target}' is assigned in the body") + for loop in self.loops: + if loop.var == target: + self.error(STRUCTURE, f"loop variable '{target}' is assigned in its loop body") + return dataclasses.replace(s, value=value), (None if state is None else state | {target}) + + def s_StoreIndex(self, s: StoreIndex, state: _State): + idx, element, proven = self.index(s.array, s.index, state, "store") + value = self.expr(s.value, state) + if element is not None: + self.slot(value, element, f"value stored into '{s.array}'") + if s.array not in self.larrays: # a local array is the function's own: not an effect + self.stored.add(s.array) + self.effects.append(f"stores into '{s.array}'") + return dataclasses.replace(s, index=idx, value=value, proven=proven), state + + def s_ExprStmt(self, s: ExprStmt, state: _State): + return dataclasses.replace(s, value=self.expr(s.value, state)), state + + def cond(self, c: object, state: _State, what: str) -> ir.Expr: + c2 = self.expr(c, state, cond=True) + self.slot(c2, Type.BOOL, f"condition of {what}") + return c2 + + def s_If(self, s: If, state: _State): + cond = self.cond(s.cond, state, "if") + then, st = self.block(s.then, state) + orelse, se = self.block(s.orelse, state) + return dataclasses.replace(s, cond=cond, then=then, orelse=orelse), _meet(st, se) + + def s_While(self, s: While, state: _State): + cond = self.cond(s.cond, state, "while") + loop = _Loop(None, None) + self.loops.append(loop) + # Assignments only add names, so the loop-head state is the entry state (the meet of the + # entry, end-of-body and continue states equals the entry state). + body, _ = self.block(s.body, state) + self.loops.pop() + forever = type(s.cond) is Const and s.cond.type is Type.BOOL and s.cond.value is True + after: _State = None if forever else state + for br in loop.breaks: + after = _meet(after, br) + return dataclasses.replace(s, cond=cond, body=body), after + + def s_ForRange(self, s: ForRange, state: _State): + start = self.expr(s.start, state) + stop = self.expr(s.stop, state) + self.slot(start, Type.I64, "range start") + self.slot(stop, Type.I64, "range stop") + step = s.step + step_ok = (type(step) is Const and step.type is Type.I64 and type(step.value) is int + and step.value != 0 and _I64_MIN <= step.value <= _I64_MAX) + if not step_ok: + self.error(STRUCTURE, f"range step must be a non-zero i64 Const, got {step!r}") + var = s.var + if var in self.arrays: + self.error(ARRAY_USE, f"loop variable '{var}' is an array parameter") + elif self.scalars.get(var) is not Type.I64: + self.error(TYPE, f"loop variable '{var}' must be a declared i64 local") + if var in self.entry: + self.error(STRUCTURE, f"entry global '{var}' is assigned by the loop") + for outer in self.loops: + if outer.var == var: + self.error(STRUCTURE, f"loop variable '{var}' is assigned in its loop body " + f"(by an inner loop)") + bounds = None + if (step_ok and step.value > 0 and type(start) is Const and start.type is Type.I64 + and type(start.value) is int and start.value >= 0 and type(stop) is Len + and (stop.array in self.arrays + or (stop.array in self.larrays + and stop.array not in _assigned(s.body)))): + bounds = stop.array + loop = _Loop(var, bounds) + if (step_ok and getattr(start, "type", None) is Type.I64 + and getattr(stop, "type", None) is Type.I64): + (slo, shi), (elo, ehi) = self.interval(start), self.interval(stop) + if step.value > 0: + lo = slo + hi = max(lo, ehi - 1) + else: + hi = shi + lo = min(hi, elo + 1) + loop.interval = (max(lo, _I64_MIN), min(hi, _I64_MAX)) + self.loops.append(loop) + body, _ = self.block(s.body, None if state is None else state | {var}) + self.loops.pop() + # Zero iterations leave `var` (and everything the body assigns) as it was: the state after + # the loop is the entry state, met with every break state (each a superset of it). + after: _State = state + for br in loop.breaks: + after = _meet(after, br) + return dataclasses.replace(s, start=start, stop=stop, body=body), after + + def s_Return(self, s: Return, state: _State): + returns = self.fn.returns + if s.value is None: + if returns is not Type.NONE: + self.error(TYPE, f"return without a value in a function returning " + f"{getattr(returns, 'value', returns)}") + return s, None + value = self.expr(s.value, state) + if isinstance(returns, Type): + self.slot(value, returns, "return value") + return dataclasses.replace(s, value=value), None + + def _jump(self, s: Break | Continue, state: _State): + if not self.loops: + self.error(STRUCTURE, f"{type(s).__name__.lower()} outside a loop") + elif type(s) is Break: + self.loops[-1].breaks.append(state) + return s, None + + s_Break = _jump + s_Continue = _jump + + # --- the function ------------------------------------------------------------------------- + + def run(self) -> Function: + fn = self.fn + params = self.declarations() + if self.larrays: + counts: dict[str, int] = {} + _assigned(fn.body, counts) + self.assigns = {n: counts.get(n, 0) for n in self.larrays} + body, end = self.block(fn.body, params) + if end is not None and fn.returns is not Type.NONE: + self.error(MISSING_RETURN, f"the end of '{fn.name}' is reachable but it returns " + f"{getattr(fn.returns, 'value', fn.returns)}") + # deopt placement and flags + if self.can_deopt and fn.pure is not True: + self.error(DEOPT, "a node that can deopt (checked i64 arithmetic, i64 negation, Unbox to " + "i64, or a call of a may_deopt function) is in an impure function; a redo " + "would repeat its effects") + if fn.may_deopt is not self.can_deopt: + if fn.pure is True or fn.may_deopt is True: + self.error(FLAG_DEOPT, f"may_deopt is {fn.may_deopt} but the body " + f"{'can' if self.can_deopt else 'cannot'} deopt") + if fn.pure is True and self.effects: + self.error(FLAG_PURE, "pure is set but the body " + "; ".join(sorted(set(self.effects)))) + for p in fn.params: + if type(p) is ArrayParam and p.name in self.arrays and p.stored is not (p.name in + self.stored): + self.error(ARRAY_STORED, f"array '{p.name}' has stored={p.stored} but the body " + f"{'stores' if p.name in self.stored else 'never stores'}" + f" into it") + return dataclasses.replace(fn, body=body) + + +def _assigned(body: object, counts: dict[str, int] | None = None) -> dict[str, int]: + """Every name assigned by an Assign statement anywhere in `body` (nested blocks included), with + the number of Assign statements. Tolerates a malformed body (the checker reports it).""" + counts = {} if counts is None else counts + if isinstance(body, tuple): + for st in body: + cls = type(st) + if cls is Assign: + counts[st.target] = counts.get(st.target, 0) + 1 + elif cls is If: + _assigned(st.then, counts) + _assigned(st.orelse, counts) + elif cls in (While, ForRange): + _assigned(st.body, counts) + return counts + + +def _exact(op: BinOpKind, a: tuple[int, int], b: tuple[int, int]) -> tuple[int, int]: + """The exact (unbounded) result interval of ADD/SUB/MUL on operand intervals.""" + if op is BinOpKind.ADD: + return a[0] + b[0], a[1] + b[1] + if op is BinOpKind.SUB: + return a[0] - b[1], a[1] - b[0] + products = (a[0] * b[0], a[0] * b[1], a[1] * b[0], a[1] * b[1]) + return min(products), max(products) + + +def _cycles(graph: dict[str, set[str]]) -> set[str]: + """Names on a cycle of `graph` (Tarjan's strongly connected components).""" + index: dict[str, int] = {} + low: dict[str, int] = {} + stack: list[str] = [] + on: set[str] = set() + result: set[str] = set() + counter = [0] + + def visit(v: str) -> None: + index[v] = low[v] = counter[0] + counter[0] += 1 + stack.append(v) + on.add(v) + for w in graph.get(v, ()): + if w not in graph: + continue + if w not in index: + visit(w) + low[v] = min(low[v], low[w]) + elif w in on: + low[v] = min(low[v], index[w]) + if low[v] == index[v]: + comp = [] + while True: + w = stack.pop() + on.discard(w) + comp.append(w) + if w == v: + break + if len(comp) > 1 or v in graph.get(v, ()): + result.update(comp) + + for v in graph: + if v not in index: + visit(v) + return result + + +def verify(module: Module) -> tuple[Module, list[Diagnostic]]: + """Prove each function of `module`; return the module of proved functions (with proven bounds + marked) and every diagnostic. Rejected functions move to `skipped` with the reason.""" + diags: list[Diagnostic] = [] + reasons: dict[str, str] = {} + functions = module.functions if isinstance(module.functions, tuple) else () + + def reject(fn_name: str, line: int, rule: str, message: str) -> None: + d = Diagnostic(fn_name, rule, message, line) + if d not in diags: + diags.append(d) + reasons.setdefault(fn_name, f"{rule}: {message}") + + named: dict[str, list[Function]] = {} + for g in functions: + if type(g) is not Function: + reject(getattr(g, "name", repr(g)), getattr(g, "source_line", 0), UNKNOWN, + f"{type(g).__name__} is not an ir Function") + continue + named.setdefault(g.name, []).append(g) + for name, gs in named.items(): + if len(gs) > 1: + for g in gs: + reject(name, g.source_line, STRUCTURE, f"function '{name}' is defined " + f"{len(gs)} times") + sigs = {name: gs[0] for name, gs in named.items() if len(gs) == 1} + + checked: dict[str, Function] = {} + callees: dict[str, set[str]] = {} + open_why: dict[str, str] = {} # a function that runs user code -> why + for name, g in sigs.items(): + c = _Checker(g, sigs) + rebuilt = c.run() + for d in c.diags: + reject(d.function, d.source_line, d.rule, d.message) + checked[name] = rebuilt + callees[name] = c.callees + if c.opens: + open_why[name] = c.opens[0] + + # Closedness (entry_globals): a fixpoint over the call graph. + changed = True + while changed: + changed = False + for name, cs in callees.items(): + if name not in open_why: + bad = sorted(c for c in cs if c in open_why) + if bad: + open_why[name] = f"calls '{bad[0]}', which runs user code ({open_why[bad[0]]})" + changed = True + for name, g in sigs.items(): + if name in open_why and isinstance(g.entry_globals, tuple) and g.entry_globals: + reject(name, g.source_line, ENTRY_OPEN, + f"has entry globals but is not closed: {open_why[name]}") + + for name in sorted(_cycles(callees)): + reject(name, sigs[name].source_line, RECURSION, + f"'{name}' is on a cycle of calls; compiled recursion has no depth limit") + + # A call into a function that is not compiled cannot be a C call: propagate to a fixpoint. + changed = True + while changed: + changed = False + for name, cs in callees.items(): + if name in reasons: + continue + bad = sorted(c for c in cs if c in reasons) + if bad: + reject(name, sigs[name].source_line, UNVERIFIED, + f"calls '{bad[0]}', which is not compiled ({reasons[bad[0]]})") + changed = True + + kept = tuple(checked[g.name] for g in functions + if type(g) is Function and g.name in checked and g.name not in reasons) + skipped = dict(module.skipped) + for name, reason in reasons.items(): + skipped.setdefault(name, reason) + return Module(module.name, kept, skipped), diags diff --git a/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth b/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth new file mode 100644 index 000000000..98776970a --- /dev/null +++ b/python-multiplatform-ksp/src/main/python/typedpython_builtins.pth @@ -0,0 +1 @@ +import builtins; builtins.compiled = getattr(builtins, "compiled", lambda f, /: f) diff --git a/python-multiplatform-ksp/src/test/python/conftest.py b/python-multiplatform-ksp/src/test/python/conftest.py new file mode 100644 index 000000000..f04ff2974 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/conftest.py @@ -0,0 +1,48 @@ +from pathlib import Path +import textwrap + +import pytest + +FIXTURES = Path(__file__).parent / "fixtures" + + +@pytest.fixture +def write(tmp_path): + """Write a dedented source file under tmp_path and return its path.""" + + def _write(name: str, source: str) -> Path: + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(source)) + return path + + return _write + + +def rules(diagnostics): + return sorted({d.rule for d in diagnostics}) + + +def errors(diagnostics): + return [d for d in diagnostics if d.severity == "error"] + + +# TP_SANITIZE=1 (tools/sanitizers.py): every extension TypedPython builds gets ASan + UBSan, and a +# sanitizer report fails the run (-fno-sanitize-recover=undefined; ASan halts). Nothing else changes. +import os # noqa: E402 + +if os.environ.get("TP_SANITIZE") == "1": + import sys as _sys + from typedpython import cbuild as _cbuild + + _SAN = ["-fsanitize=address,undefined", "-fno-sanitize-recover=undefined", "-g", + "-fno-omit-frame-pointer"] + _orig_compiler = _cbuild.compiler + + def _sanitized_compiler(): + cmd = _orig_compiler() + _SAN + if _sys.platform.startswith("linux"): # keep the ASan preload out of the compiler + cmd = ["env", "-u", "LD_PRELOAD"] + cmd + return cmd + + _cbuild.compiler = _sanitized_compiler diff --git a/python-multiplatform-ksp/src/test/python/fixtures/deps/untyped_dep.py b/python-multiplatform-ksp/src/test/python/fixtures/deps/untyped_dep.py new file mode 100644 index 000000000..af73da304 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/fixtures/deps/untyped_dep.py @@ -0,0 +1,7 @@ +# Stands in for a third-party package without stubs. It is deliberately unannotated: the gate +# must not report anything inside it, only where its untyped results flow into user code. +import json + + +def load(text): + return json.loads(text) diff --git a/python-multiplatform-ksp/src/test/python/runtime_shim/tp_shim.c b/python-multiplatform-ksp/src/test/python/runtime_shim/tp_shim.c new file mode 100644 index 000000000..2b6b07221 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/runtime_shim/tp_shim.c @@ -0,0 +1,611 @@ +/* Test extension: exposes every tp_runtime.h helper to Python so test_runtime.py can compare it + * with CPython. Every function returns (rc, value); rc == -1 raises the Python error the helper + * set. A helper that breaks the return convention (rc -1 with no error set, rc 0/1 with an error + * set) raises AssertionError, so convention violations cannot hide. */ +#define PY_SSIZE_T_CLEAN +#include +#include "tp_runtime.h" + +#ifndef TP_SHIM_NAME +#define TP_SHIM_NAME tp_shim +#endif +#define TP_CAT2(a, b) a##b +#define TP_CAT(a, b) TP_CAT2(a, b) +#define TP_STR2(a) #a +#define TP_STR(a) TP_STR2(a) + +static int rc_ok(int rc) +{ + if (rc < 0 && !PyErr_Occurred()) { + PyErr_SetString(PyExc_AssertionError, "rc < 0 but no Python error is set"); + return 0; + } + if (rc < 0) { + return 0; + } + if (PyErr_Occurred()) { + PyErr_Format(PyExc_AssertionError, "rc == %d but a Python error is set", rc); + return 0; + } + return 1; +} + +static PyObject *ret_none(int rc) { return Py_BuildValue("(iO)", rc, Py_None); } + +/* ---- int64 ---- */ +#define I64_BIN(OP) \ + static PyObject *s_##OP##_i64(PyObject *self, PyObject *args) \ + { \ + long long a, b; \ + int64_t out = 0; \ + if (!PyArg_ParseTuple(args, "LL", &a, &b)) return NULL; \ + int rc = tp_##OP##_i64((int64_t)a, (int64_t)b, &out); \ + if (!rc_ok(rc)) return NULL; \ + if (rc) return ret_none(rc); \ + return Py_BuildValue("(iL)", rc, (long long)out); \ + } +I64_BIN(add) +I64_BIN(sub) +I64_BIN(mul) +I64_BIN(floordiv) +I64_BIN(mod) + +static PyObject *s_neg_i64(PyObject *self, PyObject *args) +{ + long long a; + int64_t out = 0; + if (!PyArg_ParseTuple(args, "L", &a)) return NULL; + int rc = tp_neg_i64((int64_t)a, &out); + if (!rc_ok(rc)) return NULL; + if (rc) return ret_none(rc); + return Py_BuildValue("(iL)", rc, (long long)out); +} + +static PyObject *s_truediv_i64(PyObject *self, PyObject *args) +{ + long long a, b; + double out = 0; + if (!PyArg_ParseTuple(args, "LL", &a, &b)) return NULL; + int rc = tp_truediv_i64((int64_t)a, (int64_t)b, &out); + if (!rc_ok(rc)) return NULL; + if (rc) return ret_none(rc); + return Py_BuildValue("(id)", rc, out); +} + +static PyObject *s_i64_to_f64(PyObject *self, PyObject *args) +{ + long long a; + if (!PyArg_ParseTuple(args, "L", &a)) return NULL; + return PyFloat_FromDouble(tp_i64_to_f64((int64_t)a)); +} + +/* ---- float ---- */ +#define F64_BIN(OP) \ + static PyObject *s_##OP##_f64(PyObject *self, PyObject *args) \ + { \ + double a, b, out = 0; \ + if (!PyArg_ParseTuple(args, "dd", &a, &b)) return NULL; \ + int rc = tp_##OP##_f64(a, b, &out); \ + if (!rc_ok(rc)) return NULL; \ + if (rc) return ret_none(rc); \ + return Py_BuildValue("(id)", rc, out); \ + } +F64_BIN(truediv) +F64_BIN(floordiv) +F64_BIN(mod) + +static PyObject *s_cmp_i64_f64(PyObject *self, PyObject *args) +{ + long long a; + double b; + int op; + if (!PyArg_ParseTuple(args, "Ldi", &a, &b, &op)) return NULL; + return PyLong_FromLong(tp_cmp_i64_f64((int64_t)a, b, op)); +} + +static PyObject *s_cmp_f64_i64(PyObject *self, PyObject *args) +{ + double a; + long long b; + int op; + if (!PyArg_ParseTuple(args, "dLi", &a, &b, &op)) return NULL; + return PyLong_FromLong(tp_cmp_f64_i64(a, (int64_t)b, op)); +} + +/* ---- math ---- */ +static PyObject *s_math1(PyObject *self, PyObject *args) +{ + const char *name; + double x, out = 0; + int rc; + if (!PyArg_ParseTuple(args, "sd", &name, &x)) return NULL; + if (!strcmp(name, "sqrt")) rc = tp_math_sqrt(x, &out); + else if (!strcmp(name, "exp")) rc = tp_math_exp(x, &out); + else if (!strcmp(name, "log")) rc = tp_math_log(x, &out); + else if (!strcmp(name, "sin")) rc = tp_math_sin(x, &out); + else if (!strcmp(name, "cos")) rc = tp_math_cos(x, &out); + else if (!strcmp(name, "tan")) rc = tp_math_tan(x, &out); + else if (!strcmp(name, "fabs")) rc = tp_math_fabs(x, &out); + else { PyErr_SetString(PyExc_ValueError, "unknown math1"); return NULL; } + if (!rc_ok(rc)) return NULL; + if (rc) return ret_none(rc); + return Py_BuildValue("(id)", rc, out); +} + +static PyObject *s_math2(PyObject *self, PyObject *args) +{ + const char *name; + double x, y, out = 0; + int rc; + if (!PyArg_ParseTuple(args, "sdd", &name, &x, &y)) return NULL; + if (!strcmp(name, "atan2")) rc = tp_math_atan2(x, y, &out); + else if (!strcmp(name, "hypot")) rc = tp_math_hypot(x, y, &out); + else { PyErr_SetString(PyExc_ValueError, "unknown math2"); return NULL; } + if (!rc_ok(rc)) return NULL; + if (rc) return ret_none(rc); + return Py_BuildValue("(id)", rc, out); +} + +/* ---- boxing ---- */ +static PyObject *s_box_i64(PyObject *self, PyObject *args) +{ + long long v; + if (!PyArg_ParseTuple(args, "L", &v)) return NULL; + return tp_box_i64((int64_t)v); +} +static PyObject *s_box_f64(PyObject *self, PyObject *args) +{ + double v; + if (!PyArg_ParseTuple(args, "d", &v)) return NULL; + return tp_box_f64(v); +} +static PyObject *s_box_bool(PyObject *self, PyObject *args) +{ + int v; + if (!PyArg_ParseTuple(args, "i", &v)) return NULL; + return tp_box_bool(v); +} +static PyObject *s_unbox_i64(PyObject *self, PyObject *o) +{ + int64_t out = 0; + int rc = tp_unbox_i64(o, &out); + if (!rc_ok(rc)) return NULL; + if (rc) return ret_none(rc); + return Py_BuildValue("(iL)", rc, (long long)out); +} +static PyObject *s_unbox_f64(PyObject *self, PyObject *o) +{ + double out = 0; + int rc = tp_unbox_f64(o, &out); + if (!rc_ok(rc)) return NULL; + if (rc) return ret_none(rc); + return Py_BuildValue("(id)", rc, out); +} +static PyObject *s_unbox_bool(PyObject *self, PyObject *o) +{ + int out = 0; + int rc = tp_unbox_bool(o, &out); + if (!rc_ok(rc)) return NULL; + if (rc) return ret_none(rc); + return Py_BuildValue("(ii)", rc, out); +} + +/* ---- arrays ---- + * array_run(list, script) -> (enter_rc, results, exit_rc, exit_error) + * script items: ("get", i) | ("set", i, v) | ("len",) | ("dirty",) | ("clear",) + * A failing "get"/"set" slot lookup appends the exception instance. The struct is filled with + * garbage before enter, so an enter that leaves a field unset shows up. */ +#define ARRAY_RUN(SUF, STRUCT, CTYPE, FROMC, PARSEFMT, PARSET) \ + static PyObject *s_##SUF##_array_run(PyObject *self, PyObject *args) \ + { \ + PyObject *obj, *script; \ + if (!PyArg_ParseTuple(args, "OO", &obj, &script)) return NULL; \ + STRUCT a; \ + memset(&a, 0xAB, sizeof a); \ + int erc = tp_##SUF##_array_enter(obj, &a); \ + if (!rc_ok(erc)) return NULL; \ + PyObject *results = PyList_New(0); \ + if (!results) { tp_##SUF##_array_exit(&a); return NULL; } \ + if (erc == 0) { \ + Py_ssize_t n = PyList_GET_SIZE(script); \ + for (Py_ssize_t k = 0; k < n; k++) { \ + PyObject *op = PyList_GET_ITEM(script, k); \ + const char *kind = PyUnicode_AsUTF8(PyTuple_GET_ITEM(op, 0)); \ + PyObject *item = NULL; \ + if (!strcmp(kind, "get") || !strcmp(kind, "set")) { \ + long long i = PyLong_AsLongLong(PyTuple_GET_ITEM(op, 1)); \ + int store = kind[0] == 's'; \ + Py_ssize_t slot = -7; \ + int rc = tp_##SUF##_array_slot(&a, (int64_t)i, store, &slot); \ + if (rc < 0) { \ + item = PyErr_GetRaisedException(); \ + } else if (rc != 0 || PyErr_Occurred()) { \ + PyErr_SetString(PyExc_AssertionError, "slot rc not 0 / -1"); \ + Py_DECREF(results); tp_##SUF##_array_exit(&a); return NULL; \ + } else if (!store) { \ + item = FROMC(a.data[slot]); \ + } else { \ + PARSET v; \ + if (!PyArg_Parse(PyTuple_GET_ITEM(op, 2), PARSEFMT, &v)) { \ + Py_DECREF(results); tp_##SUF##_array_exit(&a); return NULL; \ + } \ + a.data[slot] = (CTYPE)v; \ + a.dirty[slot] = 1; \ + item = Py_NewRef(Py_None); \ + } \ + } else if (!strcmp(kind, "len")) { \ + item = PyLong_FromSsize_t(a.len); \ + } else if (!strcmp(kind, "dirty")) { \ + item = PyBytes_FromStringAndSize((const char *)a.dirty, a.len); \ + } else if (!strcmp(kind, "clear")) { \ + if (PyList_SetSlice(a.list, 0, PyList_GET_SIZE(a.list), NULL) < 0) { \ + Py_DECREF(results); tp_##SUF##_array_exit(&a); return NULL; \ + } \ + item = Py_NewRef(Py_None); \ + } \ + if (!item || PyList_Append(results, item) < 0) { \ + Py_XDECREF(item); Py_DECREF(results); \ + tp_##SUF##_array_exit(&a); \ + return NULL; \ + } \ + Py_DECREF(item); \ + } \ + } \ + int xrc = tp_##SUF##_array_exit(&a); \ + PyObject *xerr = NULL; \ + if (xrc < 0) { \ + if (!PyErr_Occurred()) { \ + PyErr_SetString(PyExc_AssertionError, "exit rc -1 without error"); \ + Py_DECREF(results); return NULL; \ + } \ + xerr = PyErr_GetRaisedException(); \ + } else if (PyErr_Occurred()) { \ + Py_DECREF(results); return NULL; \ + } \ + /* the struct must be unusable-but-safe afterwards: a second exit is a no-op */ \ + int again = tp_##SUF##_array_exit(&a); \ + PyObject *ret = Py_BuildValue("(iNiNi)", erc, results, xrc, \ + xerr ? xerr : Py_NewRef(Py_None), again); \ + return ret; \ + } +ARRAY_RUN(f64, tp_f64_array, double, PyFloat_FromDouble, "d", double) +ARRAY_RUN(i64, tp_i64_array, int64_t, PyLong_FromLongLong, "L", long long) + +static PyObject *s_array_exit_zeroed(PyObject *self, PyObject *arg) +{ + int which = (int)PyLong_AsLong(arg); + int rc; + if (which == 0) { + tp_f64_array a; + memset(&a, 0, sizeof a); + rc = tp_f64_array_exit(&a); + } else { + tp_i64_array a; + memset(&a, 0, sizeof a); + rc = tp_i64_array_exit(&a); + } + if (!rc_ok(rc)) return NULL; + return PyLong_FromLong(rc); +} + + +/* ---- local arrays ---- + * local_run(kind, init, script) -> (rc, error, results) + * kind: "f64" | "i64"; init: ("new", n, fill) | ("iota", n) [i64 only] | ("param", pylist, nocopy) + * ("copy_local", n, fill) / ("copy_param", pylist) build a source, then copy it into the array under test. + * script ops: ("get", i) | ("set", i, v) | ("len",) | ("flags",) -> (list_is_null, dirty_is_null, data_is_null) + * | ("free",) | ("exit",) -> exit rc | ("copy",) -> (list, contents of an independent copy; + * the copy is then set to [i]=v by ("copyset", i, v) which returns (src contents, copy contents)) + * Struct memory is poisoned with 0xAB before every init, so a field left unset shows up. */ +#define LOCAL_RUN(SUF, STRUCT, CTYPE, FROMC, PARSEFMT, PARSET) \ + static int loc_##SUF##_contents(const STRUCT *a, PyObject **out) \ + { \ + PyObject *l = PyList_New(a->len); \ + if (!l) return -1; \ + for (Py_ssize_t i = 0; i < a->len; i++) { \ + PyObject *v = FROMC(a->data[i]); \ + if (!v) { Py_DECREF(l); return -1; } \ + PyList_SET_ITEM(l, i, v); \ + } \ + *out = l; \ + return 0; \ + } \ + static PyObject *s_##SUF##_local_run(PyObject *self, PyObject *args) \ + { \ + PyObject *init, *script; \ + if (!PyArg_ParseTuple(args, "OO", &init, &script)) return NULL; \ + STRUCT a, src; \ + memset(&a, 0xAB, sizeof a); \ + memset(&src, 0xAB, sizeof src); \ + const char *ik = PyUnicode_AsUTF8(PyTuple_GET_ITEM(init, 0)); \ + int rc = 0; \ + int have_src = 0; \ + if (!strcmp(ik, "new")) { \ + long long n = PyLong_AsLongLong(PyTuple_GET_ITEM(init, 1)); \ + PARSET f; \ + if (!PyArg_Parse(PyTuple_GET_ITEM(init, 2), PARSEFMT, &f)) return NULL; \ + rc = tp_##SUF##_array_new(&a, (int64_t)n, (CTYPE)f); \ + } else if (!strcmp(ik, "iota")) { \ + long long n = PyLong_AsLongLong(PyTuple_GET_ITEM(init, 1)); \ + rc = TP_IOTA_##SUF(&a, (int64_t)n); \ + } else { \ + /* copy_local / copy_param: build the source, then copy it */ \ + if (!strcmp(ik, "copy_param")) { \ + int erc = tp_##SUF##_array_enter(PyTuple_GET_ITEM(init, 1), &src); \ + if (erc != 0) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_AssertionError, "enter deopt"); return NULL; } \ + } else { \ + long long n = PyLong_AsLongLong(PyTuple_GET_ITEM(init, 1)); \ + PARSET f; \ + if (!PyArg_Parse(PyTuple_GET_ITEM(init, 2), PARSEFMT, &f)) return NULL; \ + if (tp_##SUF##_array_new(&src, (int64_t)n, (CTYPE)f) != 0) { return NULL; } \ + for (Py_ssize_t i = 0; i < src.len; i++) src.data[i] = (CTYPE)(f + (PARSET)i); \ + } \ + have_src = 1; \ + rc = tp_##SUF##_array_copy(&a, &src); \ + } \ + if (rc != 0) { \ + if (rc != -1 || !PyErr_Occurred()) { \ + PyErr_SetString(PyExc_AssertionError, "local init: rc not 0/-1 or no error"); \ + return NULL; \ + } \ + PyObject *err = PyErr_GetRaisedException(); \ + int zeroed = a.list == NULL && a.len == 0 && a.data == NULL && a.dirty == NULL; \ + if (have_src) tp_##SUF##_array_free(&src); \ + return Py_BuildValue("(iN[N])", rc, err, PyBool_FromLong(zeroed)); \ + } \ + if (PyErr_Occurred()) { PyErr_SetString(PyExc_AssertionError, "error set with rc 0"); return NULL; } \ + PyObject *results = PyList_New(0); \ + Py_ssize_t ns = PyList_GET_SIZE(script); \ + for (Py_ssize_t k = 0; k < ns; k++) { \ + PyObject *op = PyList_GET_ITEM(script, k); \ + const char *kind = PyUnicode_AsUTF8(PyTuple_GET_ITEM(op, 0)); \ + PyObject *item = NULL; \ + if (!strcmp(kind, "get") || !strcmp(kind, "set")) { \ + long long i = PyLong_AsLongLong(PyTuple_GET_ITEM(op, 1)); \ + int store = kind[0] == 's'; \ + Py_ssize_t slot = -7; \ + int r = tp_##SUF##_array_slot(&a, (int64_t)i, store, &slot); \ + if (r < 0) item = PyErr_GetRaisedException(); \ + else if (!store) item = FROMC(a.data[slot]); \ + else { \ + PARSET v; \ + if (!PyArg_Parse(PyTuple_GET_ITEM(op, 2), PARSEFMT, &v)) goto fail; \ + a.data[slot] = (CTYPE)v; \ + item = Py_NewRef(Py_None); \ + } \ + } else if (!strcmp(kind, "len")) { \ + item = PyLong_FromSsize_t(a.len); \ + } else if (!strcmp(kind, "contents")) { \ + if (loc_##SUF##_contents(&a, &item) < 0) goto fail; \ + } else if (!strcmp(kind, "srccontents")) { \ + if (!have_src || loc_##SUF##_contents(&src, &item) < 0) goto fail; \ + } else if (!strcmp(kind, "flags")) { \ + item = Py_BuildValue("(iiin)", a.list == NULL, a.dirty == NULL, a.data == NULL, a.len); \ + } else if (!strcmp(kind, "srcflags")) { \ + item = Py_BuildValue("(iiin)", src.list == NULL, src.dirty == NULL, src.data == NULL, src.len); \ + } else if (!strcmp(kind, "free")) { \ + tp_##SUF##_array_free(&a); \ + item = Py_NewRef(Py_None); \ + } else if (!strcmp(kind, "exit")) { \ + item = PyLong_FromLong(tp_##SUF##_array_exit(&a)); \ + } else if (!strcmp(kind, "srcset")) { \ + long long i = PyLong_AsLongLong(PyTuple_GET_ITEM(op, 1)); \ + PARSET v; \ + if (!have_src || !PyArg_Parse(PyTuple_GET_ITEM(op, 2), PARSEFMT, &v)) goto fail; \ + src.data[i] = (CTYPE)v; \ + item = Py_NewRef(Py_None); \ + } else { \ + PyErr_SetString(PyExc_ValueError, "bad op"); \ + goto fail; \ + } \ + if (!item || PyList_Append(results, item) < 0) { Py_XDECREF(item); goto fail; } \ + Py_DECREF(item); \ + } \ + tp_##SUF##_array_free(&a); \ + if (have_src) tp_##SUF##_array_free(&src); \ + return Py_BuildValue("(iON)", 0, Py_None, results); \ + fail: \ + tp_##SUF##_array_free(&a); \ + if (have_src) tp_##SUF##_array_free(&src); \ + Py_DECREF(results); \ + return NULL; \ + } +static int TP_IOTA_f64(tp_f64_array *a, int64_t n) { (void)a; (void)n; PyErr_SetString(PyExc_ValueError, "no f64 iota"); return -2; } +static int TP_IOTA_i64(tp_i64_array *a, int64_t n) { return tp_i64_array_iota(a, n); } +LOCAL_RUN(f64, tp_f64_array, double, PyFloat_FromDouble, "d", double) +LOCAL_RUN(i64, tp_i64_array, int64_t, PyLong_FromLongLong, "L", long long) + +/* free on a zeroed struct, twice */ +static PyObject *s_local_free_zeroed(PyObject *self, PyObject *arg) +{ + if (PyLong_AsLong(arg) == 0) { + tp_f64_array a; memset(&a, 0, sizeof a); + tp_f64_array_free(&a); tp_f64_array_free(&a); + } else { + tp_i64_array a; memset(&a, 0, sizeof a); + tp_i64_array_free(&a); tp_i64_array_free(&a); + } + Py_RETURN_NONE; +} + +/* copy from a zeroed (never-initialised) source struct: must give an empty local array */ +static PyObject *s_local_copy_of_zeroed(PyObject *self, PyObject *arg) +{ + Py_ssize_t len; + if (PyLong_AsLong(arg) == 0) { + tp_f64_array s0, d; memset(&s0, 0, sizeof s0); memset(&d, 0xAB, sizeof d); + if (tp_f64_array_copy(&d, &s0) != 0) return NULL; + len = d.len; tp_f64_array_free(&d); + } else { + tp_i64_array s0, d; memset(&s0, 0, sizeof s0); memset(&d, 0xAB, sizeof d); + if (tp_i64_array_copy(&d, &s0) != 0) return NULL; + len = d.len; tp_i64_array_free(&d); + } + return PyLong_FromSsize_t(len); +} + +static PyObject *s_tuple(PyObject *self, PyObject *seq) +{ + PyObject *fast = PySequence_Fast(seq, "sequence expected"); + if (!fast) return NULL; + PyObject *t = tp_tuple(PySequence_Fast_ITEMS(fast), PySequence_Fast_GET_SIZE(fast)); + Py_DECREF(fast); + return t; +} + +static PyObject *s_any_same(PyObject *self, PyObject *seq) +{ + PyObject *fast = PySequence_Fast(seq, "sequence expected"); + if (!fast) return NULL; + int r = tp_any_same(PySequence_Fast_ITEMS(fast), PySequence_Fast_GET_SIZE(fast)); + Py_DECREF(fast); + return PyLong_FromLong(r); +} + + +/* ---- opaque objects ---- */ +static PyObject *s_global(PyObject *self, PyObject *args) +{ + PyObject *d, *name; + if (!PyArg_ParseTuple(args, "OO", &d, &name)) return NULL; + return tp_global(d, name); +} +static PyObject *s_getattr(PyObject *self, PyObject *args) +{ + PyObject *o, *name; + if (!PyArg_ParseTuple(args, "OO", &o, &name)) return NULL; + return tp_getattr(o, name); +} +/* call(callee, args_tuple, kwnames_tuple_or_None): the trailing len(kwnames) args are keywords */ +static PyObject *s_call(PyObject *self, PyObject *args) +{ + PyObject *callee, *argt, *kw; + if (!PyArg_ParseTuple(args, "OO!O", &callee, &PyTuple_Type, &argt, &kw)) return NULL; + if (kw != Py_None && (!PyTuple_Check(kw) || PyTuple_GET_SIZE(kw) > PyTuple_GET_SIZE(argt))) { + PyErr_SetString(PyExc_ValueError, "shim: kwnames must be a tuple no longer than args"); + return NULL; + } + /* vectorcall: nargs counts the positional arguments only; the keyword values follow them */ + return tp_call(callee, &PyTuple_GET_ITEM(argt, 0), + (size_t)(PyTuple_GET_SIZE(argt) - (kw == Py_None ? 0 : PyTuple_GET_SIZE(kw))), + kw == Py_None ? NULL : kw); +} +/* call_many: n calls in a row, results dropped (for refcount stability checks) */ +static PyObject *s_call_many(PyObject *self, PyObject *args) +{ + PyObject *callee, *argt; + long n; + if (!PyArg_ParseTuple(args, "OO!l", &callee, &PyTuple_Type, &argt, &n)) return NULL; + long errors = 0; + for (long i = 0; i < n; i++) { + PyObject *r = tp_call(callee, &PyTuple_GET_ITEM(argt, 0), (size_t)PyTuple_GET_SIZE(argt), NULL); + if (r == NULL) { errors++; PyErr_Clear(); } else { tp_release(&r); } + } + return PyLong_FromLong(errors); +} +static PyObject *s_getattr_many(PyObject *self, PyObject *args) +{ + PyObject *o, *name; + long n; + if (!PyArg_ParseTuple(args, "OOl", &o, &name, &n)) return NULL; + long errors = 0; + for (long i = 0; i < n; i++) { + PyObject *r = tp_getattr(o, name); + if (r == NULL) { errors++; PyErr_Clear(); } else { tp_release(&r); } + } + return PyLong_FromLong(errors); +} +static PyObject *s_binop_many(PyObject *self, PyObject *args) +{ + PyObject *a, *b; + int op; + long n; + if (!PyArg_ParseTuple(args, "OOil", &a, &b, &op, &n)) return NULL; + long errors = 0; + for (long i = 0; i < n; i++) { + PyObject *r = tp_binop_obj(a, b, op); + if (r == NULL) { errors++; PyErr_Clear(); } else { tp_release(&r); } + } + return PyLong_FromLong(errors); +} +static PyObject *s_truth(PyObject *self, PyObject *o) +{ + int rc = tp_truth(o); + if (rc < 0) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_AssertionError, "-1 w/o error"); return NULL; } + if (PyErr_Occurred()) return NULL; + return PyLong_FromLong(rc); +} +static PyObject *s_compare_bool(PyObject *self, PyObject *args) +{ + PyObject *a, *b; + int op; + if (!PyArg_ParseTuple(args, "OOi", &a, &b, &op)) return NULL; + int rc = tp_compare_bool(a, b, op); + if (rc < 0) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_AssertionError, "-1 w/o error"); return NULL; } + if (PyErr_Occurred()) return NULL; + return PyLong_FromLong(rc); +} +static PyObject *s_obj_to_f64(PyObject *self, PyObject *o) +{ + double out = 0; + int rc = tp_obj_to_f64(o, &out); + if (!rc_ok(rc)) return NULL; + return PyFloat_FromDouble(out); +} +static PyObject *s_binop_obj(PyObject *self, PyObject *args) +{ + PyObject *a, *b; + int op; + if (!PyArg_ParseTuple(args, "OOi", &a, &b, &op)) return NULL; + PyObject *r = tp_binop_obj(a, b, op); + if (r == NULL && !PyErr_Occurred()) PyErr_SetString(PyExc_AssertionError, "NULL w/o error"); + return r; +} +/* release(obj) -> True if the slot was cleared; the shim owns one extra reference meanwhile */ +static PyObject *s_release(PyObject *self, PyObject *o) +{ + PyObject *slot = Py_NewRef(o); + tp_release(&slot); + PyObject *null_slot = NULL; + tp_release(&null_slot); /* releasing an empty slot is a no-op */ + return PyBool_FromLong(slot == NULL && null_slot == NULL); +} + +static PyObject *s_info(PyObject *self, PyObject *unused) +{ + return Py_BuildValue("{s:i,s:i}", "portable", TP_RUNTIME_PORTABLE_OVERFLOW, + "py_version_hex", (int)PY_VERSION_HEX); +} + +#define M(N, F, FL) {N, (PyCFunction)(void (*)(void))F, FL, NULL} +static PyMethodDef methods[] = { + M("add_i64", s_add_i64, METH_VARARGS), M("sub_i64", s_sub_i64, METH_VARARGS), + M("mul_i64", s_mul_i64, METH_VARARGS), M("neg_i64", s_neg_i64, METH_VARARGS), + M("floordiv_i64", s_floordiv_i64, METH_VARARGS), M("mod_i64", s_mod_i64, METH_VARARGS), + M("truediv_i64", s_truediv_i64, METH_VARARGS), M("i64_to_f64", s_i64_to_f64, METH_VARARGS), + M("truediv_f64", s_truediv_f64, METH_VARARGS), M("floordiv_f64", s_floordiv_f64, METH_VARARGS), + M("mod_f64", s_mod_f64, METH_VARARGS), + M("cmp_i64_f64", s_cmp_i64_f64, METH_VARARGS), M("cmp_f64_i64", s_cmp_f64_i64, METH_VARARGS), + M("math1", s_math1, METH_VARARGS), M("math2", s_math2, METH_VARARGS), + M("box_i64", s_box_i64, METH_VARARGS), M("box_f64", s_box_f64, METH_VARARGS), + M("box_bool", s_box_bool, METH_VARARGS), + M("unbox_i64", s_unbox_i64, METH_O), M("unbox_f64", s_unbox_f64, METH_O), + M("unbox_bool", s_unbox_bool, METH_O), + M("f64_array_run", s_f64_array_run, METH_VARARGS), M("i64_array_run", s_i64_array_run, METH_VARARGS), + M("array_exit_zeroed", s_array_exit_zeroed, METH_O), + M("f64_local_run", s_f64_local_run, METH_VARARGS), M("i64_local_run", s_i64_local_run, METH_VARARGS), + M("local_free_zeroed", s_local_free_zeroed, METH_O), M("tuple_", s_tuple, METH_O), + M("local_copy_of_zeroed", s_local_copy_of_zeroed, METH_O), + M("global_", s_global, METH_VARARGS), M("getattr_", s_getattr, METH_VARARGS), + M("call", s_call, METH_VARARGS), M("call_many", s_call_many, METH_VARARGS), + M("getattr_many", s_getattr_many, METH_VARARGS), M("binop_many", s_binop_many, METH_VARARGS), + M("truth", s_truth, METH_O), M("compare_bool", s_compare_bool, METH_VARARGS), + M("obj_to_f64", s_obj_to_f64, METH_O), M("binop_obj", s_binop_obj, METH_VARARGS), + M("release", s_release, METH_O), + M("any_same", s_any_same, METH_O), M("info", s_info, METH_NOARGS), + {NULL, NULL, 0, NULL}}; + +static struct PyModuleDef moddef = {PyModuleDef_HEAD_INIT, TP_STR(TP_SHIM_NAME), NULL, -1, methods, + NULL, NULL, NULL, NULL}; + +PyMODINIT_FUNC TP_CAT(PyInit_, TP_SHIM_NAME)(void) { return PyModule_Create(&moddef); } diff --git a/python-multiplatform-ksp/src/test/python/test_any_flow.py b/python-multiplatform-ksp/src/test/python/test_any_flow.py new file mode 100644 index 000000000..fae430c44 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_any_flow.py @@ -0,0 +1,43 @@ +"""Any reaching user code from an untyped dependency (design §2.3, §4.2).""" +from conftest import FIXTURES, errors +from typedpython import gate + +DEPS = FIXTURES / "deps" + + +def test_untyped_result_flowing_into_user_code_is_an_error(write): + path = write("user.py", """ + from untyped_dep import load + + def total(text: str) -> None: + value = load(text) + print(value) + """) + found = errors(gate.check([path], search_paths=[DEPS])) + flow = [d for d in found if d.rule == "any-flow"] + assert flow, found + assert all(d.path == str(path) for d in flow) + + +def test_cast_at_the_boundary_resolves_it(write): + path = write("user.py", """ + from typing import cast + from untyped_dep import load + + def total(text: str) -> int: + value = cast(int, load(text)) + return value + 1 + """) + assert gate.check([path], search_paths=[DEPS]) == [] + + +def test_the_dependency_itself_is_not_checked(write): + path = write("user.py", """ + from typing import cast + from untyped_dep import load + + def total(text: str) -> int: + return cast(int, load(text)) + """) + found = gate.check([path], search_paths=[DEPS]) + assert not [d for d in found if d.path.endswith("untyped_dep.py")], found diff --git a/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py b/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py new file mode 100644 index 000000000..71ed65011 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_builtin_compiled.py @@ -0,0 +1,100 @@ +"""`compiled` is a builtin name (design §2.3): no import, on plain CPython and for the checker.""" +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path + +import pytest + +from conftest import errors, rules +from typedpython import gate + +PROJECT = Path(__file__).resolve().parents[3] +TMP = PROJECT / ".tmp" +PTH = "typedpython_builtins.pth" + + +def test_gate_accepts_compiled_without_an_import(write): + path = write("m.py", """ + @compiled + def f(x: int) -> int: + return x + 1 + + y: int = f(1) + """) + assert errors(gate.check([path])) == [] + + +def test_gate_still_rejects_an_unknown_builtin(write): + path = write("m.py", """ + @not_a_builtin + def f(x: int) -> int: + return x + """) + found = errors(gate.check([path])) + assert "pyrefly/unknown-name" in rules(found) + assert any("not_a_builtin" in d.message for d in found) + + +def test_compiled_keeps_the_decorated_type(write): + path = write("m.py", """ + @compiled + def f(x: int) -> int: + return x + + s: str = f(1) + """) + assert "pyrefly/bad-assignment" in rules(errors(gate.check([path]))) + + +@pytest.fixture(scope="module") +def installed(): + """The wheel, built and installed into a fresh venv; returns (python, wheel path).""" + root = TMP / "builtin-venv" + wheels = TMP / "builtin-wheel" + for d in (root, wheels): + shutil.rmtree(d, ignore_errors=True) + subprocess.run(["uv", "build", "--wheel", "--out-dir", str(wheels), str(PROJECT)], + check=True, capture_output=True) + wheel = next(wheels.glob("typedpython-*.whl")) + subprocess.run(["uv", "venv", "--python", "3.13", str(root)], check=True, capture_output=True) + python = root / "bin" / "python" + subprocess.run(["uv", "pip", "install", "--python", str(python), "--no-deps", str(wheel)], + check=True, capture_output=True) + return python, wheel + + +def py(python, *args): + return subprocess.run([str(python), *args], capture_output=True, text=True) + + +def test_wheel_puts_the_pth_at_its_root(installed): + _, wheel = installed + assert PTH in zipfile.ZipFile(wheel).namelist() + + +def test_decorator_works_with_no_import(installed): + python, _ = installed + r = py(python, "-c", "@compiled\ndef f(): return 1\nprint(f())") + assert (r.returncode, r.stdout.strip()) == (0, "1"), r.stderr + + +def test_compiled_is_the_identity(installed): + python, _ = installed + r = py(python, "-c", "def f(): pass\nprint(compiled(f) is f)") + assert r.stdout.strip() == "True", r.stderr + + +def test_the_pth_does_not_overwrite_the_runtime(installed): + python, _ = installed + site = next((python.parent.parent / "lib").glob("python3*/site-packages")) + code = ( + "import builtins, site\n" + "sentinel = object()\n" + "builtins.compiled = sentinel\n" + f"site.addsitedir({str(site)!r})\n" + "print(builtins.compiled is sentinel)\n" + ) + r = py(python, "-S", "-c", code) + assert r.stdout.strip() == "True", r.stderr diff --git a/python-multiplatform-ksp/src/test/python/test_cgen.py b/python-multiplatform-ksp/src/test/python/test_cgen.py new file mode 100644 index 000000000..eea61452b --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_cgen.py @@ -0,0 +1,1581 @@ +"""TypedPython C back end: typed IR -> C source -> built CPython extension (#41). + +The IR is built by hand here (no front end). Every end-to-end test compares the compiled function +with the interpreted one that the extension keeps in `__typedpython_interpreted__`, because the +contract (ir.py, INTENT §1.4) is "compiled code computes what CPython computes". + +End-to-end tests need the C runtime header `tp_runtime.h`. It is looked up in +`$TYPEDPYTHON_RUNTIME_DIR` when that is set, otherwise in the package's own `runtime/` directory; +when it is missing those tests skip with that reason (a pre-implementation skip, not a regression). +Generation-only tests (`test_gen_*`) never need it. + +Every build happens inside the caller's `heavy.sh` lock (the whole pytest process runs under it); +build output goes to the worktree's `.tmp/cgen-tests/`, never to /tmp. +""" +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import sys +import sysconfig +import textwrap +from pathlib import Path + +import pytest + +from typedpython import ir +from typedpython.ir import ( + And, ArrayParam, Assign, BinOp, BinOpKind, Box, Call, Compare, CompareKind, Const, ExprStmt, + ForRange, Function, If, Index, Len, Local, Module, Param, Return, StoreIndex, Type, + Global, GetAttr, CallObject, Truth, CompareObj, ObjToFloat, + UnaryOp, UnaryOpKind, Unbox, While, Break, Continue, ToFloat, MathCall, MathFunc, + NewArray, CopyArray, Tuple, +) + +PACKAGE = Path(__file__).resolve().parents[2] / "main" / "python" / "typedpython" +WORKTREE = Path(__file__).resolve().parents[4] +OUT_ROOT = WORKTREE / ".tmp" / "cgen-tests" + + +def runtime_dir() -> Path: + env = os.environ.get("TYPEDPYTHON_RUNTIME_DIR") + return Path(env) if env else PACKAGE / "runtime" + + +def need_runtime(): + header = runtime_dir() / "tp_runtime.h" + if not header.is_file(): + pytest.skip(f"C runtime header not available yet: {header} does not exist " + "(set TYPEDPYTHON_RUNTIME_DIR to the tp-runtime worktree's runtime/ dir)") + text = header.read_text() + missing = [h for h in OBJ_HELPERS if not re.search(rf"\b{h}\s*\(", text)] + if missing: + pytest.skip(f"{header} does not define the opaque-object helpers yet: {missing}") + + +# --- IR shorthands ------------------------------------------------------------------------------- + +I64, F64, BOOL, OBJ, NONE = Type.I64, Type.F64, Type.BOOL, Type.OBJ, Type.NONE + + +def c(v, t=None): + if t is None: + t = BOOL if isinstance(v, bool) else I64 if isinstance(v, int) else F64 + return Const(t, v) + + +def L(name, t): + return Local(t, name) + + +def binop(op, a, b, t=None): + if t is None: + t = F64 if (op is BinOpKind.TRUEDIV) else a.type + return BinOp(t, op, a, b) + + +def call(name, *args, kwnames=()): + """`name(*args)` for a module global (a binder function is an ordinary global).""" + return CallObject(OBJ, Global(OBJ, name), tuple(args), tuple(kwnames)) + + +def fn(name, params, returns, locals_, body, *, pure, may_deopt, line=1, entry_globals=()): + return Function(name=name, params=tuple(params), returns=returns, locals=dict(locals_), + body=tuple(body), pure=pure, may_deopt=may_deopt, source_line=line, + entry_globals=tuple(entry_globals)) + + +# The module source that every e2e module embeds; each test picks the functions it compiles. +SOURCE = textwrap.dedent('''\ + """typedpython test module — the interpreted reference.""" + calls = [] + + def record(x): + calls.append(x) + return x + + def poly(x: float, n: int) -> float: + acc = 0.0 + for i in range(n): + acc = acc * x + 0.1 + return acc + + def mul_all(a: int, n: int) -> int: + acc = 1 + for i in range(n): + acc = acc * a + return acc + + def twice_mul(a: int, n: int) -> int: + return mul_all(a, n) * 2 + + def eff(x: int) -> int: + record(x) + return x + + def scale(xs: list[float], k: float, n: int) -> None: + for i in range(n): + xs[i] = xs[i] * k + + def addfirst(dst: list[float], src: list[float]) -> None: + for i in range(len(dst)): + dst[i] = dst[i] + src[0] + + def floordiv(a: int, b: int) -> int: + return a // b + + def objmix(o, p): + q = o + p + q = record(q) + return q + o + + def count_until(limit: int, skip: int) -> int: + total = 0 + i = 0 + while True: + if i >= limit: + break + i = i + 1 + if i == skip: + continue + total = total + i + return total + + def isum(xs: list[int]) -> int: + s = 0 + for i in range(len(xs)): + s = s + xs[i] + return s + + def hyp(x: float, y: float) -> float: + return math.sqrt(x * x + y * y) + + def mixed_lt(a: int, b: float) -> bool: + return a < b and not (b < 0.0) + + def tofl(a: int) -> float: + return float(a) / 3.0 + + def neg_last(xs: list[float]) -> float: + return -xs[-1] + + # --- plain-Python stand-ins for binder proxies (a Kotlin function, a Kotlin object) --- + made = [] + + class Counter: + def __init__(self, scale): + self.scale = scale + self.reads = 0 + + @property + def factor(self): + self.reads += 1 + return self.scale + + def apply(self, v, *, offset=0.0): + return v * self.factor + offset + + def make(scale): + made.append(Counter(scale)) + return made[-1] + + def boom(x): + raise ValueError(f"bad value {x!r}") + + def target(): + return 1 + + def interop(scale: float, n: int) -> float: + obj = make(scale) + acc = 0.0 + for i in range(n): + v = obj.apply(acc, offset=0.5) + acc = float(v) * 0.5 + math.sqrt(acc + 1.0) + if obj.factor > 1.0: + acc = acc + 1.0 + return acc + float(obj.reads) + + def which(): + return target() + + def length(o): + return len(o) + + def calls_boom(o): + return boom(o) + + def truthy(o) -> bool: + if o: + return True + return False + + # --- 957315ea: proven ops, callee-only redo, entry snapshots of globals, CompareObj --- + def proven_eff(x: int) -> int: + record(x) + return x + 1 + + def growth(n: int) -> float: + acc = 1 + for i in range(n): + acc = acc * 3 + return float(acc) + + def growth_lie(n: int) -> float: + acc = 1 + for i in range(n): + acc = acc * 3 + return acc # the IR below claims float(acc): a contract violation the redo catches + + def fill(xs: list[float], n: int) -> float: + for i in range(len(xs)): + xs[i] = xs[i] * 2.0 + g = growth(n) + xs[0] = xs[0] + g + return g + + def fill_lie(xs: list[float], n: int) -> float: + xs[0] = xs[0] * 2.0 + return growth_lie(n) + + def big(n: int) -> bool: + acc = 1 + for i in range(n): + acc = acc * 3 + return acc > 1000 + + def flag(xs: list[float], n: int) -> float: + xs[0] = xs[0] + 1.0 + if big(n): + return 1.0 + return 0.0 + + def selfeq(x) -> bool: + if x == x: + return True + return False + + SCALE = 2.5 + + def scaled(x: float) -> float: + return x * SCALE + + def twice_scaled(x: float) -> float: + return scaled(x) * 2.0 + + # --- local arrays and tuple returns (fannkuch, M2) --- + def la_build(n: int) -> tuple: + a = list(range(n)) + b = [0] * n + s = 0 + for i in range(n // 2): + t = a[i] + a[i] = a[n - 1 - i] + a[n - 1 - i] = t + c = a[:] + for i in range(len(c)): + b[i] = c[i] * 3 + a[i] + s = s + b[i] + first = -1 + last = -1 + if len(a) > 0: + first = a[0] + last = a[-1] + return (len(a), len(b), first, last, s) + + def la_f64(n: int) -> tuple: + x = [0.5] * n + y = x[:] + total = 0.0 + for i in range(len(x)): + x[i] = x[i] * float(i) + 1.0 + y[i] = x[i] / 2.0 + total = total + y[i] + return (len(y), total) + + def la_overflow(n: int, k: int) -> tuple: + a = list(range(n)) + b = a[:] + acc = 1 + for i in range(len(b)): + acc = acc + k + b[i] = i + return (acc, len(a)) + + def la_index(n: int, i: int) -> int: + a = list(range(n)) + return a[i] + + def la_store(n: int, i: int, v: int) -> int: + a = [0] * n + a[i] = v + return len(a) + + def la_reassign(n: int) -> tuple: + a = list(range(n)) + a = [7] * (n + 1) + a = a[:] + return (len(a), a[0], a[-1]) + + def la_copy_param(xs: list[int]) -> tuple: + c = xs[:] + c[0] = 99 + return (xs[0], c[0], len(c)) + + def la_unit() -> tuple: + return () + ''') +SOURCE = "import math\n" + SOURCE + + +def f_poly(): + acc, x, n, i = L("acc", F64), L("x", F64), L("n", I64), L("i", I64) + return fn("poly", [Param("x", F64), Param("n", I64)], F64, + {"acc": F64, "i": I64}, + [Assign("acc", c(0.0)), + ForRange("i", c(0), n, c(1), + (Assign("acc", binop(BinOpKind.ADD, binop(BinOpKind.MUL, acc, x), c(0.1))),)), + Return(acc)], + pure=True, may_deopt=False, line=9) + + +def f_mul_all(): + acc, a, n = L("acc", I64), L("a", I64), L("n", I64) + return fn("mul_all", [Param("a", I64), Param("n", I64)], I64, {"acc": I64, "i": I64}, + [Assign("acc", c(1)), + ForRange("i", c(0), n, c(1), (Assign("acc", binop(BinOpKind.MUL, acc, a)),)), + Return(acc)], + pure=True, may_deopt=True) + + +def f_twice_mul(): + call = Call(I64, "mul_all", (L("a", I64), L("n", I64))) + return fn("twice_mul", [Param("a", I64), Param("n", I64)], I64, {}, + [Return(binop(BinOpKind.MUL, call, c(2)))], pure=True, may_deopt=True) + + +def f_eff(): + x = L("x", I64) + return fn("eff", [Param("x", I64)], I64, {}, + [ExprStmt(call("record", Box(OBJ, x))), Return(x)], + pure=False, may_deopt=False) + + +def f_scale(): + xs = "xs" + i = L("i", I64) + return fn("scale", [ArrayParam(xs, Type.F64_ARRAY, stored=True), Param("k", F64), Param("n", I64)], + NONE, {"i": I64}, + [ForRange("i", c(0), L("n", I64), c(1), + (StoreIndex(xs, i, binop(BinOpKind.MUL, Index(F64, xs, i), L("k", F64))),))], + pure=False, may_deopt=False) + + +def f_addfirst(): + i = L("i", I64) + return fn("addfirst", [ArrayParam("dst", Type.F64_ARRAY, stored=True), + ArrayParam("src", Type.F64_ARRAY, stored=False)], + NONE, {"i": I64}, + [ForRange("i", c(0), Len(I64, "dst"), c(1), + (StoreIndex("dst", i, binop(BinOpKind.ADD, Index(F64, "dst", i, proven=True), + Index(F64, "src", c(0)))),))], + pure=False, may_deopt=False) + + +def f_floordiv(): + return fn("floordiv", [Param("a", I64), Param("b", I64)], I64, {}, + [Return(binop(BinOpKind.FLOORDIV, L("a", I64), L("b", I64)))], pure=True, may_deopt=True) + + +def f_objmix(): + o, p, q = L("o", OBJ), L("p", OBJ), L("q", OBJ) + return fn("objmix", [Param("o", OBJ), Param("p", OBJ)], OBJ, {"q": OBJ}, + [Assign("q", binop(BinOpKind.ADD, o, p)), + Assign("q", call("record", q)), + Return(binop(BinOpKind.ADD, q, o))], + pure=False, may_deopt=False) + + +def f_count_until(): + i, total = L("i", I64), L("total", I64) + return fn("count_until", [Param("limit", I64), Param("skip", I64)], I64, {"total": I64, "i": I64}, + [Assign("total", c(0)), Assign("i", c(0)), + While(c(True), ( + If(Compare(BOOL, CompareKind.GE, i, L("limit", I64)), (Break(),)), + Assign("i", binop(BinOpKind.ADD, i, c(1))), + If(Compare(BOOL, CompareKind.EQ, i, L("skip", I64)), (Continue(),)), + Assign("total", binop(BinOpKind.ADD, total, i)), + )), + Return(total)], + pure=True, may_deopt=True) + + +def f_isum(): + s, i = L("s", I64), L("i", I64) + return fn("isum", [ArrayParam("xs", Type.I64_ARRAY, stored=False)], I64, {"s": I64, "i": I64}, + [Assign("s", c(0)), + ForRange("i", c(0), Len(I64, "xs"), c(1), + (Assign("s", binop(BinOpKind.ADD, s, Index(I64, "xs", i, proven=True))),)), + Return(s)], + pure=True, may_deopt=True) + + +def f_hyp(): + x, y = L("x", F64), L("y", F64) + s = binop(BinOpKind.ADD, binop(BinOpKind.MUL, x, x), binop(BinOpKind.MUL, y, y)) + return fn("hyp", [Param("x", F64), Param("y", F64)], F64, {}, + [Return(MathCall(F64, MathFunc.SQRT, (s,)))], pure=True, may_deopt=False) + + +def f_mixed_lt(): + a, b = L("a", I64), L("b", F64) + return fn("mixed_lt", [Param("a", I64), Param("b", F64)], BOOL, {}, + [Return(And(BOOL, Compare(BOOL, CompareKind.LT, a, b), + UnaryOp(BOOL, UnaryOpKind.NOT, Compare(BOOL, CompareKind.LT, b, c(0.0)))))], + pure=True, may_deopt=False) + + +def f_tofl(): + return fn("tofl", [Param("a", I64)], F64, {}, + [Return(binop(BinOpKind.TRUEDIV, ToFloat(F64, L("a", I64)), c(3.0)))], + pure=True, may_deopt=False) + + +def f_neg_last(): + return fn("neg_last", [ArrayParam("xs", Type.F64_ARRAY, stored=False)], F64, {}, + [Return(UnaryOp(F64, UnaryOpKind.NEG, Index(F64, "xs", c(-1))))], + pure=True, may_deopt=False) + + +def f_interop(): + obj, acc, v, i = L("obj", OBJ), L("acc", F64), L("v", OBJ), L("i", I64) + apply = CallObject(OBJ, GetAttr(OBJ, obj, "apply"), (Box(OBJ, acc), Box(OBJ, c(0.5))), ("offset",)) + step = binop(BinOpKind.ADD, binop(BinOpKind.MUL, ObjToFloat(F64, v), c(0.5)), + MathCall(F64, MathFunc.SQRT, (binop(BinOpKind.ADD, acc, c(1.0)),))) + return fn("interop", [Param("scale", F64), Param("n", I64)], F64, + {"obj": OBJ, "acc": F64, "v": OBJ, "i": I64}, + [Assign("obj", call("make", Box(OBJ, L("scale", F64)))), + Assign("acc", c(0.0)), + ForRange("i", c(0), L("n", I64), c(1), ( + Assign("v", apply), + Assign("acc", step), + If(CompareObj(BOOL, CompareKind.GT, GetAttr(OBJ, obj, "factor"), Box(OBJ, c(1.0))), + (Assign("acc", binop(BinOpKind.ADD, acc, c(1.0))),)), + )), + Return(binop(BinOpKind.ADD, acc, ObjToFloat(F64, GetAttr(OBJ, obj, "reads"))))], + pure=False, may_deopt=False) + + +def f_which(): + return fn("which", [], OBJ, {}, [Return(call("target"))], pure=False, may_deopt=False) + + +def f_length(): + return fn("length", [Param("o", OBJ)], OBJ, {}, [Return(call("len", L("o", OBJ)))], + pure=False, may_deopt=False) + + +def f_calls_boom(): + return fn("calls_boom", [Param("o", OBJ)], OBJ, {}, [Return(call("boom", L("o", OBJ)))], + pure=False, may_deopt=False) + + +def f_truthy(): + return fn("truthy", [Param("o", OBJ)], BOOL, {}, + [If(Truth(BOOL, L("o", OBJ)), (Return(c(True)),)), Return(c(False))], + pure=False, may_deopt=False) + + +def f_proven_eff(): + x = L("x", I64) + return fn("proven_eff", [Param("x", I64)], I64, {}, + [ExprStmt(call("record", Box(OBJ, x))), + Return(BinOp(I64, BinOpKind.ADD, x, c(1), proven=True))], + pure=False, may_deopt=False) + + +def _growth_ir(name, result): + acc = L("acc", I64) + return fn(name, [Param("n", I64)], result, {"acc": I64, "i": I64}, + [Assign("acc", c(1)), + ForRange("i", c(0), L("n", I64), c(1), (Assign("acc", binop(BinOpKind.MUL, acc, c(3))),)), + Return(ToFloat(F64, acc) if result is F64 else Compare(BOOL, CompareKind.GT, acc, c(1000)))], + pure=True, may_deopt=True) + + +def f_growth(): + return _growth_ir("growth", F64) + + +def f_growth_lie(): + return _growth_ir("growth_lie", F64) + + +def f_big(): + return _growth_ir("big", BOOL) + + +def f_fill(): + i, g = L("i", I64), L("g", F64) + return fn("fill", [ArrayParam("xs", Type.F64_ARRAY, stored=True), Param("n", I64)], F64, + {"i": I64, "g": F64}, + [ForRange("i", c(0), Len(I64, "xs"), c(1), + (StoreIndex("xs", i, binop(BinOpKind.MUL, Index(F64, "xs", i, proven=True), c(2.0)), + proven=True),)), + Assign("g", Call(F64, "growth", (L("n", I64),), redo=True)), + StoreIndex("xs", c(0), binop(BinOpKind.ADD, Index(F64, "xs", c(0)), g)), + Return(g)], + pure=False, may_deopt=False) + + +def f_fill_lie(): + return fn("fill_lie", [ArrayParam("xs", Type.F64_ARRAY, stored=True), Param("n", I64)], F64, {}, + [StoreIndex("xs", c(0), binop(BinOpKind.MUL, Index(F64, "xs", c(0)), c(2.0))), + Return(Call(F64, "growth_lie", (L("n", I64),), redo=True))], + pure=False, may_deopt=False) + + +def f_flag(): + return fn("flag", [ArrayParam("xs", Type.F64_ARRAY, stored=True), Param("n", I64)], F64, {}, + [StoreIndex("xs", c(0), binop(BinOpKind.ADD, Index(F64, "xs", c(0)), c(1.0))), + If(Call(BOOL, "big", (L("n", I64),), redo=True), (Return(c(1.0)),)), + Return(c(0.0))], + pure=False, may_deopt=False) + + +def f_selfeq(): + x = L("x", OBJ) + return fn("selfeq", [Param("x", OBJ)], BOOL, {}, + [If(CompareObj(BOOL, CompareKind.EQ, x, x), (Return(c(True)),)), Return(c(False))], + pure=False, may_deopt=False) + + +def f_scaled(): + return fn("scaled", [Param("x", F64)], F64, {}, + [Return(binop(BinOpKind.MUL, L("x", F64), L("SCALE", F64)))], + pure=True, may_deopt=False, entry_globals=[Param("SCALE", F64)]) + + +def f_twice_scaled(): + # The callee's entry-global guard can fail at the call: that is a deopt of the callee, which a + # pure caller propagates (so this caller may deopt). + return fn("twice_scaled", [Param("x", F64)], F64, {}, + [Return(binop(BinOpKind.MUL, Call(F64, "scaled", (L("x", F64),)), c(2.0)))], + pure=True, may_deopt=True) + + +INTEROP = (f_interop, f_which, f_length, f_calls_boom, f_truthy) +CONTRACT_957315EA = (f_proven_eff, f_growth, f_growth_lie, f_big, f_fill, f_fill_lie, f_flag, + f_selfeq, f_scaled, f_twice_scaled) +OBJ_HELPERS = ("tp_global", "tp_getattr", "tp_call", "tp_truth", "tp_compare_bool", "tp_obj_to_f64", + "tp_binop_obj", "tp_release") + +ALL = (f_poly, f_mul_all, f_twice_mul, f_eff, f_scale, f_addfirst, f_floordiv, f_objmix, + f_count_until, f_isum, f_hyp, f_mixed_lt, f_tofl, f_neg_last) + INTEROP + CONTRACT_957315EA + + +# --- building ------------------------------------------------------------------------------------- + +def source_file(name: str) -> Path: + d = OUT_ROOT / name + if d.exists(): + shutil.rmtree(d) + d.mkdir(parents=True) + p = d / f"{name}.py" + p.write_text(SOURCE) + return p + + +def build_module(name: str, functions=ALL, flags=None): + """Generate, build and import an extension `name` compiling `functions`.""" + from typedpython import cbuild, cgen + + need_runtime() + src = source_file(name) + module = Module(name=name, functions=tuple(f() for f in functions)) + c_source = cgen.generate(module, src) + kwargs = {} if flags is None else {"flags": flags} + so = cbuild.build(c_source, name, src.parent, runtime_dir=runtime_dir(), **kwargs) + return cbuild.load(name, so) + + +@pytest.fixture(scope="module") +def mod(): + return build_module("tp_cgen_e2e") + + +def interp(m, name): + return m.__typedpython_interpreted__[name] + + +def deopts(m): + return m.__typedpython_deopts__ + + +# --- generation only (no runtime header needed) --------------------------------------------------- + +def gen(functions=ALL, name="tp_gen_only"): + from typedpython import cgen + + src = source_file(name) + return cgen.generate(Module(name=name, functions=tuple(f() for f in functions)), src) + + +def test_gen_has_multiphase_init_and_embeds_the_source(): + text = gen() + assert "PyInit_tp_gen_only" in text and "PyModuleDef_Init" in text and "Py_mod_exec" in text + assert "__typedpython_interpreted__" in text and "__typedpython_deopts__" in text + assert "tp_impl_poly" in text and "METH_FASTCALL" in text + + +def test_gen_rejects_a_deopting_node_in_an_impure_function(): + from typedpython import cgen + + bad = fn("bad", [Param("a", I64)], I64, {}, + [ExprStmt(call("record", Box(OBJ, L("a", I64)))), + Return(binop(BinOpKind.ADD, L("a", I64), c(1)))], + pure=False, may_deopt=False) + with pytest.raises(cgen.CGenError, match="deopt"): + cgen.generate(Module("tp_bad", (bad,)), source_file("tp_bad")) + + +def test_gen_rejects_a_deopting_call_from_an_impure_caller(): + from typedpython import cgen + + caller = fn("caller", [Param("a", I64)], I64, {}, + [ExprStmt(call("record", Box(OBJ, L("a", I64)))), + Return(Call(I64, "mul_all", (L("a", I64), c(2))))], + pure=False, may_deopt=False) + with pytest.raises(cgen.CGenError, match="deopt"): + cgen.generate(Module("tp_bad2", (f_mul_all(), caller)), source_file("tp_bad2")) + + +# Every function the generated C calls must be CPython C API, a tp_runtime.h helper named in +# runtime/API.md, or a function the generated file defines itself. +C_WORDS = {"void", "if", "for", "while", "switch", "return", "sizeof", "defined", "INT64_C", "UINT64_C"} + + +def api_helpers() -> set[str]: + text = (PACKAGE / "runtime" / "API.md").read_text() + return set(re.findall(r"\b(tp_[a-z0-9_]+)\s*\(", text)) + + +def test_gen_calls_only_cpython_api_and_runtime_helpers(): + text = gen() + # strip comments and string literals so the embedded source does not count + body = re.sub(r"/\*.*?\*/", "", text, flags=re.S) + body = re.sub(r'"(?:\\.|[^"\\])*"', '""', body) + called = set(re.findall(r"\b([A-Za-z_][A-Za-z0-9_]*)\s*\(", body)) + defined = set(re.findall(r"^(?:static\s+)?[A-Za-z_][A-Za-z0-9_ \*]*?\b([A-Za-z_][A-Za-z0-9_]*)\s*\(", + body, flags=re.M)) + helpers = api_helpers() + assert "tp_add_i64" in helpers and "tp_any_same" in helpers # the scan reads API.md for real + bad = sorted(n for n in called - C_WORDS - helpers - defined + if not re.match(r"^(Py[A-Z_]|PyModuleDef_Init$)", n)) + assert bad == [], f"generated C calls functions outside the CPython API and tp_runtime.h: {bad}" + # tp_* names that are called but neither in API.md nor defined here are typos in waiting + stray = sorted(n for n in called if n.startswith("tp_") and n not in helpers and n not in defined) + assert stray == [] + # no private CPython API + assert not re.search(r"\b_Py[A-Za-z]", body) + + +# --- end to end ----------------------------------------------------------------------------------- + +def test_compiled_functions_replace_the_globals(mod): + assert type(mod.poly).__name__ == "builtin_function_or_method" + assert type(interp(mod, "poly")).__name__ == "function" + assert mod.record.__module__ == "tp_cgen_e2e" # interpreted names live in the same dict + assert interp(mod, "poly").__globals__ is mod.__dict__ + + +def test_f64_loop_returns_exactly_the_interpreted_result(mod): + before = deopts(mod) + for x, n in ((1.1, 1000), (0.37, 17), (-2.5, 9), (1e308, 3), (float("nan"), 2)): + got, want = mod.poly(x, n), interp(mod, "poly")(x, n) + assert type(got) is float + assert repr(got) == repr(want), (x, n, got, want) # bit-exact, incl. inf/nan + assert deopts(mod) == before + + +def test_i64_overflow_in_a_pure_function_deopts_to_the_big_int_result(mod): + before = deopts(mod) + assert mod.mul_all(3, 10) == 3 ** 10 + assert deopts(mod) == before + assert mod.mul_all(3, 50) == 3 ** 50 + assert deopts(mod) == before + 1 + + +def test_deopt_propagates_through_a_c_to_c_call(mod): + before = deopts(mod) + assert mod.twice_mul(7, 5) == 2 * 7 ** 5 + assert deopts(mod) == before + assert mod.twice_mul(7, 40) == 2 * 7 ** 40 + # The outermost compiled call is redone once (+1). The interpreted twice_mul then calls the + # global mul_all — the compiled one, since there is one namespace — which deopts again (+1). + assert deopts(mod) == before + 2 + + +def test_big_int_argument_takes_the_interpreted_path_before_any_effect(mod): + mod.calls.clear() + before = deopts(mod) + assert mod.eff(5) == 5 + assert mod.calls == [5] and deopts(mod) == before + big = 2 ** 70 + assert mod.eff(big) == big + assert mod.calls == [5, big] # the effect ran once, by the interpreter + assert deopts(mod) == before + 1 + assert mod.mul_all(2 ** 64, 1) == 2 ** 64 and deopts(mod) == before + 2 + + +def test_wrong_arity_raises_cpythons_own_type_error(mod): + with pytest.raises(TypeError) as want: + interp(mod, "poly")(1.0) + with pytest.raises(TypeError) as got: + mod.poly(1.0) + assert str(got.value) == str(want.value) + assert "poly() missing 1 required positional argument: 'n'" == str(got.value) + with pytest.raises(TypeError) as got3: + mod.poly(1.0, 2, 3) + assert "poly() takes 2 positional arguments but 3 were given" == str(got3.value) + + +def test_keyword_arguments_take_the_interpreted_path(mod): + before = deopts(mod) + assert repr(mod.poly(x=1.1, n=30)) == repr(interp(mod, "poly")(1.1, 30)) + assert repr(mod.poly(1.1, n=30)) == repr(interp(mod, "poly")(1.1, 30)) + assert deopts(mod) == before + 2 + with pytest.raises(TypeError) as got: + mod.poly(1.1, m=3) + with pytest.raises(TypeError) as want: + interp(mod, "poly")(1.1, m=3) + assert str(got.value) == str(want.value) + + +def test_exact_float_guard_rejects_int_and_subclass(mod): + class F(float): + def __rmul__(self, other): + return 42.0 + + before = deopts(mod) + assert mod.poly(2, 3) == interp(mod, "poly")(2, 3) # int x -> interpreted + assert mod.poly(F(2.0), 3) == interp(mod, "poly")(F(2.0), 3) == 42.1 # subclass -> interpreted + assert deopts(mod) == before + 2 + + +def test_f64_array_param_is_written_back(mod): + xs = [1.0, 2.0, 3.5] + ident = id(xs) + before = deopts(mod) + assert mod.scale(xs, 2.0, 3) is None + assert xs == [2.0, 4.0, 7.0] and id(xs) == ident + assert all(type(v) is float for v in xs) + assert deopts(mod) == before + + +def test_exception_mid_loop_keeps_the_partial_updates(mod): + xs, ys = [1.0, 2.0, 3.0], [1.0, 2.0, 3.0] + with pytest.raises(IndexError) as got: + mod.scale(xs, 10.0, 5) + with pytest.raises(IndexError) as want: + interp(mod, "scale")(ys, 10.0, 5) + assert str(got.value) == str(want.value) == "list index out of range" + assert xs == ys == [10.0, 20.0, 30.0] + + +def test_negative_index_counts_from_the_end(mod): + assert mod.neg_last([1.0, 2.5]) == -2.5 + with pytest.raises(IndexError, match="list index out of range"): + mod.neg_last([]) + + +def test_aliasing_two_array_params_takes_the_interpreted_path(mod): + a, b = [1.0, 2.0, 3.0], [1.0, 2.0, 3.0] + before = deopts(mod) + mod.addfirst(a, a) + interp(mod, "addfirst")(b, b) + assert a == b == [2.0, 4.0, 5.0] + assert deopts(mod) == before + 1 + d, s = [1.0, 2.0], [10.0] + mod.addfirst(d, s) + assert d == [11.0, 12.0] and s == [10.0] and deopts(mod) == before + 1 + + +def test_a_non_float_element_takes_the_interpreted_path(mod): + xs, ys = [1.0, 2, 3.0], [1.0, 2, 3.0] + before = deopts(mod) + mod.scale(xs, 2.0, 3) + interp(mod, "scale")(ys, 2.0, 3) + assert xs == ys and [type(v) for v in xs] == [type(v) for v in ys] + assert deopts(mod) == before + 1 + assert mod.scale((1.0,), 2.0, 0) is None and deopts(mod) == before + 2 # a tuple is not a list + + +def test_i64_array_and_its_range_guard(mod): + before = deopts(mod) + assert mod.isum([1, 2, 3, -4]) == 2 and deopts(mod) == before + assert mod.isum([2 ** 63 - 1, 1]) == 2 ** 63 and deopts(mod) == before + 1 # overflow in body + assert mod.isum([2 ** 64, 1]) == 2 ** 64 + 1 and deopts(mod) == before + 2 # element guard + assert mod.isum([True, 1]) == 2 and deopts(mod) == before + 3 # bool is not int + + +def test_integer_division_by_zero_has_cpythons_message(mod): + assert mod.floordiv(-7, 2) == -4 + with pytest.raises(ZeroDivisionError) as got: + mod.floordiv(1, 0) + with pytest.raises(ZeroDivisionError) as want: + interp(mod, "floordiv")(1, 0) + assert str(got.value) == str(want.value) + before = deopts(mod) + assert mod.floordiv(-2 ** 63, -1) == 2 ** 63 and deopts(mod) == before + 1 + + +def test_while_break_continue(mod): + for limit, skip in ((10, 3), (0, 0), (5, 9), (100, 100)): + assert mod.count_until(limit, skip) == interp(mod, "count_until")(limit, skip) + + +def test_math_and_mixed_compare_and_tofloat(mod): + for x, y in ((3.0, 4.0), (1e200, 1e200), (0.1, 0.2)): + assert repr(mod.hyp(x, y)) == repr(interp(mod, "hyp")(x, y)) + for a, b in ((2 ** 53 + 1, float(2 ** 53)), (1, 1.5), (3, 2.0), (-1, -0.5), (1, float("nan"))): + assert mod.mixed_lt(a, b) is interp(mod, "mixed_lt")(a, b), (a, b) + for a in (1, 2 ** 53 + 1, -(2 ** 62) - 3, 7): + assert repr(mod.tofl(a)) == repr(interp(mod, "tofl")(a)) + + +def test_obj_code_keeps_reference_counts_balanced(mod): + o, p = 10 ** 30, 10 ** 31 # not cached small ints + mod.calls.clear() + assert mod.objmix(o, p) == interp(mod, "objmix")(o, p) + mod.calls.clear() + ro, rp = sys.getrefcount(o), sys.getrefcount(p) + for _ in range(1000): + mod.objmix(o, p) + mod.calls.clear() + assert (sys.getrefcount(o), sys.getrefcount(p)) == (ro, rp) + with pytest.raises(TypeError) as got: + mod.objmix(1, "a") + with pytest.raises(TypeError) as want: + interp(mod, "objmix")(1, "a") + assert str(got.value) == str(want.value) + + +def test_interop_matches_the_interpreted_run_including_property_reads(mod): + for scale, n in ((1.5, 20), (0.5, 7), (2.0, 0)): + mod.made.clear() + got = mod.interop(scale, n) + got_reads = mod.made[-1].reads + want = interp(mod, "interop")(scale, n) + want_reads = mod.made[-1].reads + assert repr(got) == repr(want), (scale, n) + assert got_reads == want_reads == 2 * n # `factor` is read in apply() and in the if + assert len(mod.made) == 2 # make() ran once per call, no redo + + +def test_a_rebound_global_is_seen_and_builtins_are_found(mod): + assert mod.which() == 1 + old = mod.target + try: + mod.target = lambda: 2 + assert mod.which() == 2 + del mod.target + with pytest.raises(NameError) as got: + mod.which() + with pytest.raises(NameError) as want: + interp(mod, "which")() + assert str(got.value) == str(want.value) == "name 'target' is not defined" + finally: + mod.target = old + assert mod.length([1, 2, 3]) == 3 + + +def test_an_exception_in_the_callee_propagates_and_leaks_nothing(mod): + o = object() + with pytest.raises(ValueError) as want: + interp(mod, "calls_boom")(o) + del want + before = sys.getrefcount(o) + for _ in range(1000): + with pytest.raises(ValueError) as got: + mod.calls_boom(o) + assert str(got.value) == f"bad value {o!r}" + del got + assert sys.getrefcount(o) == before + + +def test_truth_runs_dunder_bool_and_its_errors(mod): + class B: + def __bool__(self): + raise RuntimeError("no truth") + + assert mod.truthy([1]) is True and mod.truthy([]) is False and mod.truthy(0.0) is False + with pytest.raises(RuntimeError, match="no truth"): + mod.truthy(B()) + + +def test_gen_rejects_an_effect_in_a_pure_function(): + from typedpython import cgen + + bad = fn("badpure", [Param("o", OBJ)], OBJ, {}, [Return(GetAttr(OBJ, L("o", OBJ), "x"))], + pure=True, may_deopt=False) + with pytest.raises(cgen.CGenError, match="pure"): + cgen.generate(Module("tp_bad3", (bad,)), source_file("tp_bad3")) + + +# --- 957315ea: proven ops, Call.redo, entry_globals, CompareObj without identity shortcut ---------- + +def test_gen_accepts_proven_ops_in_an_impure_function(): + from typedpython import cgen + + a = L("a", I64) + ok = fn("ok", [Param("a", I64)], I64, {}, + [ExprStmt(call("record", Box(OBJ, a))), + Return(UnaryOp(I64, UnaryOpKind.NEG, BinOp(I64, BinOpKind.SUB, a, c(1), proven=True), + proven=True))], + pure=False, may_deopt=False) + text = cgen.generate(Module("tp_proven_gen", (ok,)), source_file("tp_proven_gen")) + assert "typedpython: proven operation overflowed" in text + + +def test_gen_rejects_proven_on_an_op_that_is_not_checked_i64_arithmetic(): + from typedpython import cgen + + bad = fn("badproven", [Param("a", I64)], I64, {}, + [Return(BinOp(I64, BinOpKind.FLOORDIV, L("a", I64), c(2), proven=True))], + pure=True, may_deopt=True) + with pytest.raises(cgen.CGenError, match="proven"): + cgen.generate(Module("tp_bad4", (bad,)), source_file("tp_bad4")) + + +def test_gen_rejects_redo_of_an_i64_callee(): + from typedpython import cgen + + caller = fn("caller", [Param("a", I64)], I64, {}, + [ExprStmt(call("record", Box(OBJ, L("a", I64)))), + Return(Call(I64, "mul_all", (L("a", I64), c(2)), redo=True))], + pure=False, may_deopt=False) + with pytest.raises(cgen.CGenError, match="redo"): + cgen.generate(Module("tp_bad5", (f_mul_all(), caller)), source_file("tp_bad5")) + + +def test_gen_rejects_entry_globals_in_a_function_that_runs_user_code(): + from typedpython import cgen + + bad = fn("notclosed", [Param("o", OBJ)], F64, {}, + [ExprStmt(call("record", L("o", OBJ))), Return(L("SCALE", F64))], + pure=False, may_deopt=False, entry_globals=[Param("SCALE", F64)]) + with pytest.raises(cgen.CGenError, match="closed"): + cgen.generate(Module("tp_bad6", (bad,)), source_file("tp_bad6")) + + +def test_proven_op_in_an_impure_function_computes_the_interpreted_result(mod): + mod.calls.clear() + before = deopts(mod) + assert mod.proven_eff(5) == interp(mod, "proven_eff")(5) == 6 + assert mod.calls == [5, 5] and deopts(mod) == before + mod.calls.clear() + # A proof the IR producer got wrong: the overflow is an internal error, never a deopt (the + # effect has happened already, so a redo would run it twice). + with pytest.raises(SystemError, match="typedpython: proven operation overflowed"): + mod.proven_eff(2 ** 63 - 1) + assert mod.calls == [2 ** 63 - 1] and deopts(mod) == before + + +def test_redo_reruns_only_the_pure_callee_and_the_callers_effects_happen_once(mod): + xs, ys = [1.0, 2.0], [1.0, 2.0] + before = deopts(mod) + got = mod.fill(xs, 10) + assert deopts(mod) == before # 3**10 fits: no redo + want = interp(mod, "fill")(ys, 10) + assert repr(got) == repr(want) and xs == ys == [2.0 + 3.0 ** 10, 4.0] + + xs, ys = [1.0, 2.0], [1.0, 2.0] + before = deopts(mod) + got = mod.fill(xs, 45) # 3**40 leaves i64 inside growth + assert deopts(mod) == before + 1 # growth redone once, fill not redone + want = interp(mod, "fill")(ys, 45) + assert type(got) is float and repr(got) == repr(want) == repr(float(3 ** 45)) + # effects exactly once: xs[1] doubled once, xs[0] doubled once then grown once + assert xs == ys == [2.0 + float(3 ** 45), 4.0] + assert [type(v) for v in xs] == [float, float] + + +def test_redo_of_a_bool_callee(mod): + for n, want_r in ((3, 0.0), (7, 1.0), (45, 1.0)): + xs, ys = [1.0], [1.0] + before = deopts(mod) + got = mod.flag(xs, n) + assert deopts(mod) == before + (1 if n >= 40 else 0) + assert got == interp(mod, "flag")(ys, n) == want_r + assert xs == ys == [2.0] + + +def test_redo_result_of_the_wrong_type_is_a_system_error_naming_the_callee(mod): + xs = [1.0] + assert mod.fill_lie(xs, 10) == float(3 ** 10) and xs == [2.0] + xs = [1.0] + with pytest.raises(SystemError, match="growth_lie"): + mod.fill_lie(xs, 45) + assert xs == [2.0] # the caller's store is written back once + + +def test_redo_releases_its_temporaries(mod): + # The redo creates a boxed argument, the interpreted function object reference and a result + # float per call; a leak of any one is >= 24 bytes per call, so 2000 calls would grow by + # >= 48 KB. (Floats are not GC-tracked and are fresh objects, so refcounts cannot show this.) + import tracemalloc + + def run(k): + for _ in range(k): + xs = [1.0] + mod.fill(xs, 45) + mod.flag(xs, 45) + try: + mod.fill_lie(xs, 45) + except SystemError: + pass + + run(200) # warm free lists and caches + tracemalloc.start() + try: + base = tracemalloc.get_traced_memory()[0] + run(2000) + grown = tracemalloc.get_traced_memory()[0] - base + finally: + tracemalloc.stop() + assert grown < 16 * 1024, grown + n = 10 ** 30 + before = sys.getrefcount(n) + for _ in range(1000): + mod.selfeq(n) + assert sys.getrefcount(n) == before + + +def test_entry_global_is_read_at_each_call(mod): + old = mod.SCALE + try: + before = deopts(mod) + assert repr(mod.scaled(3.0)) == repr(interp(mod, "scaled")(3.0)) == repr(7.5) + mod.SCALE = 4.0 # rebinding between calls is seen + assert mod.scaled(3.0) == interp(mod, "scaled")(3.0) == 12.0 + assert mod.twice_scaled(3.0) == interp(mod, "twice_scaled")(3.0) == 24.0 + assert deopts(mod) == before + mod.SCALE = 3 # an int: the whole call is interpreted + got = mod.scaled(3.0) + assert deopts(mod) == before + 1 + assert type(got) is float and got == interp(mod, "scaled")(3.0) == 9.0 + d = deopts(mod) + assert mod.twice_scaled(3.0) == 18.0 + # the pure caller is redone (+1); the interpreted caller calls the compiled scaled, whose + # guard fails again (+1) + assert deopts(mod) == d + 2 + del mod.SCALE # unbound: interpreted path raises NameError + with pytest.raises(NameError) as got_e: + mod.scaled(3.0) + with pytest.raises(NameError) as want_e: + interp(mod, "scaled")(3.0) + assert str(got_e.value) == str(want_e.value) == "name 'SCALE' is not defined" + finally: + mod.SCALE = old + + +def test_compare_obj_has_no_identity_shortcut(mod): + nan = float("nan") + assert mod.selfeq(nan) is interp(mod, "selfeq")(nan) is False + + class Never: + def __eq__(self, other): + return False + + o = Never() + assert mod.selfeq(o) is interp(mod, "selfeq")(o) is False + assert mod.selfeq(1.0) is True + + +# --- local arrays and tuples (issue #41, contract 513f10f2) ----------------------------------------- + +I64A, F64A = Type.I64_ARRAY, Type.F64_ARRAY +LOCAL_ARRAY_HELPERS = ("tp_i64_array_new", "tp_i64_array_iota", "tp_i64_array_copy", + "tp_i64_array_free", "tp_f64_array_new", "tp_f64_array_copy", + "tp_f64_array_free", "tp_tuple") + + +def need_local_array_runtime(): + need_runtime() + header = runtime_dir() / "tp_runtime.h" + text = header.read_text() + missing = [h for h in LOCAL_ARRAY_HELPERS if not re.search(rf"\b{h}\s*\(", text)] + if missing: + pytest.skip(f"{header} does not define the local-array helpers yet (the tp-runtime agent " + f"is still writing them): {missing}") + + +def f_la_build(): + n, i, t = L("n", I64), L("i", I64), L("t", I64) + s, first, last = L("s", I64), L("first", I64), L("last", I64) + mirror = binop(BinOpKind.SUB, binop(BinOpKind.SUB, n, c(1)), i) + return fn("la_build", [Param("n", I64)], OBJ, + {"a": I64A, "b": I64A, "c": I64A, "i": I64, "t": I64, "s": I64, "first": I64, "last": I64}, + [Assign("a", NewArray(I64A, n, None, True)), + Assign("b", NewArray(I64A, n, c(0))), + Assign("s", c(0)), + ForRange("i", c(0), binop(BinOpKind.FLOORDIV, n, c(2)), c(1), ( + Assign("t", Index(I64, "a", i)), + StoreIndex("a", i, Index(I64, "a", mirror)), + StoreIndex("a", mirror, t), + )), + Assign("c", CopyArray(I64A, "a")), + ForRange("i", c(0), Len(I64, "c"), c(1), ( + StoreIndex("b", i, binop(BinOpKind.ADD, binop(BinOpKind.MUL, Index(I64, "c", i), c(3)), + Index(I64, "a", i))), + Assign("s", binop(BinOpKind.ADD, s, Index(I64, "b", i))), + )), + Assign("first", c(-1)), Assign("last", c(-1)), + If(Compare(BOOL, CompareKind.GT, Len(I64, "a"), c(0)), ( + Assign("first", Index(I64, "a", c(0))), + Assign("last", Index(I64, "a", c(-1))), + )), + Return(Tuple(OBJ, (Box(OBJ, Len(I64, "a")), Box(OBJ, Len(I64, "b")), Box(OBJ, first), + Box(OBJ, last), Box(OBJ, s))))], + pure=True, may_deopt=True) + + +def f_la_f64(): + n, i, total = L("n", I64), L("i", I64), L("total", F64) + return fn("la_f64", [Param("n", I64)], OBJ, + {"x": F64A, "y": F64A, "total": F64, "i": I64}, + [Assign("x", NewArray(F64A, n, c(0.5))), + Assign("y", CopyArray(F64A, "x")), + Assign("total", c(0.0)), + ForRange("i", c(0), Len(I64, "x"), c(1), ( + StoreIndex("x", i, binop(BinOpKind.ADD, binop(BinOpKind.MUL, Index(F64, "x", i), + ToFloat(F64, i)), c(1.0))), + StoreIndex("y", i, binop(BinOpKind.TRUEDIV, Index(F64, "x", i), c(2.0))), + Assign("total", binop(BinOpKind.ADD, total, Index(F64, "y", i))), + )), + Return(Tuple(OBJ, (Box(OBJ, Len(I64, "y")), Box(OBJ, total))))], + pure=True, may_deopt=False) + + +def f_la_overflow(): + i, acc = L("i", I64), L("acc", I64) + return fn("la_overflow", [Param("n", I64), Param("k", I64)], OBJ, + {"a": I64A, "b": I64A, "acc": I64, "i": I64}, + [Assign("a", NewArray(I64A, L("n", I64), None, True)), + Assign("b", CopyArray(I64A, "a")), + Assign("acc", c(1)), + ForRange("i", c(0), Len(I64, "b"), c(1), ( + Assign("acc", binop(BinOpKind.ADD, acc, L("k", I64))), + StoreIndex("b", i, i), + )), + Return(Tuple(OBJ, (Box(OBJ, acc), Box(OBJ, Len(I64, "a")))))], + pure=True, may_deopt=True) + + +def f_la_index(): + return fn("la_index", [Param("n", I64), Param("i", I64)], I64, {"a": I64A}, + [Assign("a", NewArray(I64A, L("n", I64), None, True)), + Return(Index(I64, "a", L("i", I64)))], + pure=True, may_deopt=False) + + +def f_la_store(): + return fn("la_store", [Param("n", I64), Param("i", I64), Param("v", I64)], I64, {"a": I64A}, + [Assign("a", NewArray(I64A, L("n", I64), c(0))), + StoreIndex("a", L("i", I64), L("v", I64)), + Return(Len(I64, "a"))], + pure=True, may_deopt=False) + + +def f_la_reassign(): + n = L("n", I64) + return fn("la_reassign", [Param("n", I64)], OBJ, {"a": I64A}, + [Assign("a", NewArray(I64A, n, None, True)), + Assign("a", NewArray(I64A, binop(BinOpKind.ADD, n, c(1)), c(7))), + Assign("a", CopyArray(I64A, "a")), + Return(Tuple(OBJ, (Box(OBJ, Len(I64, "a")), Box(OBJ, Index(I64, "a", c(0))), + Box(OBJ, Index(I64, "a", c(-1))))))], + pure=True, may_deopt=True) + + +def f_la_copy_param(): + return fn("la_copy_param", [ArrayParam("xs", I64A, stored=False)], OBJ, {"c": I64A}, + [Assign("c", CopyArray(I64A, "xs")), + StoreIndex("c", c(0), c(99)), + Return(Tuple(OBJ, (Box(OBJ, Index(I64, "xs", c(0))), Box(OBJ, Index(I64, "c", c(0))), + Box(OBJ, Len(I64, "c")))))], + pure=True, may_deopt=False) + + +def f_la_unit(): + return fn("la_unit", [], OBJ, {}, [Return(Tuple(OBJ, ()))], pure=True, may_deopt=False) + + +LOCALARR = (f_la_build, f_la_f64, f_la_overflow, f_la_index, f_la_store, f_la_reassign, + f_la_copy_param, f_la_unit) + + +@pytest.fixture(scope="module") +def la(): + need_local_array_runtime() + return build_module("tp_cgen_la", functions=LOCALARR) + + +def test_gen_local_array_scan_calls_only_runtime_helpers(): + text = gen(LOCALARR, name="tp_gen_la") + body = re.sub(r"/\*.*?\*/", "", text, flags=re.S) + body = re.sub(r'"(?:\\.|[^"\\])*"', '""', body) + called = set(re.findall(r"\b([A-Za-z_][A-Za-z0-9_]*)\s*\(", body)) + helpers = api_helpers() + for h in LOCAL_ARRAY_HELPERS: + assert h in helpers, f"{h} is not in runtime/API.md" + assert h in called, f"the local-array functions never call {h}" + stray = sorted(n for n in called if n.startswith("tp_") and n not in helpers + and not re.search(rf"^static [^\n]*\b{n}\s*\(", body, flags=re.M)) + assert stray == [] + + +def test_gen_local_arrays_never_go_through_the_write_back_exit(): + text = gen((f_la_build,), name="tp_gen_la2") + impl = text[text.index("static int tp_impl_la_build(PyObject *tp_module, int64_t l_n"):] + impl = impl[:impl.index("static PyObject *tp_wrap_la_build")] + assert "_array_exit" not in impl and "_array_enter" not in impl + assert impl.count("tp_i64_array_free(") >= 3 # a, b, c are freed at the exit block + assert ".dirty" not in impl and "->dirty" not in impl # stores into a local set no dirty bit + + +def test_gen_rejects_a_new_array_that_is_not_an_assign_value(): + from typedpython import cgen + + bad = fn("bad", [Param("n", I64)], I64, {"a": I64A}, + [ExprStmt(NewArray(I64A, L("n", I64), c(0))), Return(c(0))], pure=True, may_deopt=False) + with pytest.raises(cgen.CGenError, match="NewArray"): + cgen.generate(Module("tp_bad3", (bad,)), source_file("tp_bad3")) + + +def test_gen_rejects_assigning_an_array_of_the_wrong_element_type(): + from typedpython import cgen + + bad = fn("bad", [Param("n", I64)], I64, {"a": F64A}, + [Assign("a", NewArray(I64A, L("n", I64), c(0))), Return(c(0))], pure=True, may_deopt=False) + with pytest.raises(cgen.CGenError): + cgen.generate(Module("tp_bad4", (bad,)), source_file("tp_bad4")) + + +def test_gen_a_store_into_a_param_array_still_makes_a_pure_function_an_error(): + from typedpython import cgen + + bad = fn("bad", [ArrayParam("xs", I64A, stored=True)], NONE, {}, + [StoreIndex("xs", c(0), c(1))], pure=True, may_deopt=False) + with pytest.raises(cgen.CGenError, match="pure"): + cgen.generate(Module("tp_bad5", (bad,)), source_file("tp_bad5")) + + +def test_gen_a_store_into_a_local_array_is_not_an_effect(): + text = gen((f_la_store,), name="tp_gen_la3") + assert "tp_impl_la_store" in text + + +def test_local_arrays_equal_cpython_for_several_n(la): + for n in (0, 1, 2, 3, 7, 10, 64, -1, -5): + got, want = la.la_build(n), interp(la, "la_build")(n) + assert type(got) is tuple and got == want, (n, got, want) + assert deopts(la) == 0 + + +def test_f64_local_arrays_equal_cpython_bit_for_bit(la): + for n in (0, 1, 5, 100, -3): + got, want = la.la_f64(n), interp(la, "la_f64")(n) + assert repr(got) == repr(want), (n, got, want) + + +def test_a_tuple_with_no_elements_is_the_empty_tuple(la): + assert la.la_unit() == () and type(la.la_unit()) is tuple + + +def test_local_array_index_errors_equal_cpython(la): + for fname, args in (("la_index", (5, 5)), ("la_index", (5, -6)), ("la_index", (0, 0)), + ("la_index", (5, 4)), ("la_index", (5, -5)), + ("la_store", (3, 3, 1)), ("la_store", (3, -4, 1)), ("la_store", (0, 0, 1)), + ("la_store", (3, -1, 9)), ("la_store", (3, 2, 9))): + try: + want = ("ok", interp(la, fname)(*args)) + except Exception as e: + want = (type(e), str(e)) + try: + got = ("ok", getattr(la, fname)(*args)) + except Exception as e: + got = (type(e), str(e)) + assert got == want, (fname, args, got, want) + # and the two messages are really the list ones, not something of our own + with pytest.raises(IndexError, match=r"^list index out of range$"): + la.la_index(2, 2) + with pytest.raises(IndexError, match=r"^list assignment index out of range$"): + la.la_store(2, 2, 0) + + +def test_reassigning_a_local_array_replaces_it_and_copy_of_itself_is_safe(la): + for n in (0, 1, 5, 40): + assert la.la_reassign(n) == interp(la, "la_reassign")(n) + for bad in (-1,): # a == [7]*0: a[0] raises in both + with pytest.raises(IndexError) as g: + la.la_reassign(bad) + with pytest.raises(IndexError) as w: + interp(la, "la_reassign")(bad) + assert str(g.value) == str(w.value) + + +def test_copy_of_an_array_param_leaves_the_list_untouched(la): + xs = [5, 6, 7] + assert la.la_copy_param(xs) == interp(la, "la_copy_param")([5, 6, 7]) == (5, 99, 3) + assert xs == [5, 6, 7] + + +def test_overflow_deopt_with_local_arrays_returns_the_big_int_result(la): + before = deopts(la) + got, want = la.la_overflow(50, 2 ** 62), interp(la, "la_overflow")(50, 2 ** 62) + assert got == want and got[0] == 1 + 50 * 2 ** 62 and got[0] > 2 ** 63 + assert deopts(la) == before + 1 + before = deopts(la) + assert la.la_overflow(10, 2) == interp(la, "la_overflow")(10, 2) and deopts(la) == before + + +def heap_in_use() -> int: + """Bytes the C allocator has handed out right now (macOS: aggregated over all malloc zones).""" + import ctypes + + if sys.platform != "darwin": + pytest.skip("heap-in-use probe is written for macOS (malloc_zone_statistics) only") + + class Stats(ctypes.Structure): + _fields_ = [("blocks_in_use", ctypes.c_uint), ("size_in_use", ctypes.c_size_t), + ("max_size_in_use", ctypes.c_size_t), ("size_allocated", ctypes.c_size_t)] + + libc = ctypes.CDLL(None) + stats = Stats() + libc.malloc_zone_statistics(None, ctypes.byref(stats)) + return stats.size_in_use + + +def leaked_bytes(call, rounds=40) -> int: + import gc + + call() # warm caches and the allocator + gc.collect() + before = heap_in_use() + for _ in range(rounds): + call() + gc.collect() + return heap_in_use() - before + + +# Each call below allocates two 800 KB arrays (n = 100000); a leak of one array per call is 32 MB +# over 40 rounds, far above the allocator noise this bound allows. (LeakSanitizer does not exist on +# macOS, so the leak check is this heap-in-use delta; ASan below checks the memory errors.) +LEAK_LIMIT = 4 * 1024 * 1024 + + +def test_deopt_in_a_pure_function_frees_the_local_arrays(la): + before = deopts(la) + assert la.la_overflow(100_000, 2 ** 62)[1] == 100_000 # deopted: the sum is a big int + assert deopts(la) == before + 1 + assert leaked_bytes(lambda: la.la_overflow(100_000, 2 ** 62)) < LEAK_LIMIT + + +def test_ok_path_frees_the_local_arrays(la): + assert leaked_bytes(lambda: la.la_overflow(100_000, 1)) < LEAK_LIMIT + assert deopts(la) >= 0 + + +def test_error_path_frees_the_local_arrays(la): + def boom(): + try: + la.la_index(100_000, 10 ** 9) + except IndexError: + pass + assert leaked_bytes(boom) < LEAK_LIMIT + + +def test_reassigning_in_a_loop_frees_the_old_arrays(la): + assert leaked_bytes(lambda: la.la_reassign(100_000)) < LEAK_LIMIT + + +# --- sanitizers ----------------------------------------------------------------------------------- + +ASAN_SCRIPT = textwrap.dedent('''\ + import sys + sys.path.insert(0, {pkg!r}) + from typedpython import cbuild + m = cbuild.load({name!r}, {so!r}) + I = m.__typedpython_interpreted__ + assert repr(m.poly(1.1, 500)) == repr(I["poly"](1.1, 500)) + assert m.mul_all(3, 50) == 3 ** 50 + assert m.twice_mul(7, 40) == 2 * 7 ** 40 + xs = [1.0, 2.0, 3.0] + try: + m.scale(xs, 10.0, 5) + except IndexError: + pass + assert xs == [10.0, 20.0, 30.0], xs + a = [1.0, 2.0, 3.0]; m.addfirst(a, a); assert a == [2.0, 4.0, 5.0] + d = [1.0] * 1000; m.addfirst(d, [2.0]); assert d == [3.0] * 1000 + assert m.isum(list(range(10000))) == sum(range(10000)) + assert m.isum([2 ** 63 - 1, 1]) == 2 ** 63 + try: + m.floordiv(1, 0) + except ZeroDivisionError: + pass + for _ in range(200): + m.objmix(10 ** 30, 10 ** 31); m.calls.clear() + assert m.neg_last([1.0, 2.0]) == -2.0 + assert m.count_until(50, 7) == I["count_until"](50, 7) + for _ in range(50): + xs = [1.0, 2.0]; m.fill(xs, 45); m.flag(xs, 45) + try: + m.fill_lie([1.0], 45) + except SystemError: + pass + assert m.scaled(2.0) == 5.0 and m.selfeq(float("nan")) is False + for _ in range(50): + m.made.clear(); m.interop(1.5, 20) + try: + m.calls_boom(1) + except ValueError: + pass + print("ASAN-RUN-OK", m.__typedpython_deopts__) + ''') + + +def asan_run(name: str, functions, script_template: str) -> tuple[str, Path]: + """Build `functions` with ASan+UBSan, run `script_template` in a sanitized launcher; return the + combined output and the extension path (skips when the platform cannot do it).""" + from typedpython import cbuild + + need_runtime() + cc = sysconfig.get_config_var("CC") or "cc" + if sys.platform != "darwin": + pytest.skip("sanitizer preload recipe written for macOS only") + probe = subprocess.run(cc.split() + ["-print-file-name=libclang_rt.asan_osx_dynamic.dylib"], + capture_output=True, text=True) + asan_rt = Path(probe.stdout.strip()) + if probe.returncode != 0 or not asan_rt.is_file(): + pytest.skip(f"host compiler {cc!r} has no ASan runtime ({probe.stdout.strip()!r})") + src = source_file(name) + from typedpython import cgen + c_source = cgen.generate(Module(name=name, functions=tuple(f() for f in functions)), src) + flags = list(cbuild.DEFAULT_FLAGS) + ["-O1", "-g", "-fno-omit-frame-pointer", + "-fsanitize=address,undefined", + "-fno-sanitize-recover=undefined"] + so = cbuild.build(c_source, name, src.parent, runtime_dir=runtime_dir(), flags=flags) + # macOS refuses DYLD_INSERT_LIBRARIES of the sanitizer runtime into the framework python + # ("Sanitizer load violates platform policy", observed 2026-10-03 on Darwin 25.5), so the + # extension is loaded by a small sanitized launcher linked against libpython: ASan is then in + # the process from the start, as it must be. + launcher_c = src.parent / "asan_python.c" + launcher_c.write_text("#include \nint main(int argc, char **argv) " + "{ return Py_BytesMain(argc, argv); }\n") + launcher = src.parent / "asan_python" + libdir = sysconfig.get_config_var("LIBDIR") + ldver = sysconfig.get_config_var("LDVERSION") + link = subprocess.run(cc.split() + ["-fsanitize=address,undefined", "-g", + f"-I{sysconfig.get_paths()['include']}", str(launcher_c), + f"-L{libdir}", f"-lpython{ldver}", "-o", str(launcher)], + capture_output=True, text=True) + if link.returncode != 0: + pytest.skip(f"cannot link a sanitized launcher against libpython{ldver}: {link.stderr[-2000:]}") + env = dict(os.environ, PYTHONHOME=sys.base_prefix, + ASAN_OPTIONS="detect_leaks=0:abort_on_error=0:halt_on_error=1", + UBSAN_OPTIONS="halt_on_error=1:print_stacktrace=1") + env.pop("DYLD_INSERT_LIBRARIES", None) + script = script_template.format(pkg=str(PACKAGE.parent), name=name, so=str(so)) + run = subprocess.run([str(launcher), "-c", script], capture_output=True, text=True, env=env) + out = run.stdout + run.stderr + if "violates platform policy" in out or "Interceptors are not working" in out: + pytest.skip(f"ASan runtime could not be loaded: {out[-2000:]}") + assert run.returncode == 0 and "ASAN-RUN-OK" in run.stdout, out[-6000:] + return out, so + + +def assert_instrumented(so: Path, out: str) -> None: + linked = subprocess.run(["otool", "-L", str(so)], capture_output=True, text=True).stdout + assert "libclang_rt.asan" in linked, linked # the extension really is instrumented + assert "runtime error" not in out and "AddressSanitizer" not in out, out[-6000:] + + +def test_build_with_address_and_undefined_sanitizers(): + out, so = asan_run("tp_cgen_asan", ALL, ASAN_SCRIPT) + assert_instrumented(so, out) + + +ASAN_LA_SCRIPT = textwrap.dedent('''\ + import sys + sys.path.insert(0, {pkg!r}) + from typedpython import cbuild + m = cbuild.load({name!r}, {so!r}) + I = m.__typedpython_interpreted__ + for n in (0, 1, 2, 3, 7, 64, -4): + assert m.la_build(n) == I["la_build"](n), n + assert repr(m.la_f64(n)) == repr(I["la_f64"](n)), n + for n in (0, 1, 9): + assert m.la_reassign(n) == I["la_reassign"](n), n + for args in ((5, 5), (5, -6), (0, 0), (5, 4)): + for name in ("la_index",): + try: + want = I[name](*args) + except IndexError as e: + want = str(e) + try: + got = getattr(m, name)(*args) + except IndexError as e: + got = str(e) + assert got == want, (name, args, got, want) + try: + m.la_store(3, 3, 1) + except IndexError: + pass + assert m.la_unit() == () + xs = [5, 6, 7] + assert m.la_copy_param(xs) == (5, 99, 3) and xs == [5, 6, 7] + for _ in range(100): + assert m.la_overflow(1000, 2 ** 62) == I["la_overflow"](1000, 2 ** 62) + assert m.la_overflow(1000, 1) == I["la_overflow"](1000, 1) + print("ASAN-RUN-OK", m.__typedpython_deopts__) + ''') + + +def test_local_arrays_under_address_and_undefined_sanitizers(): + need_local_array_runtime() + out, so = asan_run("tp_cgen_asan_la", LOCALARR, ASAN_LA_SCRIPT) + assert_instrumented(so, out) + + +# --- building against the embedded CPython ------------------------------------------------------ + +EMBEDDED_314 = (WORKTREE.parents[1] / "python-multiplatform" / "build" / "python-standalone" + / "extracted" / "macos-aarch64" / "python" / "include" / "python3.14") + + +def test_builds_against_the_embedded_cpython_314_headers(): + """Compile only: the host 3.13 interpreter cannot import a 3.14 extension.""" + from typedpython import cbuild, cgen + + need_runtime() + if not (EMBEDDED_314 / "Python.h").is_file(): + pytest.skip(f"embedded CPython 3.14 headers not extracted: {EMBEDDED_314}") + name = "tp_cgen_314" + src = source_file(name) + c_source = cgen.generate(Module(name=name, functions=tuple(f() for f in ALL)), src) + flags = list(cbuild.DEFAULT_FLAGS) + ["-Werror=deprecated-declarations", + "-Werror=incompatible-pointer-types", + "-Werror=int-conversion"] + so = cbuild.build(c_source, name, src.parent, runtime_dir=runtime_dir(), flags=flags, + python_include=EMBEDDED_314, ext_suffix=".cpython-314-darwin.so") + assert so.name == "tp_cgen_314.cpython-314-darwin.so" and so.stat().st_size > 0 + # the 3.14 headers were really used: the same source against them sees PY_VERSION_HEX 3.14 + probe = cbuild.build('#include \n#if PY_VERSION_HEX < 0x030E0000\n#error not 3.14\n#endif\n' + 'PyMODINIT_FUNC PyInit_tp_probe(void) { return NULL; }\n', + "tp_probe", src.parent, runtime_dir=runtime_dir(), + python_include=EMBEDDED_314, ext_suffix=".cpython-314-darwin.so") + assert probe.is_file() diff --git a/python-multiplatform-ksp/src/test/python/test_cli.py b/python-multiplatform-ksp/src/test/python/test_cli.py new file mode 100644 index 000000000..4a7e003a8 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_cli.py @@ -0,0 +1,60 @@ +"""`python -m typedpython check` exit codes: 0 clean, 1 errors, 2 tool failure.""" +import json +import os +import subprocess +import sys +from pathlib import Path + +SRC = Path(__file__).parents[2] / "main" / "python" + + +def run(*args, env_extra=None): + env = dict(os.environ, PYTHONPATH=str(SRC), **(env_extra or {})) + return subprocess.run( + [sys.executable, "-m", "typedpython", "check", *map(str, args)], + capture_output=True, text=True, env=env, + ) + + +def test_clean_exits_0(write): + path = write("user.py", "def f(x: int) -> int:\n return x\n") + result = run(path) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_error_exits_1(write): + path = write("user.py", "def f(x):\n return x\n") + result = run(path) + assert result.returncode == 1, result.stdout + result.stderr + assert "implicit-any-parameter" in result.stdout + + +def test_warnings_alone_exit_0(write): + path = write("user.py", "def f(s: str) -> None:\n eval(s)\n") + assert run(path).returncode == 0 + assert run("--mode", "compiled", path).returncode == 1 + + +def test_json_output(write): + path = write("user.py", "def f(x):\n return x\n") + result = run("--format", "json", path) + data = json.loads(result.stdout) + assert {"path", "line", "column", "rule", "severity", "message"} <= set(data[0]) + + +def test_missing_pyrefly_exits_2(write): + path = write("user.py", "def f(x: int) -> int:\n return x\n") + result = run(path, env_extra={"TYPEDPYTHON_PYREFLY": "/nonexistent/pyrefly"}) + assert result.returncode == 2, result.stdout + result.stderr + + +def test_a_directory_is_checked_file_by_file(write, tmp_path): + write("proj/pkg/__init__.py", "") + write("proj/pkg/ok.py", "def f(x: int) -> int:\n return x\n") + write("proj/pkg/bad.py", "def g(x):\n return x\n") + write("proj/.venv/lib/junk.py", "def h(x):\n return x\n") # hidden: skipped + write("proj/pkg/__pycache__/stale.py", "def k(x):\n return x\n") # cache: skipped + result = run(tmp_path / "proj") + assert result.returncode == 1, result.stdout + result.stderr + assert "bad.py" in result.stdout + assert "junk.py" not in result.stdout and "stale.py" not in result.stdout diff --git a/python-multiplatform-ksp/src/test/python/test_forbidden.py b/python-multiplatform-ksp/src/test/python/test_forbidden.py new file mode 100644 index 000000000..b672a22f8 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_forbidden.py @@ -0,0 +1,54 @@ +"""The §3.1 forbidden list: warning under `checked`, error under `compiled`.""" +import pytest + +from conftest import rules +from typedpython import forbidden, gate + +FLAGGED = { + "forbidden/eval": "def f(s: str) -> None:\n eval(s)\n", + "forbidden/exec": "def f(s: str) -> None:\n exec(s)\n", + "forbidden/dynamic-attr": "def f(o: object, n: str) -> None:\n getattr(o, n)\n", + "forbidden/monkeypatch": "import json\n\ndef f() -> None:\n json.dumps = print\n", +} + + +@pytest.mark.parametrize("rule", sorted(FLAGGED)) +def test_rule_is_found(rule): + assert rule in {f.rule for f in forbidden.scan(FLAGGED[rule], "user.py")} + + +@pytest.mark.parametrize("call", ["setattr(o, n, 1)", "delattr(o, n)"]) +def test_other_dynamic_attribute_calls(call): + source = f"def f(o: object, n: str) -> None:\n {call}\n" + assert {f.rule for f in forbidden.scan(source, "user.py")} == {"forbidden/dynamic-attr"} + + +def test_monkeypatching_a_module_level_class(): + source = ( + "class Greeter:\n pass\n\n" + "def patch() -> None:\n Greeter.hello = 1\n" + ) + assert {f.rule for f in forbidden.scan(source, "user.py")} == {"forbidden/monkeypatch"} + + +@pytest.mark.parametrize("source", [ + "def f(o: object) -> None:\n getattr(o, 'name')\n", + "class C:\n def __init__(self) -> None:\n self.x = 1\n", + "def f() -> None:\n class Local:\n pass\n obj = Local()\n obj.y = 2\n", +]) +def test_static_forms_are_not_flagged(source): + assert forbidden.scan(source, "user.py") == [] + + +def test_findings_carry_a_location(): + [finding] = forbidden.scan("x = 1\neval('x')\n", "user.py") + assert (finding.line, finding.column) == (2, 1) + + +def test_severity_follows_the_mode(write): + path = write("user.py", "def f(s: str) -> None:\n eval(s)\n") + checked = gate.check([path], mode="checked") + compiled = gate.check([path], mode="compiled") + assert [d.severity for d in checked if d.rule == "forbidden/eval"] == ["warning"] + assert [d.severity for d in compiled if d.rule == "forbidden/eval"] == ["error"] + assert rules(checked) == rules(compiled) diff --git a/python-multiplatform-ksp/src/test/python/test_frontend.py b/python-multiplatform-ksp/src/test/python/test_frontend.py new file mode 100644 index 000000000..791f1bf22 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_frontend.py @@ -0,0 +1,1053 @@ +"""TypedPython front end: Python AST + Pyrefly types -> typed IR (#41, SPEC N-8). + +Each test writes a small module and asserts on the IR `frontend.lower` produces, or on the reason +a function was left interpreted. The IR's own docstrings (typedpython/ir.py) are the contract: +mixed int/float gets `ToFloat`, chains evaluate the middle operand once, an impure function has +no node that can deopt. +""" +import dataclasses +import shutil +import textwrap +from pathlib import Path + +import pytest + +from typedpython import frontend +from typedpython.ir import ( + And, ArrayParam, Assign, BinOp, BinOpKind, Box, Break, Call, CallObject, Compare, CompareKind, + CompareObj, Const, CopyArray, NewArray, Tuple, ExprStmt, While, ForRange, GetAttr, Global, If, Index, Len, Local, MathCall, + MathFunc, ObjToFloat, Param, Return, StoreIndex, ToFloat, Truth, Type, UnaryOp, UnaryOpKind, +) + +I64, F64, BOOL, OBJ = Type.I64, Type.F64, Type.BOOL, Type.OBJ +BENCH = Path("/Volumes/macMini/thisisthepy/PythonMultiplatform/.worktrees/typedpython-bench" + "/benchmarks/typedpython/py") + + +def lower(tmp_path, source, name="mod.py"): + path = tmp_path / name + path.write_text(textwrap.dedent(source).lstrip("\n")) + return frontend.lower(path) + + +def function(module, name): + found = [f for f in module.functions if f.name == name] + assert found, f"{name} was not lowered: {module.skipped.get(name)!r}" + return found[0] + + +def nodes(root, cls): + """Every IR node of class `cls` under `root` (a Function, statement or expression).""" + found, stack = [], [root] + while stack: + x = stack.pop() + if isinstance(x, (tuple, list)): + stack.extend(x) + elif dataclasses.is_dataclass(x) and not isinstance(x, type): + if isinstance(x, cls): + found.append(x) + stack.extend(getattr(x, f.name) for f in dataclasses.fields(x)) + return found + + +# --- the basic shape ----------------------------------------------------------------------------- + +def test_scalar_loop_lowers_to_for_range(tmp_path): + m = lower(tmp_path, """ + # typedpython: compiled + def total(n: int, x: float) -> float: + acc: float = 0.0 + for k in range(n): + acc += x + return acc + """) + assert m.skipped == {} + f = function(m, "total") + assert f.params == (Param("n", I64), Param("x", F64)) + assert f.returns == F64 + assert f.locals == {"acc": F64, "k": I64} + assert f.body == ( + Assign("acc", Const(F64, 0.0)), + ForRange("k", Const(I64, 0), Local(I64, "n"), Const(I64, 1), ( + Assign("acc", BinOp(F64, BinOpKind.ADD, Local(F64, "acc"), Local(F64, "x"))), + )), + Return(Local(F64, "acc")), + ) + assert f.pure and not f.may_deopt + assert f.source_line == 2 + + +def test_int_locals_in_a_pure_function_are_i64_and_may_deopt(tmp_path): + # `c` is unannotated: its type comes from Pyrefly. + m = lower(tmp_path, """ + @compiled + def f(a: int, b: int) -> int: + c = a + b + return c * 2 + """) + f = function(m, "f") + assert f.locals == {"c": I64} + assert f.body[0] == Assign("c", BinOp(I64, BinOpKind.ADD, Local(I64, "a"), Local(I64, "b"))) + assert f.pure and f.may_deopt + + +def test_unannotated_local_with_two_types_is_skipped(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(a: int) -> float: + c = a + c = 1.5 + return c + """) + assert m.functions == () + assert "`c`" in m.skipped["f"] and "line " in m.skipped["f"] + + +def test_impure_function_keeps_an_unbounded_int_local_as_a_python_int(tmp_path): + # Impure: a deopt could not be redone, so `m` is a CPython int object, not a checked i64. + m = lower(tmp_path, """ + @compiled + def g(a: list[float], n: int) -> None: + m: int = n + m += 1 + a[0] = 1.0 + """) + f = function(m, "g") + assert f.locals == {"m": OBJ} + assert f.body[:2] == ( + Assign("m", Box(OBJ, Local(I64, "n"))), + Assign("m", BinOp(OBJ, BinOpKind.ADD, Local(OBJ, "m"), Box(OBJ, Const(I64, 1)))), + ) + assert not f.pure and not f.may_deopt + + +def test_impure_function_with_bounded_range_indices_lowers_without_deopt(tmp_path): + m = lower(tmp_path, """ + @compiled + def h(a: list[float], n: int) -> None: + for i in range(n): + for j in range(i + 1, n): + a[j] += a[i] + """) + f = function(m, "h") + assert not f.pure and not f.may_deopt + assert f.params == (ArrayParam("a", Type.F64_ARRAY, True), Param("n", I64)) + + +def test_impure_function_with_unproved_int_arithmetic_is_skipped(tmp_path): + m = lower(tmp_path, """ + @compiled + def h(a: list[float], n: int) -> None: + a[n * 2] = 1.0 + """) + # `n * 2` can overflow i64 and the function is impure, so it is an int object: no native index. + assert m.functions == () + assert "line 3" in m.skipped["h"] and "n * 2" in m.skipped["h"] + + +def test_range_index_arithmetic_is_proven(tmp_path): + # ir.BinOp.proven / UnaryOp.proven: the interval proof is recorded in the node. + m = lower(tmp_path, """ + @compiled + def shift(a: list[float]) -> None: + for i in range(4): + a[i + 1] = a[i] - float(-i) + float(i * 2 - 1) + """) + f = function(m, "shift") + assert not f.pure and not f.may_deopt + ints = [b for b in nodes(f, BinOp) if b.type == I64] + assert ints and all(b.proven for b in ints) + assert BinOp(I64, BinOpKind.ADD, Local(I64, "i"), Const(I64, 1), proven=True) in ints + assert nodes(f, UnaryOp) == [UnaryOp(I64, UnaryOpKind.NEG, Local(I64, "i"), proven=True)] + + +def test_unbounded_int_arithmetic_is_not_proven(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(a: int, b: int) -> int: + return -(a + b) + """) + f = function(m, "f") + assert f.pure and f.may_deopt + assert f.body == (Return(UnaryOp(I64, UnaryOpKind.NEG, BinOp( + I64, BinOpKind.ADD, Local(I64, "a"), Local(I64, "b")))),) + assert not any(b.proven for b in nodes(f, BinOp)) and not nodes(f, UnaryOp)[0].proven + + +def test_float_arithmetic_is_never_marked_proven(tmp_path): + # `proven` is for I64 ADD/SUB/MUL/NEG only (F64 never deopts). + m = lower(tmp_path, """ + @compiled + def f(x: float) -> float: + return -(x + 1.0) + """) + f = function(m, "f") + assert not any(n.proven for n in nodes(f, BinOp) + nodes(f, UnaryOp)) + + +@pytest.mark.parametrize("expr", ["i // 2", "i / 2"]) +def test_int_division_cannot_be_recorded_as_proven_so_it_deopts(tmp_path, expr): + # ir.BinOp.proven exists for ADD/SUB/MUL only; the verifier cannot re-prove // and / on i64, + # so they count as deopting: in a pure function may_deopt, in an impure one a CPython int. + m = lower(tmp_path, f""" +@compiled +def f(n: int) -> float: + t: float = 0.0 + for i in range(10): + t += {expr} + return t +""") + assert function(m, "f").may_deopt + + +# --- conversions --------------------------------------------------------------------------------- + +def test_mixed_int_float_arithmetic_inserts_to_float(tmp_path): + m = lower(tmp_path, """ + @compiled + def mix(n: int, x: float) -> float: + return n * x + 1 + """) + f = function(m, "mix") + assert f.body == (Return(BinOp( + F64, BinOpKind.ADD, + BinOp(F64, BinOpKind.MUL, ToFloat(F64, Local(I64, "n")), Local(F64, "x")), + ToFloat(F64, Const(I64, 1)), + )),) + + +def test_int_value_into_a_float_local_is_skipped(tmp_path): + # CPython keeps the int: `y` would be an int at run time, not a float. + m = lower(tmp_path, """ + @compiled + def f(n: int) -> float: + y: float = n + return y + """) + assert m.functions == () + assert "line 3" in m.skipped["f"] + + +def test_int_true_division_gives_float(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(a: int, b: int) -> float: + return a / b + """) + assert function(m, "f").body == ( + Return(BinOp(F64, BinOpKind.TRUEDIV, Local(I64, "a"), Local(I64, "b"))),) + + +def test_float_of_int_is_to_float(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(a: int) -> float: + return float(a) + """) + assert function(m, "f").body == (Return(ToFloat(F64, Local(I64, "a"))),) + + +# --- chained comparison -------------------------------------------------------------------------- + +def test_chained_compare_of_locals(tmp_path): + m = lower(tmp_path, """ + @compiled + def between(lo: int, x: int, hi: int) -> bool: + return lo < x <= hi + """) + assert function(m, "between").body == (Return(And( + BOOL, + Compare(BOOL, CompareKind.LT, Local(I64, "lo"), Local(I64, "x")), + Compare(BOOL, CompareKind.LE, Local(I64, "x"), Local(I64, "hi")), + )),) + + +def test_chained_compare_evaluates_the_middle_once(tmp_path): + m = lower(tmp_path, """ + @compiled + def sq(v: float) -> float: + return v * v + + @compiled + def within(lo: float, x: float, hi: float) -> bool: + return lo < sq(x) < hi + """) + f = function(m, "within") + assert len(f.body) == 2 + first, ret = f.body + assert isinstance(first, Assign) and first.value == Call(F64, "sq", (Local(F64, "x"),)) + t = first.target + assert f.locals[t] == F64 and t not in ("lo", "x", "hi") + assert ret == Return(And( + BOOL, + Compare(BOOL, CompareKind.LT, Local(F64, "lo"), Local(F64, t)), + Compare(BOOL, CompareKind.LT, Local(F64, t), Local(F64, "hi")), + )) + + +# --- arrays -------------------------------------------------------------------------------------- + +def test_augmented_store_into_an_array_element(tmp_path): + m = lower(tmp_path, """ + @compiled + def bump(a: list[float], i: int, e: float) -> None: + a[i] += e + """) + assert function(m, "bump").body == (StoreIndex( + "a", Local(I64, "i"), + BinOp(F64, BinOpKind.ADD, Index(F64, "a", Local(I64, "i")), Local(F64, "e")), + ),) + + +def test_augmented_store_evaluates_a_computed_index_once(tmp_path): + m = lower(tmp_path, """ + @compiled + def bump_last(a: list[float], e: float) -> None: + a[len(a) - 1] += e + """) + f = function(m, "bump_last") + first, store = f.body + assert isinstance(first, Assign) + assert first.value == BinOp(I64, BinOpKind.SUB, Len(I64, "a"), Const(I64, 1), proven=True) + t = first.target + assert store == StoreIndex( + "a", Local(I64, t), + BinOp(F64, BinOpKind.ADD, Index(F64, "a", Local(I64, t)), Local(F64, "e")), + ) + assert not f.pure and not f.may_deopt + + +def test_list_params_become_array_params_with_stored(tmp_path): + m = lower(tmp_path, """ + @compiled + def scale(a: list[float], b: list[float], k: float, c: list[int]) -> None: + for i in range(len(a)): + a[i] = b[i] * k + """) + f = function(m, "scale") + assert f.params == ( + ArrayParam("a", Type.F64_ARRAY, True), ArrayParam("b", Type.F64_ARRAY, False), + Param("k", F64), ArrayParam("c", Type.I64_ARRAY, False), + ) + assert f.body[0].stop == Len(I64, "a") + + +def test_array_param_used_whole_is_skipped(tmp_path): + m = lower(tmp_path, """ + @compiled + def first(a: list[float]) -> float: + return a[0] + + @compiled + def caller(a: list[float]) -> float: + return first(a) + """) + assert [f.name for f in m.functions] == ["first"] + assert "`a`" in m.skipped["caller"] and "line 7" in m.skipped["caller"] + + +def test_int_stored_into_a_float_array_is_skipped(tmp_path): + m = lower(tmp_path, """ + @compiled + def zero(a: list[float]) -> None: + a[0] = 0 + """) + assert m.functions == () and "line 3" in m.skipped["zero"] + + +# --- math ---------------------------------------------------------------------------------------- + +def test_math_sqrt_of_stdlib_math(tmp_path): + m = lower(tmp_path, """ + import math + + @compiled + def norm(x: float, y: float) -> float: + return math.sqrt(x * x + y * y) + """) + assert function(m, "norm").body == (Return(MathCall(F64, MathFunc.SQRT, (BinOp( + F64, BinOpKind.ADD, + BinOp(F64, BinOpKind.MUL, Local(F64, "x"), Local(F64, "x")), + BinOp(F64, BinOpKind.MUL, Local(F64, "y"), Local(F64, "y")), + ),)),),) + + +@pytest.mark.parametrize("binding", ["import cmath as math", "import math\nmath = 3"]) +def test_shadowed_math_is_skipped(tmp_path, binding): + m = lower(tmp_path, binding + """ + +@compiled +def norm(x: float) -> float: + return math.sqrt(x) +""") + assert m.functions == () + assert "math" in m.skipped["norm"] + + +# --- calls and purity ---------------------------------------------------------------------------- + +def test_impure_caller_redoes_a_deopting_pure_callee(tmp_path): + # ir.Call.redo: on a deopt only the (pure) callee is redone; the caller does not deopt. + m = lower(tmp_path, """ + @compiled + def tri(i: int, j: int) -> float: + return 1.0 / ((i + j) * (i + j + 1) // 2 + i + 1) + + @compiled + def fill(n: int, out: list[float]) -> None: + for i in range(n): + out[i] = tri(i, i) + """) + tri = function(m, "tri") + assert tri.pure and tri.may_deopt + fill = function(m, "fill") + assert not fill.pure and not fill.may_deopt + assert nodes(fill, Call) == [Call(F64, "tri", (Local(I64, "i"), Local(I64, "i")), redo=True)] + assert nodes(fill, CallObject) == [] + + +@pytest.mark.parametrize("returns, body, use", [ + ("bool", "return i + j > 0", "if tri(i, i):\n out[i] = 1.0"), + ("None", "k = i + j", "tri(i, i)\n out[i] = 1.0"), +]) +def test_redo_call_for_bool_and_none_callees(tmp_path, returns, body, use): + m = lower(tmp_path, f""" +@compiled +def tri(i: int, j: int) -> {returns}: + {body} + +@compiled +def fill(n: int, out: list[float]) -> None: + for i in range(n): + {use} +""") + assert function(m, "tri").may_deopt + fill = function(m, "fill") + assert not fill.pure and not fill.may_deopt + [call] = nodes(fill, Call) + assert call.function == "tri" and call.redo + + +def test_impure_caller_of_an_int_returning_deopting_callee_calls_the_global(tmp_path): + # An I64 result of a redo could be a big int: such a callee is still called as an object. + m = lower(tmp_path, """ + @compiled + def add(i: int, j: int) -> int: + return i + j + + @compiled + def fill(n: int, out: list[float]) -> None: + for i in range(n): + out[i] = float(add(i, i)) + """) + fill = function(m, "fill") + assert not fill.pure and not fill.may_deopt + assert nodes(fill, Call) == [] + assert nodes(fill, CallObject) == [ + CallObject(OBJ, Global(OBJ, "add"), (Box(OBJ, Local(I64, "i")), Box(OBJ, Local(I64, "i"))))] + + +def test_pure_caller_of_a_deopting_callee_does_not_redo(tmp_path): + m = lower(tmp_path, """ + @compiled + def tri(i: int, j: int) -> float: + return 1.0 / (i + j) + + @compiled + def twice(i: int) -> float: + return tri(i, i) * 2.0 + """) + twice = function(m, "twice") + assert twice.pure and twice.may_deopt + assert [c.redo for c in nodes(twice, Call)] == [False] + + +def test_pure_caller_of_a_deopting_function_lowers(tmp_path): + m = lower(tmp_path, """ + @compiled + def add(i: int, j: int) -> int: + return i + j + + @compiled + def twice(i: int) -> int: + return add(i, i) + """) + twice = function(m, "twice") + assert twice.body == (Return(Call(I64, "add", (Local(I64, "i"), Local(I64, "i")))),) + assert twice.pure and twice.may_deopt + + +def test_caller_of_an_interpreted_function_calls_it_as_an_object(tmp_path): + m = lower(tmp_path, """ + @compiled + def bad(x: float) -> float: + try: + return x + except ValueError: + return 0.0 + + @compiled + def good(x: float) -> object: + return bad(x) + + @compiled + def typed(x: float) -> float: + return bad(x) + """) + assert [f.name for f in m.functions] == ["good"] + assert "line 3" in m.skipped["bad"] and "try" in m.skipped["bad"] + assert function(m, "good").body == ( + Return(CallObject(OBJ, Global(OBJ, "bad"), (Box(OBJ, Local(F64, "x")),))),) + # The result is an object; a float return would need a conversion CPython does not make. + assert "bad(x)" in m.skipped["typed"] and "line 14" in m.skipped["typed"] + + +# --- what is (not) lowered ----------------------------------------------------------------------- + +@pytest.mark.parametrize("body, what", [ + (" try:\n pass\n except ValueError:\n pass\n", "try"), + (" with x:\n pass\n", "with"), + (" yield x\n", "yield"), + (" global q\n", "global"), + (" y = [x]\n", "list"), + (" y = (lambda: x)\n", "lambda"), +]) +def test_unsupported_construct_is_skipped_with_its_line(tmp_path, body, what): + m = lower(tmp_path, "@compiled\ndef f(x: float) -> None:\n" + body) + assert m.functions == () + assert "line 3" in m.skipped["f"] and what in m.skipped["f"] + + +@pytest.mark.parametrize("signature", [ + "def f(x: float = 1.0) -> float:", "def f(*x: float) -> float:", "def f(x) -> float:", + "def f(x: float):", "def f(*, x: float) -> float:", "def f(x: float, /) -> float:", + "@staticmethod\ndef f(x: float) -> float:", +]) +def test_unsupported_signature_is_skipped(tmp_path, signature): + m = lower(tmp_path, f"@compiled\n{signature}\n return 1.0\n") + assert m.functions == () and "f" in m.skipped + + +def test_global_read_in_a_function_that_runs_user_code_is_a_global_node(tmp_path): + # `print(x)` runs user code, which could rebind K: K is read where it is used (LOAD_GLOBAL). + m = lower(tmp_path, """ + K: float = 2.0 + + @compiled + def f(x: float) -> object: + print(x) + return x * K + + @compiled + def g(x: float) -> float: + print(x) + return x * K + """) + f = function(m, "f") + assert f.body[1] == Return( + BinOp(OBJ, BinOpKind.MUL, Box(OBJ, Local(F64, "x")), Global(OBJ, "K"))) + assert f.entry_globals == () + assert not f.pure and not f.may_deopt + assert "x * K" in m.skipped["g"] + + +def test_scalar_globals_in_a_closed_function_are_entry_globals(tmp_path): + m = lower(tmp_path, """ + PI: float = 3.141592653589793 + SOLAR_MASS: float = 4 * PI * PI + DAYS = 365.24 + N = 3 + ON: bool = True + + @compiled + def scale(a: list[float]) -> None: + for i in range(N): + if ON: + a[i] = a[i] / SOLAR_MASS * DAYS + + @compiled + def k(x: float) -> float: + return x * SOLAR_MASS + """) + scale = function(m, "scale") + assert scale.entry_globals == (Param("N", I64), Param("ON", BOOL), Param("SOLAR_MASS", F64), + Param("DAYS", F64)) + assert nodes(scale, Global) == [] + assert Local(F64, "SOLAR_MASS") in nodes(scale, Local) + assert not scale.pure and not scale.may_deopt + k = function(m, "k") + assert k.entry_globals == (Param("SOLAR_MASS", F64),) + assert k.body == (Return(BinOp(F64, BinOpKind.MUL, Local(F64, "x"), Local(F64, "SOLAR_MASS"))),) + assert k.pure and not k.may_deopt + assert "SOLAR_MASS" not in k.locals + + +def test_closedness_follows_calls(tmp_path): + m = lower(tmp_path, """ + K: float = 2.0 + + @compiled + def noisy(x: float) -> float: + print(x) + return x + + @compiled + def quiet(x: float) -> float: + return x + 1.0 + + @compiled + def via_noisy(x: float) -> object: + return noisy(x) * K + + @compiled + def via_quiet(x: float) -> float: + return quiet(x) * K + """) + assert function(m, "via_noisy").entry_globals == () + assert Global(OBJ, "K") in nodes(function(m, "via_noisy"), Global) + assert function(m, "via_quiet").entry_globals == (Param("K", F64),) + + +@pytest.mark.parametrize("module", [ + "K: float = 2.0\nK = 3.0\n", # assigned twice + "K: float = 2.0\n\ndef reset() -> None:\n global K\n K = 1.0\n", # `global K` elsewhere + "import os\nif os.sep:\n K = 2.0\n", # not at top level + "for K in (1.0,):\n pass\n", # a loop target + "import os\nK = os.cpu_count()\n", # neither annotated nor literal + "K: str = 'a'\n", # not a scalar + "K: float = 2.0\ndel K\n", # deleted +]) +def test_a_global_that_may_be_rebound_is_never_an_entry_global(tmp_path, module): + m = lower(tmp_path, module + """ + +@compiled +def f(x: float) -> object: + return x * K +""") + f = function(m, "f") + assert f.entry_globals == () + assert Global(OBJ, "K") in nodes(f, Global) + + +def test_module_marker_compiles_every_module_level_def(tmp_path): + m = lower(tmp_path, """ + + # typedpython: compiled + def a(x: float) -> float: + return x + + def b(x: float) -> float: + return -x + + @staticmethod + def c(x: float) -> float: + return x + """) + assert [f.name for f in m.functions] == ["a", "b"] + assert set(m.skipped) == {"c"} and "decorator" in m.skipped["c"] + + +def test_without_marker_only_compiled_functions_are_lowered(tmp_path): + m = lower(tmp_path, """ + x: int = 1 + # typedpython: compiled + def a(x: float) -> float: + return x + + @compiled + def b(x: float) -> float: + return -x + """) + assert [f.name for f in m.functions] == ["b"] + assert m.skipped == {} + + +# --- objects: Kotlin interop and any Python value ------------------------------------------------- + +def test_compiled_code_calls_imported_callables_around_native_float_math(tmp_path): + (tmp_path / "kotlinlib.py").write_text(textwrap.dedent(""" + class Canvas: + pass + + def make_canvas(n: int) -> Canvas: + return Canvas() + + def draw(c: Canvas, x: float, alpha: float) -> None: + pass + """)) + m = lower(tmp_path, """ + import math + from kotlinlib import draw, make_canvas + + @compiled + def render(n: int, scale: float) -> float: + canvas = make_canvas(n) + total: float = 0.0 + hits: int = 0 + for i in range(n): + x: float = i * scale + total += math.sqrt(x) + draw(canvas, x, alpha=0.5) + hits += 1 + if hits > n: + break + return total + """) + f = function(m, "render") + assert not f.pure and not f.may_deopt + assert f.locals == {"canvas": OBJ, "total": F64, "hits": OBJ, "i": I64, "x": F64} + assert f.body[0] == Assign("canvas", CallObject( + OBJ, Global(OBJ, "make_canvas"), (Box(OBJ, Local(I64, "n")),))) + loop = f.body[3] + assert isinstance(loop, ForRange) and loop.var == "i" + assert loop.body == ( + Assign("x", BinOp(F64, BinOpKind.MUL, ToFloat(F64, Local(I64, "i")), Local(F64, "scale"))), + Assign("total", BinOp(F64, BinOpKind.ADD, Local(F64, "total"), + MathCall(F64, MathFunc.SQRT, (Local(F64, "x"),)))), + ExprStmt(CallObject(OBJ, Global(OBJ, "draw"), ( + Local(OBJ, "canvas"), Box(OBJ, Local(F64, "x")), Box(OBJ, Const(F64, 0.5)), + ), ("alpha",))), + Assign("hits", BinOp(OBJ, BinOpKind.ADD, Local(OBJ, "hits"), Box(OBJ, Const(I64, 1)))), + If(CompareObj(BOOL, CompareKind.GT, Local(OBJ, "hits"), Box(OBJ, Local(I64, "n"))), + (Break(),)), + ) + + +def test_object_truth_float_and_method_calls(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(o: object, x: float) -> float: + if o: + return float(o.scale(x)) + return x + """) + assert function(m, "f").body == ( + If(Truth(BOOL, Local(OBJ, "o")), (Return(ObjToFloat(F64, CallObject( + OBJ, GetAttr(OBJ, Local(OBJ, "o"), "scale"), (Box(OBJ, Local(F64, "x")),)))),)), + Return(Local(F64, "x")), + ) + + +def test_object_comparison_outside_a_condition_is_skipped(tmp_path): + # `a < b` on objects returns whatever __lt__ returns, which need not be a bool. + m = lower(tmp_path, """ + @compiled + def f(o: object, x: float) -> bool: + return o < x + """) + assert m.functions == () and "line 3" in m.skipped["f"] + + +@pytest.mark.parametrize("body", ["return not (o < x)", "r: bool = c and o < x\n return r"]) +def test_object_comparison_not_directly_a_condition_is_skipped(tmp_path, body): + m = lower(tmp_path, f"@compiled\ndef f(o: object, x: float, c: bool) -> bool:\n {body}\n") + assert m.functions == () and "line 3" in m.skipped["f"] + + +def test_object_comparison_as_an_operand_of_a_condition(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(o: object, x: float, c: bool) -> int: + if c and o < x: + return 1 + return 0 + """) + assert function(m, "f").body[0].cond == And( + BOOL, Local(BOOL, "c"), + CompareObj(BOOL, CompareKind.LT, Local(OBJ, "o"), Box(OBJ, Local(F64, "x")))) + + +def test_while_condition_with_a_temporary_reevaluates_it_each_iteration(tmp_path): + m = lower(tmp_path, """ + @compiled + def sq(v: float) -> float: + return v * v + + @compiled + def shrink(lo: float, x: float, hi: float) -> float: + while lo < sq(x) < hi: + x = x * 0.5 + return x + """) + loop = function(m, "shrink").body[0] + assert isinstance(loop, While) and loop.cond == Const(BOOL, True) + temp, test = loop.body[0], loop.body[1] + assert isinstance(temp, Assign) and temp.value == Call(F64, "sq", (Local(F64, "x"),)) + # The condition stays an If condition (never under `not`), and exits through its else. + assert isinstance(test, If) and isinstance(test.cond, And) + assert test.then == () and test.orelse == (Break(),) + + +@pytest.mark.parametrize("source", [ + "@compiled\ndef f(n: int) -> int:\n return f(n)\n", + "@compiled\ndef f(n: int) -> int:\n return g(n)\n\n" + "@compiled\ndef g(n: int) -> int:\n return f(n)\n", +]) +def test_recursion_is_skipped(tmp_path, source): + m = lower(tmp_path, source) + assert m.functions == () and "recurs" in m.skipped["f"] + + +def test_int_modulo_cannot_be_recorded_as_proven_so_it_deopts(tmp_path): + # ir.BinOp.proven covers ADD/SUB/MUL only and the verifier counts an unproven i64 `%` as a + # deopting node: in a pure function that is may_deopt, in an impure one a CPython int. + m = lower(tmp_path, """ + @compiled + def pure_mod(i: int, n: int) -> int: + return i % n + + @compiled + def put(a: list[float], i: int, n: int) -> None: + a[i % n] = 1.0 + """) + f = function(m, "pure_mod") + assert f.pure and f.may_deopt + assert not any(b.proven for b in nodes(f, BinOp)) + assert "i % n" in m.skipped["put"] + + +def test_locals_exclude_parameters(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(n: int) -> int: + n = n + 1 + return n + """) + assert function(m, "f").locals == {} + + +# --- benchmarks ---------------------------------------------------------------------------------- + +@pytest.mark.skipif(not BENCH.exists(), reason="benchmark worktree not present") +def test_nbody_kernels_lower(tmp_path): + path = tmp_path / "nbody.py" + shutil.copy(BENCH / "nbody.py", path) + path.write_text("# typedpython: compiled\n" + path.read_text()) + m = frontend.lower(path) + for name in ("advance", "energy", "offset_momentum"): + function(m, name) + advance = function(m, "advance") + assert not advance.pure and not advance.may_deopt + assert BinOp(I64, BinOpKind.ADD, Local(I64, "i"), Const(I64, 1), proven=True) \ + in nodes(advance, BinOp) + energy = function(m, "energy") + assert energy.pure and not energy.may_deopt + offset = function(m, "offset_momentum") + assert not offset.pure and not offset.may_deopt + assert offset.entry_globals == (Param("SOLAR_MASS", F64),) + assert nodes(offset, Global) == [] + + +@pytest.mark.skipif(not BENCH.exists(), reason="benchmark worktree not present") +def test_spectral_norm_kernels_lower(tmp_path): + path = tmp_path / "spectral_norm.py" + path.write_text("# typedpython: compiled\n" + (BENCH / "spectral_norm.py").read_text()) + m = frontend.lower(path) + eval_a = function(m, "eval_a") + assert eval_a.pure and eval_a.may_deopt + for name in ("mul_av", "mul_atv"): + f = function(m, name) + assert not f.pure and not f.may_deopt + [call] = nodes(f, Call) + assert call.function == "eval_a" and call.redo + + +# --- local fixed-size arrays and tuple returns (ir.NewArray / CopyArray / Tuple, #41 M2) --------- + +def test_list_range_local_is_an_iota_array(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(n: int) -> int: + x: list[int] = list(range(n)) + return x[0] + """) + f = function(m, "f") + assert f.locals == {"x": Type.I64_ARRAY} + assert f.body == ( + Assign("x", NewArray(Type.I64_ARRAY, Local(I64, "n"), None, True)), + Return(Index(I64, "x", Const(I64, 0))), + ) + assert f.pure and not f.may_deopt + + +def test_fill_locals_are_new_arrays_of_their_element_type(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(n: int) -> float: + a: list[int] = [0] * n + b = [1.5] * n + return b[a[0]] + """) + f = function(m, "f") + assert f.locals == {"a": Type.I64_ARRAY, "b": Type.F64_ARRAY} + assert f.body[:2] == ( + Assign("a", NewArray(Type.I64_ARRAY, Local(I64, "n"), Const(I64, 0))), + Assign("b", NewArray(Type.F64_ARRAY, Local(I64, "n"), Const(F64, 1.5))), + ) + assert f.pure + + +def test_slice_copy_of_a_local_and_of_a_parameter(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(p: list[int], n: int) -> int: + x: list[int] = list(range(n)) + y = x[:] + z: list[int] = p[:] + return y[0] + z[0] + """) + f = function(m, "f") + assert f.locals == {"x": Type.I64_ARRAY, "y": Type.I64_ARRAY, "z": Type.I64_ARRAY} + assert f.body[1] == Assign("y", CopyArray(Type.I64_ARRAY, "x")) + assert f.body[2] == Assign("z", CopyArray(Type.I64_ARRAY, "p")) + assert f.params[0] == ArrayParam("p", Type.I64_ARRAY, False) + + +@pytest.mark.parametrize("what, source, snippet", [ + ("returned", "return x", "return x"), + ("passed to a call", "print(x)\n return 0", "print(x)"), + ("compared", "return x == y", "x == y"), + ("iterated over", "for v in x:\n pass\n return 0", "x"), + ("sliced other than [:]", "w = x[1:]\n return 0", "x[1:]"), + ("appended to", "x.append(1)\n return 0", "x.append(1)"), + ("aliased", "w = x\n return 0", "w = x"), + ("copied outside an assignment", "return len(x[:])", "x[:]"), + ("rebound by a non-array value", "x = 5\n return 0", "x"), +]) +def test_local_array_escapes_skip_the_function(tmp_path, what, source, snippet): + source = source.replace("\n ", "\n ") # the f-string below is dedented + m = lower(tmp_path, f""" + @compiled + def f(n: int) -> int: + x: list[int] = list(range(n)) + y: list[int] = [0] * n + {source} + """) + assert "f" not in [fn.name for fn in m.functions], what + reason = m.skipped["f"] + assert snippet in reason, (what, reason) + assert "array" in reason, (what, reason) + + +@pytest.mark.parametrize("annotation, value", [ + ("list[str]", '["a"] * n'), + ("list[bool]", "[True] * n"), + ("list[list[int]]", "[[0]] * n"), +]) +def test_local_array_element_type_must_be_exactly_int_or_float(tmp_path, annotation, value): + m = lower(tmp_path, f""" + @compiled + def f(n: int) -> int: + x: {annotation} = {value} + return 0 + """) + assert "f" in m.skipped + assert "exactly int or float" in m.skipped["f"], m.skipped["f"] + + +def test_int_fill_of_a_float_array_is_skipped(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(n: int) -> int: + x: list[float] = [0] * n + return 0 + """) + assert "f" in m.skipped and "float" in m.skipped["f"] + + +def test_function_with_only_local_array_stores_is_pure(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(n: int) -> int: + x: list[int] = [0] * n + for i in range(n): + x[i] = i * i + x[i] += 1 + return x[n - 1] * n + """) + f = function(m, "f") + assert nodes(f, StoreIndex) and f.pure + # pure, so the unbounded int arithmetic is native with deopt, not a CPython int + assert f.may_deopt + assert f.locals["x"] == Type.I64_ARRAY and f.returns == I64 + + +def test_function_storing_into_an_array_param_is_still_impure(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(p: list[int], n: int) -> int: + x: list[int] = [0] * n + p[0] = 1 + x[0] = 2 + return x[0] + """) + f = function(m, "f") + assert not f.pure and not f.may_deopt + assert f.params[0] == ArrayParam("p", Type.I64_ARRAY, True) + + +def test_storing_a_local_array_into_nothing_but_locals_keeps_calls_impure(tmp_path): + # An object call is an effect whatever the arrays do. + m = lower(tmp_path, """ + @compiled + def f(n: int, g: object) -> int: + x: list[int] = [0] * n + x[0] = 1 + g(1) + return x[0] + """) + assert not function(m, "f").pure + + +def test_tuple_return_boxes_each_element(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(a: int, b: float) -> tuple[int, float]: + return a, b + """) + f = function(m, "f") + assert f.returns == OBJ + assert f.body == (Return(Tuple(OBJ, (Box(OBJ, Local(I64, "a")), Box(OBJ, Local(F64, "b"))))),) + assert f.pure + + +def test_tuple_return_with_object_element_and_deopting_arithmetic(tmp_path): + m = lower(tmp_path, """ + @compiled + def f(a: int, b: int, o: object) -> tuple[int, object]: + return a // b, o + """) + f = function(m, "f") + assert f.pure and f.may_deopt + [ret] = nodes(f, Return) + assert isinstance(ret.value, Tuple) and ret.value.elements[1] == Local(OBJ, "o") + + +@pytest.mark.parametrize("annotation, value, snippet", [ + ("tuple[int, float]", "a, b", "float"), + ("object", "a, b", "tuple["), + ("tuple[int, int, int]", "a, b", "declares 3"), + ("tuple[int, ...]", "a, b", "..."), +]) +def test_tuple_return_must_match_its_annotation(tmp_path, annotation, value, snippet): + m = lower(tmp_path, f""" + @compiled + def f(a: int, b: int) -> {annotation}: + return {value} + """) + assert "f" in m.skipped + assert snippet in m.skipped["f"], m.skipped["f"] + + +@pytest.mark.skipif(not BENCH.exists(), reason="benchmark worktree not present") +def test_fannkuch_lowers(tmp_path): + path = tmp_path / "fannkuch.py" + path.write_text("# typedpython: compiled\n" + (BENCH / "fannkuch.py").read_text()) + m = frontend.lower(path) + assert "fannkuch" in [f.name for f in m.functions], m.skipped.get("fannkuch") + f = function(m, "fannkuch") + assert f.pure and f.may_deopt + assert f.returns == OBJ and f.params == (Param("n", I64),) + assert f.locals["perm1"] == f.locals["count"] == f.locals["perm"] == Type.I64_ARRAY + assert len(nodes(f, NewArray)) == 2 and len(nodes(f, CopyArray)) == 1 + assert nodes(f, Tuple) diff --git a/python-multiplatform-ksp/src/test/python/test_imports.py b/python-multiplatform-ksp/src/test/python/test_imports.py new file mode 100644 index 000000000..4a9a9a876 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_imports.py @@ -0,0 +1,31 @@ +"""User code inside a package must resolve its own imports (defect found 2026-10-03). + +The gate writes its Pyrefly config into a temporary directory. Pyrefly infers the import root +from the config's location, so without help a package's relative and absolute self-imports did +not resolve: every value from them became `Unknown` and failed the gate as `any-flow`. +""" +from conftest import errors +from typedpython import gate + + +def package(write): + write("src/pkg/__init__.py", "") + write("src/pkg/b.py", "def make() -> int:\n return 1\n") + + +def test_relative_import_inside_a_package(write): + package(write) + path = write("src/pkg/a.py", "from .b import make\n\ndef f() -> int:\n x = make()\n return x\n") + assert errors(gate.check([path])) == [] + + +def test_absolute_import_of_the_own_package(write): + package(write) + path = write("src/pkg/c.py", "from pkg.b import make\n\ndef f() -> int:\n return make()\n") + assert errors(gate.check([path])) == [] + + +def test_sibling_module_without_a_package(write): + write("flat/helper.py", "def make() -> int:\n return 1\n") + path = write("flat/main.py", "from helper import make\n\ndef f() -> int:\n return make()\n") + assert errors(gate.check([path])) == [] diff --git a/python-multiplatform-ksp/src/test/python/test_incremental.py b/python-multiplatform-ksp/src/test/python/test_incremental.py new file mode 100644 index 000000000..d4effcc4e --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_incremental.py @@ -0,0 +1,211 @@ +"""Incremental compilation (SPEC N-10, #41): module cache keyed by source + dependency interface hashes.""" +from __future__ import annotations + +import dataclasses +import re +import shutil +import time +import uuid +from pathlib import Path + +import pytest + +from typedpython import ir +from typedpython.incremental import BuildCache, dependencies, interface_hash + +TMP_ROOT = Path(__file__).resolve().parents[4] / ".tmp" +TYPES = {"int": ir.Type.I64, "float": ir.Type.F64, "bool": ir.Type.BOOL, "None": ir.Type.NONE} +DEF = re.compile(r"^def (\w+)\((.*?)\) -> (\w+):", re.M) + + +@pytest.fixture +def work(): + root = TMP_ROOT / f"incr-{uuid.uuid4().hex[:8]}" + root.mkdir(parents=True) + yield root + shutil.rmtree(root, ignore_errors=True) + + +class Fakes: + """Counting stand-ins for the front end, the C back end and the C compiler.""" + + def __init__(self): + self.lowered, self.generated, self.compiled = [], [], [] + + def lower(self, path: Path) -> ir.Module: + self.lowered.append(path.name) + params_of = lambda text: tuple( + ir.Param(n.strip().split(":")[0], TYPES[n.strip().split(":")[1].strip()]) + for n in text.split(",") if n.strip()) + funcs = tuple( + ir.Function(m[1], params_of(m[2]), TYPES[m[3]], {}, (), True, False, 1) + for m in DEF.finditer(path.read_text())) + return ir.Module(path.stem, funcs) + + def generate(self, module: ir.Module, path: Path) -> str: + self.generated.append(path.name) + return f"/* {module.name} */ {path.read_text()!r}" + + def compile_c(self, c_source: str, name: str, out_dir: Path) -> Path: + self.compiled.append(name) + out = Path(out_dir) / f"{name}.so" + out.write_text("OBJ:" + c_source) + return out + + def counts(self): + return (len(self.lowered), len(self.generated), len(self.compiled)) + + def reset(self): + self.lowered.clear(); self.generated.clear(); self.compiled.clear() + + +def build(cache, fakes, root, files, **over): + args = dict(lower=fakes.lower, generate=fakes.generate, compile_c=fakes.compile_c, + compiler_version="1", platform="linux-x86_64", flags=["-O2"]) + args.update(over) + return cache.build_project([root / f for f in files], root, **args) + + +def project(root: Path): + (root / "a.py").write_text("def f(x: int) -> int:\n return x + 1\n") + (root / "b.py").write_text("from a import f\ndef g(y: int) -> int:\n return f(y)\n") + (root / "c.py").write_text("def h(z: float) -> float:\n return z\n") + return ["a.py", "b.py", "c.py"] + + +def status(report): + return {Path(m.path).name: m.status for m in report.modules} + + +def test_1_first_builds_all_second_builds_nothing(work): + files, fakes, cache = project(work), Fakes(), BuildCache(work / "cache") + r1 = build(cache, fakes, work, files) + assert fakes.counts() == (3, 3, 3) + assert set(status(r1).values()) == {"rebuilt"} + fakes.reset() + t = time.perf_counter() + r2 = build(cache, fakes, work, files) + elapsed = time.perf_counter() - t + assert fakes.counts() == (0, 0, 0) + assert set(status(r2).values()) == {"reused"} + print(f"unchanged rebuild wall time: {elapsed * 1000:.2f} ms") + assert elapsed < 0.5 + assert all(m.artifact.exists() for m in r2.modules) + + +def test_2_body_change_rebuilds_only_that_module(work): + files, fakes, cache = project(work), Fakes(), BuildCache(work / "cache") + build(cache, fakes, work, files) + fakes.reset() + (work / "a.py").write_text("def f(x: int) -> int:\n return x + 2\n") + r = build(cache, fakes, work, files) + assert fakes.compiled == ["a"] + assert status(r) == {"a.py": "rebuilt", "b.py": "reused", "c.py": "reused"} + assert "source" in next(m for m in r.modules if m.path.name == "a.py").reasons + + +def test_3_signature_change_rebuilds_dependents_not_independents(work): + files, fakes, cache = project(work), Fakes(), BuildCache(work / "cache") + build(cache, fakes, work, files) + fakes.reset() + (work / "a.py").write_text("def f(x: float) -> float:\n return x + 1\n") + r = build(cache, fakes, work, files) + assert sorted(fakes.compiled) == ["a", "b"] + assert status(r) == {"a.py": "rebuilt", "b.py": "rebuilt", "c.py": "reused"} + b = next(m for m in r.modules if m.path.name == "b.py") + assert any("dependency interface" in x and "a.py" in x for x in b.reasons), b.reasons + + +@pytest.mark.parametrize("over,component", [ + ({"compiler_version": "2"}, "compiler_version"), + ({"platform": "darwin-arm64"}, "platform"), + ({"flags": ["-O0"]}, "flags"), +]) +def test_4_toolchain_change_rebuilds_everything(work, over, component): + files, fakes, cache = project(work), Fakes(), BuildCache(work / "cache") + build(cache, fakes, work, files) + fakes.reset() + r = build(cache, fakes, work, files, **over) + assert fakes.counts() == (3, 3, 3) + assert set(status(r).values()) == {"rebuilt"} + assert all(component in m.reasons for m in r.modules), [m.reasons for m in r.modules] + + +def test_5_corrupt_entry_is_a_miss_others_reused(work): + files, fakes, cache = project(work), Fakes(), BuildCache(work / "cache") + r1 = build(cache, fakes, work, files) + victim = next(m for m in r1.modules if m.path.name == "c.py").artifact + victim.write_text("OBJ:trunc") # damaged contents + fakes.reset() + r2 = build(cache, fakes, work, files) + assert fakes.compiled == ["c"] + assert status(r2) == {"a.py": "reused", "b.py": "reused", "c.py": "rebuilt"} + assert "corrupt" in next(m for m in r2.modules if m.path.name == "c.py").reasons + # and a deleted artefact, and garbage metadata + fakes.reset() + for m in r2.modules: + if m.path.name == "a.py": + m.artifact.unlink() + if m.path.name == "b.py": + (m.artifact.parent / "meta.json").write_text("{not json") + r3 = build(cache, fakes, work, files) + assert sorted(fakes.compiled) == ["a", "b"] + assert status(r3)["c.py"] == "reused" + + +def test_6_relative_import_in_package_detected(work): + pkg = work / "pkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("") + (pkg / "base.py").write_text("def f(x: int) -> int:\n return x\n") + (pkg / "sub").mkdir() + (pkg / "sub" / "__init__.py").write_text("") + (pkg / "sub" / "leaf.py").write_text("") + (pkg / "user.py").write_text("from .base import f\nfrom .sub import leaf\nimport os, json\n") + (pkg / "deep.py").write_text("from ..pkg import base\nfrom . import base as b2\nimport pkg.sub.leaf\n") + d = dependencies(pkg / "user.py", work) + assert pkg / "base.py" in d and pkg / "sub" / "leaf.py" in d + assert not any(p.name in ("os.py", "json.py") for p in d) + d2 = dependencies(pkg / "deep.py", work) + assert pkg / "base.py" in d2 and pkg / "sub" / "leaf.py" in d2 + assert pkg / "deep.py" not in d2 + + +def test_6b_relative_dependency_drives_rebuild(work): + pkg = work / "pkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("") + (pkg / "base.py").write_text("def f(x: int) -> int:\n return x\n") + (pkg / "user.py").write_text("from .base import f\ndef g(x: int) -> int:\n return f(x)\n") + files, fakes, cache = ["pkg/base.py", "pkg/user.py"], Fakes(), BuildCache(work / "cache") + build(cache, fakes, work, files) + fakes.reset() + (pkg / "base.py").write_text("def f(x: float) -> int:\n return 1\n") + build(cache, fakes, work, files) + assert sorted(fakes.compiled) == ["pkg.base", "pkg.user"] + + +def fn(name="f", params=(ir.Param("x", ir.Type.I64),), ret=ir.Type.I64, body=(), locals_=None, line=1): + return ir.Function(name, tuple(params), ret, locals_ or {}, tuple(body), True, False, line) + + +def test_7_interface_hash(): + base = ir.Module("m", (fn(),)) + h = interface_hash(base) + body = (ir.Return(ir.Const(ir.Type.I64, 1)),) + assert interface_hash(ir.Module("m", (fn(body=body, locals_={"t": ir.Type.F64}, line=99),))) == h + assert interface_hash(ir.Module("m", (dataclasses.replace(fn(), pure=False, may_deopt=True),))) == h + assert interface_hash(ir.Module("m", (fn(params=(ir.Param("x", ir.Type.F64),)),))) != h + assert interface_hash(ir.Module("m", (fn(params=(ir.Param("y", ir.Type.I64),)),))) != h + assert interface_hash(ir.Module("m", (fn(ret=ir.Type.F64),))) != h + assert interface_hash(ir.Module("m", (fn(name="g"),))) != h + arr = lambda t, s: fn(params=(ir.ArrayParam("x", t, s),)) + assert interface_hash(ir.Module("m", (arr(ir.Type.F64_ARRAY, False),))) != \ + interface_hash(ir.Module("m", (arr(ir.Type.I64_ARRAY, False),))) + assert interface_hash(ir.Module("m", (fn(),), {"z": "why"})) != h # skipped names count + assert interface_hash(ir.Module("m", (fn(),), {"z": "why"})) == \ + interface_hash(ir.Module("m", (fn(),), {"z": "other reason"})) + # order of params matters, order of functions does too (it is the module's order) + p2 = (ir.Param("a", ir.Type.I64), ir.Param("b", ir.Type.F64)) + assert interface_hash(ir.Module("m", (fn(params=p2),))) != \ + interface_hash(ir.Module("m", (fn(params=p2[::-1]),))) diff --git a/python-multiplatform-ksp/src/test/python/test_measurement.py b/python-multiplatform-ksp/src/test/python/test_measurement.py new file mode 100644 index 000000000..45e5a15b2 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_measurement.py @@ -0,0 +1,50 @@ +"""What the gate costs on top of a plain `pyrefly check` (design §7, AGENTS.md §14). + +The extra work is the per-expression type report (`--report-pysa`) and the AST pass. The numbers +are printed so a run records them; only a loose sanity bound is asserted. +""" +import subprocess +import sys +import time + +import pytest + +from typedpython import gate + +FUNCTIONS = 300 + + +def synthetic_module() -> str: + lines = [] + for i in range(FUNCTIONS): + lines += [ + f"def f{i}(a: int, b: int) -> int:", + " total = 0", + " for k in range(a):", + " total += k * b", + " return total", + "", + ] + return "\n".join(lines) + + +@pytest.mark.measurement +def test_gate_cost_against_plain_pyrefly(write): + path = write("big.py", synthetic_module()) + lines = synthetic_module().count("\n") + 1 + + start = time.perf_counter() + subprocess.run( + [sys.executable, "-m", "pyrefly", "check", str(path), "--summary=none"], + capture_output=True, check=False, + ) + plain = time.perf_counter() - start + + start = time.perf_counter() + found = gate.check([path]) + gated = time.perf_counter() - start + + print(f"\n[measure] {lines} lines: plain pyrefly {plain * 1000:.0f} ms, " + f"gate {gated * 1000:.0f} ms, ratio {gated / plain:.2f}x") + assert found == [] + assert gated < 60 diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline.py b/python-multiplatform-ksp/src/test/python/test_pipeline.py new file mode 100644 index 000000000..a835153c9 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_pipeline.py @@ -0,0 +1,102 @@ +"""End to end: source -> frontend -> verify -> cgen -> cbuild -> import (SPEC N-8, #41).""" +import importlib.util +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from typedpython import pipeline + +SRC = Path(__file__).parents[2] / "main" / "python" + + +def interpreted(path: Path): + spec = importlib.util.spec_from_file_location(path.stem + "_interp", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +SCALAR = """\ +# typedpython: compiled + + +def total(n: int) -> float: + acc: float = 0.0 + for k in range(n): + acc += k * 0.5 + return acc + + +def fact(n: int) -> int: + r: int = 1 + for k in range(2, n + 1): + r = r * k + return r +""" + + +def test_scalar_module_compiles_and_matches_cpython(write, tmp_path): + path = write("scalar.py", SCALAR) + result = pipeline.compile_module(path, tmp_path / "out") + assert result.extension is not None, result.skipped + assert {f.name for f in result.module.functions} >= {"total", "fact"}, result.skipped + compiled = pipeline.load(result) + ref = interpreted(path) + assert compiled.total(1000) == ref.total(1000) + assert compiled.fact(30) == ref.fact(30) # promotes past i64 + assert compiled.__typedpython_deopts__ >= 1 + + +def test_unsupported_functions_stay_interpreted_with_a_reason(write, tmp_path): + path = write("mixed.py", SCALAR + """ + +def greet(name: str) -> str: + return "hi " + name +""") + result = pipeline.compile_module(path, tmp_path / "out") + assert "greet" in result.skipped and result.skipped["greet"] + assert pipeline.load(result).greet("x") == "hi x" + + +def test_a_module_with_nothing_to_compile_produces_no_extension(write, tmp_path): + path = write("plain.py", "def f(x: int) -> int:\n return x\n") + result = pipeline.compile_module(path, tmp_path / "out") + assert result.extension is None + + +def test_gate_errors_block_compilation(write, tmp_path): + path = write("bad.py", "# typedpython: compiled\n\ndef f(x):\n return x\n") + with pytest.raises(pipeline.CompileError, match="implicit-any-parameter"): + pipeline.compile_module(path, tmp_path / "out") + + +DEMO = """\ +# typedpython: compiled +import sys + + +def total(n: int) -> float: + acc: float = 0.0 + for k in range(n): + acc += k * 0.5 + return acc + + +def main() -> None: + print(f"{total(int(sys.argv[1])):.3f}") +""" + + +def test_demo_reports_identical_output(write, tmp_path): + path = write("demo_mod.py", DEMO) + env = dict(os.environ, PYTHONPATH=str(SRC)) + result = subprocess.run( + [sys.executable, "-m", "typedpython", "demo", "--out", str(tmp_path / "b"), str(path), "1000"], + capture_output=True, text=True, env=env, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "identical output" in result.stdout and "249750.000" in result.stdout + assert "compiled functions: total" in result.stdout diff --git a/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py b/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py new file mode 100644 index 000000000..9eef09304 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_pipeline_incremental.py @@ -0,0 +1,171 @@ +"""compile_project: the incremental cache wired into the real pipeline (SPEC N-10, #41). + +Real Pyrefly, real C builds. The three-module project: `a` imports `b`; `c` is independent. +""" +import importlib.util +import textwrap +import time +from pathlib import Path + +import pytest + +from typedpython import cbuild, frontend, pipeline + +HEADER = "# typedpython: compiled\n" + +B = HEADER + textwrap.dedent("""\ + def twice(n: int) -> int: + return n * 2 + """) +A = HEADER + textwrap.dedent("""\ + import b + + + def inc(n: int) -> int: + return n + 1 + """) +C = HEADER + textwrap.dedent("""\ + def half(x: float) -> float: + return x * 0.5 + """) + + +def interpreted(path: Path): + spec = importlib.util.spec_from_file_location(path.stem + "_interp", path) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + return m + + +@pytest.fixture +def project(tmp_path): + root = tmp_path / "proj" + root.mkdir() + for name, src in (("a", A), ("b", B), ("c", C)): + (root / f"{name}.py").write_text(src) + return root + + +@pytest.fixture +def counts(monkeypatch): + """Count calls into the compile steps (the work an unchanged rebuild must not do).""" + n = {"lower": 0, "build": 0} + real_lower, real_build = frontend.lower, cbuild.build + + def lower(*a, **k): + n["lower"] += 1 + return real_lower(*a, **k) + + def build(*a, **k): + n["build"] += 1 + return real_build(*a, **k) + + monkeypatch.setattr(frontend, "lower", lower) + monkeypatch.setattr(cbuild, "build", build) + return n + + +def run(project, tmp_path): + return pipeline.compile_project( + [project / "a.py", project / "b.py", project / "c.py"], project, tmp_path / "cache") + + +def status(result): + return {m.path.stem: m.status for m in result.modules} + + +def test_first_build_compiles_all_and_unchanged_rebuild_does_nothing(project, tmp_path, counts): + first = run(project, tmp_path) + assert status(first) == {"a": "rebuilt", "b": "rebuilt", "c": "rebuilt"} + assert all(m.extension is not None for m in first.modules) + assert counts == {"lower": 3, "build": 3} + + counts.update(lower=0, build=0) + start = time.perf_counter() + second = run(project, tmp_path) + elapsed = time.perf_counter() - start + print(f"\nunchanged rebuild of 3 modules: {elapsed:.3f} s (compile steps skipped)") + assert status(second) == {"a": "reused", "b": "reused", "c": "reused"} + assert counts == {"lower": 0, "build": 0} + assert [m.extension for m in second.modules] == [m.extension for m in first.modules] + + +def test_body_change_rebuilds_that_module_only(project, tmp_path, counts): + run(project, tmp_path) + (project / "b.py").write_text(B.replace("n * 2", "n * 3")) + r = run(project, tmp_path) + assert status(r) == {"a": "reused", "b": "rebuilt", "c": "reused"} + assert "source" in next(m for m in r.modules if m.path.stem == "b").reasons + + +def test_signature_change_rebuilds_dependents(project, tmp_path, counts): + run(project, tmp_path) + (project / "b.py").write_text(B.replace("n: int) -> int", "n: int, k: int) -> int") + .replace("n * 2", "n * k")) + r = run(project, tmp_path) + assert status(r) == {"a": "rebuilt", "b": "rebuilt", "c": "reused"} + a = next(m for m in r.modules if m.path.stem == "a") + assert any("dependency interface" in x for x in a.reasons), a.reasons + + +def test_compiler_version_change_rebuilds_everything(project, tmp_path, counts, monkeypatch): + run(project, tmp_path) + monkeypatch.setattr(pipeline, "COMPILER_VERSION", "something-else") + r = run(project, tmp_path) + assert status(r) == {"a": "rebuilt", "b": "rebuilt", "c": "rebuilt"} + assert all("compiler_version" in m.reasons for m in r.modules) + + +def test_rebuilt_and_reused_extensions_compute_the_interpreted_results(project, tmp_path, monkeypatch): + monkeypatch.syspath_prepend(str(project)) # a's `import b` runs at load + run(project, tmp_path) + (project / "b.py").write_text(B.replace("n * 2", "n * 3")) + r = run(project, tmp_path) + mods = {m.path.stem: m for m in r.modules} + b = cbuild.load("b", mods["b"].extension) + assert b.twice(7) == interpreted(project / "b.py").twice(7) == 21 + a = cbuild.load("a", mods["a"].extension) # reused + assert a.inc(4) == 5 + c = cbuild.load("c", mods["c"].extension) + assert c.half(5.0) == interpreted(project / "c.py").half(5.0) + + +def test_module_with_nothing_compiled_has_no_extension_and_is_reused(tmp_path): + root = tmp_path / "proj" + root.mkdir() + (root / "plain.py").write_text(HEADER + "def f(s: str) -> str:\n return s + 'x'\n") + cache = tmp_path / "cache" + first = pipeline.compile_project([root / "plain.py"], root, cache) + m = first.modules[0] + assert m.extension is None and m.status == "rebuilt" and m.skipped + second = pipeline.compile_project([root / "plain.py"], root, cache) + assert second.modules[0].status == "reused" and second.modules[0].reasons == [] + assert second.modules[0].extension is None + + +def test_interface_describes_the_verified_module_not_the_lowered_one(tmp_path, monkeypatch): + """A function the verifier rejects stays interpreted, so it must not be in the interface.""" + import dataclasses + + from typedpython import incremental, ir + + root = tmp_path / "proj" + root.mkdir() + (root / "dep.py").write_text(HEADER + "def ok(n: int) -> int:\n return n + 1\n") + real_lower = frontend.lower + seen = {} + + def lower_with_a_bad_function(path): + m = real_lower(path) + good = m.functions[0] + bad = dataclasses.replace(good, name="bad", returns=ir.Type.F64) # verify/type: wrong return + seen["lowered"] = dataclasses.replace(m, functions=(*m.functions, bad)) + return seen["lowered"] + + monkeypatch.setattr(frontend, "lower", lower_with_a_bad_function) + r = pipeline.compile_project([root / "dep.py"], root, tmp_path / "cache") + dep = r.modules[0] + assert "bad" in dep.skipped, dep.skipped + assert [f.name for f in seen["lowered"].functions] == ["ok", "bad"] + unverified = incremental.interface_hash(seen["lowered"]) + assert dep.interface != unverified diff --git a/python-multiplatform-ksp/src/test/python/test_pyrefly_kinds.py b/python-multiplatform-ksp/src/test/python/test_pyrefly_kinds.py new file mode 100644 index 000000000..f7e864dfb --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_pyrefly_kinds.py @@ -0,0 +1,69 @@ +"""Each Pyrefly Any-leak kind the gate raises to error (design §4.2).""" +import pytest + +from conftest import errors, rules +from typedpython import gate + +CASES = { + "implicit-any-parameter": """ + def f(x, y: int) -> int: + return y + """, + "unannotated-return": """ + def f(y: int): + return y + """, + "unknown-variable-type": """ + import json + def f() -> None: + d = json.loads("1") + z = d["k"] + """, + "no-any-return-implicit": """ + import json + def f() -> int: + d = json.loads("1") + return d["k"] + """, + "no-any-return-explicit": """ + from typing import Any + def f(v: Any) -> int: + return v + """, + "explicit-any": """ + from typing import Any + def f(v: Any) -> None: + pass + """, +} + + +@pytest.mark.parametrize("kind", sorted(CASES)) +def test_kind_is_an_error(write, kind): + path = write("user.py", CASES[kind]) + found = gate.check([path]) + assert f"pyrefly/{kind}" in rules(errors(found)), found + + +def test_ordinary_type_error_is_an_error(write): + path = write("user.py", """ + def f(y: int) -> int: + return y + "a" + """) + assert errors(gate.check([path])), "a plain type error must still fail the gate" + + +def test_fully_typed_code_is_clean(write): + path = write("user.py", """ + def area(w: int, h: int) -> int: + scale = 2 + return w * h * scale + + class Box: + def __init__(self, w: int) -> None: + self.w = w + + def double(self) -> int: + return self.w * 2 + """) + assert gate.check([path]) == [] diff --git a/python-multiplatform-ksp/src/test/python/test_rebinding.py b/python-multiplatform-ksp/src/test/python/test_rebinding.py new file mode 100644 index 000000000..731707041 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_rebinding.py @@ -0,0 +1,184 @@ +"""Rebinding a name to a different type, and mixed-element containers (design §3.1, #22). + +Both are forbidden-list items: warnings under `checked`, errors under `compiled`. They need the +inferred type of an assignment target, which Pyrefly's type report does not give directly; the +gate reads each target back through a probe load (see typedpython.rebinding). +""" +import pytest + +from typedpython import gate, rebinding + +REBIND = "forbidden/rebinding" +MIXED = "forbidden/mixed-container" + + +def found(write, source, mode="checked"): + return gate.check([write("user.py", source)], mode=mode) + + +def by_rule(diagnostics, rule): + return [d for d in diagnostics if d.rule == rule] + + +def test_rebinding_to_another_type_is_reported(write): + result = found(write, """ + def f() -> None: + x = 1 + x = "a" + print(x) + """) + [d] = by_rule(result, REBIND) + assert (d.line, d.severity) == (4, "warning") + assert "int" in d.message and "str" in d.message + + +def test_compiled_mode_makes_it_an_error(write): + result = found(write, """ + def f() -> None: + x = 1 + x = "a" + print(x) + """, mode="compiled") + assert [d.severity for d in by_rule(result, REBIND)] == ["error"] + + +def test_augmented_assignment_that_changes_the_type(write): + result = found(write, """ + def f() -> float: + total = 0 + total += 0.5 + return total + """) + assert by_rule(result, REBIND), result + + +def test_module_level_rebinding(write): + result = found(write, """ + limit = 10 + limit = "ten" + print(limit) + """) + assert by_rule(result, REBIND), result + + +@pytest.mark.parametrize("source", [ + # same type again + "def f() -> None:\n x = 1\n x = 2\n print(x)\n", + # a declared union governs; Pyrefly checks each assignment against it + "def f() -> None:\n x: int | str = 1\n x = 'a'\n print(x)\n", + # an annotated parameter reassigned within its type + "def f(n: int) -> int:\n n = n + 1\n return n\n", +]) +def test_not_rebinding(write, source): + assert by_rule(gate.check([write("user.py", source)]), REBIND) == [] + + +def test_mixed_list_is_reported(write): + result = found(write, """ + def f() -> None: + mixed = [1, "a"] + print(mixed) + """) + [d] = by_rule(result, MIXED) + assert d.line == 3 and "int | str" in d.message + + +def test_mixed_dict_values_are_reported(write): + result = found(write, """ + def f(a: int) -> None: + d = {"k": a, "j": "s"} + print(d) + """) + assert by_rule(result, MIXED), result + + +@pytest.mark.parametrize("source", [ + "def f() -> None:\n nums = [1, 2]\n print(nums)\n", + "def f() -> None:\n pair = (1, 'a')\n print(pair)\n", # tuples are typed per slot + "def f() -> None:\n xs: list[int | str] = [1, 'a']\n print(xs)\n", # declared on purpose +]) +def test_not_mixed(write, source): + assert by_rule(gate.check([write("user.py", source)]), MIXED) == [] + + +def test_probe_keeps_every_line_number(): + source = "def f() -> None:\n x = 1 # one\n x = 'a'\n y, z = 1, 2\n" + probed, probes = rebinding.probe(source) + assert probed.count("\n") == source.count("\n") + assert {p.name for p in probes} >= {"x", "y", "z"} + for p in probes: + line = probed.splitlines()[p.line - 1] + assert line[p.column - 1:].startswith(p.name), (p, line) + + +def test_no_candidates_means_no_second_pyrefly_run(write, monkeypatch): + calls = [] + real = gate.pyrefly.run + + def counting(*args, **kwargs): + calls.append(args) + return real(*args, **kwargs) + + monkeypatch.setattr(gate.pyrefly, "run", counting) + gate.check([write("user.py", "def f(a: int) -> int:\n b = a + 1\n return b\n")]) + assert len(calls) == 1 + + +def test_rebinding_inside_a_package_with_relative_imports(write): + write("pkg/__init__.py", "") + write("pkg/b.py", "def make() -> int:\n return 1\n") + path = write("pkg/a.py", """ + from .b import make + + def f() -> None: + x = make() + x = "s" + print(x) + """) + result = gate.check([path]) + assert by_rule(result, REBIND), result + assert not [d for d in result if d.severity == "error"], result + + +def test_pyrefly_runs_once_when_a_file_has_a_candidate(write, monkeypatch): + """The probed copy is the only Pyrefly run: errors, any-flow and types all come from it.""" + calls = [] + real = gate.pyrefly.run + + def counting(*args, **kwargs): + calls.append(args) + return real(*args, **kwargs) + + monkeypatch.setattr(gate.pyrefly, "run", counting) + result = found(write, """ + def f() -> None: + x = 1 + x = "a" + print(x) + """) + assert by_rule(result, REBIND) + assert len(calls) == 1 + + +def test_error_column_after_a_probe_points_at_the_original_source(write): + """`x = 1; y = undefined_name`: the probe goes after `x = 1`, left of the error.""" + source = "x = 1\nx = 2\nx = 1; y = undefined_name\n" + path = write("user.py", source) + result = gate.check([path]) + [d] = [d for d in result if d.rule == "pyrefly/unknown-name"] + original_column = source.splitlines()[2].index("undefined_name") + 1 + assert (d.line, d.column) == (3, original_column) + + +def test_error_left_of_a_probe_keeps_its_column(write): + source = "x = 1\nx = 2\nprint(undefined_name); x = 3\n" + path = write("user.py", source) + [d] = [d for d in gate.check([path]) if d.rule == "pyrefly/unknown-name"] + assert (d.line, d.column) == (3, source.splitlines()[2].index("undefined_name") + 1) + + +def test_probe_loads_are_not_any_flow_findings(write): + """The probe load `; x` has type Unknown here, but it is a discarded value, not a finding.""" + path = write("user.py", "x = eval('1')\nx = eval('2')\nprint(x)\n") + flows = {(d.line, d.column) for d in gate.check([path]) if d.rule == "any-flow"} + assert flows == {(3, 7)} # the user's own load of x, not the probe loads on lines 1 and 2 diff --git a/python-multiplatform-ksp/src/test/python/test_runtime.py b/python-multiplatform-ksp/src/test/python/test_runtime.py new file mode 100644 index 000000000..c213475a9 --- /dev/null +++ b/python-multiplatform-ksp/src/test/python/test_runtime.py @@ -0,0 +1,1206 @@ +"""Differential tests of tp_runtime.h against CPython (design §4.3, #41). + +`runtime_shim/tp_shim.c` is a tiny extension that exposes every helper of +`typedpython/runtime/tp_runtime.h`; each test calls the helper and CPython's own operation on the +same inputs and compares the outcome exactly: + + * results bit-for-bit (floats compared by their IEEE bits, so -0.0 and 0.0 differ; NaN is + compared as "any NaN", payload and sign of a NaN are not part of the contract); + * exceptions by exact type AND message, taken from the interpreter running the test (so the + 3.13 and 3.14 messages are each checked against the real thing); + * the return convention: 0 ok / 1 deopt / -1 error. The shim turns a convention violation (-1 + without an exception, 0/1 with an exception set) into AssertionError. + +Deopt (rc 1) is allowed only where API.md names it; each test says when it expects it. + +Environment: set TP_SANITIZE=1 (with DYLD_INSERT_LIBRARIES pointing at the ASan runtime on macOS, +or LD_PRELOAD on Linux) to build the shim with AddressSanitizer + UBSan. +""" +from __future__ import annotations + +import importlib.util +import itertools +import math +import operator +import os +import random +import shlex +import subprocess +import struct +import sys +import sysconfig +from pathlib import Path + +import pytest + +HERE = Path(__file__).resolve().parent +KSP = HERE.parents[2] +REPO_ROOT = KSP.parent +RUNTIME_DIR = KSP / "src" / "main" / "python" / "typedpython" / "runtime" +SHIM_C = HERE / "runtime_shim" / "tp_shim.c" +BUILD_DIR = REPO_ROOT / ".tmp" / "runtime_shim_build" +HEAVY = REPO_ROOT.parent.parent / ".tmp" / "heavy.sh" + +I64_MIN = -(2**63) +I64_MAX = 2**63 - 1 +INF = math.inf +NAN = math.nan + + +# --- building the shim ------------------------------------------------------------------------ + +def _lock_already_held() -> bool: + """True if an ancestor process is heavy.sh (the whole pytest run then already holds the lock; + taking it again would deadlock).""" + pid = os.getpid() + for _ in range(40): + out = subprocess.run(["ps", "-o", "ppid=,command=", "-p", str(pid)], + capture_output=True, text=True).stdout.strip() + if not out: + return False + ppid, _, cmd = out.partition(" ") + if "heavy.sh" in cmd: + return True + pid = int(ppid) + if pid <= 1: + return False + return False + + +def _run_compile(cmd: list[str]) -> None: + if HEAVY.exists() and not _lock_already_held(): + cmd = [str(HEAVY)] + cmd + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode != 0: + raise RuntimeError("shim build failed:\n" + " ".join(cmd) + "\n" + proc.stdout + proc.stderr) + + +def _build(variant: str): + suffix = sysconfig.get_config_var("EXT_SUFFIX") + sanitize = os.environ.get("TP_SANITIZE") == "1" + name = f"tp_shim_{variant}" + ("_san" if sanitize else "") + BUILD_DIR.mkdir(parents=True, exist_ok=True) + out = BUILD_DIR / (name + suffix) + cc = shlex.split(sysconfig.get_config_var("CC") or "cc") + cmd = cc + ["-std=c99", "-O1", "-g", "-Wall", "-Wextra", "-Werror", "-Wno-unused-parameter", + "-shared", "-fPIC", f"-DTP_SHIM_NAME={name}", + "-I", sysconfig.get_paths()["include"], "-I", str(RUNTIME_DIR)] + if variant == "portable": + cmd.append("-DTP_RUNTIME_FORCE_PORTABLE_OVERFLOW") + if sanitize: + cmd += ["-fsanitize=address,undefined", "-fno-sanitize-recover=undefined", + "-fno-omit-frame-pointer"] + res = subprocess.run(cc + ["-print-resource-dir"], capture_output=True, text=True).stdout.strip() + if sys.platform == "darwin" and res: + cmd += ["-Wl,-rpath," + res + "/lib/darwin"] + if sys.platform == "darwin": + cmd += ["-undefined", "dynamic_lookup"] + cmd += [str(SHIM_C), "-o", str(out)] + _run_compile(cmd) + spec = importlib.util.spec_from_file_location(name, out) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +@pytest.fixture(scope="session", params=["default", "portable"]) +def rt(request): + """The compiled shim; built once per variant per session. `portable` forces the fallback for + the overflow builtins so both code paths of the header are exercised.""" + mod = _build(request.param) + info = mod.info() + assert bool(info["portable"]) == (request.param == "portable") + return mod + + +# --- comparison helpers ----------------------------------------------------------------------- + +def norm(v): + if isinstance(v, bool): + return ("b", v) + if isinstance(v, float): + return ("nan",) if v != v else ("f", struct.pack(" bool: + return I64_MIN <= v <= I64_MAX + + +INT_VALUES = sorted({0, 1, -1, 2, -2, 7, -7, 2**31, -(2**31), 2**31 - 1, 2**32, 2**53, -(2**53), + 2**53 + 1, -(2**53) - 1, 2**53 - 1, 2**62, -(2**62), 2**62 + 1, I64_MIN, + I64_MAX, I64_MIN + 1, I64_MAX - 1, 3, -3, 10, -10, 1000003, -1000003}) + +FLOAT_VALUES = [0.0, -0.0, 1.0, -1.0, 1.5, -1.5, 0.5, -0.5, 2.5, -2.5, 3.0, -3.0, 7.0, 0.1, -0.1, + INF, -INF, NAN, 5e-324, -5e-324, 2.2250738585072014e-308, 1e-300, -1e-300, 1e300, + -1e300, 1.7976931348623157e308, -1.7976931348623157e308, 2.0**53, -(2.0**53), + 2.0**63, -(2.0**63), 1e22, 4.5e15] + + +# --- ints ------------------------------------------------------------------------------------- + +@pytest.mark.parametrize("name,pyop", [("add_i64", operator.add), ("sub_i64", operator.sub), + ("mul_i64", operator.mul)]) +def test_checked_arithmetic_matches_bigint_and_deopts_exactly_on_overflow(rt, name, pyop): + fn = getattr(rt, name) + cases = [] + for a, b in itertools.product(INT_VALUES, repeat=2): + exact = pyop(a, b) + cases.append(((a, b), ("i", exact) if fits(exact) else ("deopt",), sh(fn, a, b))) + check(cases, name) + + +def test_neg_i64_deopts_only_for_int64_min(rt): + cases = [] + for a in INT_VALUES: + cases.append(((a,), ("i", -a) if fits(-a) else ("deopt",), sh(rt.neg_i64, a))) + check(cases, "neg_i64") + assert sh(rt.neg_i64, I64_MIN) == ("deopt",) + + +def test_fuzz_checked_arithmetic_against_bigint(rt): + rnd = random.Random(20261003) + cases = [] + for _ in range(4000): + bits_a, bits_b = rnd.choice([8, 31, 32, 53, 62, 63]), rnd.choice([8, 31, 32, 53, 62, 63]) + a = rnd.randint(-(2**bits_a), 2**bits_a) + b = rnd.randint(-(2**bits_b), 2**bits_b) + a, b = max(I64_MIN, min(I64_MAX, a)), max(I64_MIN, min(I64_MAX, b)) + for name, pyop in (("add_i64", operator.add), ("sub_i64", operator.sub), + ("mul_i64", operator.mul)): + exact = pyop(a, b) + cases.append(((name, a, b), ("i", exact) if fits(exact) else ("deopt",), + sh(getattr(rt, name), a, b))) + check(cases, "fuzz") + + +def test_floordiv_i64_is_python_floor_division(rt): + cases = [] + for a, b in itertools.product(INT_VALUES, repeat=2): + ref = py(operator.floordiv, a, b) + if ref[0] == "i" and not fits(ref[1]): + ref = ("deopt",) # only MIN // -1 + cases.append(((a, b), ref, sh(rt.floordiv_i64, a, b))) + check(cases, "floordiv_i64") + assert sh(rt.floordiv_i64, I64_MIN, -1) == ("deopt",) + + +def test_mod_i64_has_the_sign_of_the_divisor(rt): + cases = [((a, b), py(operator.mod, a, b), sh(rt.mod_i64, a, b)) + for a, b in itertools.product(INT_VALUES, repeat=2)] + check(cases, "mod_i64") + assert sh(rt.mod_i64, -7, 2) == ("i", 1) + assert sh(rt.mod_i64, 7, -2) == ("i", -1) + assert sh(rt.mod_i64, I64_MIN, -1) == ("i", 0) # CPython: 0 (the C % would trap) + + +def test_int_division_by_zero_messages_are_cpythons(rt): + for fn, op in ((rt.floordiv_i64, operator.floordiv), (rt.mod_i64, operator.mod), + (rt.truediv_i64, operator.truediv)): + for a in (0, 1, -5, I64_MAX, I64_MIN): + exp, got = py(op, a, 0), sh(fn, a, 0) + assert exp[0] == "exc" and exp[1] is ZeroDivisionError + assert got == exp, (op, a, exp, got) + + +def test_fuzz_floordiv_mod_i64(rt): + rnd = random.Random(7) + cases = [] + for _ in range(4000): + a = rnd.randint(-(2**rnd.choice([4, 31, 63])), 2**rnd.choice([4, 31, 63]) - 1) + b = rnd.randint(-(2**rnd.choice([2, 9, 31, 63])), 2**rnd.choice([2, 9, 31, 63]) - 1) + a, b = max(I64_MIN, min(I64_MAX, a)), max(I64_MIN, min(I64_MAX, b)) + for name, op in (("floordiv_i64", operator.floordiv), ("mod_i64", operator.mod)): + ref = py(op, a, b) + if ref[0] == "i" and not fits(ref[1]): + ref = ("deopt",) + cases.append(((name, a, b), ref, sh(getattr(rt, name), a, b))) + check(cases, "fuzz floordiv/mod") + + +TRUEDIV_INTS = INT_VALUES + [2**53 - 2, 2**53 + 2, -(2**53) + 1, 2**53 + 3, 9007199254740993, 5, + -5, 6, 49, 355, 113] + + +def test_truediv_i64_is_correctly_rounded_or_deopts_above_2_53(rt): + cases = [] + for a, b in itertools.product(TRUEDIV_INTS, repeat=2): + ref = py(operator.truediv, a, b) + if ref[0] != "exc" and (abs(a) > 2**53 or abs(b) > 2**53): + ref = ("deopt",) + cases.append(((a, b), ref, sh(rt.truediv_i64, a, b))) + check(cases, "truediv_i64") + # the boundary itself is exact-rounded by the C division + assert sh(rt.truediv_i64, 2**53, 3) == py(operator.truediv, 2**53, 3) + assert sh(rt.truediv_i64, 2**53 + 1, 3) == ("deopt",) + assert sh(rt.truediv_i64, 1, -(2**53) - 1) == ("deopt",) + assert sh(rt.truediv_i64, 0, -5) == ("f", struct.pack(" +0.0 (sign of divisor) + assert sh(rt.mod_f64, 1.0, -1.0) == ("f", nz) + assert sh(rt.mod_f64, -7.5, 2.0) == norm(-7.5 % 2.0) + assert sh(rt.floordiv_f64, -0.0, 5.0) == norm(-0.0 // 5.0) + assert sh(rt.floordiv_f64, 1.0, -INF) == norm(1.0 // -INF) + + +# --- int/float exact comparison --------------------------------------------------------------- + +OPS = [(0, operator.lt), (1, operator.le), (2, operator.eq), (3, operator.ne), (4, operator.gt), + (5, operator.ge)] +CMP_INTS = INT_VALUES + [2**53 + 2, 2**63 - 1024, -(2**63) + 1024, 2**63 - 513, 9223372036854774784, + 4503599627370497, 9007199254740992] +CMP_FLOATS = FLOAT_VALUES + [float(2**53), float(2**53 + 2), 9.223372036854776e18, + -9.223372036854776e18, 9.223372036854775e18, 9223372036854774784.0, + 0.1 + 0.2, 2.0**62, 4503599627370496.5, -4503599627370496.5, + 9007199254740993.0, 1e19, -1e19, 2.0**63 - 1024] + + +def test_int_float_comparison_is_exact_for_every_operator(rt): + assert op_name_sanity() + cases = [] + for a, b in itertools.product(CMP_INTS, CMP_FLOATS): + for code, pyop in OPS: + cases.append(((a, b, code), int(pyop(a, b)), rt.cmp_i64_f64(a, b, code))) + cases.append(((b, a, code), int(pyop(b, a)), rt.cmp_f64_i64(b, a, code))) + check(cases, "cmp") + + +def op_name_sanity(): + import _operator # noqa: F401 + return True + + +def test_int_float_comparison_named_hard_cases(rt): + # 2**53 + 1 is not equal to float(2**53), although (double)(2**53+1) == 2**53 + assert rt.cmp_i64_f64(2**53 + 1, float(2**53), 2) == 0 + assert rt.cmp_i64_f64(2**53 + 1, float(2**53), 4) == 1 + assert rt.cmp_i64_f64(I64_MAX, 9.223372036854776e18, 0) == 1 # INT64_MAX < 2**63 + assert rt.cmp_i64_f64(I64_MAX, 9.223372036854776e18, 2) == 0 + assert rt.cmp_i64_f64(I64_MIN, -9.223372036854776e18, 2) == 1 # -2**63 == -2.0**63 + for code, _ in OPS: + assert rt.cmp_i64_f64(1, NAN, code) == (1 if code == 3 else 0) + assert rt.cmp_f64_i64(NAN, 1, code) == (1 if code == 3 else 0) + assert rt.cmp_i64_f64(I64_MAX, INF, 0) == 1 and rt.cmp_i64_f64(I64_MIN, -INF, 4) == 1 + + +# --- math ------------------------------------------------------------------------------------- + +MATH1 = ["sqrt", "exp", "log", "sin", "cos", "tan", "fabs"] +MATH_VALUES = FLOAT_VALUES + [1000.0, -1000.0, 709.0, 709.782712893384, 709.7827128933841, -745.0, + -746.0, -1e5, 1e5, 0.5 * math.pi, math.pi, 1e22, 1e15, 100.0, + -100.0, 710.0] + + +@pytest.mark.parametrize("name", MATH1) +def test_math_one_argument_matches_math_module_including_messages(rt, name): + ref_fn = getattr(math, name) + cases = [((x,), py(ref_fn, x), sh(rt.math1, name, x)) for x in MATH_VALUES] + check(cases, "math." + name) + + +@pytest.mark.parametrize("name", ["sqrt", "exp", "log", "sin", "cos", "tan"]) +def test_math_one_argument_fuzz_is_bit_exact(rt, name): + rnd = random.Random(hash(name) & 0xFFFF) + ref_fn = getattr(math, name) + xs = [rnd.uniform(-1, 1) * 10.0 ** rnd.randint(-10, 10) for _ in range(3000)] + cases = [((x,), py(ref_fn, x), sh(rt.math1, name, x)) for x in xs] + check(cases, "fuzz math." + name) + + +def test_math_named_error_cases(rt): + # the rows named in the task, spelled out: type+message are whatever this interpreter says + for name, x in [("sqrt", -1.0), ("sqrt", -0.0), ("exp", 1000.0), ("log", 0.0), ("log", -1.0), + ("log", -INF), ("sin", INF), ("cos", -INF), ("tan", INF), ("sqrt", -INF)]: + exp, got = py(getattr(math, name), x), sh(rt.math1, name, x) + assert got == exp, (name, x, exp, got) + assert sh(rt.math1, "sqrt", -0.0) == ("f", struct.pack(" 0) or +# [lo(stop)+1, hi(start)] (step < 0); Len [0, 2**60-1]; ADD/SUB/MUL/NEG/POS by interval arithmetic; +# MOD by a divisor whose interval is positive [0, hi(divisor)-1]; FLOORDIV by a positive divisor +# interval by its endpoints; everything else (params, entry globals, other locals, Index, Unbox, +# Call) the full i64 range. + +def _ranged(op, start=0, stop=5, step=1, impure=True, pure=False, may_deopt=False, params=()): + """for k in range(start, stop, step): t = in an (impure) function""" + body = ([IMPURE] if impure else []) + [ + ForRange("k", i(start) if isinstance(start, int) else start, + i(stop) if isinstance(stop, int) else stop, i(step), (Assign("t", op(li("k"))),)), + Return(), + ] + return fn(params=params, locals_={"k": I64, "t": I64}, pure=pure, may_deopt=may_deopt, + body=body) + + +def test_proven_add_of_a_small_range_var_is_accepted_in_an_impure_function(): + assert_accepted(_ranged(lambda k: padd(k, i(1)))) + + +def test_proven_add_of_two_i64_params_is_rejected(): + ps = [Param("a", I64), Param("b", I64)] + assert_rejected(PROVEN, fn(params=ps, returns=I64, body=[IMPURE, Return(padd(li("a"), li("b")))])) + assert_rejected(PROVEN, fn(params=ps, returns=I64, pure=True, + body=[Return(padd(li("a"), li("b")))])) + + +def test_unproven_i64_ops_still_deopt_in_an_impure_function(): + assert_rejected(DEOPT, _ranged(lambda k: add(k, i(1)))) + assert_rejected(DEOPT, _ranged(lambda k: UnaryOp(I64, UnaryOpKind.NEG, k))) + + +def test_a_proven_op_is_not_a_deopting_node(): + # pure function whose only i64 op is proven: it cannot deopt, so may_deopt must be False + ok = _ranged(lambda k: padd(k, i(1)), impure=False, pure=True, may_deopt=False) + assert_accepted(ok) + assert_rejected(FLAG_DEOPT, dataclasses.replace(ok, may_deopt=True)) + + +@pytest.mark.parametrize("op", [ + lambda k: psub(k, i(1)), + lambda k: pmul(k, k), + lambda k: pneg(k), + lambda k: UnaryOp(I64, UnaryOpKind.NEG, k, proven=True), + lambda k: padd(padd(k, i(1)), pmul(k, i(3))), # proven ops nest +]) +def test_other_provable_ops_are_accepted(op): + assert_accepted(_ranged(op)) + + +def test_proven_neg_of_a_param_is_rejected(): + # -(-2**63) overflows + assert_rejected(PROVEN, fn(params=[Param("n", I64)], returns=I64, body=[IMPURE, Return(pneg(li("n")))])) + + +def test_proven_on_a_node_that_has_no_overflow_is_rejected(): + ps = [Param("x", F64), Param("n", I64)] + assert_rejected(PROVEN, fn(params=ps, returns=F64, + body=[Return(BinOp(F64, BinOpKind.ADD, lf("x"), lf("x"), proven=True))])) + assert_rejected(PROVEN, fn(params=ps, returns=I64, pure=True, may_deopt=True, body=[ + Return(BinOp(I64, BinOpKind.FLOORDIV, li("n"), i(2), proven=True))])) + assert_rejected(PROVEN, fn(params=ps, returns=I64, body=[ + Return(UnaryOp(I64, UnaryOpKind.POS, li("n"), proven=True))])) + + +def test_a_non_bool_proven_flag_is_rejected(): + out, diags = run(_ranged(lambda k: BinOp(I64, BinOpKind.ADD, k, i(1), proven=1))) + assert "f" not in kept(out) + + +def test_range_var_interval_uses_the_stop_bound(): + assert_accepted(_ranged(lambda k: padd(k, i(2 ** 63 - 5)), stop=5)) # k <= 4 + assert_rejected(PROVEN, _ranged(lambda k: padd(k, i(2 ** 63 - 4)), stop=5)) # 4 + (2**63-4) + + +def test_range_var_interval_uses_the_start_bound(): + # range(-5, 5): k >= -5 + assert_accepted(_ranged(lambda k: psub(k, i(2 ** 63 - 5)), start=-5)) # -5 - (2**63-5) + assert_rejected(PROVEN, _ranged(lambda k: psub(k, i(2 ** 63 - 4)), start=-5)) + + +def test_negative_step_bounds(): + # range(10, -1, -1): k in [0, 10] + assert_accepted(_ranged(lambda k: psub(k, i(2 ** 63 - 1)), start=10, stop=-1, step=-1)) + assert_rejected(PROVEN, _ranged(lambda k: padd(k, i(2 ** 63 - 10)), start=10, stop=-1, step=-1)) + # range(n, 0, -1) with n a param: k in [1, 2**63-1]: k - 1 is provable, k + 1 is not + ps = [Param("n", I64)] + assert_accepted(_ranged(lambda k: psub(k, i(1)), start=li("n"), stop=0, step=-1, params=ps)) + assert_rejected(PROVEN, _ranged(lambda k: padd(k, i(1)), start=li("n"), stop=0, step=-1, + params=ps)) + + +def test_range_up_to_a_param_bounds_the_var_by_the_param_range(): + # range(0, n): k <= 2**63-2, so k + 1 fits but k + 2 may not + ps = [Param("n", I64)] + assert_accepted(_ranged(lambda k: padd(k, i(1)), stop=li("n"), params=ps)) + assert_rejected(PROVEN, _ranged(lambda k: padd(k, i(2)), stop=li("n"), params=ps)) + + +def test_len_bound_is_zero_to_two_to_the_62(): + a = [ArrayParam("a", F64A, stored=False)] + L = Len(I64, "a") + assert_accepted(fn(params=a, returns=I64, body=[IMPURE, Return(padd(L, i(2 ** 62 - 1)))])) + assert_rejected(PROVEN, fn(params=a, returns=I64, + body=[IMPURE, Return(padd(L, i(2 ** 62)))])) + assert_rejected(PROVEN, fn(params=a, returns=I64, body=[IMPURE, Return(pmul(L, i(2)))])) + assert_accepted(fn(params=a, returns=I64, body=[IMPURE, Return(pmul(L, i(-2)))])) # -2**63 + assert_rejected(PROVEN, fn(params=a, returns=I64, body=[IMPURE, Return(pmul(L, i(-3)))])) + + +def test_mod_and_floordiv_are_full_range_so_nothing_is_provable_from_them(): + ps = [Param("n", I64)] + m = BinOp(I64, BinOpKind.MOD, li("n"), i(7)) + assert_rejected(PROVEN, fn(params=ps, returns=I64, pure=True, may_deopt=True, + body=[Return(padd(m, i(1)))])) + + +def test_what_the_intervals_cannot_prove(): + # a local assigned a constant is still full range (locals other than loop vars are not tracked) + assert_rejected(PROVEN, fn(locals_={"x": I64}, returns=I64, + body=[IMPURE, Assign("x", i(1)), Return(padd(li("x"), i(1)))])) + # the loop var after its loop is full range + assert_rejected(PROVEN, fn(locals_={"k": I64}, returns=I64, body=[ + IMPURE, Assign("k", i(0)), ForRange("k", i(0), i(5), i(1), ()), Return(padd(li("k"), i(1)))])) + # an i64 array element is full range + assert_rejected(PROVEN, fn(params=[ArrayParam("a", I64A, stored=False)], returns=I64, + body=[IMPURE, Return(padd(Index(I64, "a", i(0)), i(1)))])) + + +def test_proven_flag_survives_verification(): + out = assert_accepted(_ranged(lambda k: padd(k, i(1)))) + loop = out.functions[0].body[1] + assert loop.body[0].value.proven is True + + +# --- 8. Call.redo --------------------------------------------------------------------------------- + +def _callee(name="g", returns=F64, pure=True, may_deopt=True): + """g(n: int) — deopts on n * 2 overflow when may_deopt; returns `returns`.""" + prod = BinOp(I64, BinOpKind.MUL, li("n"), i(2)) if may_deopt else li("n") + value = {F64: ToFloat(F64, prod), I64: prod, BOOL: lt(prod, i(0)), NONE: None}[returns] + body = ([ExprStmt(prod), Return()] if returns is NONE else [Return(value)]) + if not pure: + body = [IMPURE] + body + return fn(name, params=[Param("n", I64)], returns=returns, pure=pure, may_deopt=may_deopt, + body=body) + + +def _redo_caller(returns=F64, pure=False, may_deopt=False, redo=True): + call = Call(returns, "g", (i(3),), redo=redo) + body = [ExprStmt(call), Return()] + if not pure: + body = [IMPURE] + body + return fn("f", pure=pure, may_deopt=may_deopt, body=body) + + +@pytest.mark.parametrize("returns", [F64, BOOL, NONE]) +def test_redo_from_an_impure_caller_to_a_pure_may_deopt_callee_is_accepted(returns): + assert_accepted(_redo_caller(returns), _callee(returns=returns)) + + +def test_redo_to_an_i64_returning_callee_is_rejected(): + out, _ = assert_rejected(REDO, _redo_caller(I64), _callee(returns=I64)) + assert "g" in kept(out) + + +def test_redo_in_a_pure_caller_is_rejected(): + assert_rejected(REDO, _redo_caller(pure=True, may_deopt=True), _callee()) + assert_rejected(REDO, _redo_caller(pure=True, may_deopt=False), _callee()) + + +def test_redo_to_a_callee_that_cannot_deopt_is_rejected(): + assert_rejected(REDO, _redo_caller(), _callee(may_deopt=False)) + + +def test_redo_to_an_impure_callee_is_rejected(): + assert_rejected(REDO, _redo_caller(), _callee(pure=False, may_deopt=False)) + + +def test_redo_does_not_make_the_caller_may_deopt(): + assert_rejected(FLAG_DEOPT, _redo_caller(may_deopt=True), _callee()) + + +def test_the_same_call_without_redo_is_a_deopt_in_an_impure_caller(): + assert_rejected(DEOPT, _redo_caller(redo=False), _callee()) + + +def test_a_non_bool_redo_flag_is_rejected(): + out, _ = run(dataclasses.replace(_redo_caller(), body=( + IMPURE, ExprStmt(Call(F64, "g", (i(3),), redo=1)), Return())), _callee()) + assert "f" not in kept(out) + + +# --- 9. Function.entry_globals -------------------------------------------------------------------- + +def _closed(entry=(Param("N", I64),), extra=(), returns=F64, name="f"): + """A closed, impure function: for k in range(0, N): a[k] = a[k] * SCALE (SCALE an entry global + when listed).""" + a = ArrayParam("a", F64A, stored=True) + return fn(name, params=[a], entry_globals=entry, locals_={"k": I64}, returns=returns, body=[ + *extra, + ForRange("k", i(0), Len(I64, "a"), i(1), ( + StoreIndex("a", li("k"), BinOp(F64, BinOpKind.MUL, Index(F64, "a", li("k")), + ToFloat(F64, li("N")))),)), + Return(f(0.0)) if returns is F64 else Return(), + ]) + + +def test_entry_global_in_a_closed_function_is_accepted(): + assert_accepted(_closed()) + + +def test_entry_globals_are_assigned_at_entry(): + assert_accepted(fn(entry_globals=[Param("G", F64), Param("B", BOOL)], returns=F64, + body=[If(lb("B"), (Return(lf("G")),)), Return(f(1.0))])) + + +def test_entry_global_in_a_pure_function_is_accepted(): + assert_accepted(fn(entry_globals=[Param("G", F64)], returns=F64, pure=True, + body=[Return(lf("G"))])) + + +@pytest.mark.parametrize("node", [ + ExprStmt(CallObject(OBJ, Global(OBJ, "print"), (), ())), + ExprStmt(GetAttr(OBJ, Global(OBJ, "m"), "x")), + If(Truth(BOOL, Global(OBJ, "m")), ()), + If(CompareObj(BOOL, CompareKind.EQ, Global(OBJ, "m"), Box(OBJ, Const(I64, 1))), ()), + ExprStmt(ObjToFloat(F64, Global(OBJ, "m"))), + ExprStmt(BinOp(OBJ, BinOpKind.ADD, Global(OBJ, "m"), Global(OBJ, "m"))), +]) +def test_entry_global_in_a_function_running_user_code_is_rejected(node): + assert_rejected(ENTRY_OPEN, _closed(extra=[node])) + + +def test_the_same_open_node_without_entry_globals_is_accepted(): + # The open node alone is fine: only entry globals need the function closed. + a = ArrayParam("a", F64A, stored=False) + assert_accepted(fn(params=[a], returns=F64, body=[ + ExprStmt(CallObject(OBJ, Global(OBJ, "print"), (), ())), Return(Index(F64, "a", i(0)))])) + + +def test_global_read_and_box_keep_a_function_closed(): + assert_accepted(_closed(extra=[ExprStmt(Global(OBJ, "m")), ExprStmt(Box(OBJ, li("N")))])) + + +def test_entry_global_needs_every_callee_closed_transitively(): + h_open = fn("h", body=[IMPURE, Return()]) + h_closed = fn("h", body=[Return()]) + g = fn("g", body=[ExprStmt(Call(NONE, "h", ())), Return()]) + caller = _closed(extra=[ExprStmt(Call(NONE, "g", ()))]) + assert_accepted(caller, g, h_closed) + out, _ = assert_rejected(ENTRY_OPEN, caller, g, h_open) + assert "g" in kept(out) and "h" in kept(out) + + +def test_entry_global_must_be_a_scalar_param(): + assert_rejected(TYPE, _closed(entry=(Param("N", OBJ),))) + assert_rejected(TYPE, fn(entry_globals=[Param("A", F64A)], body=[Return()])) + assert_rejected(TYPE, fn(entry_globals=[Param("A", NONE)], body=[Return()])) + out, _ = run(fn(entry_globals=[ArrayParam("A", F64A, stored=False)], body=[Return()])) + assert "f" not in kept(out) + out, _ = run(fn(entry_globals=["N"], body=[Return()])) + assert "f" not in kept(out) + + +def test_entry_global_names_must_not_collide(): + assert_rejected(STRUCTURE, _closed(entry=(Param("a", I64),))) # an array param + assert_rejected(STRUCTURE, fn(params=[Param("N", I64)], entry_globals=[Param("N", I64)], + body=[Return()])) + assert_rejected(STRUCTURE, fn(locals_={"N": I64}, entry_globals=[Param("N", I64)], + body=[Return()])) + assert_rejected(STRUCTURE, fn(entry_globals=[Param("N", I64), Param("N", I64)], + body=[Return()])) + + +def test_entry_global_is_read_as_a_local_of_its_type(): + assert_rejected(TYPE, fn(entry_globals=[Param("G", F64)], returns=I64, + body=[Return(li("G"))])) + + +def test_assigning_an_entry_global_is_rejected(): + assert_rejected(STRUCTURE, fn(entry_globals=[Param("G", F64)], + body=[Assign("G", f(1.0)), Return()])) + assert_rejected(STRUCTURE, fn(entry_globals=[Param("N", I64)], body=[ + ForRange("N", i(0), i(3), i(1), ()), Return()])) + + +def test_entry_global_i64_is_full_range(): + assert_rejected(PROVEN, _closed(extra=[ExprStmt(padd(li("N"), i(1)))])) + + +def test_call_of_a_function_with_entry_globals_from_an_impure_caller(): + # The callee's entry guard can fail in the middle of the caller: that is a deopt there, + # unless the caller already guarded the same global (and, being closed, cannot rebind it). + g = fn("g", entry_globals=[Param("G", F64)], returns=F64, pure=True, body=[Return(lf("G"))]) + impure = fn("f", returns=F64, body=[IMPURE, Return(Call(F64, "g", ()))]) + assert_rejected(DEOPT, impure, g) + covered = fn("f", params=[ArrayParam("a", F64A, stored=True)], + entry_globals=[Param("G", F64)], returns=F64, + body=[StoreIndex("a", i(0), Call(F64, "g", ())), Return(lf("G"))]) + assert_accepted(covered, g) + wrong_type = dataclasses.replace(covered, entry_globals=(Param("G", I64),), + body=(StoreIndex("a", i(0), Call(F64, "g", ())), + Return(f(0.0)))) + assert_rejected(DEOPT, wrong_type, g) + assert_accepted(fn("f", returns=F64, pure=True, may_deopt=True, + body=[Return(Call(F64, "g", ()))]), g) + + +# --- "nothing slips through": seeded IR mutation ------------------------------------------------- + +def _base_functions(): + """Two correct functions exercising every node kind; the mutator breaks them.""" + pure = fn("dot", params=[ArrayParam("a", F64A, stored=False), + ArrayParam("w", F64A, stored=False), Param("n", I64)], + locals_={"s": F64, "k": I64, "c": I64, "t": OBJ}, returns=F64, pure=True, + may_deopt=True, line=3, body=[ + Assign("s", f(0.0)), + Assign("c", i(0)), + Assign("t", g_("TOLERANCE")), + ForRange("k", i(0), Len(I64, "a"), i(1), ( + If(And(BOOL, lt(li("k"), li("n")), lt(Index(F64, "a", li("k")), f(1e9))), + (Assign("s", add(lf("s"), BinOp(F64, BinOpKind.MUL, + Index(F64, "a", li("k")), + Index(F64, "w", i(0))), F64)),), + (Continue(),)), + Assign("c", add(li("c"), i(1))), + )), + Return(add(lf("s"), MathCall(F64, MathFunc.SQRT, (ToFloat(F64, li("c")),)), + F64)), + ]) + impure = fn("scale", params=[ArrayParam("a", F64A, stored=True), Param("x", F64), + Param("o", OBJ), Param("c", BOOL)], + locals_={"k": I64, "r": OBJ, "y": F64}, returns=F64, line=20, body=[ + Assign("r", call_obj(GetAttr(OBJ, g_("Modifier"), "padding"), + Box(OBJ, lf("x")), lo("o"), kw=("dp",))), + Assign("y", Unbox(F64, lo("r"))), + If(Truth(BOOL, lo("r")), (Assign("y", ObjToFloat(F64, lo("o"))),)), + ForRange("k", i(0), Len(I64, "a"), i(1), ( + StoreIndex("a", li("k"), BinOp(F64, BinOpKind.MUL, + Index(F64, "a", li("k")), lf("y"))), + )), + While(And(BOOL, Compare(BOOL, CompareKind.GT, lf("y"), f(1.0)), + CompareObj(BOOL, CompareKind.NE, lo("o"), lo("r"))), ( + Assign("y", BinOp(F64, BinOpKind.TRUEDIV, lf("y"), f(2.0))), + Assign("r", BinOp(OBJ, BinOpKind.ADD, lo("r"), lo("o"))), + If(UnaryOp(BOOL, UnaryOpKind.NOT, lt(lf("y"), f(100.0))), (Break(),)), + )), + Return(lf("y")), + ]) + return pure, impure + + +def _paths(node, pred, path=()): + """Every (path, node) below `node` whose node satisfies pred. A path is a tuple of + field names / tuple indices.""" + out = [] + if pred(node): + out.append((path, node)) + if dataclasses.is_dataclass(node) and not isinstance(node, type): + for fl in dataclasses.fields(node): + out += _paths(getattr(node, fl.name), pred, path + (fl.name,)) + elif isinstance(node, tuple): + for k, x in enumerate(node): + out += _paths(x, pred, path + (k,)) + return out + + +def _replace(node, path, new): + if not path: + return new + head, rest = path[0], path[1:] + if isinstance(node, tuple): + return node[:head] + (_replace(node[head], rest, new),) + node[head + 1:] + return dataclasses.replace(node, **{head: _replace(getattr(node, head), rest, new)}) + + +def _is_expr(n): return isinstance(n, ir.Expr) + + +def _replace_body(g, path, new): + return dataclasses.replace(g, body=_replace(g.body, path, new)) + + +def _breaking_mutations(rng, g): + """Return (description, mutant, rules): the mutant breaks a property by construction; rules + is the set of rule ids any one of which is an acceptable report.""" + exprs = _paths(g.body, _is_expr) + path, e = rng.choice(exprs) + kind = rng.randrange(14) + if kind == 0: # read of an undeclared/unassigned local of the right type + return "unassigned read", _replace_body(g, path, Local(e.type, "zz_unset")), { + DA, TYPE, STRUCTURE} + if kind == 1: # change the declared type of a node + other = rng.choice([t for t in (I64, F64, BOOL, OBJ) if t is not e.type]) + return "retype", _replace_body(g, path, dataclasses.replace(e, type=other)), { + TYPE, OWNERSHIP, STRUCTURE} + if kind == 2: # substitute an unknown node + return "unknown", _replace_body(g, path, Weird(e.type)), {UNKNOWN, STRUCTURE} + if kind == 3: # flip pure + return "flip pure", dataclasses.replace(g, pure=not g.pure), {FLAG_PURE, DEOPT, FLAG_DEOPT} + if kind == 4: # flip may_deopt + return "flip may_deopt", dataclasses.replace(g, may_deopt=not g.may_deopt), { + FLAG_DEOPT, DEOPT} + if kind == 5: # flip stored on the first array + k = next(j for j, p in enumerate(g.params) if isinstance(p, ArrayParam)) + p = g.params[k] + ps = g.params[:k] + (dataclasses.replace(p, stored=not p.stored),) + g.params[k + 1:] + return "flip stored", dataclasses.replace(g, params=ps), {ARRAY_STORED, FLAG_PURE} + if kind == 6: # change the return type + other = rng.choice([t for t in (I64, F64, BOOL, OBJ, NONE) if t is not g.returns]) + return "returns", dataclasses.replace(g, returns=other), {TYPE, OWNERSHIP, MISSING_RETURN} + if kind == 7: # drop a declared local + name = rng.choice(sorted(g.locals)) + loc = {k: v for k, v in g.locals.items() if k != name} + return "drop local", dataclasses.replace(g, locals=loc), {TYPE, STRUCTURE} + if kind == 8: # call an unknown function + return "unknown call", _replace_body(g, path, Call(e.type, "nowhere", ())), { + UNRESOLVED, STRUCTURE} + if kind == 9: # a store into the pure function, or an I64 add into the impure one + if g.pure: + st = StoreIndex(next(p.name for p in g.params if isinstance(p, ArrayParam)), i(0), + f(0.0)) + return "store in pure", dataclasses.replace(g, body=(st,) + g.body), { + FLAG_PURE, ARRAY_STORED} + st = ExprStmt(add(i(1), i(2))) + return "deopt in impure", dataclasses.replace(g, body=(st,) + g.body), {DEOPT} + if kind == 10: # a Break/Continue at function level + return "stray break", dataclasses.replace( + g, body=(rng.choice([Break(), Continue()]),) + g.body), {STRUCTURE} + if kind == 11: # an effecting object node into the pure function / a value-position CompareObj + if g.pure: + node = rng.choice([GetAttr(OBJ, lo("t"), "v"), call_obj(lo("t")), Truth(BOOL, lo("t")), + ObjToFloat(F64, lo("t")), BinOp(OBJ, BinOpKind.ADD, lo("t"), + lo("t"))]) + return "effect in pure", dataclasses.replace( + g, body=g.body[:3] + (ExprStmt(node),) + g.body[3:]), {FLAG_PURE} + cmp = CompareObj(BOOL, CompareKind.EQ, lo("o"), lo("o")) + return "compare_obj value", dataclasses.replace( + g, locals={**g.locals, "q": BOOL}, body=(Assign("q", cmp),) + g.body), {CONDITION_ONLY} + if kind == 12: # a scalar into an OBJ slot (unbox removed / box removed) + boxes = _paths(g.body, lambda n: type(n) in (Box, Unbox, ObjToFloat)) + if boxes: + bpath, bx = rng.choice(boxes) + return "drop box", _replace_body(g, bpath, bx.operand), {OWNERSHIP, TYPE} + return "drop box", _replace_body(g, path, Box(OBJ, e)) if e.type in (I64, F64, BOOL) \ + else _replace_body(g, path, Unbox(F64, e)), {OWNERSHIP, TYPE, STRUCTURE} + # kind 13: drop the first Assign (its local becomes possibly unassigned) — breaking because the + # local is read later, which holds for the first Assign of both base functions. + k = next(j for j, s in enumerate(g.body) if isinstance(s, Assign)) + return "drop assign", dataclasses.replace(g, body=g.body[:k] + g.body[k + 1:]), {DA} + + +def _preserving_mutations(rng, g): + """Rewrites that keep every property; the verifier must still accept them.""" + kind = rng.randrange(4) + if kind == 0: # wrap the body in `if True:` on both branches + return "if-true wrap", dataclasses.replace(g, body=(If(b(True), g.body, g.body),)) + if kind == 1: # an extra declared, assigned, unused local + return "dead local", dataclasses.replace( + g, locals={**g.locals, "zz": F64}, body=(Assign("zz", f(0.5)),) + g.body) + if kind == 2: # a dead statement after the return + return "dead tail", dataclasses.replace(g, body=g.body + (ExprStmt(f(1.0)),)) + # swap the operands of a Compare / CompareObj (types stay valid) + cmps = _paths(g.body, lambda n: type(n) in (Compare, CompareObj)) + path, c = rng.choice(cmps) + flipped = {CompareKind.LT: CompareKind.GT, CompareKind.GT: CompareKind.LT}.get(c.op, c.op) + return "flip compare", _replace_body(g, path, type(c)(BOOL, flipped, c.right, c.left)) + + +def test_base_functions_are_accepted(): + out = assert_accepted(*_base_functions()) + dot, scale = out.functions + assert [a.proven for a in _accesses(dot)] == [True, True, False] + assert [a.proven for a in _accesses(scale)] == [True, True] + + +@pytest.mark.parametrize("seed", [20261003]) +def test_no_breaking_mutation_slips_through(seed): + rng = random.Random(seed) + base = _base_functions() + missed, seen = [], set() + for n in range(200): + g = base[rng.randrange(2)] + desc, mutant, allowed = _breaking_mutations(rng, g) + seen.add(desc) + others = [h for h in base if h.name != g.name] + out, diags = run(*others, mutant) + got = rules_for(diags, g.name) + if g.name in kept(out) or not (got & allowed): + missed.append((n, desc, g.name, sorted(got))) + assert missed == [] + assert len(seen) >= 12 # the seed exercised most mutation kinds + + +@pytest.mark.parametrize("seed", [7]) +def test_preserving_rewrites_are_accepted(seed): + rng = random.Random(seed) + base = _base_functions() + rejected = [] + for n in range(100): + g = base[rng.randrange(2)] + desc, mutant = _preserving_mutations(rng, g) + out, diags = run(*[h for h in base if h.name != g.name], mutant) + if g.name not in kept(out): + rejected.append((n, desc, [dataclasses.astuple(d) for d in diags])) + assert rejected == [] + + +# --- mutation of proven / redo / entry_globals --------------------------------------------------- + +def _proof_functions(): + """Correct functions exercising proven ops, redo and entry globals; the mutator breaks them.""" + h = fn("h", params=[Param("n", I64)], returns=F64, pure=True, may_deopt=True, line=1, + body=[Return(ToFloat(F64, BinOp(I64, BinOpKind.MUL, li("n"), i(2))))]) + q = fn("q", params=[Param("n", I64)], returns=I64, pure=True, may_deopt=True, line=2, + body=[Return(add(li("n"), i(1)))]) + pc = fn("pc", returns=F64, pure=True, may_deopt=True, line=3, + body=[Return(Call(F64, "h", (i(3),)))]) + fill = fn("fill", params=[ArrayParam("a", F64A, stored=True)], + entry_globals=[Param("N", I64), Param("S", F64)], + locals_={"k": I64, "t": I64, "v": I64, "u": F64}, returns=F64, line=4, body=[ + ForRange("k", i(0), Len(I64, "a"), i(1), ( + Assign("t", psub(pmul(li("k"), i(2)), i(1))), + Assign("v", pneg(padd(li("k"), i(1)))), + StoreIndex("a", li("k"), BinOp(F64, BinOpKind.MUL, Index(F64, "a", li("k")), + ToFloat(F64, li("t")))), + )), + Assign("u", Call(F64, "h", (i(3),), redo=True)), + Return(BinOp(F64, BinOpKind.ADD, BinOp(F64, BinOpKind.ADD, lf("u"), lf("S")), + ToFloat(F64, li("N")))), + ]) + talk = fn("talk", returns=NONE, line=5, body=[IMPURE, Return()]) + return {x.name: x for x in (h, q, pc, fill, talk)} + + +def _proof_mutations(rng, base): + """(description, mutated functions by name, the function that must be rejected, the rules any + one of which is an acceptable report).""" + b = dict(base) + kind = rng.randrange(14) + if kind == 0: # widen a proven op's operand to a full-range entry global + paths = _paths(b["fill"].body, lambda n: type(n) is BinOp and n.proven) + path, node = rng.choice(paths) + wide = BinOp(I64, rng.choice([BinOpKind.ADD, BinOpKind.MUL]), node.left, li("N"), + proven=True) + b["fill"] = _replace_body(b["fill"], path, wide) + return "widen proven", b, "fill", {PROVEN} + if kind == 1: # proven set on an op that can overflow + b["q"] = _replace_body(b["q"], (0, "value"), + BinOp(I64, BinOpKind.ADD, li("n"), i(1), proven=True)) + return "proven on param add", b, "q", {PROVEN} + if kind == 2: # proven set on a node that has no i64 overflow + node = rng.choice([BinOp(F64, BinOpKind.ADD, f(1.0), f(2.0), proven=True), + BinOp(I64, BinOpKind.FLOORDIV, i(4), i(2), proven=True), + UnaryOp(I64, UnaryOpKind.POS, i(1), proven=True), + BinOp(I64, BinOpKind.ADD, i(1), i(2), proven=1)]) + t = Type.F64 if node.type is F64 else Type.I64 + b["q"] = dataclasses.replace(b["q"], returns=t, may_deopt=node.type is I64 and + getattr(node, "op", None) is BinOpKind.FLOORDIV, + body=(Return(node),)) + return "proven on wrong node", b, "q", {PROVEN, TYPE} + if kind == 3: # un-prove a proven op in the impure function: it deopts + paths = _paths(b["fill"].body, lambda n: type(n) in (BinOp, UnaryOp) and n.proven) + path, node = rng.choice(paths) + b["fill"] = _replace_body(b["fill"], path, dataclasses.replace(node, proven=False)) + return "unprove", b, "fill", {DEOPT} + if kind == 4: # un-redo + paths = _paths(b["fill"].body, lambda n: type(n) is Call) + path, node = paths[0] + b["fill"] = _replace_body(b["fill"], path, dataclasses.replace(node, redo=False)) + return "drop redo", b, "fill", {DEOPT} + if kind == 5: # redo from a pure caller + b["pc"] = dataclasses.replace(b["pc"], body=(Return(Call(F64, "h", (i(3),), redo=True)),)) + return "redo in pure", b, "pc", {REDO} + if kind == 6: # the callee stops being may_deopt / pure / F64-or-BOOL-or-NONE + how = rng.randrange(3) + h = b["h"] + if how == 0: + b["h"] = dataclasses.replace(h, may_deopt=False) + return "redo callee not may_deopt", b, "fill", {REDO} + if how == 1: + b["h"] = dataclasses.replace(h, pure=False, may_deopt=False, body=(IMPURE,) + ( + Return(ToFloat(F64, li("n"))),)) + return "redo callee impure", b, "fill", {REDO} + b["h"] = dataclasses.replace(h, returns=I64, body=(Return(BinOp( + I64, BinOpKind.MUL, li("n"), i(2))),)) + b["fill"] = _replace_body(b["fill"], (1, "value"), Call(I64, "h", (i(3),), + redo=True)) + return "redo to i64", b, "fill", {REDO, TYPE} + if kind == 7: # an open node in the function with entry globals + node = rng.choice([GetAttr(OBJ, g_("m"), "x"), pycall("f"), Truth(BOOL, g_("m")), + ObjToFloat(F64, g_("m")), BinOp(OBJ, BinOpKind.ADD, g_("m"), g_("m")), + CompareObj(BOOL, CompareKind.EQ, g_("m"), g_("m"))]) + node = ExprStmt(node) if type(node) is not CompareObj else If(node, ()) + b["fill"] = dataclasses.replace(b["fill"], body=(node,) + b["fill"].body) + return "open node", b, "fill", {ENTRY_OPEN} + if kind == 8: # entry globals on a function that is open + b["talk"] = dataclasses.replace(b["talk"], entry_globals=(Param("N", I64),)) + return "entry global on open", b, "talk", {ENTRY_OPEN} + if kind == 9: # a callee that runs user code + b["h"] = dataclasses.replace(b["h"], pure=False, may_deopt=False, body=(IMPURE,) + ( + Return(ToFloat(F64, li("n"))),)) + return "open callee", b, "fill", {ENTRY_OPEN} + if kind == 10: # entry global removed while the body reads it + keep = rng.choice([("N",), ("S",), ()]) + eg = tuple(p for p in b["fill"].entry_globals if p.name in keep) + b["fill"] = dataclasses.replace(b["fill"], entry_globals=eg) + return "drop entry global", b, "fill", {TYPE} + if kind == 11: # a bad entry global: wrong type / collision / duplicate / array + bad = rng.choice([(Param("N", OBJ), TYPE), (Param("a", I64), STRUCTURE), + (Param("k", I64), STRUCTURE), (Param("N", I64), STRUCTURE), + (Param("W", F64A), TYPE), (Param("W", NONE), TYPE)]) + b["fill"] = dataclasses.replace(b["fill"], entry_globals=b["fill"].entry_globals + (bad[0],)) + return "bad entry global", b, "fill", {bad[1]} + if kind == 12: # assign an entry global + tgt = rng.choice(["N", "S"]) + v = i(1) if tgt == "N" else f(1.0) + b["fill"] = dataclasses.replace(b["fill"], body=(Assign(tgt, v),) + b["fill"].body) + return "assign entry global", b, "fill", {STRUCTURE} + # kind 13: a function with entry globals called, unguarded, from an impure caller + caller = fn("fill2", params=[ArrayParam("a", F64A, stored=True)], returns=F64, line=6, + body=[StoreIndex("a", i(0), f(1.0)), Return(Call(F64, "pc2", ()))]) + callee = fn("pc2", entry_globals=[Param("S", F64)], returns=F64, pure=True, line=7, + body=[Return(lf("S"))]) + b["fill2"], b["pc2"] = caller, callee + return "unguarded entry-global call", b, "fill2", {DEOPT} + + +def test_proof_base_functions_are_accepted(): + assert_accepted(*_proof_functions().values()) + + +@pytest.mark.parametrize("seed", [20261004]) +def test_no_proof_breaking_mutation_slips_through(seed): + rng = random.Random(seed) + base = _proof_functions() + missed, seen = [], set() + for n in range(300): + desc, funcs, target, allowed = _proof_mutations(rng, base) + seen.add(desc) + out, diags = run(*funcs.values()) + got = rules_for(diags, target) + if target in kept(out) or not (got & allowed): + missed.append((n, desc, target, sorted(got))) + assert missed == [] + + +def test_each_proof_mutation_kind_was_exercised(): + rng = random.Random(20261004) + base = _proof_functions() + kinds = {_proof_mutations(rng, base)[0] for _ in range(300)} + assert len(kinds) == 16, sorted(kinds) # 14 kinds; kind 6 has three variants + + +# --- 10. local arrays and tuples (NewArray / CopyArray / Tuple; issue #41, N-9) ------------------- + +def na(t, n, fill=None, iota=False): return NewArray(t, n, fill, iota) +def la(name, t=F64A): return Local(t, name) +def tup(*elements): return Tuple(OBJ, tuple(elements)) + + +def _local_sum(body=None, pure=True, may_deopt=False, params=(), locals_=None, returns=F64): + """t = [0.0] * 8; for k in range(len(t)): t[k] = 1.0; s += t[k]; return s""" + body = body if body is not None else [ + Assign("t", na(F64A, i(8), f(0.0))), + Assign("s", f(0.0)), + ForRange("k", i(0), Len(I64, "t"), i(1), ( + StoreIndex("t", li("k"), f(1.0)), + Assign("s", add(lf("s"), Index(F64, "t", li("k")), F64)), + )), + Return(lf("s")), + ] + return fn(params=params, locals_={"t": F64A, "s": F64, "k": I64, **(locals_ or {})}, + returns=returns, pure=pure, may_deopt=may_deopt, body=body) + + +def _flags(function): + out = assert_accepted(function) + return [a.proven for a in _accesses(out.functions[0])] + + +# 10.1 what a local array may be and where an allocation may stand + +def test_a_pure_function_with_a_local_array_is_accepted(): + assert_accepted(_local_sum()) + + +def test_storing_into_a_local_array_is_not_an_effect(): + assert_accepted(_local_sum(pure=True)) + assert_accepted(_local_sum(pure=False)) # a conservative flag is also fine + + +def test_storing_into_an_array_param_still_makes_a_function_impure(): + g = _local_sum(params=[ArrayParam("a", F64A, stored=True)], body=[ + Assign("t", na(F64A, i(8), f(0.0))), + StoreIndex("t", i(0), f(1.0)), + StoreIndex("a", i(0), f(1.0)), + Return(f(0.0)), + ]) + assert_rejected(FLAG_PURE, g) + assert_accepted(dataclasses.replace(g, pure=False)) + + +def test_reading_a_local_array_does_not_make_the_stored_flag_wrong(): + assert_accepted(_local_sum(params=[ArrayParam("a", F64A, stored=False)])) + + +def test_may_deopt_flag_is_recomputed_with_local_arrays(): + deopting = _local_sum(body=[ + Assign("t", na(I64A, i(4), i(0))), + Assign("x", add(Index(I64, "t", i(0)), i(1))), + Return(ToFloat(F64, li("x"))), + ], locals_={"t": I64A, "x": I64}, may_deopt=True) + assert_accepted(deopting) + assert_rejected(FLAG_DEOPT, dataclasses.replace(deopting, may_deopt=False)) + assert_rejected(FLAG_DEOPT, dataclasses.replace(_local_sum(), may_deopt=True)) + assert_rejected(DEOPT, dataclasses.replace(deopting, pure=False)) + + +def test_each_allocation_form_is_accepted(): + for t, alloc, deopts in [(F64A, na(F64A, i(3), f(1.5)), False), + (I64A, na(I64A, i(3), i(7)), False), + (I64A, na(I64A, li("n"), None, True), False), + (F64A, na(F64A, add(li("n"), i(1)), f(0.0)), True)]: + assert_accepted(fn(params=[Param("n", I64)], locals_={"t": t}, returns=NONE, pure=True, + may_deopt=deopts, body=[Assign("t", alloc), Return()])) + + +@pytest.mark.parametrize("place", ["expr-stmt", "return", "call-arg", "box", "tuple", "fill", + "length", "condition-free-assign"]) +def test_an_allocation_anywhere_but_an_assign_to_a_local_array_is_rejected(place): + alloc = na(F64A, i(2), f(0.0)) + stmt = { + "expr-stmt": ExprStmt(alloc), + "return": Return(alloc), + "call-arg": ExprStmt(Call(NONE, "g", (alloc,))), + "box": Assign("o", Box(OBJ, alloc)), + "tuple": Assign("o", tup(alloc)), + "fill": Assign("t", na(F64A, i(2), alloc)), + "length": Assign("t", na(F64A, alloc, f(0.0))), + "condition-free-assign": Assign("o", alloc), + }[place] + g = fn(locals_={"t": F64A, "o": OBJ}, returns=NONE, body=[stmt, Return()] if place != "return" + else [stmt]) + out, diags = run(g, fn("g", params=[Param("x", OBJ)], returns=NONE, body=[Return()])) + assert "f" not in kept(out) + assert ESCAPE in rules_for(diags, "f"), [dataclasses.astuple(d) for d in diags] + + +def test_a_copy_anywhere_but_an_assign_to_a_local_array_is_rejected(): + for stmt in [ExprStmt(CopyArray(F64A, "a")), Return(CopyArray(F64A, "a")), + Assign("o", CopyArray(F64A, "a")), Assign("x", CopyArray(F64A, "a"))]: + g = fn(params=[ArrayParam("a", F64A, stored=False)], locals_={"o": OBJ, "x": F64}, + returns=NONE, body=[stmt, Return()]) + out, diags = run(g) + assert "f" not in kept(out) + assert ESCAPE in rules_for(diags, "f"), (stmt, [dataclasses.astuple(d) for d in diags]) + + +def test_an_allocation_into_an_array_param_is_rejected(): + assert_rejected(ARRAY_USE, fn(params=[ArrayParam("a", F64A, stored=True)], returns=NONE, body=[ + Assign("a", na(F64A, i(2), f(0.0))), Return()])) + + +def test_the_allocation_type_must_match_the_declared_local(): + assert_rejected(TYPE, fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", na(F64A, i(2), f(0.0))), Return()])) + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", CopyArray(I64A, "a")), Return()], + params=[ArrayParam("a", I64A, stored=False)])) + + +def test_the_node_type_must_be_an_array_type(): + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", NewArray(I64, i(2), i(0))), Return()])) + + +def test_a_local_array_may_only_be_assigned_an_allocation(): + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", f(1.0)), Return()])) + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", Const(NONE, None)), Return()])) + + +def test_new_array_length_must_be_i64(): + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", na(F64A, f(2.0), f(0.0))), Return()])) + assert_rejected(OWNERSHIP, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", na(F64A, g_("N"), f(0.0))), Return()])) + + +def test_new_array_fill_must_match_the_element_type(): + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", na(F64A, i(2), i(0))), Return()])) + assert_rejected(TYPE, fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", na(I64A, i(2), f(0.0))), Return()])) + assert_rejected(OWNERSHIP, fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", na(I64A, i(2), g_("zero"))), Return()])) + + +def test_new_array_fill_is_absent_exactly_when_iota(): + assert_rejected(TYPE, fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", na(I64A, i(2))), Return()])) + assert_rejected(TYPE, fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", na(I64A, i(2), i(0), True)), Return()])) + + +def test_iota_is_only_for_i64_arrays_and_must_be_a_bool(): + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", na(F64A, i(2), None, True)), Return()])) + assert_rejected(TYPE, fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", NewArray(I64A, i(2), None, 1)), Return()])) + + +def test_copy_array_source_must_be_an_array_of_the_same_type(): + ok = fn(params=[ArrayParam("a", F64A, stored=False)], locals_={"t": F64A}, returns=NONE, + pure=True, body=[Assign("t", CopyArray(F64A, "a")), Return()]) + assert_accepted(ok) + assert_rejected(ARRAY_USE, dataclasses.replace(ok, params=(Param("a", F64),))) + assert_rejected(ARRAY_USE, fn(locals_={"t": F64A, "x": F64}, returns=NONE, body=[ + Assign("x", f(1.0)), Assign("t", CopyArray(F64A, "x")), Return()])) + assert_rejected(ARRAY_USE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", CopyArray(F64A, "nowhere")), Return()])) + assert_rejected(TYPE, dataclasses.replace( + ok, params=(ArrayParam("a", I64A, stored=False),))) + + +def test_copy_of_a_local_array_is_accepted(): + assert_accepted(fn(locals_={"t": I64A, "u": I64A}, returns=NONE, pure=True, body=[ + Assign("t", na(I64A, i(4), None, True)), Assign("u", CopyArray(I64A, "t")), Return()])) + + +def test_local_array_declarations_are_checked(): + assert_rejected(TYPE, fn(params=[Param("t", I64)], locals_={"t": I64A}, returns=NONE, + body=[Return()])) + assert_rejected(ARRAY_USE, fn(params=[ArrayParam("t", I64A, stored=False)], + locals_={"t": I64A}, returns=NONE, body=[Return()])) + assert_rejected(STRUCTURE, fn(entry_globals=[Param("t", I64)], locals_={"t": I64A}, + returns=NONE, body=[Return()])) + assert_rejected(TYPE, fn(locals_={"t": Type.NONE}, returns=NONE, body=[Return()])) + + +# 10.2 no escape + +def _escapes(use, ret=NONE): + """f(): t = [0] * 2; ; with t an i64 local array; g takes an OBJ.""" + t = la("t", I64A) + stmt = use(t) + g = fn(locals_={"t": I64A, "o": OBJ, "x": I64, "a": F64A, "u": I64A}, returns=ret, body=[ + Assign("t", na(I64A, i(2), i(0))), stmt] + ([Return()] if ret is NONE else [])) + out, diags = run(g, fn("g", params=[Param("x", OBJ)], returns=NONE, body=[Return()])) + assert "f" not in kept(out) + assert ESCAPE in rules_for(diags, "f"), [dataclasses.astuple(d) for d in diags] + + +@pytest.mark.parametrize("use", [ + lambda t: ExprStmt(t), + lambda t: ExprStmt(Call(NONE, "g", (t,))), + lambda t: ExprStmt(call_obj(g_("h"), t)), + lambda t: ExprStmt(call_obj(t)), + lambda t: Assign("o", Box(OBJ, t)), + lambda t: Assign("o", tup(t)), + lambda t: Assign("o", tup(Box(OBJ, i(1)), t)), + lambda t: Assign("u", t), + lambda t: Assign("o", t), + lambda t: Assign("x", Unbox(I64, t)), + lambda t: Assign("o", GetAttr(OBJ, t, "x")), + lambda t: If(Truth(BOOL, t), ()), + lambda t: If(CompareObj(BOOL, CompareKind.EQ, t, lo("o")), ()), +]) +def test_a_local_array_used_as_a_value_is_an_escape(use): + _escapes(use) + + +def test_returning_a_local_array_is_an_escape(): + for returns in (OBJ, NONE): + g = fn(locals_={"t": I64A}, returns=returns, body=[ + Assign("t", na(I64A, i(2), i(0))), Return(la("t", I64A))]) + out, diags = run(g) + assert "f" not in kept(out) + assert ESCAPE in rules_for(diags, "f") + + +def test_a_local_array_is_not_a_loop_variable_or_an_array_param_argument(): + assert_rejected(TYPE, fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", na(I64A, i(2), i(0))), + ForRange("t", i(0), i(2), i(1), ()), Return()])) + callee = fn("g", params=[ArrayParam("a", I64A, stored=False)], returns=NONE, body=[Return()]) + caller = fn(locals_={"t": I64A}, returns=NONE, body=[ + Assign("t", na(I64A, i(2), i(0))), ExprStmt(Call(NONE, "g", (la("t", I64A),))), + Return()]) + out, diags = run(callee, caller) + assert "f" not in kept(out) and ESCAPE in rules_for(diags, "f") + + +def test_index_store_len_and_copy_source_are_the_only_uses_and_are_accepted(): + assert_accepted(fn(locals_={"t": I64A, "u": I64A, "x": I64}, returns=I64, pure=True, + may_deopt=True, body=[ + Assign("t", na(I64A, i(4), i(0))), + StoreIndex("t", i(1), i(5)), + Assign("u", CopyArray(I64A, "t")), + Assign("x", add(Index(I64, "u", i(1)), Len(I64, "t"))), + Return(li("x")), + ])) + + +def test_index_and_store_name_a_declared_array(): + assert_rejected(ARRAY_USE, fn(locals_={"x": I64}, returns=I64, body=[ + Assign("x", Index(I64, "nowhere", i(0))), Return(li("x"))])) + assert_rejected(ARRAY_USE, fn(locals_={"x": I64}, returns=NONE, body=[ + Assign("x", i(1)), StoreIndex("x", i(0), i(1)), Return()])) + + +def test_store_value_must_match_the_local_element_type(): + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", na(F64A, i(2), f(0.0))), StoreIndex("t", i(0), i(1)), Return()])) + assert_rejected(TYPE, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", na(F64A, i(2), f(0.0))), StoreIndex("t", f(0.0), f(1.0)), Return()])) + + +# 10.3 bounds + +def test_loop_over_len_of_a_local_array_is_proven(): + assert _flags(_local_sum()) == [True, True] + + +def test_loop_over_len_of_a_local_array_of_unknown_length_is_proven(): + g = _local_sum(params=[Param("n", I64)], body=[ + Assign("t", na(F64A, li("n"), f(0.0))), + ForRange("k", i(0), Len(I64, "t"), i(1), (StoreIndex("t", li("k"), f(1.0)),)), + Return(f(0.0)), + ]) + assert _flags(g) == [True] + + +def test_other_indices_into_an_array_of_unknown_length_are_unproven(): + g = _local_sum(params=[Param("n", I64)], body=[ + Assign("t", na(F64A, li("n"), f(0.0))), + StoreIndex("t", i(0), f(1.0)), + ForRange("k", i(0), li("n"), i(1), (StoreIndex("t", li("k"), f(1.0)),)), + Return(f(0.0)), + ]) + assert _flags(g) == [False, False] + + +def test_reassigning_the_array_inside_the_loop_defeats_the_loop_proof(): + g = _local_sum(body=[ + Assign("t", na(F64A, i(8), f(0.0))), + Assign("s", f(0.0)), + ForRange("k", i(0), Len(I64, "t"), i(1), ( + Assign("t", na(F64A, i(1), f(0.0))), + Assign("s", add(lf("s"), Index(F64, "t", li("k")), F64)), + )), + Return(lf("s")), + ]) + assert _flags(g) == [False] + + +def test_reassigning_the_array_in_a_nested_statement_also_defeats_it(): + g = _local_sum(params=[Param("c", BOOL)], body=[ + Assign("t", na(F64A, i(8), f(0.0))), + Assign("s", f(0.0)), + ForRange("k", i(0), Len(I64, "t"), i(1), ( + If(lb("c"), (Assign("t", na(F64A, i(1), f(0.0))),)), + Assign("s", add(lf("s"), Index(F64, "t", li("k")), F64)), + )), + Return(lf("s")), + ]) + assert _flags(g) == [False] + + +def _const_index(length, *indices, extra=()): + """t = [0.0] * length; x = t[i0] + t[i1] + ... -> the proven flags in order""" + reads = [Index(F64, "t", i(ix)) for ix in indices] + total = reads[0] + for r in reads[1:]: + total = add(total, r, F64) + return fn(locals_={"t": F64A}, returns=F64, pure=True, body=[ + *extra, Assign("t", na(F64A, i(length), f(0.0))), Return(total)]) + + +def test_constant_indices_below_a_known_length_are_proven(): + assert _flags(_const_index(8, 0, 7)) == [True, True] + + +def test_constant_indices_outside_a_known_length_are_not_proven(): + assert _flags(_const_index(8, 8, 100, -1, -8)) == [False, False, False, False] + assert _flags(_const_index(0, 0)) == [False] + assert _flags(_const_index(-3, 0)) == [False] # a negative length is an empty array + + +def test_an_index_expression_with_a_bounded_interval_is_proven(): + g = fn(locals_={"t": F64A, "s": F64, "k": I64}, returns=F64, pure=True, may_deopt=False, body=[ + Assign("t", na(F64A, i(10), f(0.0))), + Assign("s", f(0.0)), + ForRange("k", i(0), i(9), i(1), ( + Assign("s", add(lf("s"), Index(F64, "t", padd(li("k"), i(1))), F64)), + Assign("s", add(lf("s"), Index(F64, "t", psub(li("k"), i(1))), F64)), + )), + Return(lf("s")), + ]) + assert _flags(g) == [True, False] # k + 1 in [1, 9]; k - 1 in [-1, 7] may be negative + + +def test_the_length_interval_comes_from_the_length_expression(): + g = fn(locals_={"t": F64A, "j": I64, "s": F64}, returns=F64, pure=True, body=[ + Assign("s", f(0.0)), + ForRange("j", i(2), i(6), i(1), ( + Assign("t", na(F64A, li("j"), f(0.0))), + Assign("s", add(lf("s"), Index(F64, "t", i(0)), F64)), + Assign("s", add(lf("s"), Index(F64, "t", i(1)), F64)), + Assign("s", add(lf("s"), Index(F64, "t", i(2)), F64)), + )), + Return(lf("s")), + ]) + assert _flags(g) == [True, True, False] # j in [2, 5] -> the length is at least 2 + + +def test_a_twice_assigned_array_has_no_known_length(): + g = fn(params=[Param("c", BOOL)], locals_={"t": F64A}, returns=F64, pure=True, body=[ + Assign("t", na(F64A, i(8), f(0.0))), + If(lb("c"), (Assign("t", na(F64A, i(1), f(0.0))),)), + Return(Index(F64, "t", i(5))), + ]) + assert _flags(g) == [False] + + +def test_a_copy_inherits_the_length_of_a_known_source_only(): + g = fn(params=[ArrayParam("a", F64A, stored=False)], locals_={"t": F64A, "u": F64A, "v": F64A}, + returns=F64, pure=True, body=[ + Assign("t", na(F64A, i(4), f(0.0))), + Assign("u", CopyArray(F64A, "t")), + Assign("v", CopyArray(F64A, "a")), + Return(add(add(Index(F64, "u", i(3)), Index(F64, "u", i(4)), F64), + Index(F64, "v", i(0)), F64)), + ]) + assert _flags(g) == [True, False, False] + + +def test_a_proven_flag_from_the_input_is_not_trusted_for_local_arrays(): + g = fn(params=[Param("n", I64)], locals_={"t": F64A}, returns=F64, pure=True, body=[ + Assign("t", na(F64A, li("n"), f(0.0))), + Return(Index(F64, "t", i(0), proven=True)), + ]) + assert _flags(g) == [False] + + +def test_len_of_a_local_array_has_the_length_interval(): + def doubled(length): + return fn(params=[Param("n", I64)], locals_={"t": F64A, "x": I64}, returns=I64, pure=True, + body=[Assign("t", na(F64A, length, f(0.0))), + Assign("x", pmul(Len(I64, "t"), i(2 ** 60))), Return(li("x"))]) + assert_accepted(doubled(i(5))) # len == 5: 5 * 2**60 fits i64 (only if known) + assert_rejected(PROVEN, doubled(li("n"))) # unknown length: [0, 2**62] * 2**60 does not + assert_rejected(PROVEN, doubled(i(20))) # len == 20: 20 * 2**60 does not fit + + +# 10.4 definite assignment + +@pytest.mark.parametrize("use", [ + lambda: Assign("x", Index(F64, "t", i(0))), + lambda: StoreIndex("t", i(0), f(1.0)), + lambda: Assign("n", Len(I64, "t")), + lambda: Assign("u", CopyArray(F64A, "t")), + lambda: ForRange("k", i(0), Len(I64, "t"), i(1), ()), +]) +def test_a_local_array_must_be_assigned_before_any_use(use): + g = fn(params=[Param("c", BOOL)], locals_={"t": F64A, "u": F64A, "x": F64, "n": I64, "k": I64}, + returns=NONE, body=[ + If(lb("c"), (Assign("t", na(F64A, i(2), f(0.0))),)), + use(), Return()]) + assert_rejected(DA, g) + fixed = dataclasses.replace(g, body=(Assign("t", na(F64A, i(2), f(0.0))),) + g.body[1:]) + assert_accepted(dataclasses.replace(fixed, pure=True)) + + +def test_assigned_on_both_branches_a_local_array_is_defined(): + assert_accepted(fn(params=[Param("c", BOOL)], locals_={"t": F64A}, returns=F64, pure=True, + body=[If(lb("c"), (Assign("t", na(F64A, i(2), f(0.0))),), + (Assign("t", na(F64A, i(3), f(0.0))),)), + Return(Index(F64, "t", i(0)))])) + + +def test_a_local_array_assigned_in_a_loop_is_not_defined_after_it(): + assert_rejected(DA, fn(locals_={"t": F64A, "k": I64}, returns=F64, body=[ + ForRange("k", i(0), i(3), i(1), (Assign("t", na(F64A, i(2), f(0.0))),)), + Return(Index(F64, "t", i(0)))])) + + +def test_the_allocation_operands_are_read_before_the_array_is_assigned(): + assert_rejected(DA, fn(locals_={"t": F64A}, returns=NONE, body=[ + Assign("t", na(F64A, Len(I64, "t"), f(0.0))), Return()])) + + +# 10.5 tuples + +def test_a_tuple_of_boxed_scalars_and_objects_is_accepted(): + assert_accepted(fn(params=[Param("o", OBJ), Param("x", F64)], locals_={"r": OBJ}, returns=OBJ, + pure=True, body=[ + Assign("r", tup(Box(OBJ, lf("x")), lo("o"), Box(OBJ, i(3)))), Return(lo("r"))])) + + +def test_a_tuple_can_be_returned_directly_and_may_be_empty(): + assert_accepted(fn(returns=OBJ, pure=True, body=[Return(tup())])) + assert_accepted(fn(params=[Param("x", I64)], returns=OBJ, pure=True, + body=[Return(tup(Box(OBJ, li("x"))))])) + + +def test_a_tuple_of_local_array_elements_is_accepted_through_index(): + assert_accepted(fn(locals_={"t": I64A}, returns=OBJ, pure=True, body=[ + Assign("t", na(I64A, i(3), i(1))), + Return(tup(Box(OBJ, Index(I64, "t", i(0))), Box(OBJ, Len(I64, "t"))))])) + + +def test_a_scalar_tuple_element_needs_box(): + assert_rejected(OWNERSHIP, fn(params=[Param("x", F64)], returns=OBJ, body=[ + Return(tup(lf("x")))])) + assert_rejected(OWNERSHIP, fn(returns=OBJ, body=[Return(tup(i(1)))])) + + +def test_a_tuple_is_an_obj_and_only_an_obj(): + assert_rejected(TYPE, fn(returns=I64, body=[Return(Tuple(I64, ()))])) + assert_rejected(OWNERSHIP, fn(returns=I64, body=[Return(tup())])) + assert_rejected(OWNERSHIP, fn(locals_={"x": F64}, returns=NONE, body=[ + Assign("x", tup()), Return()])) + assert_rejected(TYPE, fn(returns=BOOL, body=[Return(Tuple(BOOL, ()))])) + + +def test_a_tuple_creates_a_reference(): + # an OBJ local assigned from a tuple is an owned reference, like Box/CallObject + assert_accepted(fn(locals_={"r": OBJ, "q": OBJ}, returns=OBJ, pure=True, body=[ + Assign("r", tup()), Assign("q", tup(lo("r"))), Return(lo("q"))])) + + +def test_tuple_elements_must_be_a_tuple_of_checked_expressions(): + assert_rejected(STRUCTURE, fn(returns=OBJ, body=[Return(Tuple(OBJ, [Box(OBJ, i(1))]))])) + assert_rejected(DA, fn(locals_={"o": OBJ}, returns=OBJ, body=[Return(tup(lo("o")))])) + assert_rejected(TYPE, fn(returns=OBJ, body=[Return(tup(Const(F64, 1)))])) + + +def test_a_tuple_is_not_an_effect_but_unboxing_its_elements_is_not_available(): + # a tuple creation is pure; the function may keep pure=True with a deopt-free body + assert_accepted(fn(returns=OBJ, pure=True, may_deopt=False, body=[Return(tup())])) + + +# 10.6 mutation: nothing slips through + +def _array_functions(): + """Correct functions with local arrays and tuples; the mutator breaks them.""" + fk = fn("fk", params=[Param("n", I64), ArrayParam("a", I64A, stored=False)], + locals_={"p": I64A, "q": I64A, "k": I64, "s": I64, "t": OBJ, "o": OBJ}, + returns=OBJ, pure=True, may_deopt=True, line=3, body=[ + Assign("p", na(I64A, i(8), None, True)), + Assign("q", CopyArray(I64A, "p")), + Assign("s", i(0)), + Assign("o", g_("OBJECT")), + ForRange("k", i(0), Len(I64, "p"), i(1), ( + StoreIndex("q", li("k"), add(Index(I64, "p", li("k")), i(1))), + Assign("s", add(li("s"), Index(I64, "q", li("k")))), + )), + Assign("t", tup(Box(OBJ, li("s")), Box(OBJ, Index(I64, "q", i(0))), lo("o"))), + Return(lo("t")), + ]) + ip = fn("ip", params=[ArrayParam("a", F64A, stored=True)], + locals_={"z": F64A, "k": I64, "r": OBJ}, returns=NONE, line=9, body=[ + Assign("z", na(F64A, Len(I64, "a"), f(0.0))), + ForRange("k", i(0), Len(I64, "a"), i(1), ( + StoreIndex("z", li("k"), Index(F64, "a", li("k"))), + StoreIndex("a", li("k"), BinOp(F64, BinOpKind.ADD, Index(F64, "z", li("k")), + f(1.0))), + )), + Assign("r", call_obj(g_("print"), Box(OBJ, Index(F64, "z", i(0))))), + Return(), + ]) + sink = fn("sink", params=[Param("x", OBJ)], returns=NONE, line=12, body=[Return()]) + return {x.name: x for x in (fk, ip, sink)} + + +def _array_mutations(rng, base): + """(description, functions by name, the function that must be rejected, the rules any one of + which is an acceptable report).""" + b = dict(base) + fk, ip = b["fk"], b["ip"] + kind = rng.randrange(16) + arr = rng.choice([("p", I64A), ("q", I64A)]) + leak = la(*arr) + insert = lambda g, *stmts: dataclasses.replace(g, body=g.body[:-1] + tuple(stmts) + g.body[-1:]) + if kind == 0: # return the array + b["fk"] = dataclasses.replace(fk, body=fk.body[:-1] + (Return(leak),)) + return "return array", b, "fk", {ESCAPE} + if kind == 1: # pass it to a compiled Call + b["fk"] = insert(fk, ExprStmt(Call(NONE, "sink", (leak,)))) + return "call array", b, "fk", {ESCAPE} + if kind == 2: # pass it to CallObject + b["fk"] = insert(fk, Assign("o", call_obj(g_("keep"), leak))) + return "callobject array", b, "fk", {ESCAPE} + if kind == 3: # box it + b["fk"] = insert(fk, Assign("o", Box(OBJ, leak))) + return "box array", b, "fk", {ESCAPE} + if kind == 4: # put it in a tuple + b["fk"] = insert(fk, Assign("o", tup(Box(OBJ, i(1)), leak))) + return "tuple array", b, "fk", {ESCAPE} + if kind == 5: # alias it + b["fk"] = insert(fk, Assign("q" if arr[0] == "p" else "p", leak)) + return "alias array", b, "fk", {ESCAPE} + if kind == 6: # an allocation where it may not stand + alloc = rng.choice([na(I64A, i(2), i(0)), CopyArray(I64A, "p")]) + stmt = rng.choice([ExprStmt(alloc), Assign("o", alloc), Assign("s", alloc), + Assign("o", Box(OBJ, alloc))]) + b["fk"] = insert(fk, stmt) + return "stray allocation", b, "fk", {ESCAPE} + if kind == 7: # pure flag set on an impure function / an extra effect in a pure one + if rng.random() < 0.5: + b["ip"] = dataclasses.replace(ip, pure=True) + return "pure on impure", b, "ip", {FLAG_PURE} + b["fk"] = insert(fk, StoreIndex("a", i(0), i(1))) + return "param store in pure", b, "fk", {FLAG_PURE} + if kind == 8: # pure flag cleared on a function that deopts / may_deopt flipped + if rng.random() < 0.5: + b["fk"] = dataclasses.replace(fk, pure=False) + return "drop pure", b, "fk", {DEOPT} + b["fk"] = dataclasses.replace(fk, may_deopt=False) + return "drop may_deopt", b, "fk", {FLAG_DEOPT} + if kind == 9: # an impurity added to the pure function + stmt = rng.choice([ExprStmt(pycall("print")), + Assign("o", GetAttr(OBJ, g_("m"), "x"))]) + b["fk"] = insert(fk, stmt) + return "impurity in pure", b, "fk", {FLAG_PURE} + if kind == 10: # the allocation is removed: every use reads an unassigned array + keep = [s for s in fk.body if not (type(s) is Assign and s.target == arr[0])] + b["fk"] = dataclasses.replace(fk, body=tuple(keep)) + return "drop allocation", b, "fk", {DA} + if kind == 11: # a malformed allocation + bad = rng.choice([na(I64A, i(8)), na(I64A, i(8), i(0), True), na(I64A, f(8.0), i(0)), + na(I64A, i(8), f(0.0)), NewArray(I64A, i(8), None, 1), + na(F64A, i(8), f(0.0)), NewArray(I64, i(8), i(0))]) + b["fk"] = dataclasses.replace(fk, body=(Assign("p", bad),) + fk.body[1:]) + return "bad allocation", b, "fk", {TYPE} + if kind == 12: # a bad copy source + src = rng.choice([("n", {ARRAY_USE}), ("nowhere", {ARRAY_USE})]) + b["fk"] = dataclasses.replace(fk, body=(fk.body[0], Assign("q", CopyArray(I64A, src[0])) + ) + fk.body[2:]) + return "bad copy source", b, "fk", src[1] | {DA} + if kind == 13: # a scalar tuple element without Box + b["fk"] = insert(fk, Assign("o", tup(li("s")))) + return "unboxed tuple element", b, "fk", {OWNERSHIP} + if kind == 14: # the local's declared type disagrees with the allocation + b["fk"] = dataclasses.replace(fk, locals={**fk.locals, "p": F64A}) + return "declared type", b, "fk", {TYPE} + # kind 15: a tuple typed as something else + b["fk"] = insert(fk, Assign("s", Tuple(I64, ()))) + return "tuple not obj", b, "fk", {TYPE, OWNERSHIP} + + +def test_array_base_functions_are_accepted(): + assert_accepted(*_array_functions().values()) + + +@pytest.mark.parametrize("seed", [20261005]) +def test_no_array_breaking_mutation_slips_through(seed): + rng = random.Random(seed) + base = _array_functions() + missed = [] + for n in range(400): + desc, funcs, target, allowed = _array_mutations(rng, base) + out, diags = run(*funcs.values()) + got = rules_for(diags, target) + if target in kept(out) or not (got & allowed): + missed.append((n, desc, target, sorted(got))) + assert missed == [] + + +def test_each_array_mutation_kind_was_exercised(): + rng = random.Random(20261005) + base = _array_functions() + kinds = {_array_mutations(rng, base)[0] for _ in range(400)} + assert len(kinds) == 18, sorted(kinds) # 16 kinds; kinds 7 and 8 have two variants diff --git a/python-multiplatform-ksp/tools/sanitizers.py b/python-multiplatform-ksp/tools/sanitizers.py new file mode 100644 index 000000000..c94018804 --- /dev/null +++ b/python-multiplatform-ksp/tools/sanitizers.py @@ -0,0 +1,189 @@ +"""ASan + UBSan setup for the TypedPython tests (SPEC N-8, issue #41). + + python tools/sanitizers.py env print KEY=VALUE lines (append to $GITHUB_ENV) + python tools/sanitizers.py selfcheck build a deliberately faulty extension and REQUIRE that + ASan and UBSan each report it; exit 1 otherwise + +Linux: the sanitizer runtime is LD_PRELOADed into the (unsanitized) interpreter. +macOS refuses to load the runtime into the framework Python, so selfcheck runs through a small +sanitized launcher (Py_BytesMain) instead -- the same recipe as test_cgen.py. CI uses Linux only; +the macOS path exists so the self-check logic can be exercised on a developer machine. + +TP_SANITIZE=1 makes conftest.py add the sanitizer flags to every extension TypedPython builds +(cbuild.compiler) and makes test_runtime.py build its shim sanitized. +""" +from __future__ import annotations + +import os +import shlex +import subprocess +import sys +import sysconfig +import tempfile +import textwrap +from pathlib import Path + +SAN_FLAGS = ["-fsanitize=address,undefined", "-fno-sanitize-recover=undefined", "-g", "-O1", + "-fno-omit-frame-pointer"] +# detect_leaks=0: CPython itself leaks at exit. abort_on_error=0 keeps a plain nonzero exit code. +RUN_ENV = { + "ASAN_OPTIONS": "detect_leaks=0:abort_on_error=0:halt_on_error=1:allocator_may_return_null=1", + "UBSAN_OPTIONS": "halt_on_error=1:print_stacktrace=1", +} + + +def cc() -> list[str]: + return shlex.split(sysconfig.get_config_var("CC") or "cc") + + +def asan_runtime() -> str: + """Path of the shared ASan runtime belonging to the compiler that builds the extensions.""" + compiler = cc() + is_clang = "clang" in Path(compiler[0]).name + if sys.platform == "darwin": + names = ["libclang_rt.asan_osx_dynamic.dylib"] + elif is_clang: + import platform + names = [f"libclang_rt.asan-{platform.machine()}.so"] + else: + names = ["libasan.so"] + for name in names: + out = subprocess.run(compiler + [f"-print-file-name={name}"], capture_output=True, + text=True).stdout.strip() + if out and Path(out).is_file(): + return str(Path(out).resolve()) + raise SystemExit(f"no ASan runtime {names} found for compiler {compiler}") + + +def run_env() -> dict[str, str]: + env = dict(RUN_ENV) + if sys.platform.startswith("linux"): + env["LD_PRELOAD"] = asan_runtime() + return env + + +def pyrefly_wrapper() -> str: + """Pyrefly is a prebuilt Rust binary: keep the ASan preload out of it (and out of the launcher + case, where sys.executable is not the venv python). The gate honours TYPEDPYTHON_PYREFLY.""" + binary = Path(sys.prefix) / "bin" / "pyrefly" + if not binary.is_file(): + raise SystemExit(f"{binary} not found: install the test environment first") + wrapper = Path(tempfile.mkdtemp(prefix="tp-san-")) / "pyrefly" + wrapper.write_text(f'#!/bin/sh\nunset LD_PRELOAD DYLD_INSERT_LIBRARIES\nexec "{binary}" "$@"\n') + wrapper.chmod(0o755) + return str(wrapper) + + +def cmd_env() -> None: + env = {"TP_SANITIZE": "1", "TYPEDPYTHON_PYREFLY": pyrefly_wrapper(), **run_env()} + for key, value in env.items(): + print(f"{key}={value}") + + +FAULTY_C = textwrap.dedent('''\ + #include + #include + #include + + static PyObject *oob_read(PyObject *self, PyObject *arg) { + volatile long i = PyLong_AsLong(arg); /* runtime index: the read cannot be folded away */ + char *p = (char *)malloc(8); + p[0] = 1; + volatile char c = p[i]; /* heap-buffer-overflow when i >= 8 */ + free(p); + return PyLong_FromLong(c); + } + + static PyObject *overflow(PyObject *self, PyObject *arg) { + volatile int a = INT_MAX, b = (int)PyLong_AsLong(arg); + int r = a + b; /* signed-integer-overflow when b >= 1 */ + return PyLong_FromLong(r); + } + + static PyMethodDef methods[] = { + {"oob_read", oob_read, METH_O, ""}, {"overflow", overflow, METH_O, ""}, {NULL, NULL, 0, NULL}}; + static struct PyModuleDef moddef = {PyModuleDef_HEAD_INIT, "tp_faulty", NULL, -1, methods}; + PyMODINIT_FUNC PyInit_tp_faulty(void) { return PyModule_Create(&moddef); } + ''') + +DRIVER = textwrap.dedent('''\ + import importlib.machinery, importlib.util, sys + loader = importlib.machinery.ExtensionFileLoader("tp_faulty", {so!r}) + spec = importlib.util.spec_from_file_location("tp_faulty", {so!r}, loader=loader) + m = importlib.util.module_from_spec(spec); loader.exec_module(m) + print("LOADED", flush=True) + {call} + print("RETURNED-WITHOUT-REPORT", flush=True) + ''') + + +def python_command(workdir: Path) -> list[str]: + if sys.platform != "darwin": + return [sys.executable] + launcher_c = workdir / "asan_python.c" + launcher_c.write_text("#include \nint main(int argc, char **argv) " + "{ return Py_BytesMain(argc, argv); }\n") + launcher = workdir / "asan_python" + cmd = cc() + SAN_FLAGS + [f"-I{sysconfig.get_paths()['include']}", str(launcher_c), + f"-L{sysconfig.get_config_var('LIBDIR')}", + f"-lpython{sysconfig.get_config_var('LDVERSION')}", "-o", str(launcher)] + subprocess.run(cmd, check=True) + return [str(launcher)] + + +def cmd_selfcheck() -> int: + work = Path(tempfile.mkdtemp(prefix="tp-san-selfcheck-")) + src = work / "tp_faulty.c" + src.write_text(FAULTY_C) + so = work / ("tp_faulty" + sysconfig.get_config_var("EXT_SUFFIX")) + # UBSan's object-size check would stop the planted out-of-bounds read before ASan sees it (seen + # on the first CI run). Disable it for this fault-planting build only, so the ASan case proves + # ASan itself; UBSan is proven by the signed-overflow case. + build = cc() + SAN_FLAGS + ["-fno-sanitize=object-size", "-fPIC", "-shared", + f"-I{sysconfig.get_paths()['include']}", str(src), + "-o", str(so)] + if sys.platform == "darwin": + build += ["-undefined", "dynamic_lookup"] + subprocess.run(build, check=True) + py = python_command(work) + env = dict(os.environ, **run_env()) + if sys.platform == "darwin": + env["PYTHONHOME"] = sys.base_prefix + env.pop("DYLD_INSERT_LIBRARIES", None) + failures = [] + cases = [ + ("clean call is silent", "m.oob_read(0); m.overflow(0)", None, 0), + ("ASan reports heap overflow", "m.oob_read(8)", # first byte past the end: always in the redzone + "AddressSanitizer: heap-buffer-overflow", None), + ("UBSan reports signed overflow", "m.overflow(1)", "runtime error: signed integer overflow", None), + ] + for name, call, needle, want_rc in cases: + run = subprocess.run(py + ["-c", DRIVER.format(so=str(so), call=call)], capture_output=True, + text=True, env=env) + out = run.stdout + run.stderr + if "violates platform policy" in out or "Interceptors are not working" in out: + failures.append(f"{name}: sanitizer runtime could not be loaded:\n{out[-1500:]}") + continue + if needle is None: + ok = run.returncode == 0 and "RETURNED-WITHOUT-REPORT" in out and "Sanitizer" not in out \ + and "runtime error" not in out + else: + ok = needle in out and run.returncode != 0 and "RETURNED-WITHOUT-REPORT" not in out + print(f"{'ok ' if ok else 'FAIL'} {name} (rc={run.returncode})") + if not ok: + failures.append(f"{name}:\n--- head ---\n{out[:2500]}\n--- tail ---\n{out[-800:]}") + if failures: + print("\n".join(failures), file=sys.stderr) + print("SELF-CHECK FAILED: the sanitizer setup does not detect a known fault", file=sys.stderr) + return 1 + print("SELF-CHECK OK: ASan and UBSan both report the planted faults") + return 0 + + +if __name__ == "__main__": + if sys.argv[1:] == ["env"]: + cmd_env() + elif sys.argv[1:] == ["selfcheck"]: + sys.exit(cmd_selfcheck()) + else: + sys.exit(__doc__)