From 0da916f2252aa60d350a79e139c458c7eea23fc4 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 07:15:26 +0900 Subject: [PATCH] Docs: One current design per topic, superseded designs archived, ROADMAP cut to what is left The three upcall documents merge into docs/design/upcall.md. Designs the code no longer follows (the pythonx-synthesising adapter, the Pythonic stub product, the PanamaPort plan, the 3.15t gate, old wasm and investigation narratives) move to docs/archive/ with a superseded header. Design docs that contradicted code or SPEC are corrected. ROADMAP goes from 295 KB to 29 KB: what is left, keeping the old section numbers so code citations still resolve; history is in git (4ed98143). References in code comments and docs follow the moves. --- AGENTS.md | 3 +- PROJECT.md | 4 +- README.md | 2 +- benchmarks/cost_table.py | 2 +- benchmarks/rn-benchmark/COMPARISON.md | 12 +- benchmarks/rn-benchmark/README.md | 2 +- .../rn-benchmark/RNBench/bench/appRun.js | 2 +- .../rn-benchmark/RNBench/bench/harness.js | 2 +- docs/INTENT.md | 4 +- docs/archive/README.md | 46 + .../android-ffm-design.md | 13 + ...downcall-design-android-first-diagnosis.md | 46 + .../downcall-design-benchmark-drift.md | 318 ++ .../android-unregistered-surface.md | 269 + .../gc-scheduling-investigation.md | 516 ++ .../gil-parking-investigation.md | 369 ++ .../jni-call-convention-audit.md | 340 ++ .../kotlin-extensions-in-python-pyi.md | 55 + docs/archive/pyi-generation-pythonic-stubs.md | 461 ++ .../pythonx-adapter-design.md | 27 +- docs/archive/threading-and-abi-3.15t-gate.md | 52 + docs/archive/wasm-design-experiment-log.md | 1386 +++++ .../wasm-youtrack-issue.md | 8 + docs/design/architecture.md | 73 +- docs/design/binding-policy.md | 31 +- docs/design/downcall-design.md | 549 +- docs/design/ecosystem.md | 58 +- docs/design/kotlin-extensions-in-python.md | 169 +- docs/design/marshalling-design.md | 37 +- docs/design/object-lifetime.md | 64 +- docs/design/pyi-generation-design.md | 603 +-- docs/design/threading-and-abi.md | 131 +- docs/design/typedpython.md | 6 + docs/design/upcall-async-design.md | 1218 ----- docs/design/upcall-design.md | 1018 ---- docs/design/upcall-table-design.md | 913 ---- docs/design/upcall.md | 1331 +++++ docs/guide/status.html | 2 +- .../android-unregistered-surface.md | 298 +- docs/investigations/cost-table.md | 22 +- .../gc-scheduling-investigation.md | 677 +-- .../gil-parking-investigation.md | 439 +- .../jni-call-convention-audit.md | 439 +- docs/locale/README_ko.md | 2 +- .../graal-native-image-verification.md | 6 + docs/platforms/python-version-acquisition.md | 29 +- docs/platforms/wasm-design.md | 1493 +----- docs/roadmap/ROADMAP.md | 4462 ++--------------- ksp-fixtures/app/build.gradle.kts | 2 +- .../kotlin/fixture/app/NativeSmokeTest.kt | 4 +- .../kotlin/fixture/app/AppExtras.kt | 2 +- .../fixture/app/GeneratedSuspendTest.kt | 8 +- ksp-fixtures/artifact/build.gradle.kts | 2 +- .../kotlin/fixture/artifact/ComposeSeed.kt | 2 +- .../WalkedArtifactComposeModifierTest.kt | 4 +- .../WalkedArtifactDefaultOmissionTest.kt | 2 +- ksp-fixtures/compose/build.gradle.kts | 2 +- .../compose/PythonAnchoredDraggable.kt | 2 +- .../kotlin/fixture/compose/PythonComposed.kt | 2 +- .../fixture/compose/PythonComposition.kt | 6 +- .../compose/PythonContextMenuOpenDetector.kt | 2 +- .../kotlin/fixture/compose/PythonDraggable.kt | 4 +- .../kotlin/fixture/compose/PythonLayoutId.kt | 2 +- .../fixture/compose/PythonModifierLocal.kt | 2 +- .../fixture/compose/PythonPullRefresh.kt | 2 +- .../kotlin/fixture/compose/PythonSwipeable.kt | 2 +- .../fixture/compose/ComposableRenderTest.kt | 2 +- .../fixture/compose/ComposedRenderTest.kt | 2 +- .../fixture/compose/DraggableRenderTest.kt | 2 +- .../fixture/compose/LayoutIdRenderTest.kt | 2 +- .../compose/ModifierSeededRenderTest.kt | 2 +- .../fixture/compose/PullRefreshRenderTest.kt | 2 +- .../compose/RecompositionAccumulationTest.kt | 2 +- .../fixture/compose/SwipeableRenderTest.kt | 2 +- ksp-fixtures/library/build.gradle.kts | 2 +- .../kotlin/fixture/library/Suspending.kt | 4 +- .../gradle/PythonBindingsPlugin.kt | 2 +- .../gradle/artifact/ArtifactRendering.kt | 4 +- .../gradle/artifact/ArtifactScanner.kt | 14 +- .../gradle/artifact/ComposableThunks.kt | 4 +- .../gradle/artifact/KlibScanner.kt | 4 +- .../gradle/artifact/KotlinMetadata.kt | 2 +- .../artifact/ArtifactFragmentRenderingTest.kt | 2 +- .../gradle/artifact/ArtifactScannerTest.kt | 6 +- .../gradle/artifact/ComposableBindingTest.kt | 2 +- .../gradle/artifact/DefaultOmissionTest.kt | 2 +- .../defaultsfixture/DefaultFixtures.kt | 2 +- .../python/multiplatform/ksp/BindingPolicy.kt | 6 +- .../python/multiplatform/ksp/Constants.kt | 6 +- .../multiplatform/ksp/FragmentDiscovery.kt | 4 +- .../multiplatform/ksp/FragmentScanner.kt | 6 +- .../python/multiplatform/ksp/KsAdapters.kt | 2 +- .../ksp/PythonBindingProcessorProvider.kt | 6 +- .../multiplatform/ksp/SourceRendering.kt | 4 +- .../python/multiplatform/ksp/TypeShape.kt | 2 +- .../multiplatform/ksp/SourceRenderingTest.kt | 2 +- python-multiplatform/build.gradle.kts | 2 +- .../python/native/ffi/UpcallEntry.android.kt | 4 +- .../ffi/pythonx/PythonCallables.kt | 4 +- .../ffi/pythonx/PythonxAdapter.kt | 16 +- .../multiplatform/ffi/upcall/AsyncUpcall.kt | 4 +- .../multiplatform/ffi/upcall/PendingCall.kt | 4 +- .../ffi/upcall/PythonProxySource.kt | 4 +- .../ffi/upcall/UpcallTrampoline.kt | 8 +- .../reflection/ExposedCallable.kt | 14 +- .../reflection/InstallsUpcallTable.kt | 2 +- .../reflection/PythonInternal.kt | 2 +- .../reflection/ReflectedClass.kt | 2 +- .../multiplatform/reflection/UpcallTable.kt | 6 +- .../ffi/pythonx/ComposableShapedFragment.kt | 2 +- .../ffi/pythonx/PythonxAdapterTest.kt | 6 +- .../ffi/pythonx/PythonxCallableTest.kt | 2 +- .../ffi/pythonx/PythonxComposableTest.kt | 2 +- .../ffi/pythonx/PythonxDefaultsTest.kt | 4 +- .../ffi/pythonx/PythonxTableSourceTest.kt | 4 +- .../ffi/upcall/AsyncUpcallPortabilityTest.kt | 2 +- .../ffi/upcall/GeneratedProxyCostTest.kt | 2 +- .../ffi/upcall/OwnedResultLifetimeTest.kt | 2 +- .../ffi/upcall/PendingCallTest.kt | 2 +- .../ffi/upcall/ProxyBootstrap.kt | 2 +- .../ffi/upcall/ProxyHandleLifetimeTest.kt | 2 +- .../ffi/upcall/PythonProxyInstallTest.kt | 2 +- .../ffi/upcall/PythonProxySourceTest.kt | 2 +- .../multiplatform/reflection/TestFragments.kt | 2 +- .../reflection/UpcallOverheadTest.kt | 2 +- .../native/ffi/UpcallBoundaryCostTest.kt | 2 +- .../kotlin/python/native/ffi/UpcallStub.kt | 2 +- .../ffi/upcall/AsyncCompletionProbeTest.kt | 2 +- .../ffi/upcall/AsyncUpcallCancellationTest.kt | 2 +- .../ffi/upcall/AsyncUpcallDeliveryTest.kt | 6 +- .../AsyncUpcallEarlyCancellationTest.kt | 2 +- .../upcall/AsyncioAvailabilityProbeTest.kt | 4 +- python-multiplatform/src/nativeMain/README.md | 2 +- .../kotlin/python/native/ffi/UpcallEntry.kt | 6 +- .../upcall/AsyncUpcallNativeDeliveryTest.kt | 2 +- .../AsyncUpcallNativeEarlyCancellationTest.kt | 2 +- .../multiplatform/ffi/upcall/NativeThread.kt | 2 +- python-multiplatform/src/wasmJsMain/README.md | 5 +- .../kotlin/python/native/ffi/UpcallEntry.kt | 8 +- .../ffi/WasmSelectorsImportTest.kt | 2 +- .../ffi/upcall/ProxyBootstrap.wasmJs.kt | 2 +- .../python/multiplatform/demo/ui/App.kt | 2 +- .../multiplatform/demo/NativeImageMain.kt | 2 +- .../demo/bindings/UpcallDemo.desktop.kt | 2 +- .../demo/bindings/ProxyDemo.wasmJs.kt | 4 +- 145 files changed, 7021 insertions(+), 11317 deletions(-) create mode 100644 docs/archive/README.md rename docs/{platforms => archive}/android-ffm-design.md (92%) create mode 100644 docs/archive/downcall-design-android-first-diagnosis.md create mode 100644 docs/archive/downcall-design-benchmark-drift.md create mode 100644 docs/archive/investigations/android-unregistered-surface.md create mode 100644 docs/archive/investigations/gc-scheduling-investigation.md create mode 100644 docs/archive/investigations/gil-parking-investigation.md create mode 100644 docs/archive/investigations/jni-call-convention-audit.md create mode 100644 docs/archive/kotlin-extensions-in-python-pyi.md create mode 100644 docs/archive/pyi-generation-pythonic-stubs.md rename docs/{design => archive}/pythonx-adapter-design.md (97%) create mode 100644 docs/archive/threading-and-abi-3.15t-gate.md create mode 100644 docs/archive/wasm-design-experiment-log.md rename docs/{platforms => archive}/wasm-youtrack-issue.md (78%) delete mode 100644 docs/design/upcall-async-design.md delete mode 100644 docs/design/upcall-design.md delete mode 100644 docs/design/upcall-table-design.md create mode 100644 docs/design/upcall.md diff --git a/AGENTS.md b/AGENTS.md index 1426f3666..4f1c5fc84 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -160,7 +160,8 @@ interoperability in both directions: | `binary/` | Source archives of per-platform CPython distributions | | `python_for_kotlin_binding.mermaid` | User-authored sketch of the object model (see rule 14) | | `docs/INTENT.md`, `docs/SPEC.md` | Intent and behavioural contract (rule 5) | -| `docs/design/`, `docs/platforms/`, `docs/investigations/`, `docs/roadmap/` | Design records, platform notes, investigations, progress log | +| `docs/design/`, `docs/platforms/`, `docs/investigations/`, `docs/roadmap/` | Current design records (one per topic), platform notes, investigation conclusions, the work left to do | +| `docs/archive/` | Superseded designs and investigation narratives, kept for history (see its `README.md`) | ### Source-set hierarchy diff --git a/PROJECT.md b/PROJECT.md index 65240d6a6..7602db037 100644 --- a/PROJECT.md +++ b/PROJECT.md @@ -4,7 +4,7 @@ Kotlin Multiplatform 에 CPython 을 임베딩해 **Kotlin 과 Python 이 서로 쓰게 한다. 모든 지원 플랫폼에서 같은 방식으로 동작하고, 경계를 넘는 비용을 측정하며 줄인다. - 의도(경계): [`docs/INTENT.md`](docs/INTENT.md) · 스펙(동작 계약과 상태): [`docs/SPEC.md`](docs/SPEC.md) -- 에이전트 규정: [`AGENTS.md`](AGENTS.md) · 진척 기록: [`docs/roadmap/ROADMAP.md`](docs/roadmap/ROADMAP.md) +- 에이전트 규정: [`AGENTS.md`](AGENTS.md) · 남은 작업: [`docs/roadmap/ROADMAP.md`](docs/roadmap/ROADMAP.md) --- @@ -13,7 +13,7 @@ Kotlin Multiplatform 에 CPython 을 임베딩해 **Kotlin 과 Python 이 서로 | 방향 | 이름 | 메커니즘 | 상세 | |---|---|---|---| | Kotlin → Python | **다운콜** | CPython Stable ABI 함수 호출 (`expect` 약 330개) | [`docs/design/downcall-design.md`](docs/design/downcall-design.md) | -| Python → Kotlin | **업콜** | 빌드 타임 생성 함수 테이블 + 플랫폼별 진입점 1개 | [`docs/design/upcall-design.md`](docs/design/upcall-design.md) | +| Python → Kotlin | **업콜** | 빌드 타임 생성 함수 테이블 + 플랫폼별 진입점 1개 | [`docs/design/upcall.md`](docs/design/upcall.md) | ``` ┌──────────────────────────────────────────┐ diff --git a/README.md b/README.md index fb1374959..69e143704 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ no per-platform Python → Kotlin binding code. Details: [design notes](docs/des ## 📊 Status -Honest summary — see the [roadmap](docs/roadmap/ROADMAP.md) for the full record. +Honest summary — see the [roadmap](docs/roadmap/ROADMAP.md) for what is left to do. | Area | State | |---|---| diff --git a/benchmarks/cost_table.py b/benchmarks/cost_table.py index 2f3dfae6d..6cb1c6886 100644 --- a/benchmarks/cost_table.py +++ b/benchmarks/cost_table.py @@ -4,7 +4,7 @@ Why this exists --------------- -`docs/upcall-design.md` and `docs/downcall-design.md` both carry cost tables, and both have been +`docs/design/upcall.md` and `docs/design/downcall-design.md` both carry cost tables, and both have been invalidated at least once by numbers that were transcribed by hand from a run whose conditions were not written down next to them. The two failures were: diff --git a/benchmarks/rn-benchmark/COMPARISON.md b/benchmarks/rn-benchmark/COMPARISON.md index 92b970fc3..b3a8d446f 100644 --- a/benchmarks/rn-benchmark/COMPARISON.md +++ b/benchmarks/rn-benchmark/COMPARISON.md @@ -1,7 +1,7 @@ # React Native's boundary against PythonMultiplatform's, measured the same way Reference project: `/Volumes/macMini/thisisthepy/PythonMultiplatform`, whose figures below are -quoted from `docs/upcall-design.md`, `docs/downcall-design.md` and `ROADMAP.md` at the commit +quoted from `docs/design/upcall.md`, `docs/downcall-design.md` and `ROADMAP.md` at the commit **`73c40da4`** (`develop`, 2026-08-14). **Nothing in the reference project was modified or re-run**; its numbers are read, not reproduced. @@ -13,7 +13,7 @@ its numbers are read, not reproduced. **One fourth source, added for the iOS comparison and cited separately everywhere it appears:** `docs/cost-table.md`, whose `iosSimulatorArm64` row was captured at **`958c0082b294`** at load [1.97, 2.29, 5.92], marked uncontaminated, from three runs. §4b and §1c use it rather than -`upcall-design.md` because it is the only reference-side source that publishes an iOS upcall, its +`docs/design/upcall.md` because it is the only reference-side source that publishes an iOS upcall, its same-run pure-Python callee control and its `withPython` scope figure together — the three cells the comparison needs from one execution. Its upcall figure (2253.20–2312.97 ns) and §4a's quoted 2266–2301 ns are close but **are not the same measurement**; neither is derived from the other. @@ -62,7 +62,7 @@ by reading generated code rather than documentation: | | React Native | PythonMultiplatform | |---|---|---| -| **iOS** | JSI (C++) → Objective-C message send. One crossing. The generated `RNBenchSpec.h` builds `NativeBenchSpecJSI` over `ObjCTurboModule::InitParams` and calls the ObjC method directly. | No runtime boundary at all: a `PyMethodDef` whose `ml_meth` is a `staticCFunction` in the same binary (`docs/upcall-design.md`, "cheapest of the three"). | +| **iOS** | JSI (C++) → Objective-C message send. One crossing. The generated `RNBenchSpec.h` builds `NativeBenchSpecJSI` over `ObjCTurboModule::InitParams` and calls the ObjC method directly. | No runtime boundary at all: a `PyMethodDef` whose `ml_meth` is a `staticCFunction` in the same binary (`docs/design/upcall.md` §7.4). | | **Android** | JSI (C++) → **JNI** → Kotlin. Two crossings. Confirmed with `nm` on the built `libappmodules.so`: `NativeBenchSpecJSI::NativeBenchSpecJSI(JavaTurboModule::InitParams const&)`. | C entry point in `artMain/cinterop/jni_onload.def` → **JNI** → Kotlin. Two crossings, "one shim per shape, plus a JNI upcall per call". | So on both projects Android is structurally the expensive side of the same asymmetry, for the same @@ -107,7 +107,7 @@ all. Getting this wrong is the failure mode that matters most: two tables side b comparison whether or not the cells correspond. Reference-side sources: `UpcallBoundaryCostTest` (`commonTest`, quoted through -`docs/upcall-design.md` §"One upcall, across all five platforms") and `overhead/BenchmarkTest` +`docs/design/upcall.md` §"One upcall, across all five platforms") and `overhead/BenchmarkTest` (quoted through `docs/downcall-design.md`). | row here | what it charges | reference-project counterpart | same direction? | comparable? | @@ -265,7 +265,7 @@ JS (Hermes, no JIT) --JSI--> C++ --JNI--> Kotlin, running on ART The callee of every boundary row is `BenchModule.kt` executing on ART. **ART is precisely the runtime whose tier-up the reference project measured, and it produced the largest warmup requirement -in its entire table** (`docs/upcall-design.md`, "Is 100 000 enough on a device?"): +in its entire table** (`docs/design/upcall.md` §7.2): | reference target | first repetition ÷ plateau | flat from | margin at a 100 000 warmup | |---|---|---|---| @@ -350,7 +350,7 @@ row on iOS**, which is the opposite of what §3a predicts for Android. ### 4a. Ours, quoted (not re-run) -`UpcallBoundaryCostTest`, `docs/upcall-design.md`. **Re-quoted 2026-08-14 at `73c40da4`.** The +`UpcallBoundaryCostTest`, `docs/design/upcall.md`. **Re-quoted 2026-08-14 at `73c40da4`.** The previous version of this table was taken at `bd0029d1` and is superseded: it carried four rows as *italic* 3 000-warmup readings and two ART rows with three columns reading *not recorded*. Upstream has since re-measured every device row at the 100 000 warmup and filled all five columns from one diff --git a/benchmarks/rn-benchmark/README.md b/benchmarks/rn-benchmark/README.md index 795aee8fb..748898527 100644 --- a/benchmarks/rn-benchmark/README.md +++ b/benchmarks/rn-benchmark/README.md @@ -1,7 +1,7 @@ # rn-benchmark A React Native app whose only purpose is to price the **JS ↔ native boundary** with the same -measurement method the [PythonMultiplatform](../thisisthepy/PythonMultiplatform) repository uses to +measurement method the [PythonMultiplatform](../..) repository uses to price its **Python ↔ Kotlin** boundary, so that the two can be read next to each other. The comparison itself, including what corresponds to what and what is still unmeasured, is in diff --git a/benchmarks/rn-benchmark/RNBench/bench/appRun.js b/benchmarks/rn-benchmark/RNBench/bench/appRun.js index 3cd93b190..74529578b 100644 --- a/benchmarks/rn-benchmark/RNBench/bench/appRun.js +++ b/benchmarks/rn-benchmark/RNBench/bench/appRun.js @@ -39,7 +39,7 @@ const REPS = 5; * JS (Hermes, no JIT) --JSI--> C++ --JNI--> **Kotlin, running on ART** * * The callee of every boundary row is `BenchModule.kt` executing on ART, and ART is precisely the - * runtime whose tier-up the reference project measured: its `docs/upcall-design.md` sweep found ART + * runtime whose tier-up the reference project measured: its `docs/design/upcall.md` sweep found ART * on `pmp_api36` still 9% above its plateau at 70 000 calls and only arriving at ~90 000-100 000 -- * the largest warmup requirement of any configuration it measured, and the one that set its * constant. So the RN Android boundary rows sit on the same curve, for the same reason, on the same diff --git a/benchmarks/rn-benchmark/RNBench/bench/harness.js b/benchmarks/rn-benchmark/RNBench/bench/harness.js index 70cbfd90a..7b903f115 100644 --- a/benchmarks/rn-benchmark/RNBench/bench/harness.js +++ b/benchmarks/rn-benchmark/RNBench/bench/harness.js @@ -134,7 +134,7 @@ /** * Where a sweep settles, in the same terms the reference project's device sweep table uses - * (`docs/upcall-design.md`, "Is 100 000 enough on a device?"): first repetition over the plateau, + * (`docs/design/upcall.md` §7.2): first repetition over the plateau, * the call count from which the series stays flat, and the margin a given warmup has over it. * * The plateau is the mean of the second half, so it cannot be dragged by the early readings the diff --git a/docs/INTENT.md b/docs/INTENT.md index 97f365c16..e3e9084ba 100644 --- a/docs/INTENT.md +++ b/docs/INTENT.md @@ -78,7 +78,7 @@ Behind the per-platform entry point, every platform uses the same generated func no per-platform Python → Kotlin binding code to maintain. > Inferred — confirm with the maintainer: this "one mechanism everywhere" property is a goal, not -> only an implementation detail. It is stated as a decision in `PROJECT.md` and `docs/design/upcall-design.md`. +> only an implementation detail. It is stated as a decision in `PROJECT.md` and `docs/design/upcall.md`. ## 3. What it deliberately is NOT @@ -87,7 +87,7 @@ no per-platform Python → Kotlin binding code to maintain. | A binder that renames namespaces (`androidx` → `pythonx`, or any other) | §2.2 | | A replacement for the `pythonx` packages | §2.3 — they are real packages in their own repository | | Runtime-reflection or name-lookup based binding | §2.1 — impossible under Kotlin/Native and native image | -| A hand-written wrapper per Kotlin function or class | Generic adaptation plus generated tables; per-declaration wrappers were tried in 2024 and rotted (`docs/design/pythonx-adapter-design.md`) | +| A hand-written wrapper per Kotlin function or class | Generic adaptation plus generated tables; per-declaration wrappers were tried in 2024 and rotted (`docs/archive/pythonx-adapter-design.md`) | | A Python re-implementation, transpiler, or JVM-hosted Python | It embeds CPython itself, so C extensions keep working | | A build tool, dependency resolver or packager | That is pypackpack (the work) and toolchain (the Gradle vocabulary) | | A sub-interpreter parallelism model | Rejected: extensions that do not declare multi-interpreter support fail to import; parallelism comes from free-threaded CPython | diff --git a/docs/archive/README.md b/docs/archive/README.md new file mode 100644 index 000000000..0659efaab --- /dev/null +++ b/docs/archive/README.md @@ -0,0 +1,46 @@ +# Archive + +Superseded design documents and the long narratives of investigations, kept for history. +**Nothing here is current.** Every file starts with a "Superseded by … on " header that names +the document that replaced it. When an archived file disagrees with [`../INTENT.md`](../INTENT.md), +[`../SPEC.md`](../SPEC.md), a current document in `docs/design/`, `docs/platforms/` or +`docs/investigations/`, or the code, the archived file is the one that is wrong. + +Archived on 2026-10-03 (issue #15). Documents that were merged rather than archived, such as the +three upcall documents now in [`../design/upcall.md`](../design/upcall.md), are not copied here. Their +full text is in git (`git log -- docs/design/upcall-design.md`). + +## Superseded designs + +| File | Replaced by | Why it is here | +|---|---|---| +| [`pythonx-adapter-design.md`](pythonx-adapter-design.md) | SPEC U-7/U-8, `KotlinSurface.kt`, [`../design/kotlin-extensions-in-python.md`](../design/kotlin-extensions-in-python.md) | The design in which the binder synthesised `pythonx.*` modules and renamed to snake_case. AGENTS.md §12.1–12.2 forbid both: Kotlin names stay Kotlin names, and `pythonx` is a real package in pythonx-compose. Its measurements of Compose (synthetic parameters, `pointerInput`, `draggable`, `layoutId`) are history; the behaviour is asserted by the `ksp-fixtures/compose` tests. | +| [`pyi-generation-pythonic-stubs.md`](pyi-generation-pythonic-stubs.md) | [`../design/pyi-generation-design.md`](../design/pyi-generation-design.md), SPEC B-7 | The Pythonic stub product: snake_case names, `pythonx` stubs, the stub manifest, `@overload` sets, and the `Modifier` metaclass/Protocol analysis. The plugin now emits Kotlin-name stubs only, and the Pythonic product belongs to pythonx-compose. | +| [`kotlin-extensions-in-python-pyi.md`](kotlin-extensions-in-python-pyi.md) | [`../design/pyi-generation-design.md`](../design/pyi-generation-design.md), SPEC B-7 | The original §4.5 of `kotlin-extensions-in-python.md`, which proposed Pythonic stubs. | +| [`android-ffm-design.md`](android-ffm-design.md) | [`../design/downcall-design.md`](../design/downcall-design.md), `python-multiplatform/src/androidMain/README.md` | The PanamaPort-based Android FFM plan. It was never built: AGENTS.md §12.9 forbids PanamaPort, and Android binds through JNI `RegisterNatives` (SPEC C-3). | +| [`threading-and-abi-3.15t-gate.md`](threading-and-abi-3.15t-gate.md) | [`../design/threading-and-abi.md`](../design/threading-and-abi.md) | The reasoning that gated free-threading on 3.15t. Free-threaded 3.14t now runs the desktop suite as an opt-in (`-PpythonFreeThreaded=true`, SPEC T-2). | +| [`downcall-design-android-first-diagnosis.md`](downcall-design-android-first-diagnosis.md) | [`../design/downcall-design.md`](../design/downcall-design.md) | The first diagnosis of the Android JNI argument-shift bug. The bug is fixed. | +| [`downcall-design-benchmark-drift.md`](downcall-design-benchmark-drift.md) | [`../design/downcall-design.md`](../design/downcall-design.md) ("Warmup"), [`../design/upcall.md`](../design/upcall.md) §7.2 | The investigation into benchmark figures that turned out to measure the benchmark itself (missing warmup). | +| [`wasm-design-experiment-log.md`](wasm-design-experiment-log.md) | [`../platforms/wasm-design.md`](../platforms/wasm-design.md) | The chronological lab notebook of the wasm experiment, including its "parked" phase and the conclusions that were wrong before they were tested. wasm is now integrated (`tools/wasm/build-cpython.sh`, `wasmJsTest`). | +| [`wasm-youtrack-issue.md`](wasm-youtrack-issue.md) | [`../platforms/wasm-design.md`](../platforms/wasm-design.md) | A draft Kotlin/Wasm compiler issue. Do not file it: Kotlin 2.4.20-Beta2 imports the shared memory, so the problem it describes no longer applies. | + +## Investigation narratives — `investigations/` + +The current file of the same name in [`../investigations/`](../investigations/) keeps the conclusion, +the numbers with their conditions, and how to reproduce them. The archived copy is the full original, +with its step-by-step reasoning, the hypotheses it rejected, and its corrections. + +| File | Current | +|---|---| +| [`investigations/android-unregistered-surface.md`](investigations/android-unregistered-surface.md) | [`../investigations/android-unregistered-surface.md`](../investigations/android-unregistered-surface.md). The migration is done; this copy keeps the call-graph traces. | +| [`investigations/gc-scheduling-investigation.md`](investigations/gc-scheduling-investigation.md) | [`../investigations/gc-scheduling-investigation.md`](../investigations/gc-scheduling-investigation.md) | +| [`investigations/gil-parking-investigation.md`](investigations/gil-parking-investigation.md) | [`../investigations/gil-parking-investigation.md`](../investigations/gil-parking-investigation.md). This copy ends with the cause open; the current file names it: a borrowed pointer wrapped as owned, causing a double free. | +| [`investigations/jni-call-convention-audit.md`](investigations/jni-call-convention-audit.md) | [`../investigations/jni-call-convention-audit.md`](../investigations/jni-call-convention-audit.md). This copy keeps the 71-row classification table. | + +## Not archived, for reference + +- [`../design/typedpython.md`](../design/typedpython.md) is a live design proposal from another session + and stays in `docs/design/`. A note at its top says that the stub manifest it relies on moved to + pythonx-compose. +- [`../roadmap/ROADMAP.md`](../roadmap/ROADMAP.md) now lists only the work that is left. The old + progress log is in git: `git show 4ed98143:docs/roadmap/ROADMAP.md`. diff --git a/docs/platforms/android-ffm-design.md b/docs/archive/android-ffm-design.md similarity index 92% rename from docs/platforms/android-ffm-design.md rename to docs/archive/android-ffm-design.md index d992b8ac2..5c1e10f12 100644 --- a/docs/platforms/android-ffm-design.md +++ b/docs/archive/android-ffm-design.md @@ -1,3 +1,16 @@ +> **Superseded** by [`../design/downcall-design.md`](../design/downcall-design.md) and +> [`python-multiplatform/src/androidMain/README.md`](../../python-multiplatform/src/androidMain/README.md) on 2026-10-03; kept for history. +> This was a plan to replace the Kotlin/Native JNI bridge on Android with a PanamaPort-style FFM path +> (ART method patching, hidden-API bypass, 14 NDK trampolines). It was **not** built: PanamaPort is +> not a dependency (AGENTS.md §12.9 — licence, and dependence on ART-internal struct layouts, see §2–§3 +> below), and Android binds CPython through JNI `RegisterNatives` with the calling convention chosen +> per function and per API level (measured; SPEC C-3). The Kotlin/Native Android library +> (`libmultiplatform_python3.14.so`) is still the bridge. The 14-shape census in §6 is still +> accurate and became `jvmMain/.../ShapeDowncalls.kt`. The closing "Open constraint: GraalVM Native +> Image" section is also resolved: native-image upcalls are verified manually +> (`docs/platforms/graal-native-image-verification.md`, SPEC U-6) and the desktop FFM reachability +> metadata is guarded by `ReachabilityMetadataTest.kt` (SPEC C-2). + # Android FFM Design Document (PanamaPort Analysis) This document provides an implementation design for replacing the existing Kotlin/Native JNI bridge on Android with a Panama-style FFM approach, based on an analysis of the PanamaPort reference implementation. diff --git a/docs/archive/downcall-design-android-first-diagnosis.md b/docs/archive/downcall-design-android-first-diagnosis.md new file mode 100644 index 000000000..38b98df65 --- /dev/null +++ b/docs/archive/downcall-design-android-first-diagnosis.md @@ -0,0 +1,46 @@ +# 다운콜 설계 초안의 Android 진단 (2026-08 초) + +> **Superseded** by [`../design/downcall-design.md`](../design/downcall-design.md) on 2026-10-03; kept for history. +> 아래 "확인된 버그" 는 **수정되었다** — Android 는 `RegisterNatives` 로 전부 바인딩한다 +> (`python-multiplatform/src/androidMain/README.md`, ROADMAP §2). 호출 규약 표의 전환 비용(115/35/25 ns)은 +> 외부 벤치마크에서 가져온 값이며, 이 프로젝트가 실측한 값(API 26-36)과 반대 경향이다 — 실측은 본 문서의 +> "Measured" 절과 `androidMain/README.md` 에 있다. + +### 확인된 버그: 현재 Android JNI 배선이 어긋나 있다 + +`@CName` 이 만드는 C 함수는 선언한 Kotlin 인자만 받는다: + +``` +Java_python_native_ffi_bindings_PyList_1Size(jlong list) +``` + +그런데 `androidMain/bindings.kt` 는 이를 **일반 JNI 메서드**로 선언한다. 일반 JNI 메서드를 ART 는 +이렇게 호출한다: + +``` +Java_..._PyList_1Size(JNIEnv* env, jobject thiz, jlong list) +``` + +**인자가 두 칸 밀린다.** `list` 자리에 `JNIEnv*` 가 들어간다. 확인 결과 `@CName` 익스포트 중 +`JNIEnv` 를 받는 것은 0개이고, `@CriticalNative`/`@FastNative` 도 어디에도 없다. + +즉 **Android JVM 경로는 컴파일만 되고 런타임에 동작하지 않는다.** 실행된 적이 없어 드러나지 않았다. + +### ART 의 세 가지 호출 규약 + +| | 일반 JNI | `@FastNative` | `@CriticalNative` | +|---|---|---|---| +| 전환 오버헤드 | 115 ns | 35 ns | 25 ns | +| C 시그니처 | `(JNIEnv*, jobject, args…)` | 동일 | **`(args…)` 만** | +| 객체 인자·반환 | 가능 | 가능 | **원시 타입만** | +| 정적/인스턴스 | 둘 다 | 둘 다 | **정적만** | +| JVM 으로 콜백 | 가능 | 가능 | **불가** | +| 실행 중 GC | 허용 | **차단** | **차단** | + +shape 트램폴린은 정적이고 `Long`/`Double` 만 주고받으므로 `@CriticalNative` 에 정확히 맞는다. 그리고 +우리 `@CName` 익스포트가 `JNIEnv` 를 받지 않는다는 사실이 곧 `@CriticalNative` 규약과 일치하므로, +**애노테이션을 붙이는 것이 성능 개선인 동시에 위 버그의 수정**이다. + +`@CriticalNative` 는 API 34 부터 공식 SDK 에 포함된다. minSdk 26 구간에서는 `RegisterNatives` 로 +등록해야 한다 — `RegisterNatives` 는 호출 규약이 아니라 바인딩 수단이므로 둘은 대안 관계가 아니다. + diff --git a/docs/archive/downcall-design-benchmark-drift.md b/docs/archive/downcall-design-benchmark-drift.md new file mode 100644 index 000000000..ff60c1755 --- /dev/null +++ b/docs/archive/downcall-design-benchmark-drift.md @@ -0,0 +1,318 @@ +# Downcall/upcall benchmark drift investigation (2026-08-14) + +> **Superseded** by [`../design/downcall-design.md`](../design/downcall-design.md) ("Warmup: what the +> boundary benchmarks must do") on 2026-10-03; kept for history. The sections below are the record +> of how `UpcallBoundaryCostTest`'s drift was found to be a same-process warm-up artefact, how it +> was pinned (desktop to a window, wasmJs to commit `4472f83a`), and how the warmup count was +> chosen (100 000). Figures taken at the old 3 000-call warmup are not current numbers; current +> ones are in `docs/investigations/cost-table.md` and `docs/design/upcall.md` §7. + +### Ratio consistency: mostly holds, desktop is the exception + +> **Superseded as a source of numbers, kept as the record of how the defect was found.** Every figure +> in this section and the two that follow it was taken with `UpcallBoundaryCostTest`'s 3 000-call +> warmup, which "The benchmark was measuring the benchmark" below shows was too small by more than an +> order of magnitude. The reasoning is what still stands: these three sections are what established +> that the number moved for a repository-located reason and then that the reason was the benchmark's +> own methodology. Current figures are in `docs/design/upcall.md` §7 (which absorbed `upcall-design.md`). + +`upcall-design.md`'s own upcall/downcall ratios, from the same `UpcallBoundaryCostTest`, are +reproduced by these three fresh runs (the test computes and prints its own ratio each time, so this +is a direct comparison, not a re-derivation): + +| Platform | ratio recorded in `upcall-design.md` | ratio, this pass (3 runs) | consistent? | +|---|---|---|---| +| desktop | 2.49–2.89x | 2.13–2.34x | **no — see below** | +| iOS simulator | 1.33–1.43x | 1.38–1.42x | yes, nested inside the old range | +| wasmJs | 2.56–3.36x | 2.36–2.63x | mostly — top of the new range overlaps the bottom of the old one. **Superseded: since drifted to 1.58–1.68x, pinned to `4472f83a` and explained — see "wasmJs: closed too" below** | + +**Desktop's ratio moved outside the previously recorded range, and it is the upcall side that +moved, not the downcall side.** This pass's downcall-same-shape figure (315.69–335.32 ns) sits at +the bottom of the old 315–527 ns range — consistent, just at its floor. This pass's upcall figure +(687.83–741.26 ns, read from the same `UpcallBoundaryCostTest` runs) sits *below* the old range's +floor of 861 ns entirely. So on this machine, at this commit (`65e1bf5d`), a desktop upcall through +`ctypes` measures cheaper than what `upcall-design.md`'s table recorded, while the downcall side did +not move. Two things separate the two tables and either could be the cause: `develop` has moved a +great deal since that table was captured (this session's own `git merge` pulled in 87 changed files, +including upcall-adjacent ones — see the merge log above), and the two tables were also captured on +different runs of the same machine, which is exactly the kind of thing this document elsewhere warns +is not comparable. Neither is ruled out here; this is reported as an open discrepancy, not resolved +into either explanation. + +### Closed: repository drift, not run-to-run noise (2026-08-14) + +The discrepancy above was tested directly rather than argued from a second reading. `537c1a0b` +(the commit the 861 ns floor was recorded on) and the current tip were each built in their own +freshly created `git worktree` — not reused, not incrementally recompiled — and +`:python-multiplatform:desktopTest` (the full suite, not a filtered subset — see the pitfall below) +was run three times against each, clearing `build/test-results` before every run: + +| | run 1 | run 2 | run 3 | +|---|---|---|---| +| `537c1a0b`, fresh worktree | 870.48 ns | 808.72 ns | 869.12 ns | +| current tip, fresh worktree | 697.73 ns | 680.46 ns | 686.21 ns | + +Same machine, same JDK (Temurin 21.0.12), same day. The two bands (809–870 ns vs. 680–698 ns) do +not overlap across six runs split into two isolated builds, which rules out ordinary run-to-run +variance as the explanation. Something in the repository between those two points changed the +measured number. `537c1a0b`'s figure also reproduces its own historical record (861–1313 ns) almost +exactly, which rules out a stale or miscalibrated re-measurement on this end. + +**A methodology pitfall found along the way:** an initial attempt to bisect the ~75 commits in +between used `--tests UpcallBoundaryCostTest` to skip the other ~359 tests and go faster. That +filter alone moved the number into the 1300–1400 ns range at *every* commit tried, old and new +alike — it does not merely add noise, it changes which regime the measurement lands in. Re-running +the same checkouts with the unfiltered full suite reproduced the fast band again. The cause is +below; the practical lesson is that this benchmark's absolute number is not a property of the +timed code alone, so a comparison across two runs is only valid if both used the same suite scope +this document already required (Validation runs the full suite) — a filtered re-run of one +benchmark test is not equivalent to the run it is being compared against, even on the same commit. + +**Bisection was attempted and did not converge to one commit; it converged to a window, and to a +mechanism.** Git bisect (full-suite grading, threshold at 750 ns) was run between `537c1a0b` and +the fast tip. Partway through, machine load reached 10–12 on this 8-core host — two Android +emulators and other concurrent agent activity were running, unrelated to this task and outside its +control (`ps aux` at the time confirmed it; this document's own environment notes elsewhere warn +that this machine hosts several worktrees' worth of parallel agent work). Under that load, repeated +measurements of the *same* checkout spanned both regimes (one commit read 672, 771, and 1076 ns +across three consecutive runs), which made single-commit attribution unreliable. What did hold up +under repetition was the window: commits at or before `4722cfe9` measured consistently in the slow +band (854–855 ns), and commits at or after `ec8ff389` measured consistently in the fast band (672, +700 ns, each confirmed in its own fresh clean worktree). That window — +`4722cfe9..ec8ff389` (~15 commits) — contains both commits this task flagged as candidates: +`6776329d` ("Perf: Price the layer users actually call, and stop a module read raising an exception +per access") and `d45071e7` (proxy handle lifetime). `975d3900` sits well outside this window and is +ruled out. + +**Neither candidate touches the code this benchmark measures, which points at an indirect cause.** +`git diff ..` for both `6776329d` and `d45071e7` shows changes confined to +`PythonProxySource` (the generated-proxy source emitter, its tests) and, for `d45071e7`, a doc-only +comment added to `HandleTable.kt`. Neither touches `UpcallTable.resolve`, `HandleTable.resolve`/ +`release`, the non-suspend branch of `UpcallTrampoline.invoke`, or desktop's `UpcallStub` — the +actual call chain `UpcallBoundaryCostTest` times through `TrampolineFragment`. A commit that does +not touch the timed path cannot have made the timed path itself cheaper. + +What it can do is change what runs *before* the timed path in the same process. `desktopTest` sets +no `forkEvery`, so Gradle's default applies: all ~360 tests in the suite run in one forked JVM, one +JIT compiler, one heap (confirmed by reading the `tasks.named("desktopTest")` block in +`build.gradle.kts` — no fork-per-test config exists). `6776329d`'s own commit title is "stop a +module read raising an exception per access" — replacing an exception-driven control-flow path +(expensive on the JVM: every throw captures a stack trace) with a direct one. +`GeneratedProxyCostTest`, touched by the same commit, drives `N = 10_000` iterations times 3 kept +times multiple rows of generated-proxy calls that exercise exactly that module-attribute path, and +it runs earlier in the same shared JVM process as `UpcallBoundaryCostTest`. Removing tens of +thousands of exception throws from an earlier test changes the shared process's GC pressure and +JIT tiering history by the time a later, unrelated benchmark in the same run gets timed — without +the later benchmark's own code path changing at all. That is a plausible, mechanism-level +explanation for why only desktop moved: iOS runs through XCTest's own process model and wasmJs +through a Node process with a different (V8) JIT, neither sharing HotSpot's C2 warm-up state the +way every desktop test in one Gradle-forked JVM does. + +**This is not proven to the level of a single blamed commit** — machine contention prevented that — +but it is proven to the level that matters for the table: the number moved for a real, reproducible, +repository-located reason, not for noise, and the most likely mechanism is a same-process +warm-up artifact of the benchmark's own methodology rather than a genuine drop in the upcall +boundary's per-call cost. The open discrepancy above is closed on that basis, not left open. + +**wasmJs has since drifted further, past even its own "consistent" reading above.** A fresh check +on the current tip (same machine, same session, two runs: 321.32 ns and 315.89 ns, tight) puts +wasmJs's upcall figure well below the 687–741 ns-equivalent range implied by this table's own +"this pass" ratio (2.36–2.63x against a ~192–205 ns downcall figure that still matches today's +run) — today's ratio is 1.26–1.67x. So the "yes, mostly consistent" verdict recorded for wasmJs +above was accurate for the commit it was measured on (`65e1bf5d`) and has since gone stale in the +same direction as desktop's did. iOS was not re-checked: this task's constraints rule out using the +iOS simulator, so its "yes, nested inside the old range" verdict above stands unverified rather +than reconfirmed. + +### wasmJs: closed too — one commit, and the mechanism demonstrated rather than argued (2026-08-14) + +wasmJs's drift is **repository drift, not noise**, it is pinned to the **single commit `4472f83a`**, +and — unlike desktop's, which stayed at the level of a plausible story — its mechanism was +reproduced by a controlled experiment. It is *not* a genuine change in what the wasm upcall boundary +costs. + +The measurement protocol was the one the desktop section above used, with two changes forced by +this machine: runs at the two ends were **interleaved** rather than run in two blocks, and the +timed task was re-run with `--rerun` on `wasmJsNodeTest` alone rather than `--rerun-tasks`, so a +full Kotlin recompile does not finish moments before the benchmark starts. Every run is the **whole +wasm suite** (filtering changes the measured region — see the pitfall below), `build/test-results` +is cleared before each, and all of them are 344/0/0 (258/0/0 at `537c1a0b`, whose suite was smaller). + +| commit | upcall | downcall, same shape | ratio | runs | +|---|---|---|---|---| +| `65e1bf5d` (parent) | 484.22–500.38 ns | 190.11–191.00 ns | 2.53–2.61x | 3 | +| **`4472f83a` (child)** | **306.66–315.82 ns** | **189.32–202.56 ns** | **1.52–1.66x** | 3 | +| `ec8ff389` | 482.03–507.38 ns | 197.87–204.10 ns | 2.38–2.53x | 3 | +| `4722cfe9` | 477.40–497.05 ns | 190.60–203.23 ns | 2.34–2.60x | 3 | +| `17f058ca` | 302.55–320.54 ns | 189.47–203.92 ns | 1.50–1.65x | 3 | +| `73f2af6b` | 312.50–322.11 ns | 188.95–213.49 ns | 1.46–1.66x | 3 | +| current tip | 355.82–373.21 ns | 222.07–226.88 ns | 1.58–1.64x | 4 (+4 earlier) | + +`4472f83a` and its own parent are adjacent commits measured minutes apart in one worktree, and their +upcall bands do not touch. **The downcall column does not move anywhere in this table** — it is the +upcall side alone, exactly as on desktop. + +**The window does *not* overlap desktop's.** Desktop's was `4722cfe9..ec8ff389`; both of those +endpoints measure in wasm's *slow* band and are indistinguishable from each other, so wasm did not +move across desktop's window at all. `4472f83a` sits after `ec8ff389`. The two platforms therefore +do **not** share a cause in `commonMain`'s marshalling or handle paths — which was the hypothesis +worth testing, and it is refuted rather than left open. + +**The MEMFS change is not the cause either.** `3355851c` (stdlib zip installed into MEMFS instead +of resolving out of a CPython source checkout) was the obvious environmental confound, since it +lands after the desktop window. `73f2af6b` is its immediate parent and already measures 312–322 ns — +fully in the fast band. The transition happened before it. + +**Why a commit that only adds a branch to the timed function made it measure cheaper.** +`4472f83a`'s only edit to the timed path is a `when` on `self` in `UpcallEntry.invokeMethod`, which +can only cost more, not less. What it also did was make `PythonProxySource.install()` succeed on +wasm for the first time. Before it, `GeneratedProxyCostTest` and `ProxyHandleLifetimeTest` printed +"no proxies are installable on this target, so there is nothing to measure" and drove **zero** +upcalls; after it, `GeneratedProxyCostTest` drives on the order of 270 000 calls +(`N = 10 000` × 3 kept × ~9 rows) straight through `UpcallEntry.invokeMethod` — earlier in the +**same Node process**, and therefore into the same V8 wasm tier-up state that +`UpcallBoundaryCostTest` is later timed in. The benchmark's own 3 000-iteration warmup does not +reach that state on its own. + +That was tested, not just asserted. At the **current tip**, with nothing changed but +`GeneratedProxyCostTest`'s body short-circuited so it installs nothing and calls nothing — the +pre-`4472f83a` behaviour, same commit, same build, suite still 344/0/0 — the upcall figure went +back up: + +| current tip | upcall | downcall | ratio | +|---|---|---|---| +| as it is | 355.82–373.21 ns | 222.07–226.88 ns | 1.58–1.64x | +| with `GeneratedProxyCostTest` inert | 459.67–473.79 ns | 187.24–192.16 ns | 2.36–2.53x | + +One lever, at one commit, moves the number across the whole gap and lands it back on `65e1bf5d`'s +band (2.53–2.61x) and on this document's own "this pass" reading (2.36–2.63x). So the desktop +section's suspicion — that this benchmark reports a same-process warm-up artifact rather than a +boundary cost — is **confirmed on wasmJs by direct experiment**, on a different commit and a +non-overlapping window from desktop's. What the two platforms share is the methodology, not a +code path. + +**The consequence for the table is that `UpcallBoundaryCostTest`'s absolute upcall figure is not a +property of the commit alone.** It is a property of the commit *and* of how much upcall traffic the +rest of the suite pushed through the same process first. Any future row should record the suite +composition it was taken with, or the number will "drift" again the next time an unrelated test +starts or stops exercising the boundary. + +**One thing did not reproduce and is left as observed.** `537c1a0b`, the commit +`upcall-design.md`'s 703–1075 ns wasm row was recorded on, does not reproduce that row here: a +freshly created worktree at it measured 546.45 ns once and then 1476.85–1602.25 ns on four +subsequent runs (its trampoline figure was similarly unstable, 268–612 ns, while its downcall stayed +flat at 239–247 ns). That commit's suite is 258 tests rather than 344 and predates the MEMFS change, +so it is a different environment in two ways at once, and no attempt is made here to reconcile it. +The conclusion above rests on the adjacent-commit pair `65e1bf5d`/`4472f83a` and on the tip-vs-tip +experiment, neither of which depends on `537c1a0b` at all. + +**Load, since these numbers depend on it.** Machine load average was 1.4–1.6 at the start and +stayed in the 1.7–3.2 band for every run quoted above. An earlier batch was thrown away: creating +the comparison worktree set Spotlight indexing its 16 190 files, which took load to 10.4 and made +the same commit read 546 ns then 1448–1677 ns. Those runs are not in this document. The machine also +had an Android emulator resident throughout, which is why the floor is ~1.5 rather than ~0. + +### The benchmark was measuring the benchmark, and the count that fixes it is measured (2026-08-14) + +The two sections above diagnosed `UpcallBoundaryCostTest` and stopped there: they established that +its absolute figure was a same-process warm-up artefact, pinned wasmJs's shift to one commit, and +attached a staleness note to the table. **The measurement itself was not changed**, so the next pass +would have re-derived the same non-number. This section changes it and shows the change works. + +#### What is warming, decided by experiment rather than by plausibility + +Both earlier sections reached for "JIT tiering" as a story. It is testable, and the obvious rival — +CPython-side state, meaning the specializing interpreter, free lists, the allocator, string +interning — makes an opposite prediction about one row that the report already prints. + +The timed loops were run **40 times in succession** inside one run, on both hosts, in two suite +configurations. Per 10 000 calls: + +| | rep 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | … plateau | +|---|---|---|---|---|---|---|---|---|---| +| desktop upcall, suite as-is | 801 | 744 | 704 | 705 | 676 | 602 | 593 | 537 | ~535 ns | +| desktop upcall, proxy test inert | 1307 | 716 | 696 | 660 | 548 | 539 | 558 | 554 | ~535 ns | +| desktop downcall (Kotlin-driven) | 475 | 264 | 264 | 212 | 141 | 141 | 140 | 143 | ~141 ns | +| wasmJs upcall, suite as-is | 300 | 313 | 317 | 290 | 301 | 285 | 305 | 278 | ~280 ns | +| wasmJs upcall, proxy test inert | 785 | 339 | 317 | 303 | 294 | 299 | 319 | 279 | ~280 ns | + +**The two configurations converge on the same value and they converge from opposite sides.** So the +plateau belongs to the boundary and everything before it belongs to the suite. That alone settles +what the number should be. + +**Two rows in the same sweep do not move at all**, and they are what identifies the mechanism: the +empty Python loop (8–9 ns desktop, 14–16 ns wasm) and the pure-Python callee (24–27 ns, 47–54 ns) are +flat from the very first rep, in every configuration, on both hosts. CPython-side state would move +those — they are pure interpreter work — and it does not. Only rows that cross into host code move. +It is host JIT tier-up: C2 on the JVM, V8's tiering on wasm. + +The confirmation is on desktop, where the two runtimes can be told apart: CPython is a **native +dylib** there, so its own speed cannot depend on how warm the JVM is, and indeed desktop's +pure-Python rows are identical (8.44–8.59 ns) whether the class runs alone or inside the full suite. + +#### The count, and why 100 000 + +Convergence needed 40 000 calls cold and 70 000 warm on desktop, and ~70 000 on wasm. The warm side +settles later, so it sets the requirement; 100 000 is that with margin. The Kotlin-driven rows were +on the same curve and worse off — they warmed `N / 4` = 2 500, and desktop's downcall row reads 653, +266, 266, 161 before settling at ~141 — so they use the same count now. Warming the Python-driven and +Kotlin-driven halves differently would make the headline ratio a ratio of two compilation states. + +`GeneratedProxyCostTest` needed far less, and the reason is its shape: it warms all 19 rows before +timing any, so the shared `_pm_invoke` path already received 19 × 3 000 = 57 000 calls — just under +the knee. 5 000 puts it at 95 000. Measured, its raw rows moved from 3–7% solo-vs-suite disagreement +to 1–3.4%. + +**`inline` on `Benchmark.measure` was tried and is worse.** The hypothesis was that one shared +`block()` call site goes megamorphic across every benchmark in the process. Inlining it made desktop's +downcall row need ~230 000 iterations to reach the plateau it reaches in ~50 000 through the shared +non-inlined loop, because each inlined copy is separate code that tiers up on its own. Reverted, and +recorded so it is not tried again. + +#### The check the old table could not pass + +| | full suite | proxy test short-circuited | this class alone (`--tests`) | +|---|---|---|---| +| desktop upcall, **3 000 warmup** | 673.91 ns | 536.75 ns | — | +| desktop upcall, **100 000 warmup** | 510–560 ns (11 runs) | 501–530 ns (3) | 506–550 ns (3) | +| wasmJs upcall, **3 000 warmup** | 322.03 ns | 291.67 ns | — | +| wasmJs upcall, **100 000 warmup** | 290–304 ns (7 runs) | 287–301 ns (3) | 321–353 ns (3) | + +The middle column is the lever the wasmJs section above used to prove the defect existed. It used to +move desktop by 25% and wasmJs by 10%; it now moves neither outside its own run-to-run band. **That +is the fix.** + +**The third column is honest about what is still not fixed, on one target.** Desktop passes it. wasmJs +does not, and the reason is not the boundary: filtered to this class alone, wasmJs's *pure-Python* +rows read 23.05 ns and 83.10 ns against the full suite's 15.04 ns and 47.22 ns — 55–74% slower, in +rows with no boundary in them. CPython is itself a wasm module under V8, so a Node process that lives +one second runs the interpreter's own bytecode slower than one that lives ten, and every row is +inflated together. Raising the warmup to 300 000 does not move it (solo read 318.83 and 326.21 ns +against 100 000's 320.54–337.42), which is what distinguishes a host-lifetime effect from a +call-count effect. **wasmJs's ratio columns do survive filtering** — 2.66–3.13x solo against +2.91–3.14x in the suite — so the ratio is the quantity to quote there, and the absolute column is a +full-suite figure that must be labelled as one. It is left in the table with that label rather than +deleted, because desktop's is now sound and dropping both would lose a real result. + +#### What it costs + +Measured on this machine, full suite, `build/test-results` cleared before each run: + +| | 3 000 warmup | 100 000 warmup | +|---|---|---| +| `desktopTest`, total test time | 4.93–4.97 s | 5.10–5.20 s | +| …of which `UpcallBoundaryCostTest` | 0.036 s | 0.155–0.162 s | +| …of which `GeneratedProxyCostTest` | 0.553–0.571 s | 0.625–0.661 s | +| `wasmJsNodeTest`, wall clock | 4.31 s / 5.85 s | 4.43 s / 6.11 s | + +**About +0.2 s on desktop (+4% of test time) and +0.1 s on wasm.** The warmup is bounded by the very +cost it is warming, so eight rows of 100 000 calls at ~500 ns is under a second by construction. The +wasm rows are wall clock because that runner reports 0.0 s per class in its XML. + +Test counts are unchanged throughout: **desktop 360/0/1, wasmJs 344/0/0**, on every run quoted here. + +**Load.** 1.14 at the start, 1.6–3.8 for every run quoted, `uptime` checked before each batch. No +other agent was running and no worktree was created during this pass, so the Spotlight problem the +previous section had did not arise. Two readings taken while load was transiently 6.25 (back-to-back +Gradle invocations) were timing measurements only, and are not among the per-call figures. + diff --git a/docs/archive/investigations/android-unregistered-surface.md b/docs/archive/investigations/android-unregistered-surface.md new file mode 100644 index 000000000..de644b51f --- /dev/null +++ b/docs/archive/investigations/android-unregistered-surface.md @@ -0,0 +1,269 @@ +> **Superseded** by [docs/investigations/android-unregistered-surface.md](../../investigations/android-unregistered-surface.md) on 2026-10-03 — the conclusion, numbers and reproduction steps live there; this is the full original narrative, kept for history. + +# Reachability Analysis: Android Unregistered Surface + +> **Status: superseded. Every count and every risk ranking below is historical.** +> +> The unregistered surface is empty as of the pass recorded in ROADMAP §2: 363 of 367 `external fun` +> are bound through `RegisterNatives`, and the four that are not are deliberate — `ffiAllocUtf8`, +> `ffiFreeUtf8` and `ffiReadUtf8` are hand-written JNI exports with the correct prologue, and +> `echoCriticalNamed` exists in order to be name-linked, so that a benchmark can measure that path. +> +> The claim here worth carrying forward is that the *ordering* was right and the *counts* were not. +> When the last 157 were registered, a recount found **zero** of them reachable from +> `commonMain`/`commonTest` — the earlier passes had already taken the whole reachable surface, so +> the "71 reachable" figure below had long since been consumed. The call-graph traces are kept +> because they are still the record of which public API path reaches which C function. +> +> Everything from here down is retained unedited. + +> **Earlier status: the string half of this report is closed; the counts below are stale.** +> +> Recounted from the two files: 187 registered, 178 unregistered (was 145 / 224). Declarations +> still carrying a `jstring` across the boundary went from 52 to 6, and none of those six is on +> the broken path except `ffiSymbolRaw`, which has no caller anywhere in `src/`. The +> "Critical Risk: Functions taking `String`" section below is therefore resolved in full — not +> just the four functions it happens to list. What remains is the pointer-only surface, and the +> reachability reasoning for that still holds. +> +> See ROADMAP §2 for the current numbers and `docs/design/marshalling-design.md` for the per-argument +> intern/scratch rule the migration followed. + +This report analyzes the reachability of the 304 unregistered functions in the Android JNI surface (`bindings.kt`). An unregistered function relies on name-based `@CName` linking which on Android leads to arguments arriving shifted, Kotlin `String`s being passed incorrectly, and `java.lang.Long` boxing issues. Hitting any of these functions is a latent `SIGSEGV` or similar crash. + +We traced the call graph backwards from the public API wrappers in `commonMain` (specifically under `ffi/`) through `EmbedAPI.kt` to the actual Android bindings. + +## 1. Reachable and Unregistered + +Out of the unregistered functions, exactly **71 are reachable** from the current public API. + +The list below shows the reachable functions, ordered by how likely an ordinary user is to hit them. Functions that take strings are placed at the top because they are guaranteed to crash the process due to `jstring` vs Kotlin/Native `String` mismatch. Functions used by core objects (`PyObject`, `PyDict`, `PyList`, etc.) follow. + +### Critical Risk: Functions taking `String` +These functions take a string argument and are highly likely to crash immediately if reached. + +- **`PyImport_AddModule`** + - `ffi/PyTypeChecks.kt` -> `EmbedAPI.PyImport_AddModule()` -> `bindings.PyImport_AddModule()` +- **`PyObject_DelAttrString`** + - `ffi/PyObject.kt` -> `EmbedAPI.PyObject_DelAttrString()` -> `bindings.PyObject_DelAttrString()` +- **`PyObject_SetAttrString`** + - `ffi/PyObject.kt` -> `EmbedAPI.PyObject_SetAttrString()` -> `bindings.PyObject_SetAttrString()` +- **`PyErr_SetString`** + - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyErr_SetString()` -> `bindings.PyErr_SetString()` + +### High Risk: Core Object Model (Collections and Types) +These functions manipulate common Python objects (Dicts, Lists, Tuples, Sets, etc.) and are very likely to be called during normal usage. + +- **`PyDict_Clear`** + - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyDict_Clear()` -> `bindings.PyDict_Clear()` +- **`PyDict_Contains`** + - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyDict_Contains()` -> `bindings.PyDict_Contains()` +- **`PyDict_DelItem`** + - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyDict_DelItem()` -> `bindings.PyDict_DelItem()` +- **`PyDict_GetItem`** + - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyDict_GetItem()` -> `bindings.PyDict_GetItem()` +- **`PyDict_Items`** + - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyDict_Items()` -> `bindings.PyDict_Items()` +- **`PyDict_Size`** + - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyDict_Size()` -> `bindings.PyDict_Size()` +- **`PyDict_Values`** + - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyDict_Values()` -> `bindings.PyDict_Values()` +- **`PyList_Append`** + - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PyList_Append()` -> `bindings.PyList_Append()` +- **`PyList_AsTuple`** + - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyList_AsTuple()` -> `bindings.PyList_AsTuple()` +- **`PyList_Insert`** + - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PyList_Insert()` -> `bindings.PyList_Insert()` +- **`PyList_Reverse`** + - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PyList_Reverse()` -> `bindings.PyList_Reverse()` +- **`PyList_Sort`** + - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PyList_Sort()` -> `bindings.PyList_Sort()` +- **`PyTuple_GetItem`** + - `ffi/PyType.kt` -> `EmbedAPI.PyTuple_GetItem()` -> `bindings.PyTuple_GetItem()` + - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PyTuple_GetItem()` -> `bindings.PyTuple_GetItem()` + - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyTuple_GetItem()` -> `bindings.PyTuple_GetItem()` + - `ffi/types/collections/PyTuple.kt` -> `EmbedAPI.PyTuple_GetItem()` -> `bindings.PyTuple_GetItem()` +- **`PyTuple_GetSlice`** + - `ffi/types/collections/PyTuple.kt` -> `EmbedAPI.PyTuple_GetSlice()` -> `bindings.PyTuple_GetSlice()` +- **`PyTuple_Size`** + - `ffi/PyType.kt` -> `EmbedAPI.PyTuple_Size()` -> `bindings.PyTuple_Size()` + - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PyTuple_Size()` -> `bindings.PyTuple_Size()` + - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyTuple_Size()` -> `bindings.PyTuple_Size()` + - `ffi/types/collections/PyTuple.kt` -> `EmbedAPI.PyTuple_Size()` -> `bindings.PyTuple_Size()` +- **`PySequence_Contains`** + - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PySequence_Contains()` -> `bindings.PySequence_Contains()` + - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PySequence_Contains()` -> `bindings.PySequence_Contains()` + - `ffi/types/collections/PyTuple.kt` -> `EmbedAPI.PySequence_Contains()` -> `bindings.PySequence_Contains()` +- **`PySequence_List`** + - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PySequence_List()` -> `bindings.PySequence_List()` +- **`PySequence_Tuple`** + - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PySequence_Tuple()` -> `bindings.PySequence_Tuple()` +- **`PySet_Add`** + - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PySet_Add()` -> `bindings.PySet_Add()` +- **`PySet_Clear`** + - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PySet_Clear()` -> `bindings.PySet_Clear()` +- **`PySet_Contains`** + - `ffi/types/collections/PyFrozenSet.kt` -> `EmbedAPI.PySet_Contains()` -> `bindings.PySet_Contains()` + - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PySet_Contains()` -> `bindings.PySet_Contains()` +- **`PySet_Discard`** + - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PySet_Discard()` -> `bindings.PySet_Discard()` +- **`PySet_New`** + - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PySet_New()` -> `bindings.PySet_New()` +- **`PySet_Pop`** + - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PySet_Pop()` -> `bindings.PySet_Pop()` +- **`PySet_Size`** + - `ffi/types/collections/PyFrozenSet.kt` -> `EmbedAPI.PySet_Size()` -> `bindings.PySet_Size()` + - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PySet_Size()` -> `bindings.PySet_Size()` +- **`PyFrozenSet_New`** + - `ffi/types/collections/PyFrozenSet.kt` -> `EmbedAPI.PyFrozenSet_New()` -> `bindings.PyFrozenSet_New()` +- **`PyObject_Call`** + - `ffi/PyObject.kt` -> `EmbedAPI.PyObject_Call()` -> `bindings.PyObject_Call()` +- **`PyObject_DelItem`** + - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PyObject_DelItem()` -> `bindings.PyObject_DelItem()` +- **`PyObject_GetItem`** + - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PyObject_GetItem()` -> `bindings.PyObject_GetItem()` +- **`PyObject_GetIter`** + - `ffi/PyType.kt` -> `EmbedAPI.PyObject_GetIter()` -> `bindings.PyObject_GetIter()` + - `ffi/types/collections/PyTuple.kt` -> `EmbedAPI.PyObject_GetIter()` -> `bindings.PyObject_GetIter()` +- **`PyObject_IsInstance`** + - `ffi/PyType.kt` -> `EmbedAPI.PyObject_IsInstance()` -> `bindings.PyObject_IsInstance()` + - `ffi/PyTypeChecks.kt` -> `EmbedAPI.PyObject_IsInstance()` -> `bindings.PyObject_IsInstance()` +- **`PyObject_Repr`** + - `ffi/PyObject.kt` -> `EmbedAPI.PyObject_Repr()` -> `bindings.PyObject_Repr()` +- **`PyObject_RichCompare`** + - `ffi/PyObject.kt` -> `EmbedAPI.PyObject_RichCompare()` -> `bindings.PyObject_RichCompare()` +- **`PyObject_RichCompareBool`** + - `ffi/types/basic/PyFloat.kt` -> `EmbedAPI.PyObject_RichCompareBool()` -> `bindings.PyObject_RichCompareBool()` + - `ffi/types/basic/PyInt.kt` -> `EmbedAPI.PyObject_RichCompareBool()` -> `bindings.PyObject_RichCompareBool()` + - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PyObject_RichCompareBool()` -> `bindings.PyObject_RichCompareBool()` +- **`PyObject_SetItem`** + - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PyObject_SetItem()` -> `bindings.PyObject_SetItem()` +- **`PyObject_Type`** + - `ffi/PyObject.kt` -> `EmbedAPI.PyObject_Type()` -> `bindings.PyObject_Type()` + - `ffi/PyType.kt` -> `EmbedAPI.PyObject_Type()` -> `bindings.PyObject_Type()` + - `ffi/PyTypeChecks.kt` -> `EmbedAPI.PyObject_Type()` -> `bindings.PyObject_Type()` + - `ffi/conversion/Context.kt` -> `EmbedAPI.PyObject_Type()` -> `bindings.PyObject_Type()` + - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyObject_Type()` -> `bindings.PyObject_Type()` + - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PyObject_Type()` -> `bindings.PyObject_Type()` +- **`PyModule_GetDict`** + - `ffi/types/modules/PyModule.kt` -> `EmbedAPI.PyModule_GetDict()` -> `bindings.PyModule_GetDict()` +- **`PyModule_GetFilenameObject`** + - `ffi/types/modules/PyModule.kt` -> `EmbedAPI.PyModule_GetFilenameObject()` -> `bindings.PyModule_GetFilenameObject()` +- **`PyModule_GetName`** + - `ffi/types/modules/PyModule.kt` -> `EmbedAPI.PyModule_GetName()` -> `bindings.PyModule_GetName()` +- **`PyType_GetName`** + - `ffi/PyType.kt` -> `EmbedAPI.PyType_GetName()` -> `bindings.PyType_GetName()` + - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyType_GetName()` -> `bindings.PyType_GetName()` +- **`PyType_IsSubtype`** + - `ffi/PyType.kt` -> `EmbedAPI.PyType_IsSubtype()` -> `bindings.PyType_IsSubtype()` + +### Medium Risk: Exception and Error Handling +These functions are less likely to be called during standard execution, but are critical when an error happens. A crash here masks the actual problem. + +- **`PyErr_GetRaisedException`** + - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyErr_GetRaisedException()` -> `bindings.PyErr_GetRaisedException()` +- **`PyErr_SetRaisedException`** + - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyErr_SetRaisedException()` -> `bindings.PyErr_SetRaisedException()` +- **`PyException_GetCause`** + - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyException_GetCause()` -> `bindings.PyException_GetCause()` +- **`PyException_GetContext`** + - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyException_GetContext()` -> `bindings.PyException_GetContext()` +- **`PyException_GetTraceback`** + - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyException_GetTraceback()` -> `bindings.PyException_GetTraceback()` + +### Medium Risk: Mathematics and Basic Types +- **`PyNumber_Add`** + - `ffi/types/basic/PyFloat.kt` -> `EmbedAPI.PyNumber_Add()` -> `bindings.PyNumber_Add()` + - `ffi/types/basic/PyInt.kt` -> `EmbedAPI.PyNumber_Add()` -> `bindings.PyNumber_Add()` +- **`PyNumber_And`** + - `ffi/types/collections/PyFrozenSet.kt` -> `EmbedAPI.PyNumber_And()` -> `bindings.PyNumber_And()` + - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PyNumber_And()` -> `bindings.PyNumber_And()` +- **`PyNumber_Multiply`** + - `ffi/types/basic/PyFloat.kt` -> `EmbedAPI.PyNumber_Multiply()` -> `bindings.PyNumber_Multiply()` + - `ffi/types/basic/PyInt.kt` -> `EmbedAPI.PyNumber_Multiply()` -> `bindings.PyNumber_Multiply()` +- **`PyNumber_Or`** + - `ffi/types/collections/PyFrozenSet.kt` -> `EmbedAPI.PyNumber_Or()` -> `bindings.PyNumber_Or()` + - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PyNumber_Or()` -> `bindings.PyNumber_Or()` +- **`PyNumber_Subtract`** + - `ffi/types/basic/PyFloat.kt` -> `EmbedAPI.PyNumber_Subtract()` -> `bindings.PyNumber_Subtract()` + - `ffi/types/basic/PyInt.kt` -> `EmbedAPI.PyNumber_Subtract()` -> `bindings.PyNumber_Subtract()` + - `ffi/types/collections/PyFrozenSet.kt` -> `EmbedAPI.PyNumber_Subtract()` -> `bindings.PyNumber_Subtract()` + - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PyNumber_Subtract()` -> `bindings.PyNumber_Subtract()` +- **`PyFloat_AsDouble`** + - `ffi/types/basic/PyComplex.kt` -> `EmbedAPI.PyFloat_AsDouble()` -> `bindings.PyFloat_AsDouble()` + - `ffi/types/basic/PyFloat.kt` -> `EmbedAPI.PyFloat_AsDouble()` -> `bindings.PyFloat_AsDouble()` + - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PyFloat_AsDouble()` -> `bindings.PyFloat_AsDouble()` +- **`PyFloat_FromDouble`** + - `ffi/types/basic/PyFloat.kt` -> `EmbedAPI.PyFloat_FromDouble()` -> `bindings.PyFloat_FromDouble()` +- **`PyLong_AsInt`** + - `ffi/exceptions/PyTraceback.kt` -> `EmbedAPI.PyLong_AsInt()` -> `bindings.PyLong_AsInt()` +- **`PyLong_AsLongLong`** + - `ffi/PyObject.kt` -> `EmbedAPI.PyLong_AsLongLong()` -> `bindings.PyLong_AsLongLong()` + - `ffi/types/basic/PyInt.kt` -> `EmbedAPI.PyLong_AsLongLong()` -> `bindings.PyLong_AsLongLong()` + - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PyLong_AsLongLong()` -> `bindings.PyLong_AsLongLong()` +- **`PyBool_FromLong`** + - `ffi/types/basic/PyBool.kt` -> `EmbedAPI.PyBool_FromLong()` -> `bindings.PyBool_FromLong()` +- **`PyUnicode_Concat`** + - `ffi/types/basic/PyString.kt` -> `EmbedAPI.PyUnicode_Concat()` -> `bindings.PyUnicode_Concat()` +- **`PyUnicode_Contains`** + - `ffi/types/basic/PyString.kt` -> `EmbedAPI.PyUnicode_Contains()` -> `bindings.PyUnicode_Contains()` + +### Miscellaneous Utility Functions +- **`PyCallable_Check`** + - `ffi/PyObject.kt` -> `EmbedAPI.PyCallable_Check()` -> `bindings.PyCallable_Check()` +- **`PyEval_GetBuiltins`** + - `ffi/PyTypeChecks.kt` -> `EmbedAPI.PyEval_GetBuiltins()` -> `bindings.PyEval_GetBuiltins()` + - `ffi/types/basic/PyComplex.kt` -> `EmbedAPI.PyEval_GetBuiltins()` -> `bindings.PyEval_GetBuiltins()` + - `ffi/types/basic/PyNone.kt` -> `EmbedAPI.PyEval_GetBuiltins()` -> `bindings.PyEval_GetBuiltins()` +- **`PyIter_Next`** + - `ffi/types/iteration/PyIterator.kt` -> `EmbedAPI.PyIter_Next()` -> `bindings.PyIter_Next()` +- **`Py_GetBuildInfo`** + - `ffi/Python3.kt` -> `EmbedAPI.Py_GetBuildInfo()` -> `bindings.Py_GetBuildInfo()` +- **`Py_GetCompiler`** + - `ffi/Python3.kt` -> `EmbedAPI.Py_GetCompiler()` -> `bindings.Py_GetCompiler()` +- **`Py_GetCopyright`** + - `ffi/Python3.kt` -> `EmbedAPI.Py_GetCopyright()` -> `bindings.Py_GetCopyright()` +- **`Py_GetPlatform`** + - `ffi/Python3.kt` -> `EmbedAPI.Py_GetPlatform()` -> `bindings.Py_GetPlatform()` +- **`Py_RunMain`** + - `ffi/Python3.kt` -> `EmbedAPI.Py_RunMain()` -> `bindings.Py_RunMain()` +- **`ffiReadUtf8`** (Helper used for marshalling) + - `ffi/PyObject.kt` -> `EmbedAPI.PyUnicode_AsUTF8()` -> `bindings.ffiReadUtf8()` + - `ffi/PyType.kt` -> `EmbedAPI.PyUnicode_AsUTF8()` -> `bindings.ffiReadUtf8()` + - `ffi/Python3.kt` -> `EmbedAPI.Py_GetVersion()` -> `bindings.ffiReadUtf8()` + - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyUnicode_AsUTF8()` -> `bindings.ffiReadUtf8()` + - `ffi/exceptions/PyTraceback.kt` -> `EmbedAPI.PyUnicode_AsUTF8()` -> `bindings.ffiReadUtf8()` + - `ffi/types/basic/PyString.kt` -> `EmbedAPI.PyUnicode_AsUTF8()` -> `bindings.ffiReadUtf8()` + - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PyUnicode_AsUTF8()` -> `bindings.ffiReadUtf8()` + +--- + +## 2. Unreachable Today + +There are **233** functions present in `bindings.kt` as unregistered `external fun`s that are not reached by any code in `commonMain` today (totalling 304 unregistered out of 375 overall, though exact counts from parsing show 71 reachable and 222 unreachable). + +While they should eventually be migrated to ensure the bindings layer is safe to use in its entirety, they **do not pose an immediate crash risk** for current users of the library. + +## 3. Cannot Determine + +Every path from the public `commonMain` API layer down to the underlying bindings layer could be definitively traced statically. There are **0** functions whose reachability could not be determined. The analysis explicitly covered every wrapper and `expect`/`actual` binding chain. + +## 4. Suggested Migration Order + +1. **String-taking reachable functions (The immediate crashers)** + - Because they process strings incorrectly across JNI natively, calling any of these guarantees a crash. + - `PyImport_AddModule`, `PyErr_SetString`, `PyObject_DelAttrString`, `PyObject_SetAttrString` + +2. **Core Object Model (Primitives & Collections)** + - Heavily used functions that could lead to crashes due to missing `JNIEnv` or boxing issues (`java.lang.Long` vs `jlong`). + - `PyDict_Clear`, `PyDict_Contains`, `PyDict_DelItem`, `PyDict_GetItem`, `PyDict_Items`, `PyDict_Size`, `PyDict_Values`, `PyList_Append`, `PyList_AsTuple`, `PyList_Insert`, `PyList_Reverse`, `PyList_Sort`, `PyModule_GetDict`, `PyModule_GetFilenameObject`, `PyObject_Call`, `PyObject_DelItem`, `PyObject_GetItem`, `PyObject_GetIter`, `PyObject_IsInstance`, `PyObject_Repr`, `PyObject_RichCompare`, `PyObject_RichCompareBool`, `PyObject_SetItem`, `PyObject_Type`, `PySequence_List`, `PySequence_Tuple`, `PyTuple_GetItem`, `PyTuple_GetSlice`, `PyTuple_Size`, `PyType_GetName`, `PyType_IsSubtype`, etc. + +3. **Exception and Error handling** + - Less likely to be hit in a happy path, but critical for clean failure handling. + - `PyErr_GetRaisedException`, `PyErr_SetRaisedException`, `PyException_GetCause`, `PyException_GetContext`, `PyException_GetTraceback`. + +4. **Remaining reachable functions** + - Other utility functions (e.g. maths and module utilities). + +5. **Unreachable functions** + - Deferred until the public surface is expanded to use them. diff --git a/docs/archive/investigations/gc-scheduling-investigation.md b/docs/archive/investigations/gc-scheduling-investigation.md new file mode 100644 index 000000000..63565e367 --- /dev/null +++ b/docs/archive/investigations/gc-scheduling-investigation.md @@ -0,0 +1,516 @@ +> **Superseded** by [docs/investigations/gc-scheduling-investigation.md](../../investigations/gc-scheduling-investigation.md) on 2026-10-03 — the conclusion, numbers and reproduction steps live there; this is the full original narrative, kept for history. + +# Why a pure C API embedder never collects cycles, and never merges a refcount + +Two things this document is careful about, because it corrects a previous document that was not: + +- **CPython file:line citations are now verified against the tagged source.** The v3.14.7 tarball + (the version this build pins — `gradle.properties: pythonVersion=3.14.7`) was fetched to + `/Volumes/macMini/cpython-src` for §8d and every line number in this document was re-checked + against it. Two in §1 were wrong by ~100 lines and are corrected below; the rest were right. + Nothing is vendored into this repo. Where a claim matters, §3, §4 and §8 still back it with a + measurement taken here rather than with a citation. +- **Every number in §3, §5 and §8 was measured on this machine**, on `desktopTest`, macOS arm64, + CPython 3.14.7, both the default and the `-PpythonFreeThreaded=true` builds. +- **§8 corrects §7.** A measurement probe in `GCSchedulingMeasurementTest` was leaking the list + returned by `gc.get_objects()`, which holds the whole tracked heap, and that leak is what + produced the earlier free-threaded conclusion. The correction was itself measured; the + hypothesis it replaced was measured and rejected rather than merely doubted. + +## 0. Summary + +Two symptoms that looked unrelated share one cause, and a third that looked like the same thing is +not. + +- **The cyclic collector never runs.** An embedder that only ever calls `PyObject_Call` and friends + never triggers a generational collection, however much it allocates. True on both builds. (§1) +- **Free-threaded, a completed `Py_DecRef` can leave the object alive.** The count is correct and + the decrement happened, but `tp_dealloc` has not run and the memory is not back. (§2) + + These two are the same thing: CPython schedules both jobs by setting a bit on the *eval breaker*, + and the only code that reads those bits is `_Py_HandlePending`, which the evaluation loop calls + and nothing else does. No bytecode frame, no checkpoint; no checkpoint, no collection and no + merge. +- **`CycleCollectionTest`'s free-threaded failure is *not* the same thing**, and an eval-loop + checkpoint provably does not fix it. It is deferred reference counting on the heap type, and what + materialises a deferred reference is a collection, not a checkpoint. (§3) +- **The checkpoint fixes §1 on the build with the global lock, and on the free-threaded build it + often does not.** It fires exactly as designed there and still reclaims nothing. This is *not* + caused by the FFI wrapper's per-call GIL scopes — that hypothesis was tested by holding the + checkpoint count fixed and widening the scope, and rejected. (§8) + + The cause is that free-threaded, `_Py_RunGC` re-asks `gc_should_collect` *after* the checkpoint + has read the scheduled bit, and that check gates generation 0 on the growth of the **whole + process's** memory footprint — the JVM's. Python-side allocation cannot move it, so the gate + shuts one-way as soon as the JVM's footprint settles. Confirmed by opening and closing the gate + on demand, including by faulting in 400 MB of ballast and watching Python cycles get collected + with no threshold changed. (§8d) + +## 1. `_Py_ScheduleGC` only sets a bit + +Allocation-driven collection has not been a direct call since 3.12. `_Py_ScheduleGC` sets +`_PY_GC_SCHEDULED_BIT` on the eval breaker and returns: + +| build | function | file:line (v3.14.7, verified) | +|---|---|---| +| global lock | `_Py_ScheduleGC` | `Python/gc.c:1846` | +| free-threaded | `_Py_ScheduleGC` | `Python/gc_free_threading.c:2795` | + +The bit's only reader is `_Py_HandlePending` (`Python/ceval_gil.c:1357`), which calls `_Py_RunGC` +when it is set (`Python/ceval_gil.c:1397-1399`). `_Py_HandlePending` in turn has one caller family: the +`_CHECK_PERIODIC` / `_CHECK_PERIODIC_IF_NOT_YIELD_FROM` uops in `Python/bytecodes.c`, which open +every Python-level frame (`RESUME`) and close every call instruction. There is no public entry +point to it. + +So **allocation schedules a collection that only the evaluation loop will ever perform.** An +embedder that executes no Python-level frame accumulates the scheduled bit forever and its cycles +are never broken. This is not a free-threading property; it is the same on both builds, and it is +why `gc.collect()` (or the C `PyGC_Collect`) has been the only thing that has ever collected +anything in this library's tests. + +### `Py_MakePendingCalls` is not a substitute + +It is the obvious Stable ABI candidate and it does not work. It forwards to +`_PyEval_MakePendingCalls` (`Python/ceval_gil.c:1034`), which handles `handle_signals` and +`make_pending_calls` and returns. It never looks at `_PY_GC_SCHEDULED_BIT`, and on a free-threaded +build it never looks at the merge bit either. + +This one is checked here rather than taken on trust: +`EvalCheckpointTest.testDrainPendingReleasesReclaimsWhatTheCleanerGaveBack` calls +`Py_MakePendingCalls`, asserts it returned 0, and asserts the reference count did not move. + +## 2. Free-threading: biased reference counting, and the queue only the eval loop drains + +`Py_DECREF` on a free-threaded build is not one operation. It branches on whether the decrementing +thread is the object's owner, recorded in `ob_tid`: + +- **Owner.** `ob_ref_local` is decremented non-atomically. If it reaches zero and the shared count + is zero too, `tp_dealloc` runs immediately, on this thread. +- **Not the owner.** The decrement goes to `ob_ref_shared` atomically. If that count was already at + zero — i.e. this decrement is the one that takes the object to zero — `tp_dealloc` *cannot* run + here, because the object belongs to another thread. It is pushed onto that thread's queue by + `_Py_brc_queue_object` (`Objects/object.c:411`, reached from the inline decref in + `Include/refcount.h:363`). + +The queue is drained by `_Py_brc_merge_refcounts`, whose only caller is, again, `_Py_HandlePending` +(`Python/ceval_gil.c:1388`), behind `_PY_EVAL_EXPLICIT_MERGE_BIT`. + +**And once a collection has run, a surviving object can become a non-owner case for everybody.** +On the way out, `gc_restore_tid` (`Python/gc_free_threading.c:325`) puts each survivor's owning +thread id back — *except* when its shared refcount has been merged, in which case `ob_tid` is left +at 0 (`:330`) and no thread matches any more, so every further decrement of that object takes the +shared path. (The inherited citation for this was `:205`, which is not the mechanism; merging +itself is at `:264`. The behaviour is as described, but it applies to merged survivors rather than +to all of them.) + +That is the mechanism behind the ROADMAP §9 measurement: 1000 wrappers released by the JVM cleaner, +2003 `Py_DecRef` calls demonstrably made, and `sys.getrefcount` frozen at 1002 through 50 forced +JVM GCs, 500 further C API round trips and two seconds of wall clock — then one trivial bytecode +frame released all thousand at once. + +## 3. `CycleCollectionTest`: two defects, and the one that is *not* a checkpoint problem + +`CycleCollectionTest.testHandleReleasedWhenProxyDiesWithoutCycle` was the last free-threaded +failure. It has two independent causes, and the guard assertion — "the type's count must rise by +exactly `rounds` while the instances are alive, otherwise the probe is not reading a refcount" — +caught both, exactly as it was written to. + +### 3a. The probe read the wrong bytes + +`ob_refcnt` is at offset 0 only on a build with the global lock. Free-threaded CPython lays the +header out differently: + +```c +/* Include/object.h, Py_GIL_DISABLED */ +struct _object { + uintptr_t ob_tid; /* +0, 8 bytes -- owning thread id, NOT a refcount */ + uint16_t ob_flags; /* +8 */ + PyMutex ob_mutex; /* +10 */ + uint8_t ob_gc_bits; /* +11 */ + uint32_t ob_ref_local; /* +12, 4 bytes */ + Py_ssize_t ob_ref_shared; /* +16, 8 bytes, shifted by _Py_REF_SHARED_SHIFT (2), + low 2 bits are state flags */ + PyTypeObject *ob_type; /* +24 */ +}; +``` + +`Py_REFCNT` there is `ob_ref_local + (ob_ref_shared >> 2)`. Reading eight bytes at offset 0 yields +`ob_tid` — which is why the failure reported a ten-digit number that was *identical* before and +after creating 100 instances: measured here, `6171668704` both times. The layout above was +confirmed against this build by dumping the header (see 3b); it is not taken from documentation. + +The probe now reads the right fields on each build. + +### 3b. Correcting the correction: heap types **are** deferred-reference-counted + +This section exists because two successive diagnoses of this failure were wrong in opposite +directions, and only a direct measurement settles it. + +ROADMAP §9 originally said: + +> Fixing the offset would not save the test. Its premise is that "each live instance of a heap type +> holds one reference to that type", and free-threaded CPython gives heap types **deferred +> reference counting** [...] The invariant simply does not exist there, so `tp_dealloc`'s +> obligation to release the type reference cannot be checked this way at all. + +A later source trace proposed the opposite: that this was wrong, that `_PyGC_BITS_DEFERRED` is set +only on modules, top-level functions, descriptors and immortal objects, that nothing marks a +`PyType_FromSpec` heap type or its instances as deferred, and that the real cause was queued BRC +decrements that an eval-loop checkpoint would flush. + +**Measured, both of those are partly wrong, and the practical conclusion belongs to the first.** +Dumping the proxy type's header on the free-threaded build, before and after 100 instantiations: + +| | `ob_tid` (+0) | `ob_gc_bits` (+11) | `ob_ref_local` (+12) | `ob_ref_shared` (+16) | +|---|---|---|---|---| +| fresh type | `0x16d9b70e0` | `0x41` | 2 | `0x3ffffffffffffffd` | +| 100 instances alive | `0x16d9b70e0` | `0x41` | 2 | `0x3ffffffffffffffd` | +| after `drainPendingReleases()` | `0x16d9b70e0` | `0x41` | 2 | `0x3ffffffffffffffd` | +| after `PyGC_Collect()` | `0x16d9b70e0` | `0x41` | 2 | `0x400000000000018d` | +| after 100 `Py_DecRef` → 100 `tp_dealloc` | `0` | `0x41` | 0 | `0x4000000000000007` | + +Reading that off: + +- `ob_gc_bits == 0x41` is `_PyGC_BITS_TRACKED | _PyGC_BITS_DEFERRED`. The heap type **is** marked + deferred. +- `ob_ref_shared == 0x3ffffffffffffffd` is `(0x0fffffffffffffff << 2) | 1`: a shared count of + `2^60 - 1` = `PY_SSIZE_T_MAX / 8`, which is the deferred sentinel, with the + `_Py_REF_MAYBE_WEAKREF` flag. Two independent signals agreeing. +- **Creating 100 instances changed nothing.** `PyType_GenericAlloc` increments the type through + `_Py_INCREF_TYPE`, which is a no-op for a deferred type on its owning thread. The header is + bit-identical. +- **An eval-loop checkpoint changed nothing either.** This is the decisive one: the proposed fix + does not work, and cannot, because there is no queued decrement to merge. The increments were + never made in the first place. +- **A collection is what makes deferred references real.** `ob_ref_shared` rose by exactly + `100 << 2`. This is `gc_free_threading.c` converting deferred references into counted ones as it + walks. +- **And then `tp_dealloc` gives back exactly 100.** Accounting for the flag bits moving from + `_Py_REF_MAYBE_WEAKREF` (1) to `_Py_REF_MERGED` (3) and `ob_ref_local` merging to zero, the total + goes from `1152921504606847077` back to `1152921504606846977` — the pre-instantiation value, + exactly. + +So the original ROADMAP claim was right about the *mechanism* (heap types are deferred) and wrong +about the *consequence* (that the invariant does not exist and cannot be checked). The invariant +holds on both builds. Free-threaded it is merely not **observable** until a collection has +materialised the deferred references. The later trace was right that instances are ordinary +reference-counted objects — but the test reads the *type's* count, not an instance's, so that +correction did not touch the failing assertion. + +`CycleCollectionTest` therefore takes a `PyGC_Collect()` on either side of its instantiation loop, +free-threaded only. The assertions are unchanged: the same "rises by exactly 100" and "returns to +exactly where it started" are asserted on both builds, and both now hold. + +### What this means for `abi3t` + +The conclusion drawn from the wrong premise — "so `abi3t` making `PyObject` an incomplete type +would sink this test" — does not follow either. The test does need to know the header layout, which +is a real cost of a direct probe and the one place in this repo that reads inside a `PyObject`. But +that is a property of the probe, not of the invariant, and it would be equally solvable by counting +through `sys.getrefcount` at the price of the argument's own temporary reference. The library +proper stays clean: `ProxyTypeFactory` writes only into memory it obtained from +`PyObject_GetTypeData`. + +## 4. What this repo does about §1 and §2 + +`Python3.drainPendingReleases()` reaches a checkpoint the cheapest way there is: it calls a cached, +already-compiled, zero-argument Python function whose body is `pass`, through +`PyObject_CallNoArgs`. That is one `RESUME`, hence one `_CHECK_PERIODIC`, hence one +`_Py_HandlePending` — one BRC merge, one QSBR sweep, and one scheduled collection if one was +scheduled. + +It is deliberately **not** `PyRun_SimpleString("pass")` or `exec("pass")`: + +- `PyRun_SimpleString` runs `PyErr_Print()` on failure, which clears the error indicator. This repo + has twice had unrelated tests die because a helper wiped an indicator its caller had not read + yet; `Python3.exec` is built on `PyRun_String` for exactly that reason. +- `exec("pass")` recompiles a module on every call — 23× the cached call, §5. + +`withGIL` takes one automatically at its outermost entry, behind two gates: at most one per +`Python3.autoDrainInterval` outermost scopes, and skipped entirely unless `ReleaseCounter.released` +has moved since the last one. It defaults to 32 on free-threaded builds and 0 (off) otherwise. +`reachEvalCheckpointHoldingGIL` additionally declines while `PyErr_Occurred()` is non-null, for the +same error-indicator reason. + +It is not taken from the cleaner. The queue that needs merging belongs to the thread that *owns* +the object; a cleaner owns nothing, so a checkpoint there would run Python on a cleaner thread (the +ROADMAP §1 deadlock) in order to drain an empty queue. +`EvalCheckpointTest.testCleanerActivityAloneTakesNoCheckpoint` pins the suppression. + +`PyGC_Collect` is the one Stable ABI function that reclaims on behalf of *another* thread, because +`gc_collect_internal` stops the world and walks every thread state, calling +`_PyObject_MergePerThreadRefcounts` and `merge_queued_objects` for each. It costs a heap walk +rather than a queue pop — §5 — and, as §3b shows, it is also the only thing that materialises a +deferred reference. + +## 5. Measurements + +`EvalCheckpointTest.testCheckpointCostAgainstTheAlternatives`, free-threaded desktop build, macOS +arm64. The test asserts the *ordering*, not the absolute numbers: the ordering is a property of the +design and would break the moment the checkpoint started compiling or allocating. + +| | ns/op | | +|---|---:|---| +| `withGIL { }` — attach and detach, nothing else | 170.51 | the floor | +| `Python3.drainPendingReleases()` | 302.70 | the checkpoint, ~132 ns over the floor | +| `withGIL { Py_MakePendingCalls() }` | 197.34 | cheap, and does not merge | +| `Python3.exec("pass")` | 7 037.04 | 23× the checkpoint — it recompiles | +| `withGIL { PyGC_Collect() }` | 224 625.00 | 742× the checkpoint — heap walk, but drains every thread | + +At the default `autoDrainInterval = 32` the automatic checkpoint amortises to about 4 ns on a +~170 ns outermost scope, and a workload that drops no wrappers skips it on a field compare. + +`PyGC_Collect` being ~740× the checkpoint is also the cost `CycleCollectionTest` pays for the two +collections §3b requires: ~0.45 ms for the whole test, free-threaded only. + +## 6. What is still true, and what to watch + +- The scheduled-GC half of §1 affects **both** builds. Nothing in this repo currently depends on + allocation-driven collection, and `autoDrainInterval` defaults to 0 with the global lock, so the + default configuration still never runs a generational collection on its own. That is a choice, + not an oversight, and reversing it is one assignment. +- `EvalCheckpointTest` asserts that `Py_MakePendingCalls` does *not* merge. If CPython ever changes + its mind, that test fails and says so. +- The header layout in §3a is a private detail read in exactly one place, and that read is guarded + by an assertion that fails loudly if the offsets stop being right. It already has, twice. +- The deferred sentinel gives roughly `2^60` of headroom, so nothing here is at risk of the count + actually reaching zero. But note the asymmetry `ProxyTypeFactory` lives with: its hand-written + `tp_dealloc` releases the type through the public `Py_DecRef`, which is deferred-unaware, while + the matching increment went through the deferred-aware `_Py_INCREF_TYPE` and did not happen. That + balances out once a collection has materialised the references — measured above — but it is + balanced by the collector, not by the two calls. + +## 7. GC Accumulation and Reentrancy Measurement + +Measurements taken to verify the consequences of §1 and the safety of the checkpoint confirm: + +- **Cyclic Garbage Accumulation (GIL Build)**: A workload creating 10,000 cyclic object groups entirely through the C API (never running a Python bytecode evaluation loop) with `autoDrainInterval = 0` accumulates **20,000+** cyclic garbage objects indefinitely. Because no evaluation loop runs, `_Py_ScheduleGC`'s scheduled bit is never checked by `_CHECK_PERIODIC`, and cyclic GC never occurs. +- **`autoDrainInterval` Effectiveness (GIL build)**: When running the exact same C API workload but with `autoDrainInterval = 32`, `python-multiplatform` evaluates a dummy function (`__pmp_eval_checkpoint__`) to force the evaluation loop to run periodically. This allows `_CHECK_PERIODIC` to see the scheduled bit, trigger `_Py_RunGC()`, and reclaim the cyclic garbage (leaving ~1,970 of ~20,000). **This result is specific to the build with the global lock.** Free-threaded, the same checkpoints frequently reclaim nothing at all — §8. +- **`__del__` Reentrancy Risk**: Executing `PyGC_Collect()` or processing a checkpoint can invoke `__del__` methods. A scenario where `__del__` directly calls back into Kotlin (via Panama upcalls) was executed (`GCSchedulingMeasurementTest.testReentrancyDuringCheckpoint`). The reentrancy is safe on the GIL build: it does not deadlock, it does not loop infinitely, and it successfully executes the Kotlin upcall while the garbage collection is in progress. + +## 8. Free-threaded: the checkpoint fires and reclaims nothing, and it is not the scope shape + +Everything in this section was measured here, on `desktopTest`, macOS arm64, CPython 3.14.7, both +builds. The workload is `GCSchedulingMeasurementTest.buildAndDropCycles`: 10,000 unreachable +`list` cycles, 20,000 objects, built entirely through the C API. + +### 8a. The probe was holding the evidence + +The measurement chain used to be written inline as + +```kotlin +gc.getAttr("get_objects").invoke().getAttr("__len__").invoke().toString().toInt() +``` + +which closes none of its temporaries. `gc.get_objects()` returns a list holding **a strong +reference to every tracked object in the interpreter**, so a leaked one roots the very garbage the +next measurement is trying to see. + +That leak is where §7's earlier free-threaded conclusion came from — the claim that "a trailing +explicit `gc.collect()` finds nothing further to reclaim, so the residue is not simply garbage +waiting for the next checkpoint," and the inference from it that free-threaded CPython was failing +to *reclaim* something. With the temporaries released, **an explicit `gc.collect()` reclaims the +residue essentially completely, on both builds, in every configuration tried**: 60+ repetitions, +post-collection residue between −13 and +2 objects out of ~20,000. + +So the residue is ordinary collectable cyclic garbage. What differs free-threaded is what makes the +collector *run*, not what it can take. `GCSchedulingMeasurementTest` now asserts this on both +builds (`assertResidueIsOrdinaryCollectableGarbage`), which is the opposite of what the previous +text asserted about it. + +### 8b. Rejected: "per-call `PyGILState_Ensure`/`Release` destroys the scheduled GC bit" + +The standing hypothesis was that because free-threading keeps `eval_breaker` per thread state, an +FFI wrapper that attaches and detaches around every C API call destroys and recreates the thread +state, discarding the scheduled GC bit with it — and that widening the scope would therefore fix +collection. + +**Measured and rejected.** Holding the checkpoint count fixed at 2,000 explicit +`Python3.drainPendingReleases()` calls (`autoDrainInterval = 0`, one drain every 5 rounds) and +varying *only* the enclosing scope, free-threaded, residue out of 20,000: + +| enclosing scope | reps | residue | +|---|---:|---| +| none — one `withGIL` per C API call | 5 | 20,000 ×5 | +| one per round (~12 C API calls) | 5 | 20,000 ×5 | +| one per 100 rounds | 10 | 20,000 ×10 | +| one for the whole 10,000-round workload | 10 | 11,562 / 1,280 / 1,280, then 20,000 ×7 | + +Widening the scope to 100 rounds — which gives the thread state a long life and hundreds of +allocations before it is destroyed — changes nothing whatsoever. The hypothesis predicts the +opposite, so it is wrong. + +The single-scope row does not rescue it either: **it is bimodal within one JVM.** Those ten +repetitions are consecutive and identical; it reclaimed on the first three and then never again. +The configuration latches, which is a property of accumulated interpreter state, not of the scope +shape the call happens to be made under. + +With the automatic checkpoint instead (`autoDrainInterval = 32`), free-threaded: + +| enclosing scope | checkpoints taken | residue | +|---|---:|---| +| none — one `withGIL` per C API call | 1,875 | 20,000 ×5; also 7,427 and 11,523 in two single-repetition JVMs | +| one per round | 312–313 | 20,000 ×5 | +| one per 100 rounds | 3 | 20,000 ×5 | +| one for the whole workload | **0** | 20,000 ×3 | + +The last row is a property of the gate, not of the collector: a single outermost scope decrements +`checkpointCountdown` once, so at interval 32 no automatic checkpoint is ever reached. An embedder +that wraps a long batch in one `withPython` therefore gets *fewer* automatic checkpoints, not more. + +### 8c. The contrast that is actually there + +The same sweep on the build with the global lock is stable and near-complete in **every** shape: + +| configuration | checkpoints | residue | +|---|---:|---| +| per C API call, `autoDrainInterval = 32` | 1,875 | 1,964–1,972 | +| per C API call, 2,000 explicit drains | 2,000 | 1,920 ×5 | +| one per 100 rounds, 2,000 explicit drains | 2,000 | 1,920 ×5 | +| whole workload in one scope, 2,000 explicit drains | 2,000 | 1,920–1,962 | +| one per round, `autoDrainInterval = 32` | 312–313 | 1,556–1,616 | +| one per 100 rounds, `autoDrainInterval = 32` | **3** | 1,600–3,200 | + +Three checkpoints reclaim ~90% on the GIL build. Two thousand reclaim nothing free-threaded. That +gap — not the scope shape, which is irrelevant on both builds — is the whole finding. + +### 8d. Answered: the checkpoint is fine, the collector asks a second question + +The question §8c leaves open is why a checkpoint that demonstrably runs +(`CheckpointCounter.reached` climbs by an identical, deterministic amount every run) triggers no +collection free-threaded, and why the same configuration latches from "reclaims" to "never +reclaims" inside one JVM. + +CPython v3.14.7 was fetched to `/Volumes/macMini/cpython-src` (not vendored here) and read. **The +checkpoint is not where this goes wrong.** The checkpoint reads the scheduled bit exactly as §1 +describes, on both builds. What differs is that free-threaded, `_Py_RunGC` then asks a *second* +question, and it is that one that answers no. + +#### The asymmetry, in four lines of source + +| | global lock (`Python/gc.c`) | free-threaded (`Python/gc_free_threading.c`) | +|---|---|---| +| schedules on | `generations[0].count > threshold`, and nothing else (`gc.c:1866`) | `gc_should_collect` (`:2153`) | +| re-checked when the checkpoint runs it | `gc_select_generation` — generation 0 on count alone (`gc.c:1258`) | `gc_should_collect` **again** (`:2328`) | +| process-memory gate | **none**; `gc.c` has no `last_mem` field and never calls `get_process_mem_usage` | `gc_should_collect_mem_usage` (`:2080`) | +| `long_lived_total / 4` | oldest generation only (`gc.c:1300`) | **every** generation-0 decision (`:2131`) | + +`gc_should_collect` (`gc_free_threading.c:2117`) is: + +```c +if (count <= threshold || threshold == 0 || !gc_enabled) return false; // :2123 +if (gcstate->old[0].threshold == 0) return true; // :2126 +if (count < gcstate->long_lived_total / 4) return false; // :2131 +return gc_should_collect_mem_usage(gcstate); // :2136 +``` + +and `gc_should_collect_mem_usage` (`:2080`) is, in substance: + +```c +if (deferred > threshold * 40) return true; // :2089 +if ((footprint - last_mem) > Py_MAX(last_mem / 10, 128)) return true; // :2100 +young.count = 0; deferred_count += young.count; return false; // :2109-2113 +``` + +`footprint` is `get_process_mem_usage()`, which on macOS is +`task_info(TASK_VM_INFO).phys_footprint` (`:2010`) — **the whole process**, i.e. the JVM. +`last_mem` is written in exactly one place, after a collection (`:2301`). + +So, in a JVM-hosted embedder: 20,000 small Python lists cannot move a JVM's physical footprint by +the required `last_mem / 10`, the gate returns false, and on the way out it **zeroes `young.count` +into `deferred_count`** — which means the count has to climb from zero to 2,000 again before the +question is even asked again. The only escape is `deferred_count > threshold * 40` = 80,000 +container allocations at the default threshold, and this workload makes ~20,000. Checkpoints are +irrelevant to all of it; the bit gets read and `gc_collect_main` declines at `:2328`. + +**This is also the latch.** `last_mem` only moves when a collection runs, and it moves to the +whole process's current footprint. Early in a JVM's life the footprint is still climbing steeply, +so the gate opens and collections happen; once a collection records the JVM's settled footprint, +the bar becomes a tenth of a large number that Python-side allocation cannot move, and it is shut +one-way. Nothing about the scope shape is involved, which is consistent with §8b rejecting it. + +#### Confirmed by experiment, not by reading + +The repo has been wrong before about causes derived from source reading, so each claim above was +turned into a prediction that could fail. All of it is in +`python-multiplatform/src/desktopTest/.../ref/FreeThreadedGCGateTest.kt`. + +**Which gate declines** — `gc_should_collect_mem_usage`'s failing branch zeroes `young.count` +(`:2109`); the `long_lived_total/4` branch (`:2131`) does not. So the shape of `gc.get_count()[0]` +across the workload tells them apart. Free-threaded it is a **sawtooth** +(`[1728, 1488, 976, 464, 2464, 1488, 976, 464, 2464, 1488]`) with 0 collections and residue 20,000: +the memory gate, repeatedly. + +> This probe took two attempts, and the first one was wrong in the direction that would have +> produced a confident false answer. `gc_get_count_impl` (`Modules/gcmodule.c:215`) flushes the +> per-thread allocation buffer and sets `gc->alloc_count = 0`, and `record_allocation` only +> consults `gc_should_collect` when that buffer reaches `LOCAL_ALLOC_COUNT_THRESHOLD` = 512 +> (`:69`, `:2147`). Sampling every 500 allocations therefore stopped the collector from ever being +> asked, and produced a perfectly linear trace (`226, 726, 1226, … 19726`) that reads exactly like +> proof of the *other* hypothesis. Sampling every 2,000 allocations instead gives the sawtooth. + +**The one-line falsifier** — `:2126` says `old[0].threshold == 0` short-circuits the whole chain. +Nothing in `gc.c` reads that field while scheduling. So `gc.set_threshold(2000, 0, 0)` — a setting +about *generation 1* — must switch *generation 0* collection on free-threaded and do nothing at all +with the global lock. Run A/B/A in one JVM so the latch cannot explain the difference; residue out +of 20,000, and the number of collections that actually ran: + +| arm | free-threaded | global lock | +|---|---|---| +| `(2000, 10, 10)` | 20,000 — 0 collections | 1,968 — 9 collections | +| `(2000, 0, 0)` | **1,318 — 8 collections** | 1,966 — 5 collections | +| `(2000, 10, 10)` again | 20,000 — 0 collections | 1,970 — 9 collections | + +**The `deferred_count` arithmetic** — the escape bar is `40 × threshold0`, and `deferred_count` +grows *only* in the memory gate's failing branch. Against a ~20,000-allocation workload that +predicts no collection at threshold0 = 2000 (bar 80,000) or 500 (bar 20,000), and collection at +100 (bar 4,000). Free-threaded: 20,000 / 20,000 / **1,466 with 3 collections**. The same sweep with +the global lock just scales smoothly with the threshold (9 / 36 / 172 collections), because there +is no such bar. This also proves the `long_lived_total/4` gate at `:2131` is being *passed* — +nothing else feeds `deferred_count`. + +**The latch, driven directly** — if the gate is whole-process footprint, then faulting in 400 MB of +ballast partway through the workload must open it, while changing no threshold, allocating no extra +Python container and touching nothing the collector tracks. Dropping the ballast must shut it +again. Residue out of 20,000: + +| arm | free-threaded | global lock | +|---|---|---| +| baseline | 20,000 — 0 collections | 1,972 — 9 collections | +| 400 MB faulted in midway | **9,754 — 1 collection** | 1,988 — 9 collections | +| ballast dropped | 20,000 — 0 collections | 1,968 — 9 collections | + +Reproduced identically to the object in three consecutive JVMs. **JVM memory growth, and nothing +else, collected Python cycles.** That is the finding: on a free-threaded build the Python cyclic +collector's schedule is a function of the host process's memory, which for an embedder is not +Python's memory at all. + +One experiment did **not** confirm what it was built to test and is recorded as such: scaling the +workload (10,000 / 30,000 / 60,000 cycles) at default thresholds to cross the 80,000 bar gave +1 / 2 / 2 collections, but the 10,000-cycle arm should have given 0. It ran first in its JVM, while +the footprint was still climbing, so the memory gate — the very thing being controlled for — +supplied the collections. The scaling prediction is untested, not confirmed. + +#### What this changes + +- The free-threaded case now has a **bound**, where §8 could only say the residue was bimodal and + refuse to assert one. `FreeThreadedGCGateTest` holds free-threading to the same + "reclaims more than half" bound as the GIL build *once the gate the source names is open*, and + asserts the matched no-op on the GIL build so that a CPython change to either gate fails the test + and says which one. +- `GCSchedulingMeasurementTest.measureCyclicGarbageWithAutoDrain` still asserts no bound + free-threaded in the default configuration, and that is now a documented consequence rather than + an unknown: at default thresholds the outcome depends on the JVM's footprint history, so it is + genuinely not boundable, and the two arms that reclaimed in §8b's single-scope row were the JVM + still warming up. +- §8b's rejection stands and is now explained: scope shape cannot matter because the gate never + looks at thread state. + +#### The advice does not change + +An embedder can open the gate deliberately — `gc.set_threshold(t0, 0, 0)` restores count-driven +collection free-threaded, and a small `threshold0` lowers the `40 × threshold0` bar — but both +trade throughput for it, and both are tuning of CPython internals that the Stable ABI does not +promise. The supported way to collect cycles in an embedder that runs no Python bytecode is still +the one that works identically on both builds, documented in §4: `PyGC_Collect()`, at a cost +proportional to the heap. diff --git a/docs/archive/investigations/gil-parking-investigation.md b/docs/archive/investigations/gil-parking-investigation.md new file mode 100644 index 000000000..83709d809 --- /dev/null +++ b/docs/archive/investigations/gil-parking-investigation.md @@ -0,0 +1,369 @@ +> **Superseded** by [docs/investigations/gil-parking-investigation.md](../../investigations/gil-parking-investigation.md) on 2026-10-03 — the conclusion, numbers and reproduction steps live there; this is the full original narrative, kept for history. + +# Why enabling `PyEval_SaveThread()` in `Python3.initialize()` crashes desktopTest + +Research only. No source in this repo was changed to produce this document; all CPython claims +below are cited from `python/cpython` at tag `v3.14.0` on GitHub, fetched read-only. + +## 0. Summary and confidence + +**Ruled out, with source evidence:** the failure is not "`PyGILState_Ensure` creates a second, +competing `PyThreadState`" on the initialising thread. It reuses the *same* `PyThreadState*` that +`Py_Initialize()` created. This is CPython's documented, intended mechanism for exactly this +pattern (park, then re-enter from a C thread later) — see §2 and §3. + +**Established, with source evidence, but not sufficient by itself:** `PyEval_SaveThread()` leaves +the calling OS thread's slot in the GILState TSS (`autoTSSkey`) pointing at the parked +`PyThreadState`. CPython's own comment in `pystate.c` says so explicitly: "We do not unbind the +gilstate tstate here. It will still be used in `PyGILState_Ensure()`." Reusing the same, +still-registered thread state is a supported idiom (it's what `Py_BEGIN_ALLOW_THREADS` / +`Py_END_ALLOW_THREADS` do on every existing embedding), so this fact **describes the mechanism +correctly but does not, by itself, explain a segfault.** Something else that the reuse *doesn't* +restore is the more likely proximate cause, and I could not pin that down from source reading +alone. See §5 for the two candidates and their weaknesses. + +**Confidence: medium on the mechanism (thread-state identity across the park/re-enter boundary), +low on the exact reason it becomes memory-unsafe.** The minimal experiment in §6 is designed to +close that gap cheaply, without running the full suite. + +## 1. Same thread, established rather than assumed + +`PythonTestFixture.available` is a Kotlin `by lazy` property; `Python3.initialize()` runs on +whichever thread first touches it. `desktopTest`'s Gradle configuration +(`python-multiplatform/build.gradle.kts:724`) sets no `maxParallelForks`, no +`junit.jupiter.execution.parallel.*` property exists anywhere in the repo, and no +`junit-platform.properties` exists. Gradle's `Test` task therefore runs the whole suite +sequentially inside one worker JVM, on the JVM's single test-execution thread — which is named +`"Test worker"` by Gradle, matching the thread name in today's crash report verbatim. So +`Python3.initialize()` and `EmbedApiLowLevelTest.importReturnsANewReferenceAndMissingModulesReportFailure` +run on the **same OS thread**, in the same process, with nothing else attaching in between. + +This directly determines which branch of `PyGILState_Ensure()` fires later (§3): the +"already-registered" branch, not the "first time on this OS thread" branch. + +## 2. What `PyEval_SaveThread()` does to the calling thread's state + +Source (`Python/ceval_gil.c`, v3.14.0): + +```c +PyThreadState * +PyEval_SaveThread(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + _PyThreadState_Detach(tstate); + return tstate; +} +``` + +`_PyThreadState_Detach` (`Python/pystate.c`) calls `detach_thread(tstate, _Py_THREAD_DETACHED)`: + +```c +static void +detach_thread(PyThreadState *tstate, int detached_state) +{ + assert(_Py_atomic_load_int_relaxed(&tstate->state) == _Py_THREAD_ATTACHED); + assert(tstate == current_fast_get()); + if (tstate->critical_section != 0) { + _PyCriticalSection_SuspendAll(tstate); + } + ... + tstate_deactivate(tstate); + tstate_set_detached(tstate, detached_state); + current_fast_clear(&_PyRuntime); + _PyEval_ReleaseLock(tstate->interp, tstate, 0); +} +``` + +and `tstate_deactivate`: + +```c +static inline void +tstate_deactivate(PyThreadState *tstate) +{ + ... + tstate->_status.active = 0; + + // We do not unbind the gilstate tstate here. + // It will still be used in PyGILState_Ensure(). +} +``` + +So: the `PyThreadState` object itself is **not destroyed**. It is marked inactive/detached, the +GIL is released, and the per-OS-thread "current thread state" pointer +(`current_fast_get`/`current_fast_clear`, a `thread_local` variable) is cleared to `NULL`. What is +**not** touched anywhere in this path: `tstate->gilstate_counter`, and the GILState TSS slot +(`gilstate_tss` / historically `autoTSSkey`) that `PyGILState_Ensure` consults. That slot is a +*separate* piece of per-OS-thread storage from the "current thread state" pointer, and CPython's +own comment above states outright that detaching deliberately leaves it alone. + +By the end of `Py_Initialize()` (before the parking call), the calling thread's GILState TSS slot +is already bound to the main thread's `PyThreadState`, with `gilstate_counter == 1` — confirmed by +`_PyGILState_SetTstate()`'s debug assertions (`Python/pystate.c`): + +```c +assert(runtime->gilstate.autoInterpreterState == tstate->interp); +assert(gilstate_tss_get(runtime) == tstate); +assert(tstate->gilstate_counter == 1); +``` + +(`bind_gilstate_tstate` is what performs that binding, called from `tstate_activate` — i.e. from +inside `_PyThreadState_Attach`/`PyEval_RestoreThread` — the first time the main thread is +attached during startup.) + +## 3. What `PyGILState_Ensure()` then does on that same thread + +Source (`Python/pystate.c`, v3.14.0): + +```c +PyGILState_STATE +PyGILState_Ensure(void) +{ + ... + PyThreadState *tcur = gilstate_tss_get(runtime); + int has_gil; + if (tcur == NULL) { + tcur = new_threadstate(runtime->gilstate.autoInterpreterState, + _PyThreadState_WHENCE_GILSTATE); + ... + bind_tstate(tcur); + bind_gilstate_tstate(tcur); + tcur->gilstate_counter = 0; + has_gil = 0; + } + else { + has_gil = holds_gil(tcur); + } + if (!has_gil) { + PyEval_RestoreThread(tcur); + } + ++tcur->gilstate_counter; + return has_gil ? PyGILState_LOCKED : PyGILState_UNLOCKED; +} +``` + +Because the GILState TSS slot on this OS thread is still bound (§2), `tcur` is **not** `NULL`, so +the "new thread" branch (which would create a genuinely second `PyThreadState`) is **not** taken. +`tcur` is the exact same `PyThreadState*` `Py_Initialize()` created. `holds_gil(tcur)` compares +`tcur` against `current_fast_get()`, which is `NULL` after parking, so `has_gil = 0`, and +`PyEval_RestoreThread(tcur)` re-attaches that same object (`current_fast_set`, GIL re-acquired, +`tstate_activate` — which is a no-op on the already-bound gilstate slot). `gilstate_counter` goes +1 → 2. `PyGILState_Release()` mirrors this exactly: decrements 2 → 1, and because the saved +`oldstate` was `PyGILState_UNLOCKED`, it re-parks with another internal `PyEval_SaveThread()` call. +This is self-consistent and, cycled repeatedly, returns to the same `gilstate_counter == 1`, +detached baseline every time. **Nothing in this path frees, reallocates, or otherwise clears the +frame stack, exception state, or recursion bookkeeping that the object already carried** — it is +literally the same object, reused. + +This rules out "two thread states corrupting each other's interpreter frame stack" as the +mechanism, at least on the initialising thread — there is only ever one `PyThreadState` in play +here. It does not, by itself, identify what *does* go wrong. + +## 4. Whether `PyGILState_Release` is handed an invalid state + +No. Traced above: the `PyGILState_STATE` value threaded through `withGIL`'s `tState.state` / +`PyGILState_Release` call is always the value `PyGILState_Ensure` just returned for the current, +still-registered `tcur`. There is no thread-identity mismatch and no stale/deleted state reaching +`Release` in this flow. (Contrast with the *different*, already-fixed bug documented in +`GCLeakTest.desktop.kt:6-11`: an earlier version called `PyEval_SaveThread()` from `forceGC()` on +a thread that was not attached at all. `PyEval_SaveThread`'s body dereferences +`_PyThreadState_GET()` unconditionally — on a PGO/LTO release build, `assert()`s such as +`detach_thread`'s `assert(tstate == current_fast_get())` compile out, so a `NULL` tstate reaches +`detach_thread` and is dereferenced (`tstate->critical_section`) with no check. That is a plain +null-pointer crash from calling `PyEval_SaveThread` while unattached, and it is a different bug +shape from today's: today's parking call is made while the thread *is* attached, immediately +after `Py_Initialize()`, so this specific failure mode does not apply.) + +## 5. What CPython 3.14 changed here, and the two remaining candidates + +The whole attach/detach vocabulary read above (`_PyThreadState_Attach`/`Detach`, +`tstate_activate`/`deactivate`, the `_status.active` / `_status.bound_gilstate` bitfield, +`current_fast_get`/`set`/`clear`) is the free-threading-era (PEP 703) rewrite of thread-state +lifecycle management, unified so the same source serves both GIL and `Py_GIL_DISABLED` builds. I +did not verify the exact CPython version this landed in (older CPython used a simpler +`PyEval_AcquireThread`/`ReleaseThread` pair over a single `interp->tstate_current` swap), but it is +recent relative to most public documentation and tutorials on `PyGILState_*`, which is worth +knowing before trusting older blog posts or Stack Overflow answers about this API. + +Two candidates for what actually breaks, neither confirmed: + +**Candidate A — stale per-thread C-stack recursion limits.** Each `_PyThreadStateImpl` carries +`c_stack_top` / `c_stack_soft_limit` / `c_stack_hard_limit`, computed **once** by +`_Py_InitializeRecursionLimits()` from the current machine stack pointer, and only if +`c_stack_hard_limit == 0`: + +```c +void +_PyThreadState_Attach(PyThreadState *tstate) +{ + ... + if (_tstate->c_stack_hard_limit == 0) { + _Py_InitializeRecursionLimits(tstate); + } + ... +} +``` + +Because our thread reuses the same `PyThreadState` object across every park/re-enter cycle, this +calibration happens exactly once — at whatever C call depth `Py_Initialize()`'s *first* internal +attach happened at — and is **never** recomputed on later re-entries from a structurally different +call depth (the JVM/Panama boundary is a different C stack shape than CPython's own +`Py_InitializeFromConfig` call chain). `_Py_CheckRecursiveCall` compares the live stack pointer +against this stale `c_stack_hard_limit` on every recursive C call in the eval loop +(`if (here_addr < c_stack_hard_limit) Py_FatalError(...)`). This is a real, citable staleness bug +in the reuse path — but by itself it explains a *miscalibrated guard*, and the failure it produces +when it fires is a `RecursionError` or a `Py_FatalError` with the text "Unrecoverable stack +overflow", neither of which matches today's raw `SIGSEGV` (no such message was reported), and a +single-frame `PyImport_ImportModule("sys")` call is not deep enough to plausibly exhaust several +hundred KB to MB of C stack on its own. **I could not make this candidate fit the observed crash +without more evidence, and flag it mainly because it's a real, separate latent hazard in the reuse +pattern that the fix should be aware of regardless of whether it explains today's crash.** + +**Candidate B — something the eval loop touches on the initial (never-parked) startup path that +`import sys`'s cache-hit-with-Python-level-bookkeeping path exercises differently.** +`PyImport_ImportModule("sys")` for an already-loaded module does not simply `PyDict_GetItemString` +and return — `import_ensure_initialized()` (in `Modules/import.c`, not fetched in this +investigation) can invoke Python-level machinery (attribute lookups on `__spec__`, method calls), +which is consistent with the reported crash frames (`PyObject_CallMethodObjArgs` → +`_PyFunction_Vectorcall` → `DICT_MERGE`) being genuine bytecode execution rather than a pure C +lookup. I could not determine, from source reading alone, why *this* bytecode execution — on a +thread whose `PyThreadState` is otherwise identical to the one `Py_Initialize()` used successfully +— would be unsafe. This is the open question the experiment in §6 is aimed at. + +I was not able to reduce this to one explanation. Ranked: reuse of the same `PyThreadState` across +the park/re-enter boundary is established fact and is the right place to keep looking; which piece +of state that reuse fails to restore is unresolved, with Candidate A eliminated by the mismatch in +failure shape and Candidate B unconfirmed. + +## 6. Minimal experiment (does not require the full suite) + +Add a single desktop test — not to run today, but to hand to the next attempt — that escalates in +four steps on **one thread**, printing/asserting between each so a failure localizes to a specific +step instead of reporting "0 tests run": + +```kotlin +@Test +fun parkThenReenterSameThreadIsolatesTheFailure() { + Python3.initialize(silent = true) // must NOT be parked yet at this point + val savedBeforeGuess = PyGILState_GetThisThreadState() // record identity #1, still attached + println("before park: $savedBeforeGuess") + + val parked = PyEval_SaveThread() // manual parking, mirroring the + // commented-out call in initialize() + println("PyEval_SaveThread returned: $parked") + + // Step 1: bare re-entry, no Python C API call at all inside the guard except one that + // reads state without touching bytecode. + withGIL { + val after = PyGILState_GetThisThreadState() + println("after Ensure: $after") + // EXPECT: after == savedBeforeGuess (same PyThreadState*, confirms/refutes §3 empirically + // rather than from source reading alone). A mismatch would mean the "new thread" branch + // fired -- i.e. the TSS slot was NOT still bound, contradicting the CPython source read + // here, and pointing at something specific to this embedding (e.g. the JVM's native + // thread setup interacting with TSS) rather than to CPython's generic behaviour. + } + // Step 1 crashing here would mean the attach/detach cycle itself is broken -- contradicts + // this document's source reading, and the desktop Panama bindings for PyEval_SaveThread / + // PyGILState_Ensure/Release (bindings.kt:41-50) would be the next thing to audit, since this + // project has a documented history of invoke/invokeExact and ADDRESS/JAVA_LONG mismatches. + + // Step 2: a pure-C call that cannot execute Python bytecode. + withGIL { + Py_IsInitialized() + } + // Step 2 crashing but not Step 1 would point at something broken in ordinary C API use after + // reattachment, unrelated to bytecode execution -- undermining Candidate B. + + // Step 3: the exact call that crashed today. + withGIL { + PyImport_ImportModule("sys") + } + // Step 3 crashing but not Steps 1-2 confirms it's specifically bytecode execution (not mere + // attachment, not mere C API use) that's unsafe after reattachment -- supports Candidate B + // and argues against Candidate A (see the manual-lookup step below). + + // Step 4 (only if Step 3 crashes): bypass import_ensure_initialized's Python-level path + // entirely, staying in pure C, to see whether a plain cache lookup alone is safe: + withGIL { + val modules = PySys_GetObject("modules") + PyDict_GetItemString(modules!!, "sys") + } + // If Step 4 is safe where Step 3 is not, that isolates the crash to whatever Python-level + // code import_ensure_initialized runs on a cache hit -- worth reading Modules/import.c's + // import_ensure_initialized next, specifically for anything that inspects C-stack depth, + // per-thread caches, or recursion state. +} +``` + +This is deliberately not `@Test`-annotated as ready to run blind — the next attempt should run it +alone (`--tests` filter), on a quiet tree, and stop at the first step that crashes rather than +proceeding to the next. That directly gives a call-depth-bounded bisection of the failure, which +the "audit every call site" and "run the whole suite" attempts so far have not produced. + +`PyGILState_GetThisThreadState` and `PySys_GetObject`/`PyDict_GetItemString` are already declared +in `EmbedAPI.kt` (`commonMain`); their desktop `actual`s were not verified as part of this +investigation (see §7). + +## 7. What the fix would look like, under each candidate + +- **If Candidate A (stale C-stack calibration) turns out to matter**, no amount of call-site + guarding fixes it — the fix has to either recalibrate `c_stack_hard_limit` on every reattachment + (not something this project's API surface exposes; would need a non-limited-API call or a + CPython-side change) or ensure the *first* attach happens at a call depth structurally + representative of where Python will actually run later. Concretely: call `Py_Initialize()` + itself from inside the same trampoline depth used by ordinary `withGIL` calls, rather than + directly from `Python3.initialize()`'s own stack frame, so the one-time calibration isn't + systematically shallower or deeper than steady-state use. This is a real design change, not a + one-line fix. + +- **If Candidate B (something Python-level import bookkeeping depends on) turns out to matter**, + the fix likely isn't about *this* call site specifically — `PyImport_ImportModule("sys")` is + representative of "any first bytecode execution after reattachment", not a special case — so + narrowing further requires the bisection in §6 before a fix can be scoped at all. + +- **If the experiment shows Step 1 or Step 2 already crashes** (i.e. even bare reattachment or a + pure-C call is unsafe), that overturns the "reuse is fine, something else breaks" framing of this + whole document, and the next place to look is the desktop Panama bindings for exactly these four + functions (`bindings.kt:41-50`) — this project has a documented, repeated history of exactly this + class of bug (`invoke` vs `invokeExact`, `ADDRESS` vs `JAVA_LONG`), and while a quick read during + this investigation found `PyEval_SaveThread`/`RestoreThread`/`PyGILState_Ensure`/`Release` all + using `invokeExact` and the project's `P = JAVA_LONG` pointer convention correctly, that read was + not exhaustive (the `find()` symbol-lookup helper and the underlying `MethodHandle` descriptors + it builds were not traced end to end). + +- **A structurally different fix, independent of which candidate is right:** if reusing the + initialising thread's `PyThreadState` across a park/re-enter boundary turns out to be + fundamentally what's unsafe in this embedding (as opposed to something narrower), the design + question becomes whether the initialising thread should ever call `PyGILState_Ensure` again at + all. An alternative shape: run `Py_Initialize()` on a dedicated thread that parks once and is + never reused for Python work afterward — every subsequent call, including from what is currently + "the same thread" in tests, would then go through the "first time on this OS thread" branch of + `PyGILState_Ensure`, which creates a genuinely fresh `PyThreadState` rather than resurrecting the + init-time one. That is a bigger change than uncommenting the parking line (it changes where + `Py_Initialize()`/`Py_Finalize()` are allowed to be called from, and `Python3.finalize()`'s + `PyEval_RestoreThread(mainThreadState)` call would need to run on that same dedicated thread + too), and it should not be attempted until the experiment in §6 has actually localized the fault + — building it on a guess would repeat the pattern this document was asked to stop. + +## 8. What could not be determined + +- **The exact opcode-level or memory-level reason `DICT_MERGE` faults.** Both candidates in §5 are + plausible mechanisms consistent with *some* of the evidence, but neither was confirmed against + the specific crash. This needs the experiment in §6, or a debugger session (`lldb` on the core, + or a build with `Py_DEBUG` to turn the compiled-out `assert()`s back on) — deliberately not + attempted here per the "prefer reading over running" instruction for this task. +- **Whether `import_ensure_initialized`'s Python-level path (Candidate B) is real** — I did not + fetch `Modules/import.c`, so the claim that a `sys.modules` cache hit still runs Python-level + code is inferred from general CPython import-machinery knowledge, not confirmed against 3.14 + source in this session. +- **Which CPython version introduced the current attach/detach vocabulary and the C-stack-address + recursion guard.** Both read as free-threading-era (PEP 703) work; I did not verify version + numbers, so treat "this is new" as directionally right but not dated. +- **Whether the desktop Panama bindings for the four functions in play are fully correct end to + end.** `bindings.kt:41-50` looked right on inspection (consistent `invokeExact`, consistent + `JAVA_LONG` pointer convention), but the `find()` helper and `MethodHandle` descriptor + construction underneath were not traced, and this project has a specific documented history of + bugs in exactly this area. +- **iOS.** This document is about the desktop crash only. ROADMAP §1 already separately notes that + the iOS simulator's `GCLeakTest` failure is not GIL-related and is a distinct, unexamined defect; + nothing here bears on it. diff --git a/docs/archive/investigations/jni-call-convention-audit.md b/docs/archive/investigations/jni-call-convention-audit.md new file mode 100644 index 000000000..874b23750 --- /dev/null +++ b/docs/archive/investigations/jni-call-convention-audit.md @@ -0,0 +1,340 @@ +> **Superseded** by [docs/investigations/jni-call-convention-audit.md](../../investigations/jni-call-convention-audit.md) on 2026-10-03 — the conclusion, numbers and reproduction steps live there; this is the full original narrative, kept for history. + +# JNI Calling Convention Audit + +Android binds CPython through `RegisterNatives`. Because `@CriticalNative` and `@FastNative` block the garbage collector and `@CriticalNative` lacks a `JNIEnv`, they must never be used for functions that can re-enter the Java runtime by running arbitrary Python code. + +This document classifies all 71 registered JNI functions in `jni_onload.def`. + +> ## Correction, added after verifying against the call sites +> +> **Section 1's headline is a false alarm.** It reads the registration table as the live +> configuration and concludes that six re-entrant functions are registered `@CriticalNative` and +> must be demoted. They are not live. +> +> `jni_onload.def` deliberately registers **all three conventions for each function** -- bare name +> for `@CriticalNative`, `F` suffix for `@FastNative`, `N` suffix for ordinary JNI -- and says so +> at line 105: *"Both are registered so the Kotlin side can pick per API level."* The convention +> that runs is chosen at the **call site**, not by the presence of a registration. +> +> Every one of the six already calls the ordinary variant, with a per-function reason in the +> source: +> +> ``` +> Py_Initialize -> Py_InitializeN "Runs site.py ... never on a GC-blocking path" +> Py_Finalize -> Py_FinalizeN +> PyErr_Clear -> PyErr_ClearN +> PyRun_SimpleString -> PyRun_SimpleStringN +> PyImport_ImportModule -> PyImport_ImportModuleN +> PyObject_GetAttrString -> PyObject_GetAttrStringN +> ``` +> +> The block in `EmbedAPI.android.kt` that appears to call the bare `@CriticalNative` names is +> **inside a `/** */` comment** -- dead code kept from before the migration. Reading it as live is +> what produced the finding. +> +> **Section 2 is a different matter and may be real.** Its promotion targets (`Py_IncRefN`, +> `PyEval_SaveThreadN`, and the rest) are registered *only* under `N`; there is no +> `@CriticalNative` registration to switch to. Acting on it means adding registrations as well as +> moving call sites, which is genuine work rather than a relabel. +> +> Two things to check before acting on it. The cost claim -- "~45ns on API 26-31, only ~2-4ns on +> API 34+" -- points the opposite way from what this repo measured: `@CriticalNative`'s net cost +> is near zero up to API 33 and rises sharply after (24.35ns on 34, 44.05ns on 36 hardware). If +> those measurements hold, promotion helps old devices and *hurts* new ones, which inverts the +> recommendation. And "Unresolved entries: none" overstates the confidence available from reading +> names and documentation alone; the classification below is a starting point for a human, not a +> settled answer. +> +> **What survives is the classification itself**, which is the useful part: it is the rule for +> choosing a convention when a *new* call site is written, and that is where the crash ROADMAP §3 +> warns about would actually be introduced. +> +> Section 2 was checked against those measurements and **rejected**; see *Resolution* at the end of +> this document for the judgement, the three classifications in it that are wrong, and the one +> change that was made instead. + + +## 1. Mismatches + +These functions are currently registered as `@CriticalNative` (no suffix) or `@FastNative` (F suffix), but they can execute arbitrary Python code. These are latent crashes and must be demoted to ordinary JNI. + +- **`Py_Initialize`** / **`Py_InitializeF`** + - **Re-entry path**: Imports the `site` module during startup by default, which executes module-level Python code. +- **`Py_Finalize`** / **`Py_FinalizeF`** + - **Re-entry path**: Invokes `atexit` callbacks and executes `__del__` methods of surviving objects during teardown. +- **`PyErr_Clear`** / **`PyErr_ClearF`** + - **Re-entry path**: Decrements the refcount of the current exception object, type, and traceback. If the refcount drops to zero, `__del__` is called. +- **`PyRun_SimpleString`** / **`PyRun_SimpleStringF`** + - **Re-entry path**: Parses, compiles, and executes the provided string in the `__main__` module. +- **`PyImport_ImportModule`** / **`PyImport_ImportModuleF`** + - **Re-entry path**: Executes the top-level Python code of the module being imported. +- **`PyObject_GetAttrString`** / **`PyObject_GetAttrStringF`** + - **Re-entry path**: Attribute lookup dispatches to `__getattribute__`, `__getattr__`, or descriptor `__get__` methods written in user code. + +## 2. Safe Promotions + +These functions are registered as ordinary JNI (with the `N` suffix or implicitly ordinary) but are provably leaves or non-blocking, and therefore safe to promote. Given that `@CriticalNative` is practically free on API 26-33 but costs ~24ns net on API 34+ (while `@FastNative` is ~2ns on 34+), the value of promoting these depends on the target device. + +- **`Py_IncRefN`, `Py_NewRefN`, `Py_XNewRefN`** + - **Argument for promotion**: These simply increment the `ob_refcnt` struct field. They do not trigger deallocations, do not allocate memory (no GC), and do not call Python code. Safe for `@CriticalNative`. +- **`PyGILState_ReleaseN`, `PyEval_SaveThreadN`** + - **Argument for promotion**: Unlocking the GIL mutex does not block and does not invoke Python code. Safe for `@CriticalNative`. +- **`PyGILState_GetThisThreadStateN`, `PyThreadState_GetDictN`** + - **Argument for promotion**: These only read from Thread Local Storage or read a pointer from a struct. No Python execution. Safe for `@CriticalNative`. +- **`PyEval_InitThreadsN`** + - **Argument for promotion**: Initializes locking (or is a no-op). Does not run Python code. Safe for `@CriticalNative`. +- **`asmListToArray`** + - **Argument for promotion**: This composed function only calls `PyList_Size` and `PyList_GetItem`, both of which are O(1) leaf array reads that do not mutate refcounts. Because it accesses a Java `jlongArray`, it still requires `JNIEnv`, so it cannot be `@CriticalNative`. It is safe to promote to `@FastNative`. + +## 3. Full Classification Table + +| Function Name | Current Convention | Should Be | Can Reach Python? (Mechanism) | +|---|---|---|---| +| `Py_Initialize` | Critical | Ordinary | Yes. Imports `site` module during startup. | +| `Py_IsInitialized` | Critical | Critical | No. Reads a global boolean. | +| `Py_Finalize` | Critical | Ordinary | Yes. Runs `atexit` callbacks and `__del__` on remaining objects. | +| `PyErr_Clear` | Critical | Ordinary | Yes. Decrements exception refcount; dropping to 0 runs `__del__`. | +| `PyLong_FromLongLong` | Critical | Critical | No. Primitive `malloc`, does not trigger cyclic GC. | +| `PyList_Size` | Critical | Critical | No. Reads `ob_size` struct field. | +| `PyRun_SimpleString` | Critical | Ordinary | Yes. Executes arbitrary Python string. | +| `Py_GetVersion` | Critical | Critical | No. Returns static string pointer. | +| `PyImport_ImportModule` | Critical | Ordinary | Yes. Executes top-level module code. | +| `PyObject_GetAttrString` | Critical | Ordinary | Yes. Invokes `__getattr__` or `__get__`. | +| `PyErr_Occurred` | Critical | Critical | No. Reads thread state. | +| `echo0` | Critical | Critical | No. Probe, returns argument. | +| `echo1` | Critical | Critical | No. Probe, returns argument. | +| `echo2` | Critical | Critical | No. Probe, returns argument. | +| `echoNormal` | Ordinary | Ordinary | No. Probe, requires JNIEnv. | +| `echoFast` | FastNative | FastNative | No. Probe, requires JNIEnv. | +| `PyList_SizeFast` | FastNative | FastNative | No. Leaf read. | +| `PyList_SizeNormal` | Ordinary | Ordinary | No. Leaf read. | +| `Py_InitializeF` | FastNative | Ordinary | Yes. Imports `site` module during startup. | +| `Py_IsInitializedF` | FastNative | FastNative | No. Reads a global boolean. | +| `Py_FinalizeF` | FastNative | Ordinary | Yes. Runs `atexit` callbacks and `__del__`. | +| `PyErr_ClearF` | FastNative | Ordinary | Yes. Decrements exception refcount; dropping to 0 runs `__del__`. | +| `PyLong_FromLongLongF` | FastNative | FastNative | No. Primitive `malloc`, does not trigger cyclic GC. | +| `PyList_SizeF` | FastNative | FastNative | No. Reads `ob_size` struct field. | +| `PyRun_SimpleStringF` | FastNative | Ordinary | Yes. Executes arbitrary Python string. | +| `Py_GetVersionF` | FastNative | FastNative | No. Returns static string pointer. | +| `PyImport_ImportModuleF` | FastNative | Ordinary | Yes. Executes top-level module code. | +| `PyObject_GetAttrStringF` | FastNative | Ordinary | Yes. Invokes `__getattr__` or `__get__`. | +| `PyErr_OccurredF` | FastNative | FastNative | No. Reads thread state. | +| `PyList_GetItemRaw` | Critical | Critical | No. Borrows reference from array index (leaf). | +| `ffiDirectBufferAddress` | Ordinary | Ordinary | No. Uses JNIEnv. | +| `asmExec` | Ordinary | Ordinary | Yes. Composes `PyRun_String`. | +| `asmGetAttr` | Ordinary | Ordinary | Yes. Composes `PyObject_GetAttrString` and `PyErr_Clear`. | +| `asmListToArray` | Ordinary | FastNative | No. Only calls leaf functions (`PyList_Size`, `PyList_GetItem`). | +| `Py_InitializeN` | Ordinary | Ordinary | Yes. Imports `site` module during startup. | +| `Py_FinalizeN` | Ordinary | Ordinary | Yes. Runs `atexit` callbacks and `__del__`. | +| `PyErr_ClearN` | Ordinary | Ordinary | Yes. Decrements exception refcount; dropping to 0 runs `__del__`. | +| `PyRun_SimpleStringN` | Ordinary | Ordinary | Yes. Executes arbitrary Python string. | +| `PyImport_ImportModuleN` | Ordinary | Ordinary | Yes. Executes top-level module code. | +| `PyObject_GetAttrStringN` | Ordinary | Ordinary | Yes. Invokes `__getattr__` or `__get__`. | +| `Py_DecRefN` | Ordinary | Ordinary | Yes. Reaching refcount 0 deallocates and runs `__del__`. | +| `Py_IncRefN` | Ordinary | Critical | No. Only increments struct field `ob_refcnt`. | +| `Py_NewRefN` | Ordinary | Critical | No. Only increments struct field `ob_refcnt`. | +| `Py_XNewRefN` | Ordinary | Critical | No. Only increments struct field `ob_refcnt`. | +| `PyGILState_EnsureN` | Ordinary | Ordinary | No, but blocks. Blocking while suspending JVM GC risks deadlock. | +| `PyGILState_ReleaseN` | Ordinary | Critical | No. Unlocks mutex (non-blocking). | +| `PyGILState_GetThisThreadStateN` | Ordinary | Critical | No. Reads TLS. | +| `PyEval_SaveThreadN` | Ordinary | Critical | No. Unlocks GIL mutex (non-blocking). | +| `PyEval_RestoreThreadN` | Ordinary | Ordinary | No, but blocks waiting for GIL. Risks deadlock if GC is suspended. | +| `PyEval_InitThreadsN` | Ordinary | Critical | No. Initializes locking. | +| `PyThreadState_GetDictN` | Ordinary | Critical | No. Reads struct pointer. | +| `PyTuple_NewN` | Ordinary | Ordinary | Yes (inferred). GC tracked container allocation can trigger cyclic GC, which can run `__del__`. | +| `PyTuple_SetItemN` | Ordinary | Ordinary | Yes. Overwriting an existing item drops its refcount, potentially running `__del__`. | +| `PyDict_NewN` | Ordinary | Ordinary | Yes (inferred). GC tracked container allocation can trigger cyclic GC. | +| `PyDict_SetItemStringN` | Ordinary | Ordinary | Yes. Key hashing can invoke `__hash__` / `__eq__`, replacing values decrefs. | +| `PyDict_SetItemN` | Ordinary | Ordinary | Yes. Key hashing can invoke `__hash__` / `__eq__`, replacing values decrefs. | +| `PyDict_GetItemStringN` | Ordinary | Ordinary | Yes. Dictionary lookup can invoke `__eq__` on keys. | +| `PyUnicode_AsUTF8` | Critical | Critical | No. Primitive string allocation/cache. | +| `PyUnicode_AsUTF8F` | FastNative | FastNative | No. Primitive string allocation/cache. | +| `PyUnicode_FromString` | Critical | Critical | No. Primitive string allocation/cache. | +| `PyUnicode_FromStringF` | FastNative | FastNative | No. Primitive string allocation/cache. | +| `PyImport_AddModuleRefN` | Ordinary | Ordinary | Yes. Dict lookup/insertion can invoke `__eq__` or GC. | +| `PyRun_StringN` | Ordinary | Ordinary | Yes. Executes arbitrary Python code. | +| `PyObject_StrN` | Ordinary | Ordinary | Yes. Invokes `__str__`. | +| `PyObject_IsTrueN` | Ordinary | Ordinary | Yes. Invokes `__bool__` or `__len__`. | +| `PyObject_CallNoArgsN` | Ordinary | Ordinary | Yes. Invokes `__call__`. | +| `PyObject_CallObjectN` | Ordinary | Ordinary | Yes. Invokes `__call__`. | +| `testUpcallPrimitive` | Ordinary | Ordinary | Yes (JNI upcall). Explicitly calls into Java runtime. | +| `testUpcallString` | Ordinary | Ordinary | Yes (JNI upcall). Explicitly calls into Java runtime. | +| `testUpcallUnattached` | Ordinary | Ordinary | Yes (JNI upcall in new thread). | +| `testThreadCreateFloor` | Ordinary | Ordinary | No (thread floor probe), but uses ordinary. | +| `testAttachWithoutDetach` | Ordinary | Ordinary | Yes (attaches a pthread and calls back into Java). | + +## 4. Rules of Thumb + +Based on CPython's documented semantics, the following rules apply when classifying new functions: + +1. **Any decref is re-entrant**: If a function can drop a reference count (e.g. `Py_DecRef`, `PyTuple_SetItem` dropping the overwritten item, `PyDict_SetItem`, `PyErr_Clear`), that reference can hit zero, deallocate the object, and execute arbitrary Python `__del__` methods. +2. **GC allocations are re-entrant**: Allocating GC-tracked containers (tuples, lists, dicts, instances) using functions like `PyTuple_New` or `PyDict_New` can cross the allocation threshold and trigger the cyclic garbage collector. The cyclic GC reclaims unreachable cycles, executing `__del__` on them. +3. **Blocking requires ordinary JNI**: Functions that acquire the GIL (`PyGILState_Ensure`, `PyEval_RestoreThread`) do not run Python, but they *must* stay on ordinary JNI. A thread waiting for a lock while marked `@CriticalNative` prevents the JVM from reaching a safepoint, which deadlocks the entire process if the thread holding the GIL triggers a JVM garbage collection. +4. **Dictionary lookups are re-entrant**: Even reading from a dictionary (`PyDict_GetItemString`) can invoke user code if custom keys exist that define `__eq__` or `__hash__`. +5. **Non-GC allocations are leaves**: Allocating primitives (e.g. `PyLong_FromLongLong`, `PyUnicode_FromString`) uses standard memory allocation (`malloc`) and does not participate in cyclic GC, making them pure leaves. +6. **Increments are leaves**: Functions that strictly increase reference counts (`Py_IncRef`, `Py_NewRef`) do not deallocate and thus do not trigger callbacks. + +## Report Summary + +- **Total Functions Classified**: 71 registrations. +- **Mismatches Found**: 12 (6 unique C functions exposed as both Critical and Fast). These are the dangerous ones: `Py_Initialize`, `Py_Finalize`, `PyErr_Clear`, `PyRun_SimpleString`, `PyImport_ImportModule`, and `PyObject_GetAttrString`. They can run arbitrary Python and must be moved to ordinary JNI. +- **Safe Promotions**: 8 functions (`Py_IncRefN`, `Py_NewRefN`, `Py_XNewRefN`, `PyGILState_ReleaseN`, `PyEval_SaveThreadN`, `PyGILState_GetThisThreadStateN`, `PyThreadState_GetDictN`, `PyEval_InitThreadsN`) can safely be moved from Ordinary to Critical, and `asmListToArray` to FastNative. +- **Cost of Being Wrong**: + - Promoting a re-entrant function to `@CriticalNative` is a fatal crash when upcalls exist. + - Leaving a leaf as Ordinary wastes ~45ns per call on API 26-31, but only ~2-4ns on modern API 34+. Safe promotions are therefore an optimization mostly relevant for older devices or high-volume loops, and should never override correctness. +- **Unresolved Entries**: None. All 71 registrations were resolved confidently based on documented CPython semantics (e.g. GC thresholds on container allocation). + + **This line is wrong, and it is the reason the audit is no longer the source of truth.** Header + prototypes can condemn a function and never clear one: `void Py_Initialize(void)` and + `int PyRun_SimpleString(const char*)` name no `PyObject *` and both run arbitrary Python. A + re-derivation over the table as it stands leaves **65 undecided**, and the confidence recorded + here is what put three re-entrant functions on the safe-to-promote list above. + `JniCallConventionClassificationTest` now derives the classification on every desktop build and + asserts that bucket stays non-empty, with those two functions named in it. + +--- + +# Resolution + +Written after checking section 2 against this repository's own measurements and against the +registration table as it stands at 187 entries (the audit read 71). + +**The table holds 366 today**, so this resolution is itself a snapshot — roughly 180 registrations +have never been classified by any hand-written pass. That is why the classification moved into +`JniCallConventionClassificationTest`, which re-derives it from `bindings.kt`, `jni_onload.def` and +the bundled CPython headers on every desktop build. Its re-derivation agrees with the judgement +recorded here — the eight GC-blocking call sites reproduce exactly, no function was wrongly +promoted, and all 366 prologues match their convention — so what was missing was never correctness, +only a way for it to stay correct. + +## The promotion to `@CriticalNative` is rejected + +Section 2 argues its eight promotions are worth taking because `@CriticalNative` costs "~45ns on +API 26-31, only ~2-4ns on API 34+". That is backwards. Measured here, net of the Kotlin floor +(`docs/design/downcall-design.md`, `androidMain/README.md`), the *change* a promotion buys is: + +| API | ordinary → `@FastNative` | ordinary → `@CriticalNative` | +|---|---|---| +| 26 | −6.64 | −43.47 | +| 30 | −41.33 | −73.33 | +| 31 | −21.68 | −45.99 | +| 33 | −5.79 | −7.86 | +| 34 | −5.84 | **+16.54** | +| 36 | −14.77 | **+25.73** | + +`@CriticalNative` is a **loss** on the two newest levels measured. Taking section 2 as written +would slow every one of those eight functions down on any device from Android 14 onward. + +It could be had behind the existing `preferFastNative` branch — critical below 34, ordinary above. +That is not being done, for three reasons. + +**It is not a relabel.** `@CriticalNative` receives no `JNIEnv` or `jclass`, so it cannot reuse the +`f_*N` wrappers, which all have them. Each promotion needs a new table entry pointing at the raw +CPython symbol, a new `@CriticalNative` declaration, and a branch at the call site. Compare with a +`@FastNative` promotion, which is one annotation: the C signature is already right and the +registration name does not change. + +**Three of the eight cannot be promoted at all.** They are not leaves: + +| | audit says | actually | +|---|---|---| +| `PyGILState_Release` | "unlocks mutex (non-blocking)" | the release that drops the counter to zero deletes the thread state; clearing a thread state decrefs its dict and exception state, so `__del__` runs | +| `PyThreadState_GetDict` | "reads struct pointer" | allocates the thread dict with `PyDict_New` on first call per thread — a GC-tracked allocation, so rule 2 applies | +| `PyEval_InitThreads` | "initializes locking" | a no-op retained for the stable ABI (removed from the C API in 3.13; still exported by the bundled `libpython`, confirmed with `nm -D`). Promoting a no-op buys the convention delta and nothing else | + +That leaves `Py_IncRef`, `Py_NewRef`, `Py_XNewRef`, `PyGILState_GetThisThreadState` and +`PyEval_SaveThread` as genuine leaves — and of those, only the three refcount operations are on a +path called often enough for tens of nanoseconds to accumulate. + +**The audit's own confidence claim does not hold.** "Unresolved entries: none" is not available +from reading names and documentation, which is exactly how the three above were misread. + +## What was changed instead: `PyList_GetItem` + +The static sweep found one place where the current configuration contradicts the measurements, and +it is a demotion rather than a promotion. + +`PyList_GetItemRaw` was registered `@CriticalNative` with **no `@FastNative` twin and no +`preferFastNative` branch** — the only declaration in `bindings.kt` that ignores the device axis. +It is also the per-element call of bulk list iteration, so the penalty multiplies by N. ROADMAP §5 +measured `list → LongArray`, 1000 elements, at 50065.89 ns on API 36 hardware: 50 ns per element, +against a `@CriticalNative` net cost of 44.05 ns on that device. That 50 ns also covers the loop and +the `toNativePointer` conversion, so 44.05 is an upper bound on the convention's share rather than +an attribution — but it is an upper bound of 88%, which is enough to act on. + +So a `@FastNative` twin was added (`f_PyList_GetItemRaw` in `jni_onload.def`, +`PyList_GetItemRawF` in `bindings.kt`) and the call site now branches on `preferFastNative`, the +same shape as `PyList_Size` directly above it. Predicted effect on API 36: 44.05 → 3.55 ns per +element. Nothing else was promoted or demoted. + +This is compile-verified only. `JniWiringTest.bothListGetItemConventionsAgree` asserts the two +registrations return identical pointers at every index of `sys.path`, which catches both a missing +table entry and a wrapper written without the leading `JNIEnv*, jclass`; it has not been run, +because the device was in use. + +It also narrows §5. If a large share of the 11x bulk-iteration win came from the per-element call +running under the slowest available convention on that device, then part of the remaining case for +composition was paying for a configuration mistake rather than for crossings. + +## The registrations added since the audit: 116, all ordinary, no defects + +The audit read 71 entries. The table has 187 — 74 added at `9abc8edd`, 42 at `aea09585`, none +removed. + +**Every one of the 116 is registered `N`-suffixed and declared plain `@JvmStatic external fun`.** +Not one has a `@CriticalNative` or `@FastNative` twin registered, so no call site can be reaching a +GC-blocking variant of them: there is none to reach. The failure mode ROADMAP §3 warns about cannot +be present in the new surface. Verified by extracting the table names from all three revisions and +parsing the annotations in `bindings.kt`. + +Classifying the 42 anyway, since the point of the exercise is the rule for the *next* one. All 42 +are string-carrying; 31 are re-entrant and correctly ordinary: + +| mechanism | functions | +|---|---| +| executes module or arbitrary code | `PyImport_ExecCodeModuleN`, `PyImport_ExecCodeModuleExN`, `PyImport_ExecCodeModuleWithPathnamesN`, `PyImport_ImportModuleLevelN`, `PyImport_ImportModuleNoBlockN` †, `PyImport_ImportFrozenModuleN`, `Py_CompileStringN` | +| runs a Python-level hook | `PyObject_HasAttrStringN`, `PyObject_HasAttrStringWithErrorN` (`__getattr__`), `PyMapping_GetItemStringN`, `PyMapping_SetItemStringN`, `PyMapping_HasKeyStringN`, `PyMapping_HasKeyStringWithErrorN` (`__getitem__`/`__contains__`), `PySequence_FastN` (`__iter__`), `PyUnicode_TranslateN` (`__getitem__` on the table), `PySys_AuditTupleN` (audit hooks), `PyErr_WarnExplicitN` (the `warnings` machinery) | +| Python codec lookup / error handler | `PyUnicode_AsEncodedStringN`, `PyUnicode_FromEncodedObjectN`, `PyUnicode_DecodeLocaleN`, `PyUnicode_EncodeLocaleN`, `PyUnicode_DecodeFSDefaultN` | +| drops a reference, or allocates GC-tracked | `PyDict_DelItemStringN`, `PySys_SetObjectN`, `PyErr_NewExceptionN`, `PyErr_NewExceptionWithDocN`, `PyErr_SetFromErrnoWithFilenameN`, `PyErr_SyntaxLocationN`, `PyErr_SyntaxLocationExN`, `PyUnicodeTranslateError_SetReasonN` | +| never returns | `Py_FatalErrorN` | + +† `PyImport_ImportModuleNoBlockN` has since been unregistered: CPython 3.15 removed +`PyImport_ImportModuleNoBlock`, which had been an exact alias of `PyImport_ImportModule` since 3.3 +(ROADMAP §9). The counts above are left as they were read, since this section records what the +table held at `9abc8edd`/`aea09585` rather than what it holds now. The same migration also +unregistered `PySys_ResetWarnOptionsN` and renamed `PyWeakref_GetObjectN` to `PyWeakref_GetRefN`; +none of the three changed anyone's convention, and the replacement is registered ordinary like the +function it replaces. + +The remaining 11 are leaves on the success path: `PyBytes_FromStringN`, `PyBytes_AsStringN`, +`PyByteArray_AsStringN`, `PyUnicode_EqualToUTF8N`, `PyUnicode_CompareWithASCIIStringN`, +`PyUnicode_InternFromStringN`, `PySys_GetObjectN`, `PyImport_GetMagicTagN`, `PyEval_GetFuncNameN`, +`PyEval_GetFuncDescN`, `Py_EnterRecursiveCallN`. None of them is promoted, and none should be: +every one is cold error-reporting or introspection surface, where the whole available win is under +40 ns on a call that happens once. Step 4 of the decision procedure in `androidMain/README.md` +covers exactly this case. + +## The eight live GC-blocking call sites + +Outside the `/** */` block of dead pre-migration code, `EmbedAPI.android.kt` reaches a +`@CriticalNative` or `@FastNative` binding in eight places. All eight are leaves on the success +path and all eight can allocate a GC-tracked exception on the failure path: + +| call site | success path | failure path | +|---|---|---| +| `Py_IsInitialized` | reads a global | — | +| `Py_GetVersion` | returns a static `const char*` | — | +| `PyErr_Occurred` | reads the thread state, borrowed | — | +| `PyLong_FromLongLong` | non-GC allocation | preallocated `MemoryError` | +| `PyUnicode_FromString` | non-GC allocation | `UnicodeDecodeError` on malformed UTF-8 | +| `PyUnicode_AsUTF8` | caches the UTF-8 form | `UnicodeEncodeError` on lone surrogates | +| `PyList_Size` | reads `ob_size` | `PyErr_BadInternalCall` on a non-list | +| `PyList_GetItem` | indexes the item array | `IndexError` out of range | + +That second column is second-order and conditional — the exception instance is GC-tracked, so it +can cross the collection threshold, so the cyclic collector can run a `__del__`, which with upcalls +live can be Kotlin. It is not a live defect and it is not a reason to demote these eight, which are +the measured hot path. It is the reason step 4 of the decision procedure does not pretend "provably +a leaf" is reachable: the honest promotion test is *leaf on the success path, and measured worth +it*, not *cannot run Python*. diff --git a/docs/archive/kotlin-extensions-in-python-pyi.md b/docs/archive/kotlin-extensions-in-python-pyi.md new file mode 100644 index 000000000..dbf408b54 --- /dev/null +++ b/docs/archive/kotlin-extensions-in-python-pyi.md @@ -0,0 +1,55 @@ +# Kotlin extensions in Python — the original `.pyi` section (archived) + +> **Superseded** by [`docs/design/pyi-generation-design.md`](../design/pyi-generation-design.md) and +> [SPEC.md](../SPEC.md) B-7 on 2026-10-03; kept for history. +> +> This is §4.5 of `docs/design/kotlin-extensions-in-python.md` as it stood before 2026-10-03. It +> proposed that this repository's Gradle plugin emit a Pythonic stub — extension-as-method, the +> `Modifier` metaclass, `@overload` sets, `Dp | float` for allow-listed value classes. The plugin now +> emits Kotlin-named stubs with boundary-type annotations only, and the Pythonic stub product belongs +> to pythonx-compose. The three measured bullets (`@overload` is mandatory for 33 of 130 `Modifier` +> names; `Dp | float` versus the proxy alone; defaults as `= ...`) remain valid inputs for that +> product. Section references below are to `docs/design/kotlin-extensions-in-python.md` unless a +> path is given; the metaclass and Protocol measurements are in +> [`pyi-generation-pythonic-stubs.md`](pyi-generation-pythonic-stubs.md) §4.3–§4.4. + +### 4.5 `.pyi` + +Generation belongs to the Gradle plugin (`docs/design/ecosystem.md` §5b), and it reads the same metadata +as the binder, so the stub and the binding cannot drift. Extension-as-method appears as an ordinary +method, and the metaclass of §4.2 is expressible: + +> **The stub below is the one that was measured to fail.** mypy resolves `Modifier.padding` to the +> instance method and rejects every class-object call; see §4.2's note and +> `docs/archive/pyi-generation-pythonic-stubs.md` §4.3. It is kept here because the three bullets that follow it are +> about `@overload`, `Dp | float` and defaults, and all three survive unchanged into the shape that +> does work — `padding: ClassVar[_Modifier_padding]` where `_Modifier_padding` is a Protocol whose +> `__call__` carries exactly these overloads (`docs/archive/pyi-generation-pythonic-stubs.md` §4.4). What changes is +> where the overloads are written, not which overloads there are. + +```python +class _ModifierMeta(type): + @overload + def padding(cls, all: Dp | float) -> Modifier: ... + @overload + def padding(cls, horizontal: Dp | float = ..., vertical: Dp | float = ...) -> Modifier: ... + @overload + def padding(cls, padding_values: PaddingValues) -> Modifier: ... + +class Modifier(metaclass=_ModifierMeta): + @overload + def padding(self, all: Dp | float) -> Modifier: ... + ... +``` + +Three things follow from the measurements: + +- **`@overload` is mandatory, not optional.** 33 of 130 `Modifier` names carry more than one + overload, and they are the load-bearing ones (§3.1). A generator that emits one signature per + name will mistype `padding`, `size`, `background`, `border` and `clickable`. +- **`Dp | float` is how the allowlist shows up in the stub**, and a type on the reject list is + stubbed as the proxy alone. The stub is then the documentation for §4.4's asymmetry. +- **Defaults must be carried.** 58% of `Modifier` extension parameters declare one; a stub with + everything required would be wrong about most of the API. Metadata's `declaresDefaultValue` is + the source; the *value* is not in metadata, so stubs emit `= ...`. + diff --git a/docs/archive/pyi-generation-pythonic-stubs.md b/docs/archive/pyi-generation-pythonic-stubs.md new file mode 100644 index 000000000..96fb668a2 --- /dev/null +++ b/docs/archive/pyi-generation-pythonic-stubs.md @@ -0,0 +1,461 @@ +# Generating `.pyi` — the Pythonic stub product (archived) + +> **Superseded** by [`docs/design/pyi-generation-design.md`](../design/pyi-generation-design.md) and +> [SPEC.md](../SPEC.md) B-7 on 2026-10-03; the Pythonic stub product belongs to pythonx-compose; kept +> for history. +> +> These are the parts of `pyi-generation-design.md` (as of 2026-10-02) that designed a second, +> **Pythonic** stub product inside this repository's Gradle plugin: stubs under `pythonx.*`, +> snake_case names, `Dp | float` unions, keyword-only `content`, `@overload` sets, `Modifier` as a +> class of callback-Protocol attributes, and a `pythonx-map.toml` manifest mapping `androidx.*` to +> `pythonx.*`. It was implemented (`PythonNames.kt`, `PythonTypes.kt`, `StubManifest.kt`, +> `renderPythonicStubs`) and removed in `238119b7`, because the binder must never export a Kotlin +> namespace under another name (`docs/INTENT.md` §2.2, `AGENTS.md` §12.1). The plugin now emits only +> Kotlin-named stubs. The measurements below (mypy 2.3.0 runs, the metaclass failure, overload +> ordering, `py.typed` placement) are still valid inputs for whoever builds the Pythonic stubs in +> pythonx-compose. +> +> Section numbers are the original ones, so references such as "§4.4" in the code's history still +> resolve here. Paths in backticks are relative to the repository root. Cross-references to sections +> that were not moved (§0–§2, §3.2, §3.3, §4.1, §4.5, §6.2, §6.3) point at +> `docs/design/pyi-generation-design.md`. + +--- + +## Original introduction + +`pythonx` adapts Kotlin generically rather than wrapping function by function (`docs/design/ecosystem.md` +§5b): a module `__getattr__` finds the corresponding binding, adapts it once and caches it. Nothing +an editor can see is ever enumerated. `.pyi` is what pays that back — the stubs carry the fully +enumerated Pythonic surface the adapter will produce, so an IDE sees everything while the runtime +enumerates nothing. Generation belongs to the Gradle plugin, the way PyREPL did it; that is settled +and this file does not revisit it. + +What this file does is answer the three questions that were open under that decision: **what the +generator reads**, **what it can honestly write**, and **where the result goes**. + +Everything under "Observed" was read from working copies or run as a command on 2026-08-14, with the +path or the command given. Everything under "Judged" is a decision, and the alternatives it was +chosen over are named. Where a claim could not be checked it says so rather than being smoothed over. + + +--- + +## 3. Kotlin → Python type mapping + +### 3.1 The table + +`source` names where the fact comes from: `Km` = `kotlin-metadata-jvm`, `KS` = KSP, `desc` = JVM +descriptor, `list` = the hand-maintained allowlist of §3.4. + +| Kotlin | `.pyi` | source | note | +|---|---|---|---| +| `Boolean` | `bool` | Km/KS | | +| `Byte` `Short` `Int` `Long` | `int` | Km/KS | the boundary carries every one as `Long`; Python has one integer type, so the narrowing is invisible and correct | +| `Float` `Double` | `float` | Km/KS | | +| `String` | `str` | Km/KS | | +| `ByteArray` | `bytes` | Km/KS | | +| `Unit` | `None` | Km/KS | return position only | +| `Nothing` | `typing.Never` | Km/KS | a function returning it never returns; `Never` is the exact Python spelling | +| `T?` | `T \| None` | `KmType.isNullable`, `KSType.isMarkedNullable` | **not bindable today** — `resolveKotlinType` declines nullable outright. §3.3 | +| `List` `MutableList` | `list[T]` | Km/KS type args | not bindable today (the type gate) | +| `Set` | `set[T]` | | | +| `Map` | `dict[K, V]` | | | +| `Collection` `Iterable` | `Iterable[T]` | | read-only shape; `Sequence` would over-promise indexing | +| `Array` | `list[T]` | | judged: the marshaller's Python view is a list | +| `IntArray` `LongArray` `…` | `list[int]` etc. | desc/Km | `ByteArray` is the exception above | +| `() -> Unit` | `Callable[[], None]` | Km classifier `kotlin/Function0` | §3.5 | +| `(A) -> B` | `Callable[[A], B]` | `kotlin/FunctionN` type args | | +| `suspend (…) -> T` | — | `KmFunction.isSuspend`, `KSType.isSuspendFunctionType` | declined by both producers; must not be stubbed | +| value class on the allowlist | `Dp \| float` | Km + `list` | §3.4 | +| value class off the allowlist | `TextUnit` | Km | §3.4 | +| any other class/interface | its own stub name | Km/KS | requires that class to have a stub in the same namespace, which for a `TypeTag.OBJECT` handle is a promise the runtime does not yet keep — §7 | +| a generic type *parameter* (`fun f(x: T)`) | — | | `BindingPolicy` rejects these before the binder sees them; stubbing what cannot be called would be a lie. §7 | + +> Archive note: this table maps the **declared Kotlin type**; it was implemented as `PythonTypes.kt` +> for the Pythonic product and removed with it. The current stubs annotate the **boundary** type +> instead — see `docs/design/pyi-generation-design.md` §3.1. + +### 3.4 Value classes — how the allowlist shows up + +`docs/design/kotlin-extensions-in-python.md` §4.4 decided this and measured why a machine rule fails: the +rule "coerce iff the value class has a public constructor taking exactly its underlying type" +selects 36 of 111 and **admits `Color`**, whose public `ULong` constructor stores while the factory +a Kotlin author actually writes (`Color(Int)`) shifts by 32. Whether a wrapper packs is a semantic +fact with no bytecode witness. + +In a stub the distinction is one union member: + +| Kotlin parameter | `.pyi` | why | +|---|---|---| +| `Dp` (on the allowlist) | `Dp \| float` | public constructor, wraps a plain `Float`, and it *is* the identity on that float. 62% of value-class parameter occurrences on `Modifier` | +| `TextUnit` (rejected) | `TextUnit` | packed `Long`; raw `16` decodes as `Unspecified` — silently no value at all | +| `Color` (rejected despite passing the constructor test) | `Color` | `Color(int)` is `ULong(value) shl 32`; raw `0xFFFF0000` is transparent black through the constructor and red through the factory | +| every `packedValue` class (`Offset`, `Size`, `DpSize`, `IntSize`, `IntOffset`, `DpOffset`, `TransformOrigin`, `CornerRadius`, `Constraints`) | the proxy alone | a user-meaningful number is not the stored number, and none of these failures raises | + +**Measured, §4.4's asymmetry is expressible and enforced.** In the mypy run of §4.4, +`Text("hi", font_size=16)` produced + + error: Argument "font_size" to "Text" has incompatible type "int"; expected "TextUnit" + +while `Modifier.padding(16)` in the same file checked clean. So the stub is not merely documentation +of the asymmetry — a checker acts on it, and the user finds out at edit time rather than seeing a +label render with no font size. + +What a stub cannot carry is §4.4's *third* clause, the error message that names the constructor to +call (`"expected TextUnit; write sp(16)"`). mypy's own message names the type but not the remedy. +Judged: accept it. The runtime raises the full message; the stub gets the user to the runtime less +often. + +**Where the allowlist lives** is `docs/design/kotlin-extensions-in-python.md` §6's open question, and §5.3 +here proposes an answer that falls out of the package mapping: the same manifest. + +### 3.5 Function types, and Compose's `content` + +`() -> Unit` → `Callable[[], None]`, mechanically, from `kotlin/Function0`'s type arguments. 66 of +the 177 chainable `Modifier` extensions take at least one lambda, and every `@Composable` container +takes `content`. + +Two things the stub should do and one it should not: + +- **`content` is keyword-only in Python** even though it is the last positional parameter in Kotlin. + Compose's trailing-lambda syntax has no Python equivalent, and `Column(modifier, lambda: ...)` + reads badly. Emitting `*, content: Callable[[], None]` forces `content=`. Checked in §4.4's run: + `Column(modifier=Modifier.size(4), content=lambda: Text("x"))` resolves clean. +- **`@Composable` on a lambda parameter is not expressible and should not be faked.** A + `@Composable () -> Unit` and a `() -> Unit` are the same Python `Callable[[], None]`. Kotlin's + restriction — a composable lambda may only be invoked in a composable context — has no Python + counterpart, and inventing a distinct alias would give a checker a rule it cannot enforce. + `docs/design/ecosystem.md` §5b already decided the composer is threaded as an ordinary value. +- **Do not stub the synthetic parameters.** `$composer` and `$changed` are in the JVM descriptor and + are not in `KmFunction`. `docs/design/kotlin-extensions-in-python.md` §4.6: metadata arity is the correct + source, and any code deriving arity from the descriptor is wrong for all 500 composables. + +### 3.6 Overloads + +Python has `@overload`, and `docs/design/kotlin-extensions-in-python.md` §4.5 already calls it mandatory: +33 of `Modifier`'s 130 names carry more than one, and they are the load-bearing ones +(`padding`, `size`, `background`, `border`, `clickable`, …). + +**Measured** (`/Volumes/macMini/tmp/pyicheck/ovstub.pyi`, all four real `Modifier.padding` +overloads from §3 of that document, as a callback protocol): + +```python +class _padding(Protocol): + @overload + def __call__(self, paddingValues: PaddingValues) -> Modifier: ... + @overload + def __call__(self, all: Dp | float) -> Modifier: ... + @overload + def __call__(self, horizontal: Dp | float = ..., vertical: Dp | float = ...) -> Modifier: ... + @overload + def __call__(self, start: Dp | float = ..., top: Dp | float = ..., + end: Dp | float = ..., bottom: Dp | float = ...) -> Modifier: ... +``` + +All four coexist. `p(8)`, `p(horizontal=8, vertical=4)`, `p(start=1, bottom=2)` and +`p(PaddingValues())` each resolve to `Modifier`; `p("bad")` is rejected with all four variants +listed. mypy issued **no** overlap diagnostic. + +That last fact is the one to be careful about. mypy reports overlapping overloads only when their +*return types* are incompatible, and every `Modifier` extension returns `Modifier`. So variants 2 and +3 genuinely overlap — `p(8)` matches `all` and also matches `horizontal` positionally — and the +checker silently takes the **first** match. + +**Judged: overload order is part of the generated output and must be deterministic and +arity-ascending.** In the run above, arity-ascending put `all: Dp` before `horizontal, vertical`, and +`p(8)` selected `all` — the right answer, but only because of the order. Two further consequences: + +- The stub's order must be **the same order the Python-side dispatcher resolves in**. If the runtime + dispatcher (`docs/design/kotlin-extensions-in-python.md` §6, unsolved) picks differently, the stub lies + about which Kotlin function runs. The two orders must come from one place in the generator. +- **Defaults are emitted as `= ...`.** 254 of 435 `Modifier` extension parameters declare one and + metadata carries `declaresDefaultValue` but not the *value*. `= ...` is the correct stub spelling + for "has a default I cannot name" and is what makes the keyword-only overloads above work. + +Kotlin parameter names go through the same snake_case conversion as everything else +(`paddingValues` → `padding_values`), which `pythonx-compose` already does by hand: `text.py` maps +`font_size` → `fontSize`, `letter_spacing` → `letterSpacing`, and 14 more. + +## 4. Extension functions — and the metaclass proposal does not survive contact + +This is the section the Compose surface turns on: 604 public top-level extension functions over 136 +receivers, and 177 chainable ones on `Modifier` alone. + +(§4.1, Placement, stayed in `docs/design/pyi-generation-design.md`.) + +### 4.2 The proposal under test + +`docs/design/kotlin-extensions-in-python.md` §4.2 and §4.5 propose that `Modifier` be one class whose class +object also behaves like an instance, via a metaclass, so that `Modifier.padding(16)`, +`m.padding(16)` and `def f(m: Modifier)` all work: + +```python +class _ModifierMeta(type): + @overload + def padding(cls, all: Dp | float) -> Modifier: ... + ... + +class Modifier(metaclass=_ModifierMeta): + @overload + def padding(self, all: Dp | float) -> Modifier: ... + ... +``` + +### 4.3 Observed: it does not work, at run time or in a checker + +**Run time** (CPython 3.13, `/Volumes/macMini/tmp/pyicheck`): + +```python +class Meta(type): + def padding(cls, v): return ("META", v) +class Modifier(metaclass=Meta): + def padding(self, v): return ("INSTANCE", self, v) + +Modifier.padding # -> +Modifier.padding(16) # -> TypeError: Modifier.padding() missing 1 required positional argument: 'v' +``` + +This is `type.__getattribute__`'s documented order: a metaclass attribute wins **only if it is a data +descriptor**; otherwise the class's own MRO is searched first, and a plain function on the metaclass +is a non-data descriptor. So `Modifier.padding` is the *unbound instance method*, and `16` is being +bound to `self`. It raises here only because the arities differ by one; it would bind silently for +any pair that happened to match. + +**mypy 2.3.0**, on exactly the stub of §4.2: + + reveal_type(Modifier.padding) + note: Revealed type is "Overload(def (self: Modifier, all: Dp | float) -> Modifier, ...)" + + Modifier.padding(16) + error: No overload variant of "padding" of "Modifier" matches argument type "int" + note: Revealed type is "Any" + +The checker reproduces the runtime rule: the metaclass declaration is invisible, `Modifier.padding` +is the instance method, and every class-object spelling fails. Ten errors in a twenty-line file, and +the only lines that passed were the instance ones. + +**The fix that works at run time does not fix the checker.** A `property` on the metaclass *is* a +data descriptor and does win: + +```python +class Meta2(type): + @property + def padding(cls): return cls.EMPTY.padding +Modifier2.padding(16) # -> ('INSTANCE', , 16) correct +``` + +as does a hybrid descriptor in the class body with no metaclass at all: + +```python +class hybrid: + def __init__(self, fn): self.fn = fn + def __get__(self, obj, owner): return self.fn.__get__(obj if obj is not None else owner.EMPTY, owner) +Modifier3.padding(16) # -> ('BOUND-TO', , 16) correct +Modifier3().padding(16) # -> correct +``` + +But declaring the metaclass side as `@property -> _PaddingCall` in the stub changes nothing for +mypy — it still resolves the class's own method and produces the same twelve errors. **mypy does not +model metaclass data-descriptor precedence.** + +### 4.4 Judged: callback-Protocol attributes, and the stub need not mirror the runtime mechanism + +The shape that satisfies all three spellings is to declare each operation on the class as an +**attribute whose type is a Protocol with an overloaded `__call__`**: + +```python +class _Modifier_padding(Protocol): + @overload + def __call__(self, all: Dp | float) -> Modifier: ... + @overload + def __call__(self, *, horizontal: Dp | float = ..., vertical: Dp | float = ...) -> Modifier: ... + +class Modifier: + padding: ClassVar[_Modifier_padding] + size: ClassVar[_Modifier_size] + background: ClassVar[_Modifier_background] +``` + +**Measured** — the same twenty-line user file that produced ten errors against §4.2 produces exactly +the two intended ones: + +| expression | revealed / result | +|---|---| +| `Modifier.padding` | `_Modifier_padding` | +| `Modifier.padding(16)` | `Modifier` | +| `Modifier.padding(dp(16))` | `Modifier` | +| `Modifier.padding(horizontal=8, vertical=4)` | `Modifier` | +| `Modifier.padding(16).background(Color(0xFFFF0000))` | `Modifier` | +| `Modifier.size(4).padding(2)` | `Modifier` | +| `m.padding(16)` where `m: Modifier` | `Modifier` | +| `takes_modifier(Modifier.padding(16))` where `def takes_modifier(m: Modifier)` | clean | +| `Text("hi", font_size=16)` | **error** — `int` is not `TextUnit` (intended, §3.4) | +| `Modifier.padding("nope")` | **error**, with all overload variants listed (intended) | + +`ClassVar` is not load-bearing for inference — a plain `padding: _Modifier_padding` gives byte-identical +results — but it is the honest annotation, and it stops a checker accepting `m.padding = something`. + +The point that makes this legitimate rather than a trick: **a `.pyi` completely replaces the `.py` +for a checker, so the stub's mechanism and the runtime's mechanism do not have to agree — only the +types do.** The runtime uses the hybrid descriptor of §4.3, which binds `self` correctly for both +spellings; the stub says "a callable attribute", which is what a checker can act on. The one +inaccuracy is that a checker believes `Modifier.padding` and `m.padding` are the same object, and +nothing observable depends on that. + +**Rejected:** + +| alternative | why not | +|---|---| +| the metaclass of §4.2 as written | measured not to work, in both mypy and CPython (§4.3) | +| metaclass whose members are `@property -> Protocol` | works at run time, does not work in mypy — the class's own member still wins. Would give a *correct runtime* with a *wrong IDE*, which is the worst of the two | +| bind the module-level name `Modifier` to the companion instance | already rejected in §4.2 of the other document and still right: it destroys `Modifier` as an annotation, and the stubs exist so that annotation works | +| drop the `Modifier.padding(...)` spelling, require `Modifier().padding(...)` | changes the API away from Kotlin's for a reason that is an implementation detail of Python attribute lookup | +| emit both the metaclass **and** the Protocol attributes | the class attribute shadows the metaclass in both engines, so the metaclass is dead text that a reader will believe | + +**Cost, stated:** one Protocol class per `(receiver, name)` pair. For `Modifier` that is 130 extra +class definitions in `pythonx/compose/ui/__init__.pyi`; across the 507 distinct `(receiver, name)` +pairs in corpus A, ~507 if every receiver is stubbed. They are generated, never read by a human, and +`.pyi` files are not executed — but a 500-class stub file is a real indexing cost for an IDE and it +has not been measured. + +**Not verified: PyCharm.** Callback protocols are standard typing and PyCharm supports `Protocol`, +but whether its inference resolves `Modifier.padding(16)` through a `ClassVar[Protocol]` was not +tested. **This is the single most load-bearing unverified claim in this document** — §4.4 is chosen +over §4.2 on mypy evidence alone, and if PyCharm follows the runtime MRO rule the way mypy does then +§4.4 works there too, while if it has its own metaclass handling the conclusion could differ. +Verify before implementing. + +(§4.5, Member extensions, stayed in `docs/design/pyi-generation-design.md`.) + +--- + +## 5. Package mapping + +### 5.1 Observed: the shape of `pythonx-compose` + +`/Volumes/macMini/thisisthepy/pythonx-compose/pythonx/` contains **only** `compose/`, and there is +**no `pythonx/__init__.py`** — `pythonx` is an implicit namespace package, so several distributions +can contribute subpackages under it. `pythonx/compose/` has `__init__.py`, `layout/`, `lite/`, +`material3/`, `native/`, `runtime/`, `test/`, `ui/`, `ui/unit/`, `wrapper/`. + +### 5.2 Observed: it is not a rule + +`unzip -l` over the Compose Multiplatform desktop jars vendored at +`pythonx/compose/lite/release/app/` shows that the `org.jetbrains.compose.*` artefacts contain +`androidx.compose.*` **packages**: + + foundation-layout-desktop-1.6.11-*.jar -> androidx/compose/foundation/layout + ui-unit-desktop-1.6.11-*.jar -> androidx/compose/ui/unit + material-desktop-1.6.11-*.jar -> androidx/compose/material + +so the Kotlin namespace is `androidx.compose.*` on desktop and Android alike, and one mapping covers +both. Against the actual directories: + +| `pythonx` module | Kotlin package | mechanical? | +|---|---|---| +| `pythonx.compose.material3` | `androidx.compose.material3` | yes | +| `pythonx.compose.runtime` | `androidx.compose.runtime` | yes | +| `pythonx.compose.ui` | `androidx.compose.ui` | yes | +| `pythonx.compose.ui.unit` | `androidx.compose.ui.unit` | yes | +| `pythonx.compose.layout` | `androidx.compose.foundation.layout` | **no — `foundation.` is dropped** | +| `pythonx.compose.wrapper`, `.native`, `.lite`, `.test` | none | no counterpart | + +The `layout` entry is not inferred. `pythonx/compose/layout/arrangement.py` line 7 reads +`jclass("androidx.compose.foundation.layout.Arrangement")`, and `pythonx/compose/layout/__init__.py` +re-exports `Arrangement` from it. + +### 5.3 Judged: a rule plus a manifest the Python package owns + +**`androidx.` → `pythonx.` is the default rule; deviations come from a manifest, and the manifest +belongs to the Python package, not to the plugin.** + +The reason is ownership, not convenience. Which Kotlin package a `pythonx` module wraps is +`pythonx-compose`'s design decision — the plugin has no basis on which to invent +`pythonx.compose.layout` for `androidx.compose.foundation.layout`, and hard-coding Compose's +particular renames into a general-purpose Gradle plugin would make every other library's mapping +unreachable. + +So: a data file inside the Python distribution — e.g. `pythonx/compose/pythonx-map.toml` — declaring + + [modules] + "pythonx.compose.layout" = "androidx.compose.foundation.layout" + "pythonx.compose.ui.unit" = "androidx.compose.ui.unit" + + [value-classes] + raw-primitive-allowed = ["androidx.compose.ui.unit.Dp"] + +The plugin reads it from the resolved Python package and emits `pythonx.*` stubs accordingly; with +no manifest it emits only the Kotlin-FQN stubs. This also gives +`docs/design/kotlin-extensions-in-python.md` §6's open question ("whether the allowlist should be data or +code, and where a downstream consumer adds to it") a location: the same manifest, in the same +package, next to the code whose surface it describes. + +**Consequence: there are two stub products, from one model.** + +| product | namespace | content | mechanical? | +|---|---|---|---| +| Kotlin-FQN stubs | `androidx.compose.material3`, `junit.runner`, … | 1:1 with the modules `PythonProxySource` injects into `sys.modules`; Kotlin names, no snake_case, no extension-as-method | yes | +| Pythonic stubs | `pythonx.compose.material3`, … | snake_case, extension-as-method (§4.4), value-class allowlist (§3.4), keyword-only `content` (§3.5) | needs the manifest | + +The first is the direct descendant of PyREPL's output and is the only thing an IDE could ever see for +those modules, since they exist only as runtime `sys.modules` entries and never as files. The second +is what a user actually imports. + +**Not verified:** no such manifest exists in `pythonx-compose` today, and this proposal has not been +agreed with that repository. It is a judgement about where the information has to live, not a +description of anything. + +--- + +## 6. Where the files go — the parts about the Pythonic product + +From §6.1 (measurements 3 and 4 stayed in `docs/design/pyi-generation-design.md`): + +Five mypy runs in `/Volumes/macMini/tmp/pyicheck`, each reported above or below: + +1. **A `.pyi` beside a `.py` wins.** The fixture's `pythonx/compose/ui/__init__.py` contains nothing + but `def __getattr__(name): raise AttributeError(name)` — the shape the real adapter will have — + and every name in §4.4's table resolved from `__init__.pyi`. This is the deployment that matters: + the dynamic adapter ships, the stub ships beside it, and the checker never sees the dynamism. +2. **`py.typed` is mandatory once the package is installed.** With `pythonx/` copied into the venv's + `site-packages` and `pythonx/compose/py.typed` present, everything resolved. Deleting **only** + `py.typed`: + + error: Skipping analyzing "pythonx.compose.ui": module is installed, but missing library + stubs or py.typed marker [import-untyped] + note: Revealed type is "Any" (every line) + + The marker goes in the top-level *regular* package of the distribution — here `pythonx/compose/`, + because `pythonx` is a namespace package (§5.1). That placement was the one tested and it works. + +From §6.2: + +- **For a published `pythonx` wheel the stubs belong inside the wheel** — `.pyi` beside `.py` plus + `py.typed`, both verified in §6.1 — built by `pypackpack`'s bundle stage, not regenerated per + consumer build. The plugin's per-build generation covers the consumer's own Kotlin and the + third-party jars *its* build resolves; a shipped `pythonx-compose` covers Compose once. + +Two audiences, two destinations, one generator. + +--- + +## 7. Open — the items that belonged to the Pythonic product + +- **PyCharm.** §4.4 rests entirely on mypy. Verify `ClassVar[Protocol]` resolution, `@overload` + ordering, and `.pyi`-beside-`.py` precedence in PyCharm before implementing. If PyCharm does model + metaclass data descriptors, §4.2's shape becomes viable again there and the two engines would want + different stubs — which would be a genuinely new problem. +- **The 500-class stub cost.** ~507 callback Protocols for corpus A's `(receiver, name)` pairs, and + 130 for `Modifier` alone in one file. Not measured for IDE indexing time or for mypy's own runtime. +- **Whether any real Compose overload set collapses.** §3.6 shows four `padding` overloads coexisting, + but two Kotlin overloads can map to identical Python signatures once `Dp | float` widening is + applied. The generator must detect that and drop or qualify, and how often it happens across the + 33 multi-overload `Modifier` names is unknown — it needs the scan to be run. +- **Stub order vs dispatcher order.** §3.6: the `@overload` order and the Python-side dispatcher's + resolution order must come from one place. The dispatcher does not exist + (`docs/design/kotlin-extensions-in-python.md` §6). + *Archive note (2026-10-03): the dispatcher now exists — `_Overloads` in the binding layer + `python_multiplatform.binding` (`PythonxAdapter.kt`) serves the base name of an overload set.* +- **klib.** §0. The `pythonx.*` surface on iOS and androidNative has no producer yet. +- **The manifest of §5.3** does not exist and has not been agreed with `pythonx-compose`. diff --git a/docs/design/pythonx-adapter-design.md b/docs/archive/pythonx-adapter-design.md similarity index 97% rename from docs/design/pythonx-adapter-design.md rename to docs/archive/pythonx-adapter-design.md index bd0584b09..2a41f03f3 100644 --- a/docs/design/pythonx-adapter-design.md +++ b/docs/archive/pythonx-adapter-design.md @@ -1,5 +1,30 @@ # `pythonx` — adapting generated Kotlin bindings without wrapping them one at a time +> **Superseded** by [SPEC.md](../SPEC.md) U-7/U-8, +> [`KotlinSurface.kt`](../../python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/KotlinSurface.kt) +> and [`docs/design/kotlin-extensions-in-python.md`](../design/kotlin-extensions-in-python.md) on +> 2026-10-03; kept for history. +> +> **What is superseded:** the binder serving `pythonx` itself — it synthesised `pythonx.*` modules +> (with `__path__ = []`, so the real on-disk package could not load) and renamed members and +> parameters to snake_case (§3's rule). Since `238119b7` the binder serves Kotlin-named modules under +> Kotlin names only, with keyword arguments by Kotlin parameter name and Kotlin defaults; its layer is +> `python_multiplatform.binding` +> ([`PythonxAdapter.kt`](../../python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapter.kt)), +> and `pythonx` is a real package in the pythonx-compose repository (`AGENTS.md` §12.1–12.2). +> Wherever this file says `pythonx._adapt`, `pythonx._bind_composable` or "the `pythonx` adapter", +> read "the binding layer" for mechanics and "pythonx-compose" for naming. +> +> **Still true, and where it lives now:** the binding-layer mechanics — lazy finder plus module +> `__getattr__` (§2.3, §4.1), extension functions as receiver methods through a hybrid descriptor +> (§4.2), the value-class allowlist (§4.4, now an empty runtime seam `allow_raw_primitive` that +> pythonx-compose fills), the composer stack and the `$default` mask computed per call (§5.6) — are +> in `PythonxAdapter.kt`, whose KDoc still cites this file's section numbers. The `@Composable` +> measurements (§5.2, §5.6) are summarised in `docs/design/kotlin-extensions-in-python.md` §2.3 and +> §4.6 and pinned by `ComposableBindingTest` and `ksp-fixtures/compose`. The `Modifier` and material3 +> render proofs (§9, §10) are the `ksp-fixtures/compose` render tests cited by SPEC B-5/B-6. The +> naming conventions (§3) and the `Modifier` metaclass (§4.3) belong to pythonx-compose. + The 2024 `pythonx-compose` wrote a Python class per Compose component. 37 files, 28 of them empty, and the one that mattered most — `pythonx/compose/ui/modifier.py` — is a copy of the Button wrapper in which `padding()` composes nothing and `fill_max_size()` returns `self`. The chain was never @@ -715,7 +740,7 @@ awaitPointerEvent() }`, which is ordinary Kotlin the compiler already knows how machine for. Nothing about Python has to suspend at all if the loop that suspends is hand-written Kotlin and Python is only ever asked a synchronous question at each iteration — exactly the shape `PythonCallables.PythonFunction` already crosses for every *other* `Function0..Function5` slot in this -codebase (`docs/design/pythonx-adapter-design.md` §4.2's `content=`, among others). +codebase (§4.2's `content=` in this file, among others). That is `fixture.compose.pythonPointerInput` (`ksp-fixtures/compose/src/desktopMain/kotlin/fixture/ compose/PythonPointerInput.kt`): a hand-written, ordinarily-compiled `suspend` Kotlin function — diff --git a/docs/archive/threading-and-abi-3.15t-gate.md b/docs/archive/threading-and-abi-3.15t-gate.md new file mode 100644 index 000000000..aad63ff2b --- /dev/null +++ b/docs/archive/threading-and-abi-3.15t-gate.md @@ -0,0 +1,52 @@ +# Why free-threading was gated on 3.15t (2026-08-10 reasoning) + +> **Superseded** by [`../design/threading-and-abi.md`](../design/threading-and-abi.md) on 2026-10-03; kept for history. +> The conclusion below ("free-threading starts at 3.15t") was wrong for this project: 3.14t runs the whole +> desktop suite with `-PpythonFreeThreaded=true` (ROADMAP §9, SPEC T-2). The premise that blocked 3.14t +> (no Stable ABI on free-threaded builds) does not bind here because `Py_LIMITED_API` is not defined. + +## Why 3.15t and not 3.14t + +Free-threaded builds **do not support the Limited API or the Stable ABI** in 3.13 or 3.14. All ~330 +bindings here target the Stable ABI, so a free-threaded 3.14 build cannot be used without abandoning +it and recompiling per Python version — which would defeat the version-parameterised acquisition +pipeline. + +[PEP 803](https://peps.python.org/pep-0803/) — **Final**, targeting 3.15 — introduces `abi3t`, a +Stable ABI variant for free-threaded builds, with `abi3t` / `abi3.abi3t` wheel tags and forward +compatibility across all later versions. 3.15 is therefore the first release where free-threading +and the Stable ABI coexist. + +### What abi3t requires of us + +| Requirement | Our status | +|---|---| +| `PyObject` / `PyVarObject` become incomplete types; no field access | **Compatible.** Verified: `PyObject` appears only as `CPointer` and is never dereferenced. The only `.pointed` uses in the tree are commented-out `JNIEnv` code in `JniExport.kt`. | +| Extensions may not embed `PyObject` in their own structs | Compatible — nothing does. | +| `PyModExport` hook (PEP 793) instead of static `PyModuleDef` | Not yet relevant. We embed CPython rather than building an extension module. **It becomes relevant for the upcall work**, where Kotlin classes are exposed to Python. | +| No backwards compatibility with 3.14 or earlier | Was accepted at the time; no longer applies, 3.14t is used on desktop (ROADMAP §9). | + + +## Prebuilt availability (snapshot), as of the 20260807 python-build-standalone release + +| Platform | GIL | Free-threaded | +|---|---|---| +| macOS arm64 / x86_64 | ✅ | ✅ (incl. `3.15.0rc1`) | +| Linux x86_64 | ✅ | ✅ (incl. `3.15.0rc1`) | +| Windows x86_64 | ✅ | ✅ (incl. `3.15.0rc1`) | +| Android (python.org) | ✅ | ❌ none published | +| iOS (Python-Apple-support) | ✅ | ❌ none published | + +Desktop free-threaded builds exist today, including 3.15 release candidates. **Mobile has none**, so +enabling free-threading before mobile artefacts appear would split the threading model across +platforms — and with it the object-lifetime and thread-state design layered on top. + +Switching the default (originally) required all three of the following; ROADMAP §9 shows 1 is not needed for 3.14t on desktop, while 2 still holds: + +1. CPython 3.15 final, for `abi3t` +2. Free-threaded mobile artefacts from python.org and Python-Apple-support +3. Adequate free-threaded wheel coverage for the C extensions users care about + +`gradle.properties` already carries `pythonFreeThreaded`, defaulting to `false`, so the switch is a +configuration change once those hold. + diff --git a/docs/archive/wasm-design-experiment-log.md b/docs/archive/wasm-design-experiment-log.md new file mode 100644 index 000000000..e78d3d494 --- /dev/null +++ b/docs/archive/wasm-design-experiment-log.md @@ -0,0 +1,1386 @@ +> **Superseded** by [`../platforms/wasm-design.md`](../platforms/wasm-design.md) on 2026-10-03; kept for history. +> This is the chronological lab notebook of the wasm experiment (feasibility analysis, "Why WASM is +> parked", then a series of corrections as each hypothesis was measured). Its early sections say wasm +> is **parked / not implemented / blocked on Kotlin's toolchain**, that CPython is pinned to 3.13 and +> Kotlin to 2.0.20, and that a C shim and an `EXPORTED_FUNCTIONS` list are needed — **all of that was +> later disproved or overtaken**; read the sections in order and treat any "unverified" or "parked" +> claim as superseded by the sections after it and by the integrated implementation +> (`python-multiplatform/src/wasmJsMain/`, `wasmJsTest/`, `tools/wasm/build-cpython.sh`; SPEC §0, C-6, M-3, U-5). +> Paths such as `wasm-experiment/` and `/Volumes/macMini/wasm-build/` refer to the retired experiment +> and the pre-move build location; the build now lives in `tools/wasm/build-cpython.sh` under `.caches/`. + +# WebAssembly (WASM) Feasibility and Design + +This document details the feasibility and architectural design for supporting WebAssembly (WASM) in this Kotlin Multiplatform library, specifically addressing the challenges of embedding CPython 3.13 in a Kotlin/Wasm environment. + +## 1. Kotlin/Wasm Target Reality + +Based on the project's Kotlin version **2.0.20**: +- **Supported Targets:** Kotlin provides both `wasmJs` (for the browser and Node.js with JS host) and `wasmWasi` (for standalone Wasm environments). Both are classified as Alpha in 2.0.20 (though `wasmJs` matures in later 2.x releases). +- **JS Interop:** Kotlin/Wasm uses the WebAssembly Garbage Collection (WasmGC) proposal. It interoperates with JavaScript via `external` declarations and the `JsAny` type hierarchy. + - **Strings:** Translated seamlessly between Kotlin `String` and JavaScript `string`. They are not passed as `const char*` raw pointers. + - **Pointers & Opaque Handles:** WebAssembly linear memory pointers (like `PyObject*`) are simply 32-bit integers (`Int` or JS `Number`). They cannot be directly mapped to Kotlin's WasmGC objects. + - **Callbacks:** Kotlin lambdas or `@JsExport` functions are exposed as JavaScript function objects, not raw C function pointers. + +## 2. The Two Candidate Approaches for `wasmJs` + +Since Kotlin/Wasm and a WASM CPython build are separate modules that do not share linear memory, they must communicate through the JS host environment. + +### (a) Pyodide's High-Level JS API +This involves using `pyodide.runPython`, `pyodide.globals`, and `PyProxy` objects. +- **API Coverage:** Almost **none** of the ~330 `expect` C-API functions can be implemented this way. Pyodide abstracts away the C API. +- **Object Model:** This approach completely destroys the `PyObject`/`PyType` model the library is built on. WASM would have to use `JsAny` wrappers of `PyProxy` instead of `NativePointer`. +- **Verdict:** **Impractical.** It would result in a fundamentally different architecture and semantics for WASM compared to Android, Desktop, and Native. + +### (b) Raw CPython C Symbols Exported from a WASM Build +This involves reaching Emscripten's `EXPORTED_FUNCTIONS` (e.g., `Module._PyList_Size`), passing memory addresses as 32-bit integers. +- **API Coverage:** Can implement the entire `expect` surface, mapping 1-to-1 with the Stable ABI. +- **Object Model:** Preserves the exact same `NativePointer` and `PyObject` semantics as other platforms. +- **Per-Call Cost:** High. Every call must cross from Kotlin WasmGC -> JS -> Emscripten Linear Memory. However, since the signature census shows only 14 simple ABI shapes, this crossing can be tightly optimized. + +## 3. Does Pyodide Export What We Need? + +**No, a stock Pyodide build does not export the full Stable ABI.** +Pyodide relies on Emscripten's dead-code elimination. Symbols not used by Pyodide's internal bindings or explicitly listed in `EXPORTED_FUNCTIONS` are stripped. Furthermore: +- **CPython 3.13 Status:** The `wasm32-emscripten` build was dropped from official CPython support in 3.13. (Pyodide maintains it out-of-tree). `wasm32-wasi` was promoted to Tier 2 support. +- **Custom Build Required:** To use the raw C API approach, we would need to generate a custom Emscripten build (or a custom Pyodide distribution) that explicitly adds our ~330 required C API symbols to `EXPORTED_FUNCTIONS` (or use `EMSCRIPTEN_KEEPALIVE`). + +## 4. Pointer Width and `NativePointer` + +WASM32 uses **32-bit pointers**. +- The current abstraction `data class NativePointer(val address: Any)` uses `Any`. Passing a 32-bit integer pointer into `Any` will box it (as a `Double` on JS or a boxed primitive in WasmGC), adding allocation overhead. +- The `toRawValue(): Long` signature assumes 64-bit addresses. Since a 32-bit unsigned integer fits perfectly into a 64-bit `Long`, the abstraction mathematically holds. However, Kotlin/Wasm code implementing `toRawValue()` will need to cast the JS/Wasm 32-bit `Int` to a Kotlin `Long` carefully to avoid sign-extension bugs. + +## 5. Upcalls on WASM (Python calling Kotlin) + +Python expects a raw C function pointer (an integer address pointing to executable memory). We can preserve the "routing by data" design from other platforms, but the plumbing is different: +1. We write our ~4 static trampoline shapes in Kotlin and export them to JS using `@JsExport`. +2. In JS glue code, we use Emscripten's `addFunction(jsFunction, signature)` to inject the JS function into Emscripten's WebAssembly function table. +3. `addFunction` returns an integer (the table index). This integer *is* the C function pointer. +4. We pass this integer to CPython (e.g., in `tp_new` or `PyCFunction_NewEx`). When CPython calls it, Emscripten routes the call out to JS, which calls our Kotlin WasmGC function. The context pointer (`self` or `closure`) is passed as an integer, allowing us to route to the correct Kotlin object. + +## 6. `wasmWasi` + +**Separate implementations are required.** +Pyodide and Emscripten rely entirely on JavaScript glue code. If the project targets non-browser WASM via `wasmWasi` (pairing with CPython's Tier 2 `wasm32-wasi`), there is no JS host. +Currently, Kotlin/Wasm cannot dynamically link directly to an arbitrary C WebAssembly module without a JS host bridge, as the Wasm Component Model and native static linking are not yet mature enough in the Kotlin toolchain. Thus, `wasmWasi` is genuinely a second, entirely different port that is currently **blocked** by Kotlin/Wasm tooling limitations. + +## 7. Recommendation and Staged Plan + +**Recommendation:** Pursue **Approach (b) — Raw C Symbols via JS Bridge** for the `wasmJs` target. It is the only path that preserves the multiplatform object model. Ignore `wasmWasi` until Wasm Component Model linking matures in Kotlin. + +**Caveat:** WASM will have permanently higher per-call overhead than Native/JVM due to the WasmGC ↔ JS ↔ Emscripten boundary crossings. + +### Staged Plan +1. **Toolchain Customization:** Create a custom Pyodide/Emscripten build script that passes all ~330 `EmbedAPI` functions to Emscripten's `EXPORTED_FUNCTIONS`. +2. **Downcalls (Kotlin to Python):** Write a Kotlin `external` module that binds to `Module._Py*` functions on the Emscripten JS object. Implement `EmbedAPI.wasmJs.kt` to call these. +3. **Upcalls (Python to Kotlin):** Use Emscripten's `addFunction` via JS interop to register 4 static Kotlin `@JsExport` trampolines, returning function pointers to hand to Python. +4. **Pointer Optimization:** If boxing in `NativePointer(val address: Any)` proves too slow in WasmGC, consider refactoring the `expect` class to use a concrete `Long` across all platforms (which JVM/Native already use natively) to avoid allocations. + +--- + +## Addendum: correcting two framing errors + +### Kotlin/Native once had a wasm target, and it was removed + +The analysis above assumed Kotlin/Wasm is the only route. That is true today, but not for a +fundamental reason. + +Kotlin/Native used to ship a `wasm32` target built through LLVM, producing linear-memory +WebAssembly with a C ABI — the same memory model an Emscripten build of CPython uses. That target +could in principle have used `cinterop`, exactly as the iOS target does today. JetBrains +**deprecated it in Kotlin 1.8.20 and removed it in 1.9.20**, in favour of the Kotlin/Wasm +toolchain, which skips LLVM and is built on WasmGC. + +The consequence is that the JS bridge is forced by *tooling*, not by WebAssembly itself: + +- Kotlin/Wasm is WasmGC-based, so its memory model differs from C's linear memory. +- Kotlin/Wasm has no `cinterop` at all. `wasmJs` offers JS interop; `wasmWasi` offers WASI syscalls. + +This project targets Kotlin 2.0.20, so the `wasm32` target is long gone and pinning to 1.9.10 is +not viable — the rest of the toolchain requires 2.0.x. + +### The dependency is CPython's own WASM support, not Pyodide + +Pyodide is a downstream distribution carrying its own patches. A library that intends to be a +standard multiplatform binding should build CPython from source for a WASM target rather than +depend on Pyodide. + +Where CPython's own support actually stands: + +| Target | CPython 3.13 | CPython 3.14 | +|---|---|---| +| `wasm32-wasi` | Tier 2 (officially supported) | Tier 2 | +| `wasm32-emscripten` | Not a PEP 11 platform | Tier 3, via PEP 776 | + +[PEP 776](https://peps.python.org/pep-0776/) is Active, targets 3.14, and formalises Emscripten at +Tier 3. Three of its details bear directly on this design: + +1. **Only static linking is supported.** The PEP states it is only supported to statically link the + interpreter; dynamic linking is unsupported because `EM_JS` functions behave differently in + dynamic builds. CPython therefore cannot be dynamically loaded into another WebAssembly module. +2. **The build produces an `.mjs` plus `.wasm` pair.** A JavaScript runtime layer is intrinsic to + the Emscripten build; it is not something an integration can route around. +3. **No official python.org distribution is mandated** — binaries continue to come from downstream, + i.e. Pyodide. Avoiding Pyodide as a *dependency* therefore means building CPython from source + ourselves, not that an upstream binary exists to use instead. + +Because Kotlin/Wasm cannot statically link C code, and CPython cannot be dynamically linked into +another module, the browser architecture is necessarily three parts: the CPython Emscripten module, +the Kotlin/Wasm module, and JavaScript coordinating them. That holds regardless of which approach +is taken. + +`wasm32-wasi` is the better-supported target on the CPython side, but there is no JavaScript host to +act as glue, and Kotlin/Wasm's `wasmWasi` cannot link a C module. Bridging the two awaits the WASM +Component Model. + +### Practical consequence + +Official Emscripten support arrives in **CPython 3.14**. This project is pinned to 3.13, where +Emscripten has no PEP 11 status at all. WASM support therefore implies moving the embedded +interpreter to 3.14 or later. + +--- + +# Why WASM is parked, and what it would look like + +Recorded so the reasoning survives. Nothing here is implemented. + +## The blocker is Kotlin's toolchain, not WebAssembly + +Every other platform calls CPython directly — cinterop on iOS and androidNative, Panama on +desktop, JNI on Android. Kotlin/Wasm cannot: `wasmJs` offers only JS interop and `wasmWasi` +only WASI syscalls, and neither can link a C library. + +Kotlin/Native once had a `wasm32` target that went through LLVM and emitted linear-memory +wasm — the same memory model Emscripten's CPython uses, which is exactly what would have made +cinterop work. It was deprecated in 1.8.20 and removed in 1.9.20. So the JS bridge is a +consequence of a toolchain decision, not of WebAssembly. + +**The deeper barrier is memory, not calls.** Two wasm modules can call each other directly if +the host wires module B's exports to module A's imports at instantiation — JS is needed once, +not per call. But Kotlin/Wasm is WasmGC: its objects live on a managed heap, and it has no way +to read another module's linear memory. CPython's entire API is pointers into exactly that +memory. So JS is not a convenience in the call path — it is where the marshalling has to +happen. + +JetBrains has a **Multiple Memory** proposal under consideration that would address this, and a +C-interop example PR against `kotlin-wasm-examples` was opened in March 2024. Neither has a +committed date, and neither appears on the published roadmap, so neither can be planned around. + +## emscripten, not wasi + +| | `wasm32-emscripten` | `wasm32-wasi` | +|---|---|---| +| CPython support | Tier 3 from 3.14 (PEP 776) | **Tier 2** | +| bridge between the two modules | **JS exists** | none — needs Component Model, which Kotlin does not emit | +| C extension modules | **dlopen works** | no dlopen in preview1; extensions must be linked in | +| matches this project | **browser; the sample already targets `wasmJs`** | server/edge | + +The decisive row is the third. This project exists to share libraries, and on WASI a user could +not add a compiled package after the fact — the interpreter would have to be rebuilt. That +guts the goal on that platform, which outweighs WASI's better support tier. + +The second row is nearly as decisive: WASI has no JS host, so there is no mechanism at all to +connect a Kotlin/Wasm module to a CPython module there. + +## The shape it would take + +Put our C code inside CPython's module, so it can handle pointers, and keep the boundary to +integers. + + +- Emscripten module (one linear memory) -----+ + | CPython + our C shim | pointers handled here + +---------------------+-----------------------+ + | i32 only + +-----+-----+ + | JS glue | string copying only + +-----+-----+ + | + +--------+---------+ + | Kotlin/Wasm | + +------------------+ + +Pointers are 32-bit on wasm32, so a `PyObject*` fits in a Kotlin `Int`. The shim exposes +composed operations, the same ones measured on Android: + + int pmp_getattr(int obj, int namePtr); // lookup + error handling, one crossing + int pmp_exec(int codePtr); // __main__ + globals + run + int pmp_list_items(int list, int outPtr, int cap); // bulk: fills a caller buffer + int pmp_str_utf8(int obj, int outLenPtr); + void pmp_decref(int obj); + int pmp_scratch(void); // reusable buffer address, once + +**Interning applies here and matters more.** A Kotlin string lives on the WasmGC heap and a C +string in linear memory, so copying costs a JS hop — but attribute and module names are +repeated literals, so caching the linear-memory address makes every call after the first cost +nothing. This is the same fix measured on Android at 2238 ns → 148 ns, against a hop that is +more expensive here than JNI's. + +**Bulk is where composition earns its keep on this platform.** It was rejected on desktop and +superseded by interning on Android; here it is the only mitigation left, because +`Module.HEAP32.subarray(p, p+n)` fetches N items in one hop where per-element access would cost +N. + +Nothing above changes the layering: `wasmJsMain` supplies `actual`s for the same EmbedAPI +`expect`s, and the object model does not move. + +## What survives if the toolchain improves + +If Multiple Memory lands, the C shim stays — composition is wanted for its own sake — and only +the JS glue is replaced by direct memory access. The `actual` signatures do not change. That +bounds the throwaway work to a few dozen lines of glue, which is the argument for this shape +over waiting indefinitely. + +## Unresolved + +- `NativePointer` holds `Any`, so a wasm32 `Int` boxes on every conversion, and + `toRawValue(): Long` needs sign-extension care. +- CPython must be built with our shim and an explicit `EXPORTED_FUNCTIONS` list; a stock + Pyodide build strips the C API by dead-code elimination. That is a second build pipeline. +- Upcalls: `addFunction(jsFn, sig)` gives a callable function pointer, but re-entering + Kotlin/Wasm from it goes through JS. Unmeasured, and likely worse than every other platform. +- Lifetime: whether Kotlin/Wasm offers a GC hook equivalent to `Cleaner`/`createCleaner` has not + been checked. Without one, only explicit `close()` works. + +## Start condition + +Not now. `wasmJsMain`'s `actual`s cannot be written until it is settled what they implement — +§1/§4 (GIL and automatic release) and §7 (upcall shape) are both in flight, and building against +them now means building twice. + +## Packaging: a side module, against our own CPython + +Two independent choices, easily conflated: + +1. **How our C code is packaged** — statically linked into CPython, or a side module loaded by + `dlopen` at import time. +2. **Where CPython comes from** — built by us with Emscripten, or Pyodide's distribution. + +### Side module, on both counts of its own + +| | statically linked | **side module** | +|---|---|---| +| changing one binding | rebuild all of CPython | rebuild our module | +| release cadence | tied to CPython's | independent | +| shape | a CPython fork, effectively | an ordinary extension module (`PyInit_*`) | +| runtime | no indirection | dynamic-link indirection (small) | + +Having to rebuild the interpreter to change a binding is the same trap this project already hit +elsewhere — a `.def` edit needing a forced link and staging pass before it reached a device. An +extension module is what CPython already has a mechanism for; we are not doing anything unusual. + +### Our own CPython, and no attempt at Pyodide compatibility + +The earlier draft argued for keeping Pyodide compatibility in reach, on the grounds that Pyodide +*is* the package ecosystem on the web. That argument does not survive contact with how Pyodide +wheels are tagged: `cp312-cp312-pyodide_2024_0_wasm32` pins the Python version, the Emscripten +version **and** Pyodide's own ABI together. What you get is not "the Python ecosystem" but +"packages someone built for exactly this Pyodide release". + +So the choice is not ecosystem-versus-no-ecosystem. It is whose tag to be locked to, and +targeting Pyodide would mean the web platform tracking their release cadence while every other +platform is pinned to 3.14 from source. That is a real cost for a conditional benefit. + +The loss from building our own is narrower than it first appears: + +| | on our own build | +|---|---| +| pure-Python wheels (`py3-none-any`) — no ABI tag | work unchanged | +| C extensions (numpy, pandas, lxml, cryptography) | must be built for our ABI | + +And the second row is true of Pyodide too, for anything outside the list they happen to have +built. Pyodide's advantage is a prebuilt catalogue, not a different rule. + +**Decision: our own Emscripten build of CPython 3.14, packaged against by a side module.** +Compiled third-party extensions are a known limitation on this platform, recorded rather than +solved. + +### Correction: PEP 783 unbinds the tag from Pyodide + +The section above is right about the decision and wrong about its cost. It was written from the +`pyodide_2024_0_wasm32` tag and concluded that any wheel compatibility means being locked to a +Pyodide release. That stopped being true. + +[PEP 783 — Emscripten Packaging](https://peps.python.org/pep-0783/) is **Accepted**. It replaces +the Pyodide-owned tag with a standard platform tag series that PyPI accepts: + +``` +pyodide_2024_0_wasm32 Pyodide's tag, Pyodide's platform +pyemscripten_2026_0_wasm32 standard platform tag (PEP 783) +``` + +Three things follow, and they change the trade-off rather than the conclusion. + +**The platform is versioned per Python feature release, not per distribution release.** The +current series is `pyemscripten_2024_0` (3.12), `pyemscripten_2025_0` (3.13), `pyemscripten_2026_0` +(3.14). The earlier worry — "the web platform tracks Pyodide's cadence while every other platform +is pinned to 3.14" — does not apply: the platform version *is* the Python version. Pinning to 3.14 +from source and claiming `pyemscripten_2026_0` are the same act. The patch component exists as +an escape hatch the PEP hopes never to use. + +**Compatibility is a build recipe, not a distribution.** The PEP states that the tags "can be used +by Python interpreters compiled and linked with the specified version of Emscripten and with the +specified ABI-sensitive flags". So our own build can claim the platform. The ABI-sensitive surface +the PEP enumerates: + +| | | +|---|---| +| Emscripten compiler version | pinned per platform version | +| statically linked libraries | fixed set | +| stack unwinding ABI | fixed selection | +| dependency lookup handling | specified | +| `-pthread` | prohibited | +| `-sWASM_BIGINT` | required | + +`-sWASM_BIGINT` is worth noting for us specifically: it is what makes an i64 cross the JS boundary +as a BigInt rather than being split into two i32s, which is the same `NativePointer` width question +§4 above raises. The platform requires the setting we would have wanted anyway. + +**The full specification currently lives in Pyodide's documentation**, which the PEP references +normatively while noting the wording was chosen to be "more forwards compatible to a future where +the definition of the platform moves upstream". So Pyodide still *authors* the spec even though it +no longer *owns* the tag. That is a real residual coupling, but it binds a flag list, not a runtime. + +#### What this changes + +| | earlier draft | with PEP 783 | +|---|---|---| +| what the tag pins | Pyodide release | Python version + a published flag set | +| our own build + C-extension wheels | mutually exclusive | **compatible, if we match the flags** | +| Pyodide as a dependency | the price of wheels | not required | +| pure-Python wheels | work unchanged | work unchanged | + +The decision stands — our own Emscripten CPython 3.14 — but the recorded limitation was +overstated. Compiled extensions are not lost by building our own; they are lost only if we build +with flags that diverge from `pyemscripten_2026_0`. **Matching that flag set should be a build +constraint from the start**, because retrofitting it means rebuilding the interpreter. + +Evidence that the path is real rather than paper: [pypa/packaging #804](https://github.com/pypa/packaging/pull/804) +implements the tag handling, [pypi/warehouse #19804](https://github.com/pypi/warehouse/pull/19804) +adds PyPI support, [maturin #3163](https://github.com/pyo3/maturin/pull/3163) adds it to a build +backend, and [pydantic publishes Emscripten wheels to PyPI under it](https://pydantic.dev/articles/emscripten-wheels-pydantic). + +#### Resolved: the flag list, read off the source rather than the rendered page + +`pyodide.org/en/stable/development/abi.html` still returns 403 to automated fetching. The same +documents are in the repository and `raw.githubusercontent.com` serves them, which is how the +following was read. Sources are `github.com/pyodide/pyodide` at tag **`314.0.4`** — +`docs/development/abi/314.md`, `docs/development/abi/flags.md`, `Makefile.envs` — and +`github.com/pyodide/pyodide-build` `pyodide_build/config.py`. + +`Makefile.envs` at that tag pins the whole platform in four lines: + +``` +PYODIDE_VERSION ?= 314.0.4 +PYODIDE_ABI_VERSION ?= 2026_0 +PYVERSION ?= 3.14.2 +PYODIDE_EMSCRIPTEN_VERSION ?= 5.0.3 +``` + +| | `pyemscripten_2026_0` (3.14) | for comparison, `2025_0` (3.13) | +|---|---|---| +| Emscripten | **5.0.3** | 4.0.9 | +| unwinding | `-fwasm-exceptions -sSUPPORT_LONGJMP=wasm`, at **compile and link** | same | +| `-sWASM_BIGINT` | required, but **default since Emscripten 4.0.0** — not typed explicitly | same | +| `-pthread` | prohibited; "if `-pthread` is used, the resulting libraries will not load" | same | +| dependency lookup | full `RPATH` support, and `RPATH` is the *only* mechanism | same | +| Rust | stable `1.93.0`; no nightly, no custom sysroot | nightly + custom sysroot | +| new static libs | `sqlite3` 3.39.0, `lzma`/xz 5.2.2 | — | + +Statically linked into the main binary: `zlib`, `bzip2`, `sqlite3`, `lzma`, `zstd`, `libffi`, +`libhiwire`, and the Emscripten JS libraries (`libGL`, `libegl.js`, `libwebgl.js`, `libhtml5*.js`, +`libsdl.js`, `libwebsocket.js`, `libeventloop.js`, `liblz4`, the filesystem backends). **OpenSSL is +not** — Pyodide builds it as a shared side module (`libssl.so`/`libcrypto.so`, OpenSSL 1.1.1w) +found through `RPATH`. + +Extension modules are side modules. Pyodide documents `-sSIDE_MODULE=2` plus an explicit +`-sEXPORTED_FUNCTIONS=["_PyInit_…"]` as the recommendation, though its own C default is +`-sSIDE_MODULE=1`. That is exactly the shape the packaging decision above already chose for our +shim. + +**PEP 783 itself does not enumerate any of this.** It lists the ABI-sensitive *surface* and then +defers normatively to Pyodide's documentation for the values. It does specify how the tag is +derived, and that is the part that bites: `packaging` reads +`sysconfig.get_config_var("PYEMSCRIPTEN_PLATFORM_VERSION")`. **That variable does not exist +anywhere in CPython 3.14.2** — `grep -r PYEMSCRIPTEN` over the whole checkout returns nothing. So a +stock CPython Emscripten build cannot advertise the platform even if it matches the flags; the +variable is something Pyodide's build injects. + +--- + +## Correction: the JS layer is not mandatory + +Everything above assumes Kotlin/Wasm can only reach CPython through JavaScript — that crossings +cost a JS hop and that string data must be copied by JS between two isolated memories. Reading +the Kotlin compiler shows both halves of that are wrong on the `wasmJs` target. This section +records what was checked and where it stops. + +### 1. Kotlin can call another wasm module's exports directly + +`@WasmImport(module, name)` and `@WasmExport(name)` exist in the stdlib +(`libraries/stdlib/wasm/src/kotlin/wasm/Annotations.kt`), gated behind +`@ExperimentalWasmInterop` since 2.2. The annotation doc says the function is imported "without +type adapters". The compiler's own codegen test `callingWasmDirectly.kt` states the intent +outright: + +> Here we pass export of another Wasm module to our import directly without JS layer. +> This enables strict type check without JS conversions. + +So `pmp_getattr(i32, i32) -> i32` can be a direct wasm-to-wasm call. JS is still involved in +*instantiating* the modules and wiring the import, but not in the call. This is exactly the +narrow integer interface the shape above already proposes — it just costs less than assumed. + +The constraint is the one that makes it work: no type adapters means wasm primitive types only. +No strings, no objects. Which is what the composed shim signature already looks like. + +### 2. On `wasmJs`, Kotlin *imports* its linear memory + +`WasmCompiledModuleFragment.createAndExportMemory` branches on a flag that +`wasmCompiler.kt` sets as `importWasmMemoryInsteadOfExport = isWasmJsTarget`: + +| target | what the module does with its memory | +|---|---| +| `wasmWasi` | **defines and exports** it — `WasmExport.Memory("memory", …)`, and the source comments that the name is a WASI ABI convention | +| `wasmJs` | **imports** it, as `WasmImportDescriptor("intrinsics", "memory")`, declared with 0 initial pages | + +That inverts the premise. A `wasmJs` module does not own a memory it defines; the host hands one +in. And `kotlin.wasm.unsafe.Pointer` is an `i32` into precisely that memory. + +So the question "can Kotlin read CPython's `char*`" reduces to "can the host supply Emscripten's +memory as that import" — at the wasm level, both are just a `WebAssembly.Memory` in an import +object, and Emscripten exposes its own as `Module.wasmMemory`. **If that works, string +marshalling does not merely get cheaper, it disappears** — which matters more here than anywhere +else, because every measurement on Android and desktop found marshalling, not crossings, to be +the dominant cost. + +### 3. The blocker is the allocator, not the memory + +`kotlin/wasm/unsafe/MemoryAllocation.kt` creates the top-level allocator as +`ScopedMemoryAllocator(0, parent = null)` and extends it with `wasm_memory_grow`. **It starts at +address 0 and assumes it owns everything.** In Kotlin's own memory that is correct. In a memory +shared with Emscripten, address 0 is the null page and the low region holds Emscripten's static +data, so `withScopedMemoryAllocator` would hand out addresses CPython already owns and corrupt +the heap. + +This does not sink the idea, but it bounds it: + +- **Reading and writing CPython-owned buffers** — addresses that CPython allocated and handed + back as `i32` — is safe. +- **Allocating from Kotlin inside the shared memory** is not, unless the allocator can be given + a base address it does not currently accept. + +Whether a `Pointer` can be constructed from an arbitrary `Int` returned by the shim, rather than +only from `allocate()`, has not been checked and decides whether the safe half is usable at all. + +### 4. The impossible direction stays impossible, and is already handled + +None of this changes that a WasmGC reference cannot be stored in linear memory — that is a +spec-level guarantee, not a maturity gap, so CPython can never hold a Kotlin object pointer in a +`PyObject` field. The answer there remains an index into a Kotlin-side table, which is what +`docs/design/object-lifetime.md` already specifies for Python→Kotlin. Kotlin/Wasm even blesses the +mechanism: `JsReference` passes a Kotlin object as an opaque reference. + +### What this changes + +| | as recorded above | after reading the compiler | +|---|---|---| +| Kotlin → CPython call | JS hop | **direct wasm call** (`@WasmImport`) | +| string marshalling | JS copies between two memories | **possibly none** — if the memory is shared | +| Kotlin allocating in that memory | assumed fine | **unsafe** — allocator starts at 0 | +| CPython holding a Kotlin object | impossible | impossible (unchanged) | + +The composed-shim design does not change shape. What changes is its cost, and the reason to build +it: composition was proposed here to amortise JS hops, and if the hops are gone it has to justify +itself on crossing count alone — the same argument that closed desktop composition in ROADMAP §6. + +### Unverified, in the order that decides the design + +1. Can the host supply Emscripten's `wasmMemory` as Kotlin's `("intrinsics", "memory")` import? + The generated JS glue builds that import object; whether it can be overridden is unknown. +2. Can `Pointer` be constructed from an arbitrary `Int`? If not, the read-only half is unusable + and only `@WasmImport` survives. +3. Is `("intrinsics", "memory")` stable? It is an internal name, not public API. +4. Does `@WasmImport` work when the exporting module is an Emscripten `MAIN_MODULE`, whose + exports are ordinary wasm functions but whose instantiation Emscripten's own glue controls? + +None of these needs code in this repo to answer; all four are answerable with a small standalone +experiment, and together they decide whether WASM looks like iOS or like the JS-bridge design. + +--- + +## Measured: what the released toolchain actually does + +The section above was written from the Kotlin compiler at master and got one thing wrong in a way +that matters. `wasm-experiment/` builds a trivial `wasmJs` module on Kotlin 2.4.10, runs it, and +parses the emitted binary. Findings, in the order they change the design. + +### The memory import is master-only; the released direction is the opposite + +`importWasmMemoryInsteadOfExport = isWasmJsTarget` is in the compiler source but **not in any +released version**. Parsing the 2.4.10 output: the only length-prefixed `intrinsics` import is +`intrinsics.tag` (the exception tag), there is no memory import, and the export section contains +`memory`. Same on 2.2.20. + +So Kotlin does not import a memory today. It exports one. + +**That inverts the plan, and the inverted plan is better** — because it needs nothing unreleased +on either side. Emscripten has `-sIMPORTED_MEMORY`, which makes the CPython module import +`env.memory` rather than define it. Instantiate Kotlin first, take `exports.memory`, grow it to +Emscripten's initial size, hand it over as `env.memory`. Both halves are shipped features. + +### Kotlin's linear memory is empty, and stays empty + +This is what makes handing it over safe, and it is measured rather than assumed: + +``` +pages at startup 0 +pages after string interop 0 +pages after collections 0 +pages after exception 0 +pages after explicit allocate 2 (allocated at 0x0) +``` + +The module declares `min_pages = 0`, and all 40 of its data segments are **passive** — they +initialise WasmGC arrays through `array.new_data` and are never placed in linear memory. String +interop was the obvious suspect for a hidden consumer and it is not one; Kotlin strings are +WasmGC arrays and cross to JS as `externref`. + +So a `wasmJs` module's linear memory is unused space that happens to be attached to it. Giving it +to Emscripten costs Kotlin nothing. + +### The allocator hazard is real but avoidable + +The last probe line confirms the source reading: `withScopedMemoryAllocator` grew the memory and +allocated at **address 0**, exactly on top of where Emscripten keeps its static data. In a shared +memory this corrupts the heap on first use. + +The mitigation is not a workaround, it is the natural design: **CPython allocates, Kotlin only +reads.** `Pointer(addr)` has a public constructor and its `loadInt`/`storeByte` members compile to +plain `i32.load`/`i32.store` with no allocator involvement — confirmed by compiling and running +it. Every address we would ever dereference comes back from a C call. + +`withScopedMemoryAllocator` must therefore never appear in `wasmJsMain`. That belongs in the +source-set README as a hard rule, in the same category as desktop's "always `invokeExact`". + +### What the picture looks like now + +| | earlier draft | measured | +|---|---|---| +| Kotlin → CPython call | JS hop | direct wasm call (`@WasmImport`) | +| who owns the shared memory | CPython, imported by Kotlin (master-only) | **Kotlin exports it, Emscripten imports it** | +| Kotlin reading `char*` | impossible | `Pointer(addr).loadByte()`, no JS, no allocator | +| string marshalling | dominant cost | **none, if the memory is shared** | +| CPython holding a Kotlin object | impossible | impossible (unchanged — handle index) | + +If this holds end to end, WASM stops being the outlier and lands close to the iOS shape: no +boundary for data, a direct call for control, and a handle table for the one direction WasmGC +forbids. The composition argument weakens accordingly — it was proposed here to amortise JS hops, +and without them it must justify itself on crossing count alone, which is what closed desktop +composition in ROADMAP §6. + +### Still unverified + +1. **Does `@WasmImport` bind to an Emscripten module's export?** The mechanism is proven by the + compiler's own `callingWasmDirectly.kt` codegen test, but against a hand-built module, not one + whose instantiation Emscripten's glue controls. Needs `emsdk`, which is not installed here. +2. **Does Emscripten accept a memory that starts at 0 pages and is grown by JS before + instantiation?** `-sIMPORTED_MEMORY` normally pairs with a memory JS created at the right size. +3. **Does anything in a larger Kotlin program touch linear memory?** The probe covers strings, + collections and exceptions on a trivial module. Coroutines, `ByteArray` interop and the JS + `ArrayBuffer` bridges are untested. + +--- + +## Proven end to end: Kotlin and CPython-side C can share one memory + +The section above listed three unverified items and said the first needed `emsdk`. It is +installed, both experiments are built and run, and both pass. `wasm-experiment/` reproduces them. + +### Test A — `@WasmImport` binds to an Emscripten export + +Kotlin called `add_two` compiled by `emcc` and got 42. The compiler emits a real wasm import, +`(import "./probeA-wrapper.mjs" "add_two" (func …))`, and wires the ES module into the import +object. The value supplied prints as `function 2() { [native code] }` — a raw wasm export, not a +JS wrapper — so the call is wasm-to-wasm with no JS frame, which is what the compiler's own +`callingWasmDirectly.kt` test intends. + +### Test B — Emscripten imports the memory Kotlin exports + +``` +C: get_static_message() -> 0x400 +Kotlin: readCStringAt(same address) -> "hello-from-the-cpython-side" PASS +C: alloc_message() -> 0x10628 +Kotlin: writeCStringAt(addr, "written-by-kotlin") +C: str_len(addr) -> 17, UTF8ToString(addr) -> "written-by-kotlin" PASS +``` + +Both directions, static data and `malloc`'d heap. **No copying and no JS in the data path.** +Kotlin dereferences a C address with `Pointer(addr)` and gets the bytes C put there. + +### The single blocker, and it is one value + +Emscripten refused Kotlin's memory at first: + +``` +LinkError: memory import has no maximum limit, expected at most 4294967295 +``` + +Kotlin emits `WasmLimits(0, null)` — no maximum — and wasm requires a supplied memory's limits to +sit inside the importer's, so an unbounded memory can never satisfy any Emscripten import. +Patching section 5 to `{min: 0, max: 32768}` made everything link and both directions work. +Nothing else about the pairing needed changing. + +**That is a compiler-chosen constant, not a design problem.** It is worth a YouTrack issue; a +`-Xwasm-memory-maximum`-style knob, or simply emitting the wasm32 ceiling, would remove the need +to post-process the binary. Until then a build step can patch it, which is ugly but bounded — the +memory section is six bytes and nothing in the format holds an absolute offset. + +### The constraint that shapes the design + +A and B cannot be combined in one instantiation graph while Kotlin owns the memory: + +``` +Kotlin @WasmImport needs Emscripten's exports at instantiation time +Emscripten -sIMPORTED_MEMORY needs Kotlin's memory before that +``` + +Wasm imports are supplied up front, so that is a cycle. Three ways out: + +1. **Kotlin imports the memory instead of exporting it.** No cycle — Emscripten instantiates + first, then Kotlin receives both the memory and the functions. This is + `importWasmMemoryInsteadOfExport = isWasmJsTarget`, present at master and in no release. It is + the configuration this design wants, and the second reason to talk to JetBrains. +2. **JS trampolines for calls, shared memory for data.** Available today. Costs a JS hop per + call and keeps the data path free — and the data path is the half that dominated every + measurement on Android and desktop, so this captures most of the win. +3. Copy through JS, as originally designed. + +### What §10 looks like now + +| | original design | proven | +|---|---|---| +| control path | JS hop per call | direct wasm call, **or** JS trampoline if sharing memory | +| data path | JS copies between two memories | **none — one memory** | +| string marshalling | dominant cost, needs interning | **absent** | +| composition | needed to amortise JS hops | **weakly motivated** — same argument that closed §6 | +| CPython holding a Kotlin object | impossible | impossible — handle index, per `object-lifetime.md` | + +WASM stops being the outlier. With option 2 it is roughly Android's shape with a cheaper data +path; with option 1 it is close to iOS. + +### Rules this produces for `wasmJsMain` + +- **`withScopedMemoryAllocator` must never be called.** It allocates at address 0, on top of + Emscripten's static data. Measured: the probe grew the memory to 2 pages and handed back `0x0`. + CPython allocates; Kotlin only dereferences addresses it was given. +- `Pointer(addr)` is the only way in, and its public constructor plus raw `i32.load`/`i32.store` + members make that free. + +### Resolved + +- **Linear memory is strictly unused.** The probe was expanded to include large `ByteArray` allocations (10MB), large object graphs (100,000 items), `Uint8Array` JS ArrayBuffer bridges, and coroutines (`kotlinx.coroutines.GlobalScope.launch`). **None of them touched linear memory.** The page count stayed exactly at 0 until `withScopedMemoryAllocator` was explicitly called. This confirms that a shared linear memory design is perfectly safe and won't be corrupted by normal Kotlin runtime behavior. +- **JS trampoline cost is ~2.7x.** A microbenchmark running 10,000,000 iterations found a direct `@WasmImport` call took ~48.7 ms, while routing the call through a JS trampoline took ~132.9 ms (2.73x slower). This shows the JS boundary imposes measurable overhead, but since we eliminate copying overhead for strings and data, this cost is a worthwhile tradeoff to achieve the shared memory model. +- **Unbounded Memory bug.** Kotlin's compiler emits `WasmLimits(0, null)`, causing Emscripten to reject the imported memory. This is confirmed to be the only blocker. A draft YouTrack issue has been written to `docs/platforms/wasm-youtrack-issue.md` requesting a compiler flag (or a default Wasm32 ceiling) to fix this. + +### Still open + +- Upcalls. `addFunction` still routes through JS to re-enter WasmGC, and nothing here changes + that; §7 still needs measuring before `wasmJsMain` is written. + +### Correction: the coroutine reading was contaminated + +The result above was nearly reported wrong. `runPromise` resolves after `main()` returns, so the +coroutine's page count was taken *after* the `withScopedMemoryAllocator` demo had already grown +the memory to 2 pages. The probe printed `pages inside coroutine 2` and that number said nothing +about coroutines — it was the allocator's growth, observed late. + +Gating the allocator demo off and re-running gives the reading that means something: + +``` +pages at startup 0 +pages after string interop 0 +pages after collections 0 +pages after exception 0 +pages after large ByteArray 0 (10 MiB) +pages after large obj graph 0 (100k strings) +pages after ArrayBuffer bridge 0 +pages observed inside coroutine 0 <- uncontaminated +``` + +So the conclusion holds, and now for coroutines too: **nothing in ordinary Kotlin/Wasm touches +linear memory.** `MEASURE_ALLOCATOR` in the probe is off by default for this reason, and the +coroutine line must be read as meaningless in any run where it is on. + +--- + +## Resolved upstream: 2.4.20-Beta2 imports the memory + +The YouTrack issue drafted in `wasm-youtrack-issue.md` should not be filed. The thing it asks for +has already shipped, and in a better form than the request. + +`importWasmMemoryInsteadOfExport = isWasmJsTarget` was read on the compiler's `master` branch and +recorded here as unreleased, on the evidence that 2.2.20 and 2.4.10 both export a memory. That was +right about those two versions and wrong about the conclusion: **2.4.20-Beta2 imports it.** The +experiment builds against it and the binary says so plainly: + +``` +imports intrinsics.memory (kind 0x02, memory) + intrinsics.memory (kind 0x03, global) + intrinsics.tag +memory definitions 0 <- defines none of its own +non-function exports none <- no longer exports one +``` + +And the generated glue hands it in as an ordinary JavaScript value: + +```js +intrinsics: { + memory: new WebAssembly.Memory({ initial: 0 }), + tag: wasmTag +}, +``` + +Replacing that one expression with Emscripten's `Module.wasmMemory` is the whole integration. No +binary patching, no compiler flag, no `max` limit to negotiate — `patch-memory-max.py` becomes +history rather than a build step. + +### It also removes the constraint that shaped the design + +The instantiation cycle is gone. While Kotlin exported the memory, Emscripten had to import it +while Kotlin simultaneously needed Emscripten's exports through `@WasmImport`, and wasm supplies +imports up front, so the two could not be satisfied together. The recorded workaround was JS +trampolines for the calls, measured at 2.73x a direct call. + +With Kotlin importing, the order resolves: instantiate Emscripten first, then hand Kotlin both the +memory and the functions. **Direct wasm-to-wasm calls and a shared linear memory at the same +time** — which is the configuration this design wanted and had written off as unavailable. + +### What that leaves + +The last structural obstacle to §10 is gone. Remaining work is ordinary: pin the Kotlin version at +2.4.20-Beta2 or later (the library targets 2.0.20 today, so this is a real upgrade with its own +cost), build CPython for `pyemscripten_2026_0`, write the composed shim, and supply +`Module.wasmMemory` in place of the placeholder above. + +Two facts from earlier still hold and still constrain the shape. Kotlin never touches its linear +memory on its own — measured across strings, collections, exceptions, a 10 MiB `ByteArray`, a +100k-object graph, the `ArrayBuffer` bridge and coroutines — so handing it to Emscripten costs +nothing. And `withScopedMemoryAllocator` must never appear in `wasmJsMain`, because it allocates +from address 0 upward, on top of Emscripten's static data. + +--- + +## Test C: run, and the cycle is gone + +The section above predicted this from reading the binary. `wasm-experiment/` now builds and runs +it. `native/run.sh` reproduces; Test C is `native/combined-test.mjs`. + +### The binary confirms the shape + +Parsing `wasm-experiment.wasm` built on 2.4.20-Beta2: + +``` +import './probeA-wrapper.mjs' . 'add_two' kind=0 func <- direct wasm call +import 'intrinsics' . 'tag' kind=4 tag +import 'intrinsics' . 'memory' kind=3 global externref +import 'intrinsics' . 'memory' kind=2 memory min=0 max=none +memory definitions 0 +non-function exports none +``` + +`max=none` on the *import* is what makes this direction work. A wasm import must be satisfied by +a memory whose limits sit inside the declared ones, so an unbounded import accepts anything — +including Emscripten's bounded memory. The old direction was unsatisfiable for exactly the +mirror-image reason, and `patch-memory-max.py` existed to force it. It is no longer used. + +### The integration is one substitution + +`patch-import-object.py` rewrites one expression in the generated glue: + +```js +intrinsics: { memory: new WebAssembly.Memory({ initial: 0 }), tag: wasmTag } + -> memory: .wasmMemory +``` + +Ordering comes free from ES modules. `wasm-experiment.import-object.mjs` imports the wrapper, the +wrapper has a top-level `await factory()`, so Emscripten is fully instantiated before Kotlin's +import object is built. Emscripten is compiled exactly as it would be alone — no +`-sIMPORTED_MEMORY`, no binary patching. Kotlin is the side that adapts. + +### Both halves hold at once + +``` +Emscripten's memory object Memory, 258 pages +Kotlin's intrinsics.memory Memory, 258 pages PASS same buffer +Kotlin called cAddTwo(40, 2) -> 42 PASS @WasmImport, memory shared +C: get_static_message() -> 0x400 +Kotlin: readCStringAt(that address) -> "hello-from-the-cpython-side" PASS +Kotlin: writeCStringAt(malloc'd addr, "written-by-kotlin") +C: str_len -> 17, UTF8ToString -> "written-by-kotlin" PASS +``` + +The value handed to the import still prints as `function 2() { [native code] }` — a raw wasm +export, no JS wrapper — so the control path is wasm-to-wasm with the memory shared. + +### And it survives the memory growing + +Not previously checked, and it is not optional: CPython links with `-sALLOW_MEMORY_GROWTH +-sINITIAL_MEMORY=20971520`, so the memory *will* grow underneath Kotlin in production. A JS +TypedArray view detaches when that happens. Kotlin holds the memory as a wasm import rather than +a view, and does not: + +``` +C: malloc(64 MiB) -> 0x10638, 258 pages -> 1026 pages + PASS -- the shared memory actually grew + PASS -- Kotlin still reads the pre-growth address correctly + PASS -- Kotlin writes into memory that did not exist when it was instantiated +``` + +### Measured: the control path + +10,000,000 calls of the same trivial C function, one process, Node 24, Emscripten 5.0.3. + +| | ns/call | +|---|---| +| direct `@WasmImport` | **5.0** (stable across runs) | +| JS trampoline | 13.6 – 16.9 (2.7x – 3.4x) | + +So removing the cycle is worth about 9–12 ns per crossing. That is the whole benefit of option 1 +over option 2, and it is now available. + +### Measured: the data path — and the earlier conclusion was wrong + +Everything above said string marshalling "does not merely get cheaper, it disappears". **It does +not.** Both loops below run in Kotlin and end with a Kotlin `String` built from the same C +address; only the route differs. + +| ns per read | 27 bytes | 4000 bytes | +|---|---|---| +| shared: scan to NUL, build nothing | 193 | 1 649 | +| shared: `ByteArray` copy, no decode | 399 | 4 875 | +| shared: `ByteArray` + `decodeToString()` | 405 | **58 271** | +| shared: `CharArray` + `concatToString()` (ASCII) | **103** | 5 875 | +| shared: `StringBuilder.append(Char)` per byte | 311 | 51 348 | +| copied: `UTF8ToString` through JS | 125 | **3 827** | + +Two things account for it, and neither is the boundary: + +1. **Kotlin/Wasm strings already *are* JS strings.** The generated glue compiles with + `builtins: ['js-string']`. `UTF8ToString` produces a JS string, and handing that to Kotlin is + not a conversion. The "copy through JS" path was assumed to pay for a copy that no longer + exists. +2. **`ByteArray.decodeToString()` is pathological on long input** — 53 µs of the 58 µs figure at + 4000 bytes, about 13 ns per byte, against 0.2 ns per byte at 27 bytes. `concatToString()` on a + `CharArray` costs a tenth of it. Worth a separate report upstream. + +The honest rule that comes out of this is not "marshalling disappears" but: + +- **`char*` → `String`: use the ASCII `CharArray` + `concatToString()` route, and fall back to + `UTF8ToString` for anything long or non-ASCII.** Never `decodeToString()` on a large buffer. +- **`char*` → `ByteArray` (the `PyBytes` case): shared memory, unambiguously.** No decode, and no + JS-side equivalent that avoids a copy. +- Shared memory is still what makes writing *into* C buffers free, which has no JS equivalent at + all short of `Module.HEAPU8.set`. + +### Measured: bulk, which is what the composed shim was for + +1000 `i32` out of a C-owned array, 20,000 repetitions: + +| | ns per element | +|---|---| +| shared memory, `Pointer.loadInt()` | **0.7** | +| through JS, `Module.HEAP32[a >> 2]` | 7.1 | + +So the composition arithmetic is now fully determined. Walking a list without a shim means N +direct `PyList_GetItem` calls at **5.0 ns** of crossing each. With a shim it is one crossing plus +N shared reads at **0.7 ns**. **Composition buys ~4.3 ns per element and nothing else** — the +CPython-side work is identical either way. + +ROADMAP §6 closed desktop composition at a 10% saving. Test D below measures the same question +against the real interpreter and answers it more sharply. + +--- + +## Test D: it works against real CPython, and it closes the composition question + +Tests A and C run against a 4 KB toy compiled by `emcc`. Test D runs against **CPython 3.14.2 +built for `wasm32-emscripten`** — 10 MB of wasm, 8191 exports, `-sMAIN_MODULE`, a growable memory, +instantiated by Emscripten's own glue. The build is `Tools/wasm/emscripten` (PEP 776, Tier 3 from +3.14) under Emscripten 5.0.3; see the build notes below. + +``` +python.wasm exports 8191 +CPython's memory 320 pages +Kotlin's intrinsics.memory 320 pages PASS same buffer + +Kotlin: pyExec("answer = 6 * 7; greeting = ...") -> 0 PASS +Kotlin: pyGlobalInt("answer") -> 42 PASS +Kotlin: pyGlobalString("greeting") -> "hello-from-cpython-42" PASS +Kotlin: pyExec("bytearray(48 MiB)") -> 0, 320 -> 934 pages PASS + Kotlin still reads CPython correctly after the growth PASS + a pre-growth string still reads correctly PASS +``` + +Kotlin wrote the Python source **directly into CPython's heap** with `Pointer.storeByte` on an +address `malloc` returned, called `PyRun_SimpleString` as a direct wasm import, then dereferenced +`PyUnicode_AsUTF8`'s `char*` to get the string back. No copying, no JS in either direction, and it +survives the interpreter growing its own memory by 600 pages underneath. + +### The one build change CPython needed, and it is not ABI-sensitive + +The Stable ABI symbols are all there — `-sMAIN_MODULE` is `LINKABLE`, so Emscripten exports +everything and emits `EXPORTED_FUNCTIONS is not valid with LINKABLE set` for CPython's own list. +**This retires the recorded worry that "CPython must be built with an explicit +`EXPORTED_FUNCTIONS` list; a stock Pyodide build strips the C API by dead-code elimination."** The +supported CPython build already exports the whole surface. + +What it does *not* do is let JavaScript reach them. `Module` carries 36 keys and none of the +interesting ones, so there is nothing to hand to `@WasmImport`. One flag fixes it: + +``` +-sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY,wasmExports,wasmMemory + ^^^^^^^^^^^^^^^^^^^^^^ +``` + +After a relink, `Module.wasmExports.PyRun_SimpleString` prints as `function 4426() { [native +code] }` — a raw wasm export — and `Module.wasmMemory` is the `WebAssembly.Memory` Kotlin's +`intrinsics.memory` needs. Both are JS-glue settings and neither appears in PEP 783's +ABI-sensitive list, so this costs nothing against the platform tag. + +### Composition, measured against the interpreter + +Reading one global out of `__main__`, 200,000 times, on the same shared-memory direct-call setup. +Each row adds a pure-Kotlin optimisation: + +| | ns | | +|---|---|---| +| naive | 259.5 | 2 `malloc` + 2 `free` + 2 string writes + 4 API calls | +| + interned C strings | 185.4 | names allocated once — the Android fix, for free here | +| + module/dict hoisted | **65.2** | 2 API calls — the floor | +| one crossing (`PyErr_Occurred`) | **2.9** | direct `@WasmImport`, real interpreter | + +**75% of the naive cost is removable without shipping a single line of C.** What is left is 65 ns +of two CPython calls, of which the crossings are 5.8 ns — 9% of the floor, 2% of the naive figure. +A composed `pmp_getattr` could merge those two calls into one and save **2.9 ns**. + +The bulk case ends the same way. Composition there buys the difference between a crossing and a +shared-memory read, 2.9 ns against 0.7 ns per element, and CPython does identical work either way. +The constraint that kept the case alive — `PySequence_Fast_ITEMS` is a macro over +`PyListObject->ob_item` and `abi3t` makes `PyObject` incomplete, so shared memory does *not* let +Kotlin walk a list's storage — turns out not to matter, because N calls at 2.9 ns is cheap. + +**Verdict: no composed shim on WASM.** Same answer as ROADMAP §6 gave desktop, reached the same +way, and with a wider margin: desktop's composition was worth 10%, this is worth 1–2% after the +free wins are taken. The signatures sketched earlier — `pmp_getattr`, `pmp_exec`, `pmp_list_items`, +`pmp_str_utf8`, `pmp_decref`, `pmp_scratch` — were motivated by amortising JS hops, and the hops +are gone. What replaces them is ordinary `wasmJsMain` Kotlin: `@WasmImport` per Stable ABI +function, an interning cache for repeated names, and `Pointer` reads for everything else. + +That also removes the second build pipeline. No extension module to build, version and ship +against a specific interpreter — which was the largest recurring cost in the plan. + +### What is still not answered + +Upcalls. `addFunction` re-entering WasmGC still goes through JS and is still unmeasured; §7's +shape has to settle before `wasmJsMain` is written. Nothing in Tests C or D touches it. + +--- + +## Building CPython 3.14.2 for Emscripten: what worked + +`/Volumes/macMini/wasm-build/build-cpython-emscripten.sh` reproduces it. The driver is CPython's +own `Tools/wasm/emscripten` (`build` runs configure-build-python → make-build-python → libffi → +mpdecimal → configure-host → make-host). It succeeded end to end on the first attempt against +Emscripten 5.0.3, and `python.sh --version` prints `Python 3.14.2`. + +``` +sysconfig.get_platform() emscripten-5.0.3-wasm32 +sysconfig.get_config_var('PYEMSCRIPTEN_PLATFORM_VERSION') None +Checked 114 modules (85 built-in, 6 shared, 17 n/a, 1 disabled, 5 missing, 0 failed on import) +python.wasm 10 MB python.mjs 570 KB +``` + +### What the stock build passes, and how far it is from `pyemscripten_2026_0` + +Read off the actual command lines, not the documentation: + +| | stock CPython 3.14.2 | `pyemscripten_2026_0` | +|---|---|---| +| Emscripten | 5.0.3 (we pinned it) | **5.0.3** — matches | +| `-sWASM_BIGINT` | passed explicitly in `LDFLAGS_NODIST` | required; default since Emscripten 4.0 — matches | +| `-pthread` | absent; `configure` even reports `emcc accepts -pthread... no` | prohibited — matches | +| `-fPIC` | yes (required by `MAIN_MODULE`) | yes — matches | +| dynamic linking | `--enable-wasm-dynamic-linking` → `-sMAIN_MODULE`; extensions built `-shared -sSIDE_MODULE=1` | `MAIN_MODULE=1` / side modules — matches | +| **unwinding** | **nothing** — no `-fwasm-exceptions`, no `-sSUPPORT_LONGJMP=wasm` | **required at compile and link** — **DIVERGES** | +| **static libs** | zlib, bzip2, sqlite3, libffi, mpdecimal; **no lzma, no zstd** | + lzma 5.2.2, zstd — **DIVERGES** | +| **OpenSSL** | **absent** — "Could not build the ssl module" | shared side module, OpenSSL 1.1.1w — **DIVERGES** | +| platform variable | `PYEMSCRIPTEN_PLATFORM_VERSION` is **not defined anywhere in CPython 3.14.2** | `packaging` reads it to emit the tag — **DIVERGES** | + +So the stock Tier 3 build is close but cannot claim the tag, and the gap is real work rather than a +flag flip: the unwinding ABI has to be added at both compile and link, `lzma`/`zstd`/OpenSSL have to +be built as ports or side modules, and something has to define `PYEMSCRIPTEN_PLATFORM_VERSION`. + +A practical trap for whoever does it: the driver hardcodes `CFLAGS=-DPY_CALL_TRAMPOLINE +-sUSE_BZIP2` in its `configure` argv and appends user arguments *after*, and autoconf takes the +last assignment — so passing `CFLAGS=…` to add `-fwasm-exceptions` silently drops +`-DPY_CALL_TRAMPOLINE`. The full string has to be repeated. + +**Not verified:** that the ABI-matching build succeeds, or that a PyPI `pyemscripten_2026_0` wheel +loads into it. Only the stock build was run. + +### Environment note + +Installing Emscripten 5.0.3 through `emsdk install` **replaces `~/emsdk/upstream` in place** — the +previously active 6.0.6 now reports as not installed and would have to be re-downloaded. The active +version is a global, shared setting, not per-project. + +--- + +## Proven: the tag claim is real — a PyPI `pyemscripten_2026_0` wheel loads + +The section above listed four divergences between the stock Tier 3 build and +`pyemscripten_2026_0` and marked the whole thing "not verified". It has now been built and +tested against an actual wheel. `/Volumes/macMini/wasm-build/build-cpython-abi.sh` reproduces +it, into `cpython314-abi/` so the stock build survives for comparison. + +**Result: `pydantic_core-2.48.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl`, downloaded from +PyPI, imports and runs.** + +``` +python 3.14.2 emscripten-5.0.3-wasm32 +IMPORTED pydantic_core 2.48.0 + .../pydantic_core/_pydantic_core.cpython-314-wasm32-emscripten.so +validate_python(42) -> 42 +raised ValidationError +``` + +The `.so` is 4.4 MB of Rust/PyO3. The `ValidationError` matters as much as the `42`: it is an +exception propagating out of the side module, so the unwinding ABI is exercised and not merely +linked. + +### Only one of the four divergences actually gated it + +| | matched? | did it gate the wheel? | +|---|---|---| +| unwinding ABI | **yes, this build** | **yes — this was the whole gate** | +| `PYEMSCRIPTEN_PLATFORM_VERSION` | **yes, via a one-line CPython patch** | no — gates *claiming* the tag, not loading | +| lzma, zstd | still missing | no | +| OpenSSL | still missing | no | + +The negative control is exact. The same wheel on the stock build: + +``` +ImportError: could not load dynamic lib: .../_pydantic_core...so +LinkError: WebAssembly.Instance(): Import #204 "env" "__cpp_exception": + tag import requires a WebAssembly.Tag +``` + +Reading the binaries says why before running anything. The wheel's `.so` imports a **wasm tag**: + +``` +tag env.__cpp_exception <- the side module needs it +memory env.memory +table env.__indirect_function_table +``` + +and the two interpreters differ in exactly that: + +``` +cpython314-abi/python.wasm section id 13 (tag), exports __cpp_exception, __c_longjmp +cpython314/python.wasm no tag section, exports neither +``` + +So `-fwasm-exceptions -sSUPPORT_LONGJMP=wasm` is not a nice-to-have that affects code +generation quality — it is a link-level contract, and it is checked. `lzma`/`zstd`/OpenSSL are +ABI-sensitive in the Pyodide flag list but only bind wheels that link *them*; they cost the +stdlib modules `_lzma`, `_zstd`, `_hashlib`, `_ssl` and nothing else here. + +The build-driver trap recorded earlier is real and was avoided by repeating the whole string. +The result is visible in the generated Makefile: + +``` +CONFIGURE_CFLAGS= -DPY_CALL_TRAMPOLINE -sUSE_BZIP2 -fwasm-exceptions -sSUPPORT_LONGJMP=wasm +CONFIGURE_LDFLAGS= -fwasm-exceptions -sSUPPORT_LONGJMP=wasm +``` + +### `PYEMSCRIPTEN_PLATFORM_VERSION`, and a trap that silently produces a wrong tag + +`packaging` 26.3 implements PEP 783 as literally as the PEP describes: + +```python +def _emscripten_platforms(): + v = sysconfig.get_config_var("PYEMSCRIPTEN_PLATFORM_VERSION") + if v: + yield f"pyemscripten_{v}_wasm32" +``` + +Before defining it, the interpreter loads the wheel but cannot claim the tag — its platform tag +is `emscripten_5_0_3_wasm32` and no `pyemscripten_*` tag is produced at all. + +The obvious way to define it is a Makefile variable, because that is what `sysconfig +--generate-posix-vars` reads. **That silently produces the wrong tag.** `sysconfig` coerces any +value that parses as an integer, and Python accepts underscores inside integer literals: + +```python +int('2026_0') == 20260 int('2025_0') == 20250 int('2024_0') == 20240 +``` + +so the config var comes out as the integer `20260` and `packaging` emits +`pyemscripten_20260_wasm32` — a plausible-looking tag that matches nothing. Every version in +the PEP's series has this shape, so it is not specific to 2026. + +`sysconfig` already has the opt-out — `_ALWAYS_STR`, which holds the two Apple deployment +targets for the same reason. Adding one name to it is the whole fix: + +```python +_ALWAYS_STR = { + 'IPHONEOS_DEPLOYMENT_TARGET', + 'MACOSX_DEPLOYMENT_TARGET', + 'PYEMSCRIPTEN_PLATFORM_VERSION', # int('2026_0') == 20260 +} +``` + +After which the interpreter's first tag is exactly the wheel's: + +``` +config var = '2026_0' +first tag = cp314-cp314-pyemscripten_2026_0_wasm32 +wheel tag = cp314-cp314-pyemscripten_2026_0_wasm32 MATCH +``` + +This is worth reporting upstream: CPython defines no way to set the variable, and the one +mechanism a packager would naturally reach for corrupts it without any error. + +### What this changes for the plan + +The earlier decision — our own Emscripten CPython 3.14, no Pyodide dependency — stands, and the +recorded limitation shrinks again. "Compiled third-party extensions are a known limitation on +this platform" is now false for anything published under the PEP 783 tag. The remaining +limitation is narrower and ordinary: packages that need `lzma`, `zstd` or OpenSSL need those +built, and packages not published for the tag still have to be built. + +--- + +## Upcalls: measured, and better than every prediction in this document + +§5 and §7 step 3 specify one mechanism — `@JsExport` the trampoline, wrap it in a JS closure, +hand it to `addFunction` — and "Unresolved" writes it off as "goes through JS, unmeasured, and +likely worse than every other platform". Tests E and F measure it, and measure an alternative +this document does not consider. + +### There is no JS in the call path, because a funcref is a funcref + +`@WasmExport` puts a Kotlin function in the **wasm** export section with no type adapters, so +what JS receives is an exported wasm function, not a closure: + +``` +String(raw.kotlin_upcall_add) -> function 5097() { [native code] } +``` + +`WebAssembly.Table.prototype.set` accepts any exported wasm function whatever instance produced +it, so it can go straight into Emscripten's `__indirect_function_table`, and Emscripten's +`call_indirect` reaches it directly. JS appears **once, at registration** — to put a funcref in +a table — and never in a call. + +``` +table.grow(1) -> 2; table.set(2, kotlin_upcall_add) accepted +C: call_fp(2, 40, 2) -> 42 +``` + +Test E, 10,000,000 indirect calls issued from inside C so no JS is in the timed loop: + +| | ns/call | crossing | +|---|---|---| +| C → C (control, same module) | 0.7 | — | +| **C → Kotlin, `table.set`** | **3.1** | **2.4 ns** | +| C → Kotlin, `addFunction` + JS closure (the design's mechanism) | 10.9 | 10.2 ns | + +The design's mechanism is **3.5x** the direct one. And the direct upcall at ~3 ns is the mirror +of the direct downcall at ~5 ns — the two directions cost the same order, which is the thing +this document assumed could not be true. + +`addFunction` handed the wasm export directly works too, and deduplicates onto the entry +`table.set` already installed — so it is the same path, not a third one. + +### Re-entrancy holds: the WasmGC heap is live inside a frame CPython pushed + +The routing-by-data design needs the upcall to find a Kotlin object from a context integer and +write into linear memory. Both work from inside a C-pushed frame: + +``` +Kotlin registry[7] = "routed-through-a-cpython-frame" +C: call_fp(route, ctx=7, out=0x10628) -> 30 +C: UTF8ToString(out) -> "routed-through-a-cpython-frame" +``` + +An unrouted context returns cleanly rather than trapping, and allocating on the WasmGC heap +inside the upcall is fine. + +**Signatures must match exactly.** `call_indirect` is statically typed and does not coerce: +calling a `(i32)->i32` export through a `(i32,i32)->i32` slot raises `RuntimeError: null +function or function signature mismatch`. That is a runtime trap, not a compile error, so the +Kotlin trampoline's arity is a correctness requirement rather than a style question. + +### Test F: real CPython calls Kotlin, through a real `PyMethodDef` + +A toy's `call_indirect` is not what CPython does. CPython is built `-DPY_CALL_TRAMPOLINE`, and +`pycore_emscripten_trampoline.h` routes every `PyCFunction`, `getset` getter and setter through +`_PyEM_TrampolineCall`, which dispatches on arity. So the mechanism has to survive that. + +It does. A Kotlin `@WasmExport` of shape `(PyObject*, PyObject*) -> PyObject*`, put in CPython's +own table and registered in a `PyMethodDef` that Kotlin builds in linear memory: + +``` +CPython's __indirect_function_table: Table, length 13518 +table.grow(1) -> 13518; table.set(13518, kotlin_pycfunction) accepted +Kotlin: installPyCallback("kotlin_cb", fp=13518) -> 0 + +Kotlin: pyExec("cb_result = kotlin_cb(41)") -> 0 +Kotlin-side upcall counter: 0 -> 1 +Kotlin: pyGlobalInt("cb_result") -> 42 +type(kotlin_cb).__name__ = "builtin_function_or_method" +``` + +Python sees an ordinary builtin. Variadic arguments arrive intact, 1000 calls in a Python loop +all return correctly, and a bad argument raises rather than corrupting anything. The return +value is built by Kotlin calling `PyLong_FromLong` — **a downcall issued from inside an +upcall**, so both directions are live on one stack. + +### But CPython's own wasm trampoline never installs, and that is an upstream defect + +`Python/emscripten_trampoline.c` tries to replace its JS trampoline with a wasm one that +dispatches using `__builtin_wasm_test_function_pointer_signature` (a wasm-gc `ref.test`). A +four-argument Kotlin export matches none of the four shapes it tests, so it discriminates: + +* wasm trampoline → all four `ref.test`s fail → `SystemError: Handler takes too many arguments` +* JS fallback → `wasmTable.get(func)(arg1, arg2, arg3)` → called anyway, fourth argument zero + +``` +kotlin_cb_4(1) -> "returned 9906512" +``` + +It was called. **The JS fallback is live**, and instrumenting the swallowed `catch` says why: + +``` +[instr] fell back: LinkError: WebAssembly.Instance(): Import #0 "env" "memory": + memory import must be a WebAssembly.Memory object +[instr] wasmTable is undefined | wasmMemory is undefined +``` + +The `EM_JS` initialiser runs while `python.mjs` is still evaluating, and needs `wasmTable` and +`wasmMemory` — which in this configuration are **exports** of `python.wasm`, assigned only after +instantiation. The `LinkError` is swallowed by a bare `catch (e) {}` and the JS fallback is kept +silently. + +This is an instantiation cycle, not an ordering slip — the same shape as the one Kotlin's memory +import dissolved, pointing the other way. The trampoline module imports `env.memory` and +`env.__indirect_function_table` from the very module whose import object needs the trampoline. +`-sIMPORTED_MEMORY` was tried: it fixes the memory half and the failure moves to the table, + +``` +[instr] fell back: LinkError: Import #1 "env" "__indirect_function_table": + table import requires a WebAssembly.Table +[instr] wasmTable undefined wasmMemory object +``` + +and Emscripten has no imported-table setting to finish the job. So on this toolchain pair the +wasm trampoline cannot install at all. + +**The consequence is symmetric, which is why it does not change the design.** Every +`PyCFunction` call in the build crosses a JS frame — a C extension's exactly as much as +Kotlin's. Measured like for like, `abs()` (a C builtin declared `METH_O`) against a Kotlin +`METH_O` function with the identical wasm signature: + +| | ns/call | +|---|---| +| pure Python `def py_cb(x): return x+1` | 59 | +| C builtin `abs(x)` — `METH_O` | 73 | +| **Kotlin `kotlin_cb_o(x)` — `METH_O`** | **87** | +| | **14 ns attributable to the crossing** | +| Kotlin, `METH_VARARGS` | 161 | +| Kotlin, `METH_VARARGS` + tuple read | 183 | + +The `METH_VARARGS` rows carry CPython's argument-tuple construction; that cost is identical for +a C extension and is not the crossing. 14 ns against `abs` matches Test E's `addFunction` +figure, as it should — both are the JS-frame path. + +### What this means for §7 + +| platform | how Python re-enters the host | +|---|---| +| iOS | a plain function call | +| Android | a JNI callback | +| desktop | a Panama upcall stub | +| **WASM, slot reached by `call_indirect`** | **a plain `call_indirect` — ~3 ns** | +| **WASM, `PyMethodDef` today** | the same JS trampoline every C extension pays — ~14 ns over C | + +**§7's design holds on WASM, and the caveat it carries is wrong.** "WASM will have permanently +higher per-call overhead than Native/JVM due to the WasmGC ↔ JS ↔ Emscripten boundary" describes +a boundary that is not in the path. Routing by data through a small set of static trampoline +shapes works unchanged; the context integer, the WasmGC lookup and the linear-memory write all +survive re-entry. + +Two things change in step 3 of the staged plan: + +* **`@WasmExport` and `WebAssembly.Table.set`, not `@JsExport` and `addFunction`.** Same number + of moving parts, 3.5x cheaper, and JS touched once per trampoline rather than once per call. +* **Prefer slots CPython calls directly.** Type slots installed through `PyType_FromSpec` are + invoked as ordinary `call_indirect`; `PyMethodDef` entries and getset descriptors go through + `_PyEM_TrampolineCall`. Only the first gets the ~3 ns figure while the upstream defect stands. + (The slot/trampoline split is read from `pycore_emscripten_trampoline.h`, which wraps exactly + `_PyCFunction_TrampolineCall`, `descr_get_trampoline_call` and `descr_set_trampoline_call`; + the ~3 ns figure for the direct path is Test E's measurement, not a measurement of a type slot.) + +### Still open + +Nothing structural. `wasmJsMain` can be written against §7 as amended. What is *not* answered: + +- Whether a Kotlin exception thrown inside an upcall unwinds safely through CPython's frames. + Every trampoline here returns normally or returns `NULL`. +- Reference-count ownership across the boundary, which is a contract question rather than a + toolchain one. +- ~~Whether Kotlin/Wasm offers a GC hook equivalent to `Cleaner`/`createCleaner`.~~ **Answered.** + The stdlib does not, but the *host* does, and `toJsReference()` reaches it without pinning the + object it hands over — measured at `alive 0 / 200` with JS holding only a `WeakRef` and a + `FinalizationRegistry` entry. `registerCleaner` is built on that. The second finding cost more: + a collection is observable only across a **macrotask** boundary, because a `WeakRef` outlives the + job that created it and the registry callback is delivered as a task, and + `FinalizationRegistry.prototype.cleanupSome()` — which would have made it synchronous — has been + removed from V8. See `src/wasmJsMain/README.md` and ROADMAP §10 for the numbers. + + +### Verification status of the wheel claim + +**Demonstrated, reproducibly, as Test G in `native/run.sh`.** The wheel is the real PyPI file -- +`pydantic_core-2.48.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl`, 1,403,190 bytes, sha256 +`4fc45a49334c54541cbc97bf416d9300b4b1d3b2840dfb079b1123dc3f9ef5a6`, matching +`https://pypi.org/pypi/pydantic-core/json` exactly. It contains a 4,408,639-byte +`_pydantic_core.cpython-314-wasm32-emscripten.so` (Rust/PyO3, the compiled extension). + +Against the ABI build's own `python.sh` (no Kotlin involved -- this is a CPython Emscripten +loader question, not a Kotlin one): + +``` +pydantic_core imported OK, version: 2.48.0 +validate_python("42") -> 42 +raised: ValidationError +``` + +Against the stock build, same wheel, same `python.sh` harness, as the negative control: + +``` +ImportError: could not load dynamic lib: .../_pydantic_core.cpython-314-wasm32-emscripten.so +LinkError: WebAssembly.Instance(): Import #204 "env" "__cpp_exception": tag import requires a WebAssembly.Tag +``` + +identical to what the original report quoted. Comparing the two `python.wasm` binaries directly: +the stock build has no tag section and no `__cpp_exception` symbol, the ABI build has two tags and +the symbol -- and that difference is exactly what separates the pass above from the failure. The +unwinding ABI (`-fwasm-exceptions -sSUPPORT_LONGJMP=wasm`) is both present only in the ABI build +and the thing that gates loading a compiled `pyemscripten_2026_0` wheel; PEP 783's premise that +matching the ABI unlocks compiled extensions holds up on this tree, for this wheel. + +Scope of what this settles: one wheel (pydantic-core), one path (import + a validator call + +exception raise). It does not check every C-API surface a compiled extension might touch, nor +whether other wheels on this tag rely on something the ABI build's flag set doesn't cover. diff --git a/docs/platforms/wasm-youtrack-issue.md b/docs/archive/wasm-youtrack-issue.md similarity index 78% rename from docs/platforms/wasm-youtrack-issue.md rename to docs/archive/wasm-youtrack-issue.md index cb4d26795..ef7342c16 100644 --- a/docs/platforms/wasm-youtrack-issue.md +++ b/docs/archive/wasm-youtrack-issue.md @@ -1,3 +1,11 @@ +> **Superseded** by [`../platforms/wasm-design.md`](../platforms/wasm-design.md) on 2026-10-03; kept for history. +> **Do not file this.** Kotlin 2.4.20-Beta2 (the version the build pins, `gradle/libs.versions.toml`) +> makes a `wasmJs` module **import** `intrinsics.memory` instead of exporting an unbounded one, so the +> blocker below no longer exists: the build substitutes Emscripten's `wasmMemory` for that import +> (`patchKotlinWasmOutputForCPython` in `python-multiplatform/build.gradle.kts`) and no binary patching +> of a memory section is needed. The `Kotlin 2.0.20 / 2.4.10` behaviour described here is accurate for +> those versions only. + # YouTrack Issue Draft: Kotlin/Wasm (wasmJs) emits unbounded memory, preventing Emscripten interoperability ## Title diff --git a/docs/design/architecture.md b/docs/design/architecture.md index 8852a8f71..db08d8c25 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -1,7 +1,8 @@ # 아키텍처 -전체 그림과 방향별 상세는 [`../PROJECT.md`](../../PROJECT.md) 에서 시작한다. 이 문서는 소스셋 계층, -타입 래퍼 구조, 검증 수단을 다룬다. +전체 그림과 방향별 상세는 [`PROJECT.md`](../../PROJECT.md) 에서 시작한다. 이 문서는 소스셋 계층, +타입 래퍼 구조, 계층 규칙, 테스트 계층을 다룬다. 의도·사양의 기준은 [`INTENT.md`](../INTENT.md), +[`SPEC.md`](../SPEC.md) 이고, 이 문서는 설계 기록이다. ## 소스셋 계층 @@ -21,19 +22,22 @@ commonMain `wasmJsMain` 은 **중간 소스셋 없이 `commonMain` 바로 아래에 둔다.** 공유할 상대가 없고(JNI·Panama· cinterop 어느 쪽과도 겹치지 않는다), 바로 아래에 적힌 `expect inline fun` 제약을 피하기 위해서이기도 -하다. `EmbedAPI.kt` 의 `inline` 132개가 그대로 `actual inline` 이 된다. +하다. `EmbedAPI.kt` 의 `expect inline fun`(현재 131개)이 그대로 `actual inline` 이 된다. `jvmMain` 은 정식 중간 소스셋이므로 **여기에 `expect` 를 두고 두 리프에서 `actual` 을 제공하는 것이 가능하다.** 실험으로 확인했다. 단, **`expect inline fun` 과 조합하면 Kotlin 2.0.20 에서 `Internal error in file lowering` 컴파일러 -크래시가 난다.** 마이그레이션 대상 함수에서 `inline` 을 떼면 해결된다. `EmbedAPI.kt` 의 `inline` 147개 -중 132개는 컴파일러가 이미 "인라인 이득 없음" 경고를 내고 있어, 떼는 데 실질적 손해가 없다. +크래시가 난다.** 마이그레이션 대상 함수에서 `inline` 을 떼면 해결된다. `EmbedAPI.kt` 의 `inline` 중 +다수는 컴파일러가 이미 "인라인 이득 없음" 경고를 내고 있어, 떼는 데 실질적 손해가 없다. (크래시는 +Kotlin 2.0.20 에서 관찰됐다. 빌드는 이후 Kotlin 2.4.20-Beta2 로 올라갔고(`gradle/libs.versions.toml`), +이 크래시를 그 버전에서 다시 확인하지는 않았다 — 그래서 AGENTS.md §16 은 여전히 JVM 계열 통합 전에 +확인하라고 한다.) ## FFI 계층 ``` -commonMain/kotlin/python/native/ffi/EmbedAPI.kt expect 315개 (C 심볼 310개) +commonMain/kotlin/python/native/ffi/EmbedAPI.kt expect 약 314개 (그중 `inline` 131개) ├── nativeMain/.../EmbedAPI.native.kt cinterop (iOS, androidNative) ├── androidMain/.../EmbedAPI.android.kt + bindings.kt JNI ├── desktopMain/.../EmbedAPI.desktop.kt + bindings.kt + Panama.kt Panama @@ -45,7 +49,8 @@ commonMain/kotlin/python/native/ffi/EmbedAPI.kt expect 315개 (C 심볼 310 desktop·native 와 동일하다. 포인터는 `NativePointer` (value class, `address: Any`) 로 통일한다. 플랫폼별 실체는 `Long`(JVM), -`CPointer<*>`(Native), `Int`(wasm32) 다. +`CPointer<*>`(Native), `Int`(wasm32) 다. 데스크톱 FFM 은 포인터를 `ADDRESS` 가 아니라 `JAVA_LONG` 으로 +넘기고 `invokeExact` 만 쓴다 (`desktopMain/README.md`). `Any` 래핑 때문에 변환마다 박싱이 일어난다. `@HighOverheadNativeCall` 애노테이션이 비싼 변환 지점을 표시한다. @@ -64,11 +69,12 @@ desktop·native 와 동일하다. ``` PyObject 모든 래퍼의 기반. NativePointer 보유, PyAutoCloseable 상속 ├── PyType 인스턴스 캐시 보유 - ├── PyException CPython 에러 인디케이터 ↔ Kotlin throwable - ├── 기본 타입 PyInt PyFloat PyBool PyString PyNone PyComplex - ├── 컬렉션 PyList PyTuple PyDict PySet PyFrozenSet - ├── PyIterator / PyIterable - └── PyModule + ├── PyException CPython 에러 인디케이터 ↔ Kotlin throwable (exceptions/) + ├── 기본 타입 PyInt PyFloat PyBool PyString PyBytes PyNone PyComplex (types/basic) + ├── 컬렉션 PyList PyTuple PyDict PySet PyFrozenSet (types/collections) + ├── PyIterator / PyIterable, 제너레이터·코루틴 (types/iteration) + ├── PyModule / PyPackage (types/modules) + └── 호출 가능·코드·디스크립터·유틸리티 (PyFunction, PyMethod, PyCode, PyProperty, PyRange, PyWeakRef ...) conversion/ Context, ConversionStrategy, PyProxy, PyValue ``` @@ -86,22 +92,17 @@ Kotlin 제약 때문에 다이어그램대로 갈 수 없던 지점들: ### JVM 시그니처 충돌 주의 -`PyObject` 의 타입 프로퍼티는 `pyType` 이다. `type` 으로 두면 JVM 에서 `getType()` 접근자가 생겨 -공개 메서드 `getType()` 과 **같은 시그니처로 충돌**하고, 모든 서브클래스가 Android·Desktop 에서 +`PyObject` 의 타입 프로퍼티는 `Type` (대문자 T, `val Type: PyType`) 이다. 소문자 `type` 프로퍼티와 +별도의 `getType()` 메서드를 함께 두면 JVM 에서 접근자 `getType()` 이 **같은 시그니처로 충돌**하고, 모든 서브클래스가 Android·Desktop 에서 컴파일에 실패한다. Kotlin/Native 에는 그 규칙이 없어 JVM 타깃을 실제로 빌드하기 전까지 드러나지 않았다. ## 검증 수단 -| 명령 | 커버 범위 | -|---|---| -| `:python-multiplatform:iosSimulatorArm64Test` | commonMain + nativeMain + iosMain, **실제 CPython 구동** | -| `:python-multiplatform:compileKotlinAndroidNativeArm64` | commonMain + nativeMain + artMain. Xcode 불필요 | -| `:python-multiplatform:compileKotlinDesktop` | commonMain + jvmMain + desktopMain | -| `:python-multiplatform:compileDebugKotlinAndroid` | commonMain + jvmMain + androidMain | - -**iOS 시뮬레이터 테스트만이 코드를 실제로 실행한다.** 나머지는 컴파일 검증이다. Desktop 코드는 아직 -단 한 줄도 실행된 적이 없다. +명령 목록과 어떤 소스셋을 덮는지는 AGENTS.md §15 가 기준이다 (`compileKotlinAndroidNativeArm64`, +`compileKotlinIosSimulatorArm64`, `compileKotlinDesktop`, 그리고 `ksp-fixtures` 세 모듈을 각각 따로). +테스트는 컴파일 검증만이 아니다 — 데스크톱(`desktopTest`), iOS 시뮬레이터, 안드로이드 기기, wasm(Node) +에서 실제 CPython 을 구동하는 스위트가 있다 (SPEC §0 의 "Test path" 열). `Python.framework` 에는 표준 라이브러리가 없어 `Py_Initialize()` 가 `Failed to import encodings module` 로 프로세스를 죽인다. 빌드가 배포 아카이브에서 stdlib 를 풀어 @@ -196,6 +197,11 @@ expect abstract class PyAutoCloseable(pointer: NativePointer) { } ``` +> 이 스니펫은 그 커밋 시점의 모양이다. 이후 `PyAutoCloseable` 은 `expect` 가 아닌 **commonMain 의 +> 일반 추상 클래스**가 되었고(`python/multiplatform/ref/PyAutoCloseable.kt`), 플랫폼 차이는 +> `expect interface PlatformCleaner` + `expect fun registerCleaner(pointer, closeAction)` 로 옮겨졌다. +> 아래 표가 현재의 플랫폼별 메커니즘이다. + **2. Gave it an abstract `clean()`, so the cleanup action comes from the subclass.** Before, the base held a hardcoded (and commented-out) `//PyDecRef(this.pointer)`; after, it calls `clean()` and `PyObject` supplies it. @@ -213,16 +219,22 @@ bodies — which is how the commented logic became code. | platform | lifetime mechanism after the split | |---|---| -| Kotlin/Native | explicit `close()` | +| Kotlin/Native (iOS, androidNative) | `kotlin.native.ref.createCleaner` (`close()` 도 가능, 한 번만 실행) | | Android API 33+ | `java.lang.ref.Cleaner` | -| Android API 26-32 | `PhantomReference` + `ReferenceQueue` + daemon thread | +| Android API 26-32 (minSdk 26) | `PhantomReference` + `ReferenceQueue` + daemon thread | | Desktop | `java.lang.ref.Cleaner` | +| wasmJs | JS 쪽 약한 참조 기반 레지스트리 (`PyAutoCloseable.wasmJs.kt`) | + +(minSdk 가 24 였던 시절의 서술이다. 현재 minSdk 는 26 이고 API 26-32 가 `PhantomReference` 경로를 쓴다.) The split is sound and stays. What was lost was incidental: `PyObject.android.kt` had been the only place an `actual external fun` could go, and that was the hook for routing an operation through a single composed JNI call. Measurement later put a number on it — 5.5x on one `getAttr`, 11x on a 1000-element list conversion. Recovering it does not mean reverting the -commit; it means expressing composed operations in the FFI layer, as above. +commit; it means expressing composed operations in the FFI layer, as above. (The `PyObject_GetAttrComposed` +snippet is illustrative, not a function in the tree. For string-carrying calls, interning later beat +composition and replaced it — `marshalling-design.md`; composition remains only where crossing count +scales with N.) ## Test layering: low-level and assembled @@ -241,17 +253,18 @@ Android, everything under `androidInstrumentedTest` that reaches through `bindin ### Why both, everywhere -`commonTest` was entirely assembled and ran only on iOS and desktop. Android's instrumented -tests were entirely low-level. So no test anywhere called the real object model on a device — +At the time, `commonTest` was entirely assembled and ran only on iOS and desktop. Android's +instrumented tests were entirely low-level (today `commonTest` also runs on Android and wasm; SPEC §0). So no test anywhere called the real object model on a device — and it turned out `Python3.exec` crashes the process on its first call there, because several functions it reaches still take a Kotlin `String` straight across JNI. Fourteen green Android tests said nothing about that. A test that does not walk the path callers walk proves only that some other path works. -The Android assembled tests are `@Ignore`d until ROADMAP §2 lands, because they crash the +The Android assembled tests were `@Ignore`d until ROADMAP §2 landed, because they crashed the instrumentation process rather than failing, and a crashed runner takes every other test with -it. They are the acceptance check for that work. +it. They were the acceptance check for that work; §2 is closed (213 tests per emulator, zero +failures, per ROADMAP §2) and `AssembledApiTest.kt` no longer carries an `@Ignore` on a test. ### Two things to know when adding tests here diff --git a/docs/design/binding-policy.md b/docs/design/binding-policy.md index fc1bc837b..9603149ec 100644 --- a/docs/design/binding-policy.md +++ b/docs/design/binding-policy.md @@ -1,7 +1,13 @@ # 노출 정책 — 무엇이 Python 에서 보이는가 Kotlin 의 무엇을 함수 테이블에 등록해 Python 에 노출할지에 대한 규칙. 테이블 자체의 구조는 -[`upcall-design.md`](upcall-design.md). +[`upcall.md`](upcall.md). + +이 문서의 표는 **KSP 프로세서가 소스에서 읽는 선언**(`python-multiplatform-ksp` 의 `BindingPolicy.kt`)에 +대한 규칙이다. 미리 컴파일된 jar 를 읽는 아티팩트 워커(`python-multiplatform-gradle-plugin`)는 +같은 블랙리스트 정신을 따르되, 소스 없이 바이트코드/Kotlin 메타데이터에서 읽으므로 일부 행이 +다르다 — 아래 해당 행과 §"결정 이력" 에 표시했다. Kotlin 이름은 어느 쪽이든 그대로 노출되며 +(AGENTS.md §12.1) 이 정책에는 이름을 바꾸는 규칙이 없다. ## 원칙: 블랙리스트 @@ -46,7 +52,9 @@ class InternalUtility { } // 클래스 통째로 제외 | **abstract / sealed 클래스의 생성자** | ❌ | "Cannot create an instance of an abstract class" — 생성된 파일이 컴파일되지 않는다 | | `private` / `protected` | ❌ | | | **`internal`** | ❌ | JVM 바이트코드에서는 name-mangling 된 public 이지만 의미론적으로는 비공개다. KSP 의 `Modifier.INTERNAL` 로 걸러낸다 | -| **확장 함수** | ❌ | 수신자 없이 호출할 수 없고 Python 호출 규약과 맞지 않는다 | +| **확장 함수 (KSP, 소스)** | ❌ | 생성되는 엔트리는 `Array` 위의 람다이고 수신자를 둘 자리가 없다 (`BindingPolicy.isExposedFunctionShape` 이 `extensionReceiver != null` 을 거른다). 아티팩트 워커는 다르다 — 아래 | +| **확장 함수 (아티팩트 워커, 컴파일된 jar)** | ✅ 수신자 프록시의 메서드로 | `Modifier.padding(16).size(24)` 처럼 체인이 이어진다 (SPEC U-7, `docs/design/kotlin-extensions-in-python.md`) | +| **소스 안의 `@Composable`** | ❌ (KSP) | 생성된 호출이 `@Composable` 컨텍스트 밖이라 컴파일되지 않는다. 컴파일된 jar 의 composable 은 워커가 바인딩한다 (SPEC B-6, `docs/design/ecosystem.md` §4) | ## 라이브러리 자신은 통째로 제외한다 @@ -60,9 +68,9 @@ class InternalUtility { } // 클래스 통째로 제외 | 대상 | 처리 | 근거 | |---|---|---| -| **`suspend` 함수** | 자동 제외 | Python 에 대응 개념이 없고, 숨은 `Continuation` 인자가 있어 호출 자체가 불가능하다 | +| **`suspend` 함수** | **노출한다** — 비동기 호출 규약으로 | 생성 본문이 `PendingCall.start { }` 로 코루틴을 시작하고, 경계는 값(중단 없이 끝남) 또는 `asyncio.Future`(중단됨)를 돌려준다. Python 에서 `await` 한다 (SPEC U-5, [`upcall.md`](upcall.md)). 처음에는 "자동 제외"였다 — 아래 정정 참조. `suspend` **함수 타입** 값(`suspend (Long) -> Long`)은 불투명 `OBJECT` 핸들로 나간다 | | **`inline` + `reified`** | 자동 제외 | 컴파일 후 reified 타입 정보가 사라진다. **실제 규칙은 이보다 넓다** — 타입 파라미터가 있는 선언은 `reified` 여부와 무관하게 전부 제외한다 | -| **값 클래스(value class)** | 등록하되 호출 시 실패 허용 | JVM 에서 언박싱된 형태로 노출되어 케이스가 갈린다. 완벽히 걸러내려면 복잡도가 크다 | +| **값 클래스(value class)** | KSP: 별도 규칙 없음(`BindingPolicy.kt` 에 값 클래스 분기가 없다). 아티팩트 워커: 왕복을 지원하고, 허용 목록의 것(예: `Dp`)만 원시 값으로 쓸 수 있다 | JVM 에서 언박싱된 형태로 노출되어 케이스가 갈린다 (SPEC B-4, `ComposableValueClassSlotTest.kt`) | `suspend` 와 `reified` 는 KSP 에서 선언 수준으로 판별되므로 걸러내는 비용이 낮다. @@ -73,9 +81,11 @@ class InternalUtility { } // 클래스 통째로 제외 특히 다음 둘은 재고 여지가 있다: -- **확장 함수 제외** — 수신자를 첫 인자로 받는 Python 함수로 매핑하는 것은 원리적으로 가능하다. 지금은 - 단순함을 택했다. -- **값 클래스** — "등록하고 실패하게 둔다"는 사용자에게 불친절하다. 실제 사용 양상을 보고 다시 정한다. +- **확장 함수 제외 (KSP)** — 소스 경로에서는 제외 상태 그대로다. 다만 이것이 풀린 것은 아티팩트 + 워커 쪽이다: 확장 함수는 수신자 프록시의 메서드로 노출된다(SPEC U-7). KSP 가 생성한 프록시에서의 + 동작은 아직 테스트로 단언되지 않았다. +- **값 클래스** — 초안의 "등록하고 실패하게 둔다"는 구현되지 않았다. 워커가 값 클래스를 다루고 + (SPEC B-4), KSP 는 따로 걸러내지 않는다. ## 구현 시점의 정정 (2026-08) @@ -90,3 +100,10 @@ class InternalUtility { } // 클래스 통째로 제외 붙는 이유이기도 하다. - **컴파일러 생성 멤버는 `copy` 와 `componentN` 이 실제로 KSP 에 보인다.** `equals`/`hashCode`/ `toString` 은 보이지 않았다. 관찰된 사실이며, 둘 다 제외한다. +- **`suspend` 는 제외 대상에서 빠졌다.** 초안은 "숨은 `Continuation` 인자 때문에 호출이 불가능하다"며 + 자동 제외했다. 비동기 업콜(`PendingCall`)이 생긴 뒤 이 수식어는 제외가 아니라 **본문 형태를 + 고르는 신호**가 되었다 (`BindingPolicy.isSuspending`, `GeneratedSuspendTest`). +- **추상·봉인 클래스는 클래스로는 노출하되 생성자만 뺀다** (`BindingPolicy.isConstructible`; `inner` + 클래스도 같다 — 바깥 인스턴스를 둘 자리가 없다). 표의 "abstract / sealed 클래스의 생성자" 행의 구현. +- **`internal set` 은 노출하지 않는다.** `private set` 은 생성 코드가 컴파일되지 않아서, `internal set` + 은 컴파일은 되지만 노출 대상이 모듈 공개 API 이기 때문이다 (`BindingPolicy.isExposedSetter`). diff --git a/docs/design/downcall-design.md b/docs/design/downcall-design.md index 661f7c06c..fce470e97 100644 --- a/docs/design/downcall-design.md +++ b/docs/design/downcall-design.md @@ -1,11 +1,19 @@ # 다운콜 설계 — Kotlin → Python Kotlin 코드가 CPython Stable ABI 함수를 호출하는 방향. 반대 방향은 -[`upcall-design.md`](upcall-design.md). +[`upcall.md`](upcall.md). + +> **현재 상태 (2026-10-03 기준, 코드로 확인).** 플랫폼별 다운콜 경로는 각 `EmbedAPI..kt` + +> `bindings.kt` 가 직접 구현한다: 데스크톱은 Panama `MethodHandle` + `invokeExact` (포인터는 +> `JAVA_LONG`), Android 는 `RegisterNatives` 로 등록한 JNI (함수별·API 레벨별 호출 규약), Native 는 +> cinterop. 아래 "shape 트램폴린" 14종은 **어휘(vocabulary)로 구현되어 있으나**(`jvmMain/.../ShapeDowncalls.kt` +> 의 `expect` + 데스크톱·Android·Native 구현) 330개 `actual` 을 `jvmMain` 한 벌로 합치는 통합에는 +> **쓰이지 않았다** — 소스에서 shape 함수를 부르는 곳은 데스크톱 `ProxyTypeFactory` 의 `tp_traverse`/ +> `tp_free` 호출 두 곳뿐이다. 이 문서의 shape 설계 절은 그 계획과 근거의 기록이다. ## 시그니처 shape 조사 결과 -`EmbedAPI.kt` 의 약 330개 `expect` 선언을 전수 분석한 결과다. +`EmbedAPI.kt` 의 약 330개 `expect` 선언을 전수 분석한 결과다 (조사 시점의 수. 현재 `expect` 선언은 314줄로 센다). | 항목 | 결과 | |---|---| @@ -40,13 +48,14 @@ Kotlin/Native 가 헤더를 읽어 직접 호출 코드를 낸다. **오버헤 리플렉션은 초기화 때 한 번만 돌고 이후는 캐시된 `MethodHandle` 을 쓴다. -**알려진 성능 문제**: 호출부 305곳이 `MethodHandle.invoke(...) as Long` 형태다. `invoke` 는 호출 -지점에서 `asType` 변환을 하고, Kotlin 에서는 반환 타입이 `Any!` 로 보이므로 **호출마다 박싱/언박싱이 -발생**한다. Panama 의 핵심 장점(JIT 인라인 가능, 할당 없는 다운콜)이 사라진다. `invokeExact` 는 호출 -지점의 정적 시그니처가 핸들 타입과 정확히 일치해야 하는데, shape 14종으로 고정하면 그 조건을 맞추기 -훨씬 쉬워진다. +**성능 문제는 해결되었다.** 초기에는 호출부 305곳이 `MethodHandle.invoke(...) as Long` 이었고 호출마다 +`asType` 변환과 박싱이 일어나 `PyList_Size` 한 번이 약 1016 ns 였다. 포인터를 `ADDRESS` 대신 +`ValueLayout.JAVA_LONG` 으로 기술하고 호출부를 `invokeExact` 로 바꿔 **2.65 ns** 가 되었다 (아래 +"Fixed: 1015.95 ns -> 2.65 ns"). 현재 `desktopMain/.../bindings.kt` 에는 `invokeExact` 315곳이 있고 +`.invoke(` 는 0곳이다 (AGENTS.md §16: 무조건 `invokeExact`). 리플렉션은 핸들을 **만들 때** 심볼당 한 번만 +돈다. `Linker.Option.critical()` 은 쓰지 않는다 (`downcallHandle(..., emptyOptions)`). -### Android(ART) — shape 트램폴린 (미구현) +### Android(ART) — shape 트램폴린 (구현됨, 단 주 경로는 아니다) ART 는 런타임에 임의 시그니처의 네이티브 호출을 만들 수 없다. 그래서 shape 마다 트램폴린을 **미리 컴파일**해두고, 대상 함수 포인터를 첫 인자로 받는다: @@ -60,16 +69,16 @@ fun downcallJ_J(fn: Long, a0: Long): Long = 주소는 런타임에 무엇이든 될 수 있고, **고정되는 것은 시그니처 형태뿐**이다. -도달 방법은 두 단계로 나눈다: +도달 방법은 두 단계로 계획했다: -| 단계 | 경로 | 비용 | +| 단계 | 경로 | 상태 | |---|---|---| -| 1 | 일반 JNI → 트램폴린 → 간접 호출 | JNI 전환 비용 포함 | -| 2 | ART 엔트리 포인트 패치 + `@CriticalNative` → 트램폴린 | 간접 분기 1회만 | +| 1 | 일반 JNI → 트램폴린 → 간접 호출 | **구현됨.** `androidMain/.../bindings.kt` 의 `downcall*` `external fun` 14개가 `jni_onload.def` 에서 등록되고, Kotlin/Native 트램폴린(`nativeMain/.../EmbedAPI.native.kt`)이 함수 포인터를 직접 호출한다. `ShapeDowncalls.android.kt` 가 감싼다 | +| 2 | ART 엔트리 포인트 패치 + `@CriticalNative` → 트램폴린 | **하지 않았다.** `@CriticalNative` 는 API 34+ 에서 오히려 비싸다고 실측됐고(아래), 호출 규약은 함수별로 고르는 것으로 정리됐다 | -1단계는 에뮬레이터 없이 컴파일 검증이 되고, 2단계는 같은 트램폴린 코드를 그대로 쓰면서 도달 방법만 -바꾼다. `@CriticalNative` 는 `JNIEnv` 도 `jclass` 도 GC 스레드 상태 전환도 없어 ART 에서 가장 빠른 -네이티브 규약이다. +실제 Android 바인딩은 shape 를 거치지 않고 **함수마다** `RegisterNatives` 로 등록한 JNI 를 쓴다 +(`androidMain/README.md`). shape 트램폴린은 `ffiSymbol` 로 얻은 임의 주소를 부르는 일반 수단으로 +남아 있다. **LLVM JIT 은 필요 없다 — 다만 근거는 아래와 같다.** @@ -86,7 +95,8 @@ ART 에는 FFM 을 위한 VM 지원이 없다. HotSpot 은 `downcallHandle` 호 stub_descriptor = stub_descriptor.insertArgumentLayouts(0, WORD); // leading function pointer ``` -우리 `downcallII_I(fn, a0, a1)` 과 같은 구조다. 차이는 **생성 시점**뿐이다: +우리 `downcallII_I(fn, a0, a1)` 과 같은 구조다. 차이는 **생성 시점**뿐이다 (PanamaPort 는 의존성으로 쓰지 +않는다 — AGENTS.md §12.9. 아래는 설계 근거로 읽은 분석이다; [`android-ffm-design.md`](../archive/android-ffm-design.md)): | | PanamaPort | 이 프로젝트 | |---|---|---| @@ -102,12 +112,11 @@ stub_descriptor = stub_descriptor.insertArgumentLayouts(0, WORD); // leading fu 부트스트랩 수단이며 — LLVM API 같은 내부 네이티브 함수를 바인딩한다 — `Core` 의 Linker 경로에서는 쓰이지 않는다.) -## JVM 통합 — Desktop + Android 를 한 벌로 - -현재 JVM 쪽은 330개 `actual` 이 **두 벌** 있다. 여기에 플랫폼별 `bindings` 선언까지 더하면 손으로 -유지하는 선언이 1,300개에 가깝다. 바인딩 하나를 추가하려면 네 계층을 맞춰야 한다. +## JVM 통합 — Desktop + Android 를 한 벌로 (계획; 미수행) -shape 를 도입하면 플랫폼별 표면이 **14개로 줄고** 330개 `actual` 은 `jvmMain` 에 한 벌만 남는다: +계획은 이랬다. 당시 JVM 쪽은 330개 `actual` 이 **두 벌** 있고 플랫폼별 `bindings` 선언까지 더하면 손으로 +유지하는 선언이 1,300개에 가까웠다. shape 를 도입하면 플랫폼별 표면이 14개로 줄고 330개 `actual` 은 +`jvmMain` 에 한 벌만 남는다: ``` commonMain expect fun PyList_Size(list: NativePointer): Long @@ -119,77 +128,53 @@ jvmMain actual fun PyList_Size(list) = downcallJ_J(sym.PyList_Size, list.to ``` **인터페이스나 추상 클래스를 쓰지 않는다.** `expect`/`actual` 이 같은 일을 하는 언어 기능이고, -인터페이스를 두면 호출마다 가상 디스패치가 붙는다. `expect`/`actual` 은 컴파일 타임에 확정된다. - -검증된 절차: +인터페이스를 두면 호출마다 가상 디스패치가 붙는다. -1. 대상 함수의 commonMain `expect` 에서 `inline` 제거 -2. `androidMain`/`desktopMain` 의 `actual` 삭제 -3. `jvmMain` 에 `actual` 하나 + shape 함수 호출 -4. shape 함수를 `internal expect` 로 두고 각 플랫폼이 `internal actual` 제공 +실험으로 확인한 절차(함수 하나 `PyList_Size` 로 4개 타깃 컴파일): commonMain `expect` 에서 `inline` +제거 → 플랫폼 `actual` 삭제 → `jvmMain` 에 `actual` 하나 → shape 함수를 `internal expect`. -함수 하나(`PyList_Size`)로 4개 타깃 전부 컴파일되는 것까지 실험으로 확인했다. +**현재 상태: 이 통합은 진행되지 않았다.** `jvmMain` 에는 shape 어휘(`ShapeDowncalls.kt`)와 문자열 +마샬링 `expect`(`internedUtf8` 등)만 있고, 330개 `actual` 은 여전히 `desktopMain` 과 `androidMain` 에 각각 +있다. 이유는 아래 실측들이다: 데스크톱은 `invokeExact` 로 2.65 ns 에 도달해 shape 가 필요 없었고, Android 는 +함수별 호출 규약 선택이 shape 단일화보다 비용을 크게 좌우했다. `expect inline fun` + 중간 소스셋 조합의 +컴파일러 크래시 제약(AGENTS.md §16)도 그대로 남아 있다. ## GraalVM Native Image Native Image 는 **FFM 다운콜을 빌드 타임에 등록**해야 한다. `reflect-config.json` 으로는 안 되고, -커스텀 `Feature` 의 `duringSetup` 단계에서 `RuntimeForeignAccess.registerForDowncall(descriptor, options)` -를 호출해야 하며, `-H:+ForeignAPISupport` 로 활성화하는 **실험적** 기능이다. +`RuntimeForeignAccess.registerForDowncall(descriptor, options)` 를 쓰는 등록(여기서는 reachability +메타데이터)이 필요하며, `-H:+ForeignAPISupport` 로 활성화하는 **실험적** 기능이다. 데스크톱이 연결하는 +모든 `FunctionDescriptor` 가 그 메타데이터에 선언되어 있는지는 `ReachabilityMetadataTest.kt` 가 지킨다 +(SPEC C-2). `JAVA_LONG`/shape 로 시그니처 종류가 한정되는 것이 등록을 단순하게 한다. -여기서도 shape 14종이 유리하다 — `Feature` 하나에서 `registerForDowncall` 을 14번 부르면 끝난다. -임의 시그니처를 런타임에 만드는 구조였다면 애초에 불가능하다. - -우선순위는 낮다. 상세는 [`android-ffm-design.md`](../platforms/android-ffm-design.md). +이것은 더 이상 "우선순위 낮음"이 아니다 — 업콜이 네이티브 이미지에서 동작해야 한다는 것은 이 저장소의 +결정이다 (AGENTS.md §13). 절차와 마지막 검증 기록은 +[`graal-native-image-verification.md`](../platforms/graal-native-image-verification.md) (SPEC U-6: 수동 검증, +이미지 자체를 테스트하는 자동화는 아직 없음). ## 스레드 상태 -다운콜은 전부 GIL(또는 free-threaded 빌드의 스레드 상태)을 요구한다. -[`threading-and-abi.md`](threading-and-abi.md) 를 본다. +다운콜은 전부 GIL(또는 free-threaded 빌드의 스레드 상태)을 요구한다 — `withGIL { }` +(`python/multiplatform/ffi/GILScope.kt`). [`threading-and-abi.md`](threading-and-abi.md) 를 본다. --- ## 개정: Android 는 shape 트램폴린만으로 풀리지 않는다 위의 shape 설계는 **Desktop 에는 유효하지만 Android 에는 그대로 적용되지 않는다.** 설계 논의에서 -드러난 사실들을 기록한다. - -### 확인된 버그: 현재 Android JNI 배선이 어긋나 있다 - -`@CName` 이 만드는 C 함수는 선언한 Kotlin 인자만 받는다: - -``` -Java_python_native_ffi_bindings_PyList_1Size(jlong list) -``` +드러난 사실과, 그 결과 실제로 택한 방식을 기록한다. -그런데 `androidMain/bindings.kt` 는 이를 **일반 JNI 메서드**로 선언한다. 일반 JNI 메서드를 ART 는 -이렇게 호출한다: +### 과거의 JNI 배선 버그 (수정됨) -``` -Java_..._PyList_1Size(JNIEnv* env, jobject thiz, jlong list) -``` - -**인자가 두 칸 밀린다.** `list` 자리에 `JNIEnv*` 가 들어간다. 확인 결과 `@CName` 익스포트 중 -`JNIEnv` 를 받는 것은 0개이고, `@CriticalNative`/`@FastNative` 도 어디에도 없다. +초기에는 `@CName` 으로 만든 C 함수(`JNIEnv`/`jclass` 를 받지 않음)를 일반 JNI 메서드처럼 이름으로 +연결해서, ART 가 넘기는 `JNIEnv*`·`jclass` 때문에 **인자가 두 칸 밀렸다.** 지금은 `JNI_OnLoad` +(`artMain`)가 `RegisterNatives` 로 모든 `external fun` 을 등록한다 (367개 중 363개; 예외 4개는 의도적 — +ROADMAP §2). 이름 기반 연결은 API 26 에서 런타임을 abort 시키기도 한다. 초안의 진단·표는 +[`../archive/downcall-design-android-first-diagnosis.md`](../archive/downcall-design-android-first-diagnosis.md). -즉 **Android JVM 경로는 컴파일만 되고 런타임에 동작하지 않는다.** 실행된 적이 없어 드러나지 않았다. - -### ART 의 세 가지 호출 규약 - -| | 일반 JNI | `@FastNative` | `@CriticalNative` | -|---|---|---|---| -| 전환 오버헤드 | 115 ns | 35 ns | 25 ns | -| C 시그니처 | `(JNIEnv*, jobject, args…)` | 동일 | **`(args…)` 만** | -| 객체 인자·반환 | 가능 | 가능 | **원시 타입만** | -| 정적/인스턴스 | 둘 다 | 둘 다 | **정적만** | -| JVM 으로 콜백 | 가능 | 가능 | **불가** | -| 실행 중 GC | 허용 | **차단** | **차단** | - -shape 트램폴린은 정적이고 `Long`/`Double` 만 주고받으므로 `@CriticalNative` 에 정확히 맞는다. 그리고 -우리 `@CName` 익스포트가 `JNIEnv` 를 받지 않는다는 사실이 곧 `@CriticalNative` 규약과 일치하므로, -**애노테이션을 붙이는 것이 성능 개선인 동시에 위 버그의 수정**이다. - -`@CriticalNative` 는 API 34 부터 공식 SDK 에 포함된다. minSdk 26 구간에서는 `RegisterNatives` 로 -등록해야 한다 — `RegisterNatives` 는 호출 규약이 아니라 바인딩 수단이므로 둘은 대안 관계가 아니다. +**호출 규약은 per-API-level 이면서 per-function 이다.** 전환 비용은 아래 "Measured" 절의 실측을 따른다 +(API 34 를 경계로 `@CriticalNative` 와 `@FastNative` 가 뒤집힌다). `@CriticalNative` 는 `RegisterNatives` +로만 등록하고, 정적·원시 타입만, 그리고 JVM 으로 되돌아올 수 없는 함수에만 쓴다. ### critical 계열은 업콜과 공존할 수 없다 @@ -227,30 +212,21 @@ Android 쪽이 더 빠를 수는 없다. 다만 **얼마나 다른지는 측정 우리 desktop 구현은 현재 `Linker.Option.critical()` 을 쓰지 않는다 — `Panama.kt` 가 `downcallHandle(..., emptyOptions)` 로 기본 경로를 탄다. -### 미결정: 조립 단위의 범위 - -경계 통과가 비싸고 뜨거운 호출에는 critical 도 못 쓰므로, **통과 횟수 자체를 줄이는 것**이 확실한 -접근이다. 원래 설계 의도가 이것이었다 — `bindings` 의 저수준 함수들을 `Python3.kt` 같은 **바인더 -레벨 연산으로 조립**해 `artMain` 에서 내보내고, `androidMain` 은 그 굵은 단위를 호출한다. -`artMain/JniExport.kt` 의 `initialize`/`finalize`/`internalIsInitialized` 가 그 형태로 남아 있다. +### 조립 단위의 범위 — 정해졌다 -조립 단위는 길고 업콜 가능성이 있으므로 일반 JNI 를 쓴다. 통과가 적으면 개당 115 ns 도 감당된다. -반대로 shape 트램폴린은 짧고 콜백이 없으므로 `@CriticalNative` 를 쓴다. **둘은 대립이 아니라 층이 -다르다** — 조립이 통과 횟수를 줄이고, critical 이 남은 통과를 싸게 한다. +경계 통과가 비싸고 뜨거운 호출에는 critical 도 못 쓰므로 **통과 횟수 자체를 줄이는 것**이 접근이었다 — +`bindings` 의 저수준 함수들을 바인더 레벨 연산으로 조립해 `artMain` 에서 내보내고 `androidMain` 은 그 +굵은 단위를 한 번 부른다 (`artMain` 의 `asmGetAttr` 등; 조립 단위는 길고 업콜 가능성이 있으므로 일반 +JNI, shape 계열은 짧고 콜백이 없으므로 규약을 함수별로 선택). -정해야 할 것은 **조립 단위를 어느 층위로 잡는가**이다. 지금 객체 모델(`PyObject`, `PyType`, 컬렉션)이 -`commonMain` 에 있고 그 안에서 원시 `expect` 를 호출하므로, Android 에서 조립하려면 그 연산 자체가 -플랫폼별로 갈라져야 한다. 후보: - -1. 바인더 API 를 `expect`/`actual` 로 올리고, Android 는 `artMain` 조립 함수를 호출하는 얇은 - 포워더가 된다 — 로직은 Kotlin/Native 에 한 벌 -2. `commonMain` 에 로직을 두되 Android 만 별도 경로 — 중복 발생 -3. 조립을 `Python3` 수준으로 한정하고 `PyObject`/컬렉션은 잘게 둔다 — 절충 - -이 결정 전에는 `EmbedAPI.jvm.kt` 통합을 진행할 수 없다. Android 가 조립 단위를 쓰면 잘게 쪼갠 330개 -`expect` 를 구현하지 않게 되기 때문이다. - ---- +초안은 세 후보를 두고 결정을 미뤘다(바인더 API 를 `expect`/`actual` 로 올림 / `commonMain` 에 두고 Android +만 별도 경로 / `Python3` 수준만 조립). **택한 것은 어느 것도 아니다**: 조립은 **FFI 층의 `expect`/`actual`** +(`EmbedAPI`)에 두고 객체 모델은 공통으로 둔다 — 객체 모델은 `bindings` 를 참조하지 않는다 +(`architecture.md` "Consequence for composed operations", `commonMain/README.md`). 그리고 **문자열을 +나르는 연산에서는 조립보다 interning 이 더 싸고 일반적**이라 그쪽으로 갔다 +([`marshalling-design.md`](marshalling-design.md)); 조립이 남는 곳은 통과 횟수가 N 에 비례하는 일괄 +연산(1000원소 리스트 변환 11배)이다. 데스크톱에서는 조립을 하지 않기로 닫혔다 (ROADMAP §6: 마샬링이 +호출의 10.5%). ## Measured: what the JNI calling convention actually costs (2026-08-11) @@ -308,10 +284,10 @@ measure on physical devices at both API levels.** ### Consequence for the design If this holds on hardware, the current design is optimal at minSdk and worst-case on modern -Android, which is backwards from where the users are. The fix would be to pick the convention -per API level. The annotation is compile-time, so that means declaring both variants and -dispatching on `Build.VERSION.SDK_INT` — a branch costing ~1-2ns to avoid ~22ns. Not done yet, -because it should not be built on emulator numbers alone. +Android, which is backwards from where the users are. The fix is to pick the convention per API +level. The annotation is compile-time, so that means declaring both variants and dispatching on +`Build.VERSION.SDK_INT` — a branch costing ~1-2ns to avoid ~22ns. (It held on hardware, below, and +was then built: `bindings.preferFastNative`.) ### Re-measured on physical hardware — the inversion is real @@ -358,9 +334,9 @@ for one declaration. The fix is to declare both variants and dispatch on still receives JNIEnv and jclass, so its exports need those leading parameters — meaning two export sets in artMain, not one. -Not yet implemented. The crossover API level is also still unknown: it lies somewhere between -26 and 34, and picking the threshold well needs measurements at 28/30/31/32 that have not -been taken. +**Implemented** (`bindings.preferFastNative = SDK_INT >= 34`, `androidMain/.../bindings.kt`; the +crossover is pinned in the next section, and `dispatchPicksTheFasterConventionOnThisDevice` re-measures +it on whatever device runs it). ### The crossover, pinned: Android 14 @@ -404,7 +380,10 @@ so a changed `.def` or a cleaned `build/` produces an APK holding a stale — or missing — `libmultiplatform_python3.14.so` and no `libpython3.14.so` beside it. The symptom is `UnsatisfiedLinkError`, which reads like a code error and is not one. -Until the task dependencies are fixed, run this before any instrumented test: +The copy tasks are now wired as dependencies of the assets/jniLibs merge tasks +(`python-multiplatform/build.gradle.kts`, `copyAndroidPythonBinaries` / `copyAndroidPythonAssets`); +I did not verify whether `connectedDebugAndroidTest` now forces the `linkAndroidNative*` tasks too. +If an instrumented run fails with `UnsatisfiedLinkError`, run this first: ./gradlew :python-multiplatform:linkAndroidNativeArm64 :python-multiplatform:linkAndroidNativeX64 \ :python-multiplatform:copyAndroidPythonBinaries :python-multiplatform:copyAndroidPythonAssets @@ -445,6 +424,9 @@ calls, where the transition genuinely dominates, are exactly the ones that keep ## Desktop vs Android, measured +> The first two tables below are the **pre-fix** desktop numbers (reflective `invoke`); the desktop +> row is corrected by "Fixed: 1015.95 ns -> 2.65 ns" and "Desktop vs Android, corrected" further down. + Same benchmark shape on both sides: same warmup, same iteration count, best-of-7, a pure-Kotlin identity call subtracted as the floor, and the same real C API call — `PyList_Size` on `sys.path`. @@ -611,11 +593,11 @@ per-crossing cost is highest, which is modern Android, which is where the users ### Where this leaves the design `PyObject` was `expect`/`actual` with an Android `actual` marked `external` — the hook for -exactly this — until commit 0fae961a (2025-12-20) folded it into a single `commonMain` class -along with `PyObject.desktop.kt` and `PyObject.native.kt`. That commit's real subject was -moving `PyAutoCloseable` to per-platform implementations, which was right and should stand; -losing the composition hook was collateral. Restoring `expect`/`actual` on `PyObject` while -keeping the `PyAutoCloseable` split is what reopens this path. +exactly this — until commit 0fae961a (2025-12-20) folded it into a single `commonMain` class. +That commit's real subject was moving `PyAutoCloseable` to per-platform implementations, which was +right and should stand; losing the composition hook was collateral. The resolution was **not** to +restore `expect`/`actual` on `PyObject`: composition lives in the FFI layer (`architecture.md`), and +for string-carrying operations interning replaced it (`marshalling-design.md`). Windows cannot host a Kotlin/Native assembly library, so a composed desktop path would cover macOS and Linux only, with Windows staying on direct Panama calls. At 2.65ns per crossing @@ -623,7 +605,7 @@ that is a much smaller loss on desktop than the same gap would be on Android. ## One downcall, across all platforms -The upcall side has "One upcall, across all five platforms" in `upcall-design.md` — one shared +The upcall side has §7.1 "One upcall, per platform" in [`upcall.md`](upcall.md#71-one-upcall-per-platform) — one shared test, run on every target, min–max over several runs, quoted rows marked as quotes. The downcall side had no equivalent: the benchmarks that exist are real and each one answers a question, but they are scattered across five files with no shared shape, so "how expensive is a downcall on @@ -634,7 +616,7 @@ platform X" had no single table to read. | File | Source set | What it measures | Runs where | |---|---|---|---| | `commonTest/.../overhead/Benchmark.kt` | `commonTest` | Warmup-then-time harness (`run`/`measure`), not a benchmark itself | wherever the tests below run | -| `commonTest/.../overhead/BenchmarkTest.kt` | `commonTest` | Pointer boxing, raw refcount churn, string marshalling (`PyUnicode_FromString`/`AsUTF8`, three lengths), integer marshalling, `PyObject` wrapper cost, `PyObject_GetAttrString` | compiles for all six run paths below; **confirmed green** on desktop / iosSimulatorArm64 / wasmJsNode this run; reaches androidInstrumented and androidNativeArm64/X64 through the same `commonTest` dependency edge but neither ran here (no emulator this session) | +| `commonTest/.../overhead/BenchmarkTest.kt` | `commonTest` | Pointer boxing, raw refcount churn, string marshalling (`PyUnicode_FromString`/`AsUTF8`, three lengths), integer marshalling, `PyObject` wrapper cost, `PyObject_GetAttrString` | compiles for all six run paths below; **confirmed green** on desktop / iosSimulatorArm64 / wasmJsNode when the table was first cut; reaches androidInstrumented and androidNativeArm64/X64 through the same `commonTest` dependency edge but neither ran here (no emulator this session) | | `desktopTest/.../DesktopOverheadBenchmark.kt` | `desktopTest` | Panama transition cost (`PyList_Size` net of a pure-Kotlin floor); the string-marshalling-share-of-`exec` test is `@Ignore`d (destabilises the interpreter, see the file's own comment) | desktop only | | `wasmJsTest/.../WasmMarshallingOverheadTest.kt` | `wasmJsTest` | `internedUtf8` vs `scratchUtf8` vs the old malloc/copy/free route, at two string lengths, both standalone and through a real `PyObject_GetAttrString` | wasmJs only | | `androidInstrumentedTest/.../JniOverheadBenchmark.kt` | `androidInstrumentedTest` | JNI calling-convention transition cost: ordinary / `@FastNative` / `@CriticalNative`, on a trivial echo and on `PyList_Size` | Android/ART only, needs a device or emulator | @@ -644,7 +626,7 @@ Two more files live next to these and share the word "overhead" or "Upcall" in t **not** downcall benchmarks — noted here only so a future reader does not double-count them: `commonTest/.../reflection/UpcallOverheadTest.kt` prices the upcall dispatch table (`UpcallTable.resolve` vs a cached handle), and `androidInstrumentedTest/.../UpcallOverheadTest.kt` -prices the Android upcall attach. Both are upcall-side and already covered by `upcall-design.md`. +prices the Android upcall attach. Both are upcall-side and already covered by `upcall.md`. `BenchmarkTest` reaches every one of the six execution paths KGP registers for this module (`desktopTest`, `iosSimulatorArm64Test`, `iosX64Test`, `wasmJsNodeTest`, `androidInstrumentedTest`/ @@ -653,7 +635,7 @@ prices the Android upcall attach. Both are upcall-side and already covered by `u compile — on three of the six this pass: desktop, the iOS simulator, and wasmJs (see Validation below). `iosX64Test` has no practical run path on Apple Silicon and nobody targets it; the Android/ART and androidNative rows need a connected device or emulator, which this pass did not -have available (AGENTS.md's "에뮬레이터는 쓰지 마라" for this task). +have available. ### A shared table, picked to match the upcall table's columns @@ -674,7 +656,7 @@ run's `BenchmarkTest` XML was not preserved before the second run overwrote it `UpcallBoundaryCostTest` numbers were captured from console output before that happened, which is why that row has three runs and this one has two). -androidNative and Android/ART rows are **quoted from `upcall-design.md`**, not re-measured — marked +androidNative and Android/ART rows are **quoted from `upcall.md`**, not re-measured — marked † below, same convention that document uses for its own quoted rows. Cells with no prior measurement of this exact quantity are left blank rather than approximated from a different one (e.g. `CompositionBenchmark`'s composed `getAttr` is a different quantity — four crossings folded @@ -708,320 +690,43 @@ into one — not a bare `PyUnicode_FromString`). > **The `PyUnicode_FromString, 8 chars` column has been updated.** It is sourced from > `overhead/BenchmarkTest` with warmup now 100,000 (was 100), for the reason "The benchmark was -> measuring the benchmark" explains in `upcall-design.md`. The old figures here were 3–7x the warmed +> measuring the benchmark" explains in `upcall.md`. The old figures here were 3–7x the warmed > cost because they were swept at the old `100`-call warmup on desktop (1132–1146 ns cold vs 167–458 ns warm). > The new column records the current cost at 100,000 warmup, from commit `958c0082b294`. The first three > columns also come from `UpcallBoundaryCostTest` (already fixed at 100,000 warmup since `7e9c6b8c`), so > all four columns are now consistent in warmup and source. -### Ratio consistency: mostly holds, desktop is the exception - -> **Superseded as a source of numbers, kept as the record of how the defect was found.** Every figure -> in this section and the two that follow it was taken with `UpcallBoundaryCostTest`'s 3 000-call -> warmup, which "The benchmark was measuring the benchmark" below shows was too small by more than an -> order of magnitude. The reasoning is what still stands: these three sections are what established -> that the number moved for a repository-located reason and then that the reason was the benchmark's -> own methodology. Current figures are in `upcall-design.md`'s table. - -`upcall-design.md`'s own upcall/downcall ratios, from the same `UpcallBoundaryCostTest`, are -reproduced by these three fresh runs (the test computes and prints its own ratio each time, so this -is a direct comparison, not a re-derivation): - -| Platform | ratio recorded in `upcall-design.md` | ratio, this pass (3 runs) | consistent? | -|---|---|---|---| -| desktop | 2.49–2.89x | 2.13–2.34x | **no — see below** | -| iOS simulator | 1.33–1.43x | 1.38–1.42x | yes, nested inside the old range | -| wasmJs | 2.56–3.36x | 2.36–2.63x | mostly — top of the new range overlaps the bottom of the old one. **Superseded: since drifted to 1.58–1.68x, pinned to `4472f83a` and explained — see "wasmJs: closed too" below** | - -**Desktop's ratio moved outside the previously recorded range, and it is the upcall side that -moved, not the downcall side.** This pass's downcall-same-shape figure (315.69–335.32 ns) sits at -the bottom of the old 315–527 ns range — consistent, just at its floor. This pass's upcall figure -(687.83–741.26 ns, read from the same `UpcallBoundaryCostTest` runs) sits *below* the old range's -floor of 861 ns entirely. So on this machine, at this commit (`65e1bf5d`), a desktop upcall through -`ctypes` measures cheaper than what `upcall-design.md`'s table recorded, while the downcall side did -not move. Two things separate the two tables and either could be the cause: `develop` has moved a -great deal since that table was captured (this session's own `git merge` pulled in 87 changed files, -including upcall-adjacent ones — see the merge log above), and the two tables were also captured on -different runs of the same machine, which is exactly the kind of thing this document elsewhere warns -is not comparable. Neither is ruled out here; this is reported as an open discrepancy, not resolved -into either explanation. - -### Closed: repository drift, not run-to-run noise (2026-08-14) - -The discrepancy above was tested directly rather than argued from a second reading. `537c1a0b` -(the commit the 861 ns floor was recorded on) and the current tip were each built in their own -freshly created `git worktree` — not reused, not incrementally recompiled — and -`:python-multiplatform:desktopTest` (the full suite, not a filtered subset — see the pitfall below) -was run three times against each, clearing `build/test-results` before every run: - -| | run 1 | run 2 | run 3 | -|---|---|---|---| -| `537c1a0b`, fresh worktree | 870.48 ns | 808.72 ns | 869.12 ns | -| current tip, fresh worktree | 697.73 ns | 680.46 ns | 686.21 ns | - -Same machine, same JDK (Temurin 21.0.12), same day. The two bands (809–870 ns vs. 680–698 ns) do -not overlap across six runs split into two isolated builds, which rules out ordinary run-to-run -variance as the explanation. Something in the repository between those two points changed the -measured number. `537c1a0b`'s figure also reproduces its own historical record (861–1313 ns) almost -exactly, which rules out a stale or miscalibrated re-measurement on this end. - -**A methodology pitfall found along the way:** an initial attempt to bisect the ~75 commits in -between used `--tests UpcallBoundaryCostTest` to skip the other ~359 tests and go faster. That -filter alone moved the number into the 1300–1400 ns range at *every* commit tried, old and new -alike — it does not merely add noise, it changes which regime the measurement lands in. Re-running -the same checkouts with the unfiltered full suite reproduced the fast band again. The cause is -below; the practical lesson is that this benchmark's absolute number is not a property of the -timed code alone, so a comparison across two runs is only valid if both used the same suite scope -this document already required (Validation runs the full suite) — a filtered re-run of one -benchmark test is not equivalent to the run it is being compared against, even on the same commit. - -**Bisection was attempted and did not converge to one commit; it converged to a window, and to a -mechanism.** Git bisect (full-suite grading, threshold at 750 ns) was run between `537c1a0b` and -the fast tip. Partway through, machine load reached 10–12 on this 8-core host — two Android -emulators and other concurrent agent activity were running, unrelated to this task and outside its -control (`ps aux` at the time confirmed it; this document's own environment notes elsewhere warn -that this machine hosts several worktrees' worth of parallel agent work). Under that load, repeated -measurements of the *same* checkout spanned both regimes (one commit read 672, 771, and 1076 ns -across three consecutive runs), which made single-commit attribution unreliable. What did hold up -under repetition was the window: commits at or before `4722cfe9` measured consistently in the slow -band (854–855 ns), and commits at or after `ec8ff389` measured consistently in the fast band (672, -700 ns, each confirmed in its own fresh clean worktree). That window — -`4722cfe9..ec8ff389` (~15 commits) — contains both commits this task flagged as candidates: -`6776329d` ("Perf: Price the layer users actually call, and stop a module read raising an exception -per access") and `d45071e7` (proxy handle lifetime). `975d3900` sits well outside this window and is -ruled out. - -**Neither candidate touches the code this benchmark measures, which points at an indirect cause.** -`git diff ..` for both `6776329d` and `d45071e7` shows changes confined to -`PythonProxySource` (the generated-proxy source emitter, its tests) and, for `d45071e7`, a doc-only -comment added to `HandleTable.kt`. Neither touches `UpcallTable.resolve`, `HandleTable.resolve`/ -`release`, the non-suspend branch of `UpcallTrampoline.invoke`, or desktop's `UpcallStub` — the -actual call chain `UpcallBoundaryCostTest` times through `TrampolineFragment`. A commit that does -not touch the timed path cannot have made the timed path itself cheaper. - -What it can do is change what runs *before* the timed path in the same process. `desktopTest` sets -no `forkEvery`, so Gradle's default applies: all ~360 tests in the suite run in one forked JVM, one -JIT compiler, one heap (confirmed by reading the `tasks.named("desktopTest")` block in -`build.gradle.kts` — no fork-per-test config exists). `6776329d`'s own commit title is "stop a -module read raising an exception per access" — replacing an exception-driven control-flow path -(expensive on the JVM: every throw captures a stack trace) with a direct one. -`GeneratedProxyCostTest`, touched by the same commit, drives `N = 10_000` iterations times 3 kept -times multiple rows of generated-proxy calls that exercise exactly that module-attribute path, and -it runs earlier in the same shared JVM process as `UpcallBoundaryCostTest`. Removing tens of -thousands of exception throws from an earlier test changes the shared process's GC pressure and -JIT tiering history by the time a later, unrelated benchmark in the same run gets timed — without -the later benchmark's own code path changing at all. That is a plausible, mechanism-level -explanation for why only desktop moved: iOS runs through XCTest's own process model and wasmJs -through a Node process with a different (V8) JIT, neither sharing HotSpot's C2 warm-up state the -way every desktop test in one Gradle-forked JVM does. - -**This is not proven to the level of a single blamed commit** — machine contention prevented that — -but it is proven to the level that matters for the table: the number moved for a real, reproducible, -repository-located reason, not for noise, and the most likely mechanism is a same-process -warm-up artifact of the benchmark's own methodology rather than a genuine drop in the upcall -boundary's per-call cost. The open discrepancy above is closed on that basis, not left open. - -**wasmJs has since drifted further, past even its own "consistent" reading above.** A fresh check -on the current tip (same machine, same session, two runs: 321.32 ns and 315.89 ns, tight) puts -wasmJs's upcall figure well below the 687–741 ns-equivalent range implied by this table's own -"this pass" ratio (2.36–2.63x against a ~192–205 ns downcall figure that still matches today's -run) — today's ratio is 1.26–1.67x. So the "yes, mostly consistent" verdict recorded for wasmJs -above was accurate for the commit it was measured on (`65e1bf5d`) and has since gone stale in the -same direction as desktop's did. iOS was not re-checked: this task's constraints rule out using the -iOS simulator, so its "yes, nested inside the old range" verdict above stands unverified rather -than reconfirmed. - -### wasmJs: closed too — one commit, and the mechanism demonstrated rather than argued (2026-08-14) - -wasmJs's drift is **repository drift, not noise**, it is pinned to the **single commit `4472f83a`**, -and — unlike desktop's, which stayed at the level of a plausible story — its mechanism was -reproduced by a controlled experiment. It is *not* a genuine change in what the wasm upcall boundary -costs. - -The measurement protocol was the one the desktop section above used, with two changes forced by -this machine: runs at the two ends were **interleaved** rather than run in two blocks, and the -timed task was re-run with `--rerun` on `wasmJsNodeTest` alone rather than `--rerun-tasks`, so a -full Kotlin recompile does not finish moments before the benchmark starts. Every run is the **whole -wasm suite** (filtering changes the measured region — see the pitfall below), `build/test-results` -is cleared before each, and all of them are 344/0/0 (258/0/0 at `537c1a0b`, whose suite was smaller). - -| commit | upcall | downcall, same shape | ratio | runs | -|---|---|---|---|---| -| `65e1bf5d` (parent) | 484.22–500.38 ns | 190.11–191.00 ns | 2.53–2.61x | 3 | -| **`4472f83a` (child)** | **306.66–315.82 ns** | **189.32–202.56 ns** | **1.52–1.66x** | 3 | -| `ec8ff389` | 482.03–507.38 ns | 197.87–204.10 ns | 2.38–2.53x | 3 | -| `4722cfe9` | 477.40–497.05 ns | 190.60–203.23 ns | 2.34–2.60x | 3 | -| `17f058ca` | 302.55–320.54 ns | 189.47–203.92 ns | 1.50–1.65x | 3 | -| `73f2af6b` | 312.50–322.11 ns | 188.95–213.49 ns | 1.46–1.66x | 3 | -| current tip | 355.82–373.21 ns | 222.07–226.88 ns | 1.58–1.64x | 4 (+4 earlier) | - -`4472f83a` and its own parent are adjacent commits measured minutes apart in one worktree, and their -upcall bands do not touch. **The downcall column does not move anywhere in this table** — it is the -upcall side alone, exactly as on desktop. - -**The window does *not* overlap desktop's.** Desktop's was `4722cfe9..ec8ff389`; both of those -endpoints measure in wasm's *slow* band and are indistinguishable from each other, so wasm did not -move across desktop's window at all. `4472f83a` sits after `ec8ff389`. The two platforms therefore -do **not** share a cause in `commonMain`'s marshalling or handle paths — which was the hypothesis -worth testing, and it is refuted rather than left open. - -**The MEMFS change is not the cause either.** `3355851c` (stdlib zip installed into MEMFS instead -of resolving out of a CPython source checkout) was the obvious environmental confound, since it -lands after the desktop window. `73f2af6b` is its immediate parent and already measures 312–322 ns — -fully in the fast band. The transition happened before it. - -**Why a commit that only adds a branch to the timed function made it measure cheaper.** -`4472f83a`'s only edit to the timed path is a `when` on `self` in `UpcallEntry.invokeMethod`, which -can only cost more, not less. What it also did was make `PythonProxySource.install()` succeed on -wasm for the first time. Before it, `GeneratedProxyCostTest` and `ProxyHandleLifetimeTest` printed -"no proxies are installable on this target, so there is nothing to measure" and drove **zero** -upcalls; after it, `GeneratedProxyCostTest` drives on the order of 270 000 calls -(`N = 10 000` × 3 kept × ~9 rows) straight through `UpcallEntry.invokeMethod` — earlier in the -**same Node process**, and therefore into the same V8 wasm tier-up state that -`UpcallBoundaryCostTest` is later timed in. The benchmark's own 3 000-iteration warmup does not -reach that state on its own. - -That was tested, not just asserted. At the **current tip**, with nothing changed but -`GeneratedProxyCostTest`'s body short-circuited so it installs nothing and calls nothing — the -pre-`4472f83a` behaviour, same commit, same build, suite still 344/0/0 — the upcall figure went -back up: - -| current tip | upcall | downcall | ratio | -|---|---|---|---| -| as it is | 355.82–373.21 ns | 222.07–226.88 ns | 1.58–1.64x | -| with `GeneratedProxyCostTest` inert | 459.67–473.79 ns | 187.24–192.16 ns | 2.36–2.53x | - -One lever, at one commit, moves the number across the whole gap and lands it back on `65e1bf5d`'s -band (2.53–2.61x) and on this document's own "this pass" reading (2.36–2.63x). So the desktop -section's suspicion — that this benchmark reports a same-process warm-up artifact rather than a -boundary cost — is **confirmed on wasmJs by direct experiment**, on a different commit and a -non-overlapping window from desktop's. What the two platforms share is the methodology, not a -code path. - -**The consequence for the table is that `UpcallBoundaryCostTest`'s absolute upcall figure is not a -property of the commit alone.** It is a property of the commit *and* of how much upcall traffic the -rest of the suite pushed through the same process first. Any future row should record the suite -composition it was taken with, or the number will "drift" again the next time an unrelated test -starts or stops exercising the boundary. - -**One thing did not reproduce and is left as observed.** `537c1a0b`, the commit -`upcall-design.md`'s 703–1075 ns wasm row was recorded on, does not reproduce that row here: a -freshly created worktree at it measured 546.45 ns once and then 1476.85–1602.25 ns on four -subsequent runs (its trampoline figure was similarly unstable, 268–612 ns, while its downcall stayed -flat at 239–247 ns). That commit's suite is 258 tests rather than 344 and predates the MEMFS change, -so it is a different environment in two ways at once, and no attempt is made here to reconcile it. -The conclusion above rests on the adjacent-commit pair `65e1bf5d`/`4472f83a` and on the tip-vs-tip -experiment, neither of which depends on `537c1a0b` at all. - -**Load, since these numbers depend on it.** Machine load average was 1.4–1.6 at the start and -stayed in the 1.7–3.2 band for every run quoted above. An earlier batch was thrown away: creating -the comparison worktree set Spotlight indexing its 16 190 files, which took load to 10.4 and made -the same commit read 546 ns then 1448–1677 ns. Those runs are not in this document. The machine also -had an Android emulator resident throughout, which is why the floor is ~1.5 rather than ~0. - -### The benchmark was measuring the benchmark, and the count that fixes it is measured (2026-08-14) - -The two sections above diagnosed `UpcallBoundaryCostTest` and stopped there: they established that -its absolute figure was a same-process warm-up artefact, pinned wasmJs's shift to one commit, and -attached a staleness note to the table. **The measurement itself was not changed**, so the next pass -would have re-derived the same non-number. This section changes it and shows the change works. - -#### What is warming, decided by experiment rather than by plausibility - -Both earlier sections reached for "JIT tiering" as a story. It is testable, and the obvious rival — -CPython-side state, meaning the specializing interpreter, free lists, the allocator, string -interning — makes an opposite prediction about one row that the report already prints. - -The timed loops were run **40 times in succession** inside one run, on both hosts, in two suite -configurations. Per 10 000 calls: - -| | rep 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | … plateau | -|---|---|---|---|---|---|---|---|---|---| -| desktop upcall, suite as-is | 801 | 744 | 704 | 705 | 676 | 602 | 593 | 537 | ~535 ns | -| desktop upcall, proxy test inert | 1307 | 716 | 696 | 660 | 548 | 539 | 558 | 554 | ~535 ns | -| desktop downcall (Kotlin-driven) | 475 | 264 | 264 | 212 | 141 | 141 | 140 | 143 | ~141 ns | -| wasmJs upcall, suite as-is | 300 | 313 | 317 | 290 | 301 | 285 | 305 | 278 | ~280 ns | -| wasmJs upcall, proxy test inert | 785 | 339 | 317 | 303 | 294 | 299 | 319 | 279 | ~280 ns | - -**The two configurations converge on the same value and they converge from opposite sides.** So the -plateau belongs to the boundary and everything before it belongs to the suite. That alone settles -what the number should be. - -**Two rows in the same sweep do not move at all**, and they are what identifies the mechanism: the -empty Python loop (8–9 ns desktop, 14–16 ns wasm) and the pure-Python callee (24–27 ns, 47–54 ns) are -flat from the very first rep, in every configuration, on both hosts. CPython-side state would move -those — they are pure interpreter work — and it does not. Only rows that cross into host code move. -It is host JIT tier-up: C2 on the JVM, V8's tiering on wasm. - -The confirmation is on desktop, where the two runtimes can be told apart: CPython is a **native -dylib** there, so its own speed cannot depend on how warm the JVM is, and indeed desktop's -pure-Python rows are identical (8.44–8.59 ns) whether the class runs alone or inside the full suite. - -#### The count, and why 100 000 - -Convergence needed 40 000 calls cold and 70 000 warm on desktop, and ~70 000 on wasm. The warm side -settles later, so it sets the requirement; 100 000 is that with margin. The Kotlin-driven rows were -on the same curve and worse off — they warmed `N / 4` = 2 500, and desktop's downcall row reads 653, -266, 266, 161 before settling at ~141 — so they use the same count now. Warming the Python-driven and -Kotlin-driven halves differently would make the headline ratio a ratio of two compilation states. - -`GeneratedProxyCostTest` needed far less, and the reason is its shape: it warms all 19 rows before -timing any, so the shared `_pm_invoke` path already received 19 × 3 000 = 57 000 calls — just under -the knee. 5 000 puts it at 95 000. Measured, its raw rows moved from 3–7% solo-vs-suite disagreement -to 1–3.4%. - -**`inline` on `Benchmark.measure` was tried and is worse.** The hypothesis was that one shared -`block()` call site goes megamorphic across every benchmark in the process. Inlining it made desktop's -downcall row need ~230 000 iterations to reach the plateau it reaches in ~50 000 through the shared -non-inlined loop, because each inlined copy is separate code that tiers up on its own. Reverted, and -recorded so it is not tried again. - -#### The check the old table could not pass - -| | full suite | proxy test short-circuited | this class alone (`--tests`) | -|---|---|---|---| -| desktop upcall, **3 000 warmup** | 673.91 ns | 536.75 ns | — | -| desktop upcall, **100 000 warmup** | 510–560 ns (11 runs) | 501–530 ns (3) | 506–550 ns (3) | -| wasmJs upcall, **3 000 warmup** | 322.03 ns | 291.67 ns | — | -| wasmJs upcall, **100 000 warmup** | 290–304 ns (7 runs) | 287–301 ns (3) | 321–353 ns (3) | - -The middle column is the lever the wasmJs section above used to prove the defect existed. It used to -move desktop by 25% and wasmJs by 10%; it now moves neither outside its own run-to-run band. **That -is the fix.** - -**The third column is honest about what is still not fixed, on one target.** Desktop passes it. wasmJs -does not, and the reason is not the boundary: filtered to this class alone, wasmJs's *pure-Python* -rows read 23.05 ns and 83.10 ns against the full suite's 15.04 ns and 47.22 ns — 55–74% slower, in -rows with no boundary in them. CPython is itself a wasm module under V8, so a Node process that lives -one second runs the interpreter's own bytecode slower than one that lives ten, and every row is -inflated together. Raising the warmup to 300 000 does not move it (solo read 318.83 and 326.21 ns -against 100 000's 320.54–337.42), which is what distinguishes a host-lifetime effect from a -call-count effect. **wasmJs's ratio columns do survive filtering** — 2.66–3.13x solo against -2.91–3.14x in the suite — so the ratio is the quantity to quote there, and the absolute column is a -full-suite figure that must be labelled as one. It is left in the table with that label rather than -deleted, because desktop's is now sound and dropping both would lose a real result. - -#### What it costs - -Measured on this machine, full suite, `build/test-results` cleared before each run: - -| | 3 000 warmup | 100 000 warmup | -|---|---|---| -| `desktopTest`, total test time | 4.93–4.97 s | 5.10–5.20 s | -| …of which `UpcallBoundaryCostTest` | 0.036 s | 0.155–0.162 s | -| …of which `GeneratedProxyCostTest` | 0.553–0.571 s | 0.625–0.661 s | -| `wasmJsNodeTest`, wall clock | 4.31 s / 5.85 s | 4.43 s / 6.11 s | - -**About +0.2 s on desktop (+4% of test time) and +0.1 s on wasm.** The warmup is bounded by the very -cost it is warming, so eight rows of 100 000 calls at ~500 ns is under a second by construction. The -wasm rows are wall clock because that runner reports 0.0 s per class in its XML. - -Test counts are unchanged throughout: **desktop 360/0/1, wasmJs 344/0/0**, on every run quoted here. - -**Load.** 1.14 at the start, 1.6–3.8 for every run quoted, `uptime` checked before each batch. No -other agent was running and no worktree was created during this pass, so the Spotlight problem the -previous section had did not arise. Two readings taken while load was transiently 6.25 (back-to-back -Gradle invocations) were timing measurements only, and are not among the per-call figures. +### Warmup: what the boundary benchmarks must do + +The first version of the shared table was taken with `UpcallBoundaryCostTest`'s 3 000-call warmup, +and its absolute upcall figure then *drifted* between commits (desktop 861 ns → ~690 ns, wasmJs +~490 ns → ~310 ns) without any change to the timed code. The investigation (full record in +[`../archive/downcall-design-benchmark-drift.md`](../archive/downcall-design-benchmark-drift.md)) +established: + +- The figure was a **same-process warm-up artefact**, not a boundary cost. Anything that pushed + upcall traffic through the shared JVM / Node process earlier in the suite (a proxy cost test + starting to install proxies; one commit on wasmJs, `4472f83a`) moved the number by 25% on desktop + and ~35% on wasmJs. Confirmed by direct experiment (short-circuiting `GeneratedProxyCostTest` + moved it back). +- It is **host JIT tier-up** (C2 on the JVM, V8 on wasm), not CPython-side state: the pure-Python + rows (empty loop 8–9 ns desktop; callee 24–27 ns) are flat from the first repetition, only rows + that cross into host code move. 40 repetitions plateau at ~535 ns (desktop upcall), ~141 ns + (desktop Kotlin-driven downcall), ~280 ns (wasmJs upcall). +- **Convergence needs 40 000 calls cold and ~70 000 warm, so the warmup is 100 000** (and + `GeneratedProxyCostTest` 5 000 per row, because it shares `_pm_invoke` across 19 rows). Cost: + about +0.2 s on desktop (+4% of suite time). +- With 100 000 warmup the "proxy test short-circuited" lever no longer moves desktop (501–530 vs + 510–560 ns) or wasmJs (287–301 vs 290–304 ns) outside run-to-run noise. That is the fix. +- On wasmJs the **absolute** column still does not survive filtering (`--tests` this class alone: pure + Python rows 55–74% slower, because a short-lived Node process runs CPython-under-V8 colder); the + **ratio** columns do (2.66–3.13x solo vs 2.91–3.14x in the suite). Quote wasmJs absolutes as + full-suite figures. +- `inline` on `Benchmark.measure` was tried and is worse (needs ~230 000 iterations for the same + plateau). Do not retry. +- Rules that follow: a measurement is **full suite, single task, quiet machine** (`uptime` first); + filtering with `--tests` changes the measured region; record the suite composition with the + number; never run two measurement sessions concurrently (load 10–12 made one commit read 672–1076 ns). ### What is deliberately not in the shared table diff --git a/docs/design/ecosystem.md b/docs/design/ecosystem.md index 913d97516..fa04ccad2 100644 --- a/docs/design/ecosystem.md +++ b/docs/design/ecosystem.md @@ -143,9 +143,10 @@ vendored material and prebuilt CPython artefacts. `UI.ipynb` imports `pythonx.compose.runtime`, `pythonx.compose.material3`, `pythonx.compose.ui` and `pythonx.compose.layout`, and documents signatures like `Text(text, color, font_size)`, -`Button(onclick, …)` — quoted as the notebook writes it, which is **not** the convention: the -library spells that parameter `on_click`, and `docs/design/pythonx-adapter-design.md` §3 settles it in the -library's favour — plus `Column/Row/Spacer/TextField`. It also +`Button(onclick, …)` — quoted as the notebook writes it; the 2024 library spells that parameter +`on_click` (`docs/archive/pythonx-adapter-design.md` §3 measured both). Which Python spelling +`pythonx` uses is pythonx-compose's decision: this repository's binder exposes the Kotlin name +`onClick` and renames nothing (`docs/INTENT.md` §2.2) — plus `Column/Row/Spacer/TextField`. It also requires `main.App` as a live object, `App.messages.getValue()/setValue()`, and `main.App.update(NewComposable)` for hot-swapping the UI from a Jupyter cell. @@ -171,12 +172,13 @@ handling at all. Every widget in `pycomposeui` is `@Composable`. Nothing in `pyt leave chaquopy until this is designed. *(read from source; not runtime-verified.)* This is the smallest and most blocking item, and it is entirely inside this repository. -> **Closed — see `docs/design/pythonx-adapter-design.md` §5.6.** `Text('hi')` written in Python now draws +> **Closed — see `docs/archive/pythonx-adapter-design.md` §5.6.** `Text('hi')` written in Python now draws > through the real `androidx.compose.material3.Text` (`:ksp-fixtures:compose`'s > `ComposableRenderTest`: 71 non-background pixels against 0 for an empty body). The paragraph above > is right that a *generated Kotlin* entry for a widget cannot compile, and that is not the route > taken: an artefact composable's call site is emitted as **bytecode**, with `$composer`, `$changed` -> and `$default` exposed as ordinary slots, and `pythonx` computes the mask. `BindingPolicy` does now +> and `$default` exposed as ordinary slots, and the binding layer (`python_multiplatform.binding`, +> `PythonxAdapter.kt`) computes the mask. `BindingPolicy` does now > have `@Composable` handling and what it does is **decline** — a composable in the consumer's *own > source* has no compiled signature to call yet, which is a different problem from one in a jar. @@ -190,6 +192,13 @@ this exists: generated modules are injected into `sys.modules` under their Kotli name. What is missing is a `sys.meta_path` finder so a name resolves on demand rather than only after an eager `install()`, the `pythonx` prefix, and generated `.pyi` beside it. +> **Current (2026-10-03):** the finder exists — the binding layer `python_multiplatform.binding` +> installs a `sys.meta_path` finder that makes Kotlin-named modules importable on demand +> (`_Finder`/`_Loader` in `PythonxAdapter.kt`; SPEC U-8). The `pythonx` prefix is **not** this +> repository's to provide: the binder creates no `pythonx` module, and `pythonx` is a real package in +> pythonx-compose (`AGENTS.md` §12.2). The plugin's `.pyi` are emitted under Kotlin module paths only +> (SPEC B-7). + **4. One CPython acquisition path, not two.** This repository's `stagePythonHome` downloads from python-build-standalone into a Gradle cache; ppp's `python install` downloads from this repository's own GitHub releases into `~/.pypackpack/python/`. They disagree on source and @@ -214,30 +223,30 @@ This section documents the current state, language/build system/distribution art ### `PythonMultiplatform` (this repository) - **Observed Current State**: - - Embedded CPython FFI binder (`python-multiplatform/src/commonMain/kotlin/.../EmbedAPI.kt`, platform implementations in `EmbedAPI.desktop.kt`, `bindings.kt`, JNI/Panama/cinterop). **Correction, 2026-08-17: this line used to say "CPython 3.13" — stale.** `gradle.properties`'s `pythonVersion` (the single source of truth, read into `Versions.currentVersion` at `python-multiplatform/src/commonMain/kotlin/python/multiplatform/Versions.kt`) is `3.14.7`, and every platform's *observed runtime* agrees: desktop prints `runtime : 3.14.7` (`docs/roadmap/ROADMAP.md:2063`), Android prints `3.14.7` on both `pmp_api36` and `pmp_api26` (`docs/roadmap/ROADMAP.md:2073`), wasmJs prints `3.14.2` — its own, separately-pinned Emscripten build, not this line's concern (`docs/platforms/wasm-design.md`). See the version-mismatch check below. + - Embedded CPython FFI binder (`python-multiplatform/src/commonMain/kotlin/.../EmbedAPI.kt`, platform implementations in `EmbedAPI.desktop.kt`, `bindings.kt`, JNI/Panama/cinterop). **Correction, 2026-08-17: this line used to say "CPython 3.13" — stale.** `gradle.properties`'s `pythonVersion` (the single source of truth, read into `Versions.currentVersion` at `python-multiplatform/src/commonMain/kotlin/python/multiplatform/Versions.kt`) is `3.14.7`, and every platform's *observed runtime* agrees: desktop prints `runtime : 3.14.7` (`4ed98143:docs/roadmap/ROADMAP.md:2063`), Android prints `3.14.7` on both `pmp_api36` and `pmp_api26` (`4ed98143:docs/roadmap/ROADMAP.md:2073`), wasmJs prints `3.14.2` — its own, separately-pinned Emscripten build, not this line's concern (`docs/platforms/wasm-design.md`). See the version-mismatch check below. - Kotlin object model hierarchy (`python-multiplatform/src/commonMain/kotlin/.../PyObject.kt` and wrappers). - - Code generators: KSP processor (`python-multiplatform-ksp/`) producing `FunctionTableFragment`s, and Gradle plugin (`python-multiplatform-gradle-plugin/src/main/kotlin/.../ArtifactBindingGenerator.kt`) scanning resolved dependency JARs via ASM for `ArtifactTable` fragments. + - Code generators: KSP processor (`python-multiplatform-ksp/`) producing `FunctionTableFragment`s, and Gradle plugin (`python-multiplatform-gradle-plugin/src/main/kotlin/.../artifact/ArtifactScanner.kt`, `PythonArtifactBindingsTask.kt`; *corrected 2026-10-03 — there is no `ArtifactBindingGenerator.kt`*) scanning resolved dependency jars (ASM + `kotlin-metadata-jvm`) and klibs for `ArtifactTable` fragments, plus `.pyi` stubs under Kotlin module paths (`stubs/PythonStubsTask.kt`). - Multiplatform target support: Desktop JVM/Panama, Android JNI, iOS Native Cinterop, androidNativeArm64 (`:python-multiplatform:compileKotlinAndroidNativeArm64`). - CPython binary download and SHA-256 lockfile / Sigstore verification (`python-checksums.properties`, `python-multiplatform/build.gradle.kts`). - **Language / Build System / Distribution**: - - Kotlin Multiplatform (Kotlin 2.0+), Java 21, C/C++ FFI. + - Kotlin Multiplatform (Kotlin `2.4.20-Beta2` pinned in `gradle/libs.versions.toml` as of 2026-10-03), Compose Multiplatform 1.11.1 for the sample and fixtures, Java 21, C/C++ FFI. - Gradle (`build.gradle.kts`, `settings.gradle.kts`, included build `python-multiplatform-gradle-plugin`). - Maven artifact publication (`id("maven-publish")` in `python-multiplatform/build.gradle.kts`, published under group `io.github.thisisthepy:python-multiplatform`). - **Goal in Ecosystem**: - Core language boundary and binder between Kotlin Multiplatform and CPython. - Manages low-level FFI, object reference handles, upcall/downcall function tables, GIL lifecycle, and proxy injection into CPython `sys.modules`. -- **Gap to Goal**: - - `@Composable`-capable callable shape: `ExposedCallable` typed `(Array) -> Any?` cannot invoke `@Composable` functions taking synthetic `$composer` / `$changed` parameters. Needs dedicated `CallableKind` or opt-in annotation handling. - - Python-side `sys.meta_path` finder for lazy import resolution of Kotlin namespaces (`pythonx.*` or FQCNs) upon import. - - Opaque Kotlin object round-tripping for Compose `Composer` (`HandleTable` and `ObjectReference` exist, but runtime hand-off to Python needs verification). - - Lifetime and storage of Python callables passed into Kotlin across Compose recompositions (`content=lambda: ...`, `on_click=...`). +- **Gap to Goal** (status notes added 2026-10-03, read from code and SPEC, not re-run): + - `@Composable`-capable callable shape: `ExposedCallable` typed `(Array) -> Any?` cannot invoke `@Composable` functions taking synthetic `$composer` / `$changed` parameters. Needs dedicated `CallableKind` or opt-in annotation handling. — *Closed on desktop by bytecode thunks (`ComposableThunks.kt`), SPEC B-6; Android, iOS and wasm planned (SPEC N-2).* + - Python-side `sys.meta_path` finder for lazy import resolution of Kotlin namespaces ~~(`pythonx.*` or FQCNs)~~ upon import. — *Exists for Kotlin FQNs in `python_multiplatform.binding` (SPEC U-8). `pythonx.*` is not the binder's to resolve (`AGENTS.md` §12.2).* + - Opaque Kotlin object round-tripping for Compose `Composer` (`HandleTable` and `ObjectReference` exist, but runtime hand-off to Python needs verification). — *Done on desktop: one hand-written `@Composable` pushes the live composer as a handle onto the binding layer's stack (`push_composer`), proven by the `ksp-fixtures/compose` render tests.* + - Lifetime and storage of Python callables passed into Kotlin across Compose recompositions (`content=lambda: ...`, `onClick=...`). — *Partly addressed: `PythonCallableScope` (`PythonCallables.kt`) holds the callables a composition hands to Kotlin and is closed by a Compose-side `RememberObserver` in `onForgotten`; several hand-written fixture wrappers (`pythonDraggable`, `pythonComposed`, the swipe/anchor wrappers) still leak their callbacks by design, pinned by `DraggableLeakTest`. Not verified here beyond reading the sources.* - Retire `stagePythonHome` in favor of `pypackpack`'s Python distribution management. - Hand `stageWasmBrowserRuntime` logic to `toolchain`. - **Dependency Direction**: - `PythonMultiplatform` has no dependencies on other repos in the ecosystem. - `pythonx-compose` and consumer applications depend on `PythonMultiplatform` runtime and its Gradle/KSP bindings plugin. - **Unverified**: - - Behavior of WASM browser runtime (`wasm-experiment/`) under production web bundlers. + - Behavior of WASM browser runtime under production web bundlers. (*2026-10-03:* the former `wasm-experiment/` is folded into the library — `wasmJsMain`/`wasmJsTest`, CPython built by `tools/wasm/build-cpython.sh` — commit `6e54f617`; the bundler question is unchanged.) **Version check (2026-08-17): "3.14 runtime vs. 3.13 header, never checked" — could not find the sentence, checked the claim anyway.** The literal sentence was searched for across every `.md` file @@ -245,17 +254,17 @@ in this repository (`grep -rn` for Korean and English phrasings of "nobody/no on verified", "mismatch", combined with "header"/"헤더" and "runtime"/"런타임") and in the git history of this file; it does not appear anywhere, verbatim or paraphrased. The nearest things on record are this section's own now-corrected "CPython 3.13" line above (stale since the version became -configurable) and `docs/roadmap/ROADMAP.md:2033`, which already fixed a test fixture that hardcoded the string +configurable) and `4ed98143:docs/roadmap/ROADMAP.md:2033`, which already fixed a test fixture that hardcoded the string "CPython 3.13" while `pythonVersion` had moved to 3.14.7. Neither is the sentence quoted, and neither left the underlying question — do headers and runtime actually agree, per platform — checked. That question was checked here, freshly, platform by platform: | platform | header source | runtime source | agree? | how checked | |---|---|---|---|---| -| desktop | none — Panama binds by symbol name at runtime (`java.lang.foreign.Linker`/`SymbolLookup`), no C header is compiled against | `libpython3.14.dylib`/`.so`/`.dll` extracted from the `python-build-standalone` archive pinned to `pythonVersion` (`python-checksums.properties`: `macos-aarch64-3.14.7-20260807`, etc.) | n/a — no header exists to disagree with the runtime | read `build.gradle.kts`'s desktop `jvm()` block; no `cinterop`/`headers()` call anywhere in it; `docs/roadmap/ROADMAP.md:1034-1038` records the same conclusion independently | +| desktop | none — Panama binds by symbol name at runtime (`java.lang.foreign.Linker`/`SymbolLookup`), no C header is compiled against | `libpython3.14.dylib`/`.so`/`.dll` extracted from the `python-build-standalone` archive pinned to `pythonVersion` (`python-checksums.properties`: `macos-aarch64-3.14.7-20260807`, etc.) | n/a — no header exists to disagree with the runtime | read `build.gradle.kts`'s desktop `jvm()` block; no `cinterop`/`headers()` call anywhere in it; `4ed98143:docs/roadmap/ROADMAP.md:1034-1038` records the same conclusion independently | | iOS / androidNative (Kotlin/Native `cinterop`) | `$targetExtractDir/include/python$libVersion/Python.h` (Android) or the extracted `Python.xcframework/.../Headers` (iOS) — **read from the same extraction the runtime library comes from**, not from any vendored copy | same archive, same extraction, `libVersion`/`targetExtractDir` computed from the one `pythonVersion`/`libVersion` variables | **yes, by construction** — both paths are derived from the same Gradle variable, so they cannot independently drift for a given build | read `python-multiplatform/build.gradle.kts:1093-1128` (`targetIncludePath`, the `cinterops.create("python")` block); confirmed no `defFile`/hardcoded header path overrides it | | Android JNI (`artMain`, `jni_onload.def`) | none — hand-written `extern` prototypes in `jni_onload.def`, no `#include `, pointers passed as `jlong` | `libpython$libVersion.so` linked with `-lpython$libVersion` from the same extraction tree | n/a for headers; symbol-level agreement already checked by earlier work (`nm`/`objdump` against the shipped `.so`, cited in `docs/roadmap/ROADMAP.md` around the 3.15 migration) | read `python-multiplatform/src/artMain/cinterop/jni_onload.def` and its `build.gradle.kts` cinterop block (no `headers()`/`includeDirs()` call) | -| wasmJs | Emscripten's own CPython 3.14.2 build, entirely separate toolchain (`docs/platforms/wasm-design.md`) | same 3.14.2 build (`Embedded CPython version: 3.14.2`, `docs/platforms/wasm-design.md:1287`; `runtime : 3.14.2` sample output, `docs/roadmap/ROADMAP.md:1488`) | yes, and deliberately a different minor-patch than the 3.14.7 native default — documented and reasoned about at length in `docs/platforms/wasm-design.md`, not an oversight | +| wasmJs | Emscripten's own CPython 3.14.2 build, entirely separate toolchain (`docs/platforms/wasm-design.md`) | same 3.14.2 build (`Embedded CPython version: 3.14.2`, `docs/archive/wasm-design-experiment-log.md` "Building CPython 3.14.2 for Emscripten"; `runtime : 3.14.2` sample output, `4ed98143:docs/roadmap/ROADMAP.md:1488`) | yes, and deliberately a different minor-patch than the 3.14.7 native default — documented and reasoned about at length in `docs/platforms/wasm-design.md`, not an oversight | **What is genuinely stale, and does not affect any of the above:** two artefacts in this repository still carry CPython 3.13 and are not read by the default build. `python-multiplatform/src/nativeInterop/cinterop/include/patchlevel.h` @@ -326,7 +335,7 @@ them is a repository-size decision, not a correctness one, and is left open. **`Py_LIMITED_API` status, checked directly:** `grep -rn "Py_LIMITED_API"` across `.kts`/`.def` build files finds it only inside vendored CPython header guards (`#if defined(Py_LIMITED_API) ...`) — it is never passed as a compiler define anywhere (no `-DPy_LIMITED_API`, no `compilerOpts` setting it). This -independently confirms `docs/roadmap/ROADMAP.md:1031-1038`'s own finding, reached from the free-threading +independently confirms `4ed98143:docs/roadmap/ROADMAP.md:1031-1038`'s own finding, reached from the free-threading investigation rather than this one: the project does **not** build against the Limited API / Stable ABI in the `Py_LIMITED_API`-macro sense. "abi3" as used elsewhere in this repository's docs means a self-imposed rule about which C API functions this project chooses to call (ones stable across @@ -560,7 +569,7 @@ inferring from diff stats: - Replace string-based type name parsing in `remember_saveable` with `PythonMultiplatform`'s `PyValue` type tags. - **Dependency Direction**: - `pythonx-compose` -> `PythonMultiplatform` (depends on `PythonMultiplatform`'s FFI binding surface and Python proxy module at runtime). - - Uses `.pyi` stubs emitted by `toolchain` / `PythonMultiplatform` plugin for editor autocompletion. + - ~~Uses `.pyi` stubs emitted by `toolchain` / `PythonMultiplatform` plugin for editor autocompletion.~~ *Corrected 2026-10-03:* `PythonMultiplatform`'s plugin emits stubs for the Kotlin-named modules only (SPEC B-7). The Pythonic `pythonx.*` stubs, and the `pythonx-map.toml` manifest that would drive them, are `pythonx-compose`'s own product; it builds its API from `inspect.signature` and `python_multiplatform.describe` (`KotlinSurface.kt`). - **Unverified**: - Full hot-reload integration with Jupyter notebook server outside `UI.ipynb` static cells. @@ -674,6 +683,8 @@ static. **`pythonx.*` is ours.** Whatever we wrap or add lives under that prefix. Kotlin fully-qualified names point at the original; `pythonx` points at our Pythonic layer. The two namespaces do not mix. +*(2026-10-03: "ours" means the ecosystem's — `pythonx` is a real package in pythonx-compose; this +repository's binder creates no `pythonx` module and renames nothing, `AGENTS.md` §12.1–12.2.)* **The wrapping happens in Python, not in Kotlin.** `pythonx.compose.material3` is Python code that uses the generated `androidx.compose.material3` bindings and presents a Pythonic API over them. @@ -695,9 +706,12 @@ the composer and change flags are threaded the same way every time, and a `conte adapted the same way every time. So the layer should resolve on demand — a module `__getattr__` that finds the corresponding binding, adapts it once and caches it — rather than enumerate. -That trades away IDE completion, which is exactly why the plugin generates `.pyi` from the same -metadata: the stubs carry the Pythonic names and signatures the adapter will produce, so an editor -sees a fully enumerated surface while the runtime enumerates nothing. +That trades away IDE completion, which is exactly why stubs are generated from the same metadata. +*(Corrected 2026-10-03:)* this repository's plugin generates stubs for the Kotlin-named modules, +under Kotlin names (SPEC B-7); the stubs that carry the Pythonic names and signatures belong to +`pythonx` itself, in pythonx-compose +([`docs/archive/pyi-generation-pythonic-stubs.md`](../archive/pyi-generation-pythonic-stubs.md) is the +design that was removed from here). **`JClass`/`JavaClass`, `KClass`/`KotlinClass`, `ObjcClass` work only where the platform has the thing they name.** No stubs, no substitutes, no forced uniformity across targets — a target that diff --git a/docs/design/kotlin-extensions-in-python.md b/docs/design/kotlin-extensions-in-python.md index 87870e84e..e73849eb2 100644 --- a/docs/design/kotlin-extensions-in-python.md +++ b/docs/design/kotlin-extensions-in-python.md @@ -22,17 +22,24 @@ in place with what replaced it, because the old measurement is why the replaceme | §3, the metadata-kind (`k=1`) gate | open problem | closed by `15fc5a62`, 25 minutes after this file was committed | | §3/§6, the type gate | open problem | closed by `f455ef41`; `TypeTag.OBJECT` was never the missing piece (§3.2) | | §3.1/§6, overload dispatch | unsolved, costing 33 of 130 `Modifier` names | rule chosen and implemented (`ArtifactScanner.disambiguateOverloads`); §3.1 records it | -| §4.2, the `Modifier` metaclass | proposed | **measured not to work**, at run time *and* in mypy. See `docs/design/pyi-generation-design.md` §4.3–§4.4 | - -Still true and untouched: all of §2 except the one column, §4.1, §4.3, §4.4, §4.6, §5, and two of -§6's five open items. +| §4.2, the `Modifier` metaclass | proposed | **measured not to work**, at run time *and* in mypy. See [`docs/archive/pyi-generation-pythonic-stubs.md`](../archive/pyi-generation-pythonic-stubs.md) §4.3–§4.4 | +| §1, the Compose version | 1.6.11 | the measurements here were taken on 1.6.11 and are kept as such; the build now pins Compose Multiplatform **1.11.1** (`gradle/libs.versions.toml`, `compose-plugin`), with material3 1.9.0 | +| §3.2, 43 `Modifier` extensions declined for a function-typed parameter | "a real boundary limit" | function-typed slots bind now (SPEC B-5); `ArtifactScanner`'s KDoc records **157 `Modifier` extensions bound and 8 declined** after `functionSlotOrNull` (not re-measured for this revision) | +| §3.1/§6, where the overload dispatcher lives | "in `pythonx`" | in **this repository's** binding layer, `python_multiplatform.binding` (`_Overloads` in `PythonxAdapter.kt`), which serves the base name (`padding`) and selects among the `name__` entries (SPEC U-8) | +| §4.1, primitive receivers as `pythonx.compose.ui.unit.to_dp(x)` | snake_case, under `pythonx` | the binder serves Kotlin names only (`docs/INTENT.md` §2.2): such an extension is a module-level function on its Kotlin-named module, receiver first. A Pythonic spelling is pythonx-compose's | +| §4.4, the allowlist | decided, not built | built at run time in the binding layer: `_VALUE_CLASS_ALLOWLIST` starts **empty** and pythonx-compose fills it through `allow_raw_primitive` (`PythonxAdapter.kt`) | +| §4.5, `.pyi` with `@overload` and `Dp \| float` | proposed for this plugin | the plugin emits Kotlin-named stubs with boundary types only (SPEC B-7, `docs/design/pyi-generation-design.md`); the Pythonic stub is pythonx-compose's. §4.5 is replaced below; the original is in [`docs/archive/kotlin-extensions-in-python-pyi.md`](../archive/kotlin-extensions-in-python-pyi.md) | +| §4.6, composables | "remain blocked" | callable from Python inside a composition (SPEC B-6); see §4.6's update | + +Still true and untouched: all of §2 except the one column, §4.3, §5, and two of §6's five open +items. §4.1's placement and §4.4's three rules hold, with the corrections above. --- ## 1. What was measured, and with what -The build pins Compose Multiplatform **1.6.11** (`gradle/libs.versions.toml`, `compose-plugin`), -so the primary corpus is the 19 `org.jetbrains.compose.*` desktop jars the Gradle cache had +The build pinned Compose Multiplatform **1.6.11** at the time (`gradle/libs.versions.toml`, +`compose-plugin`; it pins **1.11.1** today, and nothing here was re-measured on it), so the primary corpus is the 19 `org.jetbrains.compose.*` desktop jars the Gradle cache had resolved. As a cross-check the same tool was run over the 19 `classes.jar` extracted from the Jetpack AAR set the cache also holds (`androidx.compose.ui` 1.6.7, `foundation` 1.6.7, `material3` 1.2.1). @@ -115,7 +122,7 @@ Metadata arity excludes the synthetic parameters; the JVM descriptor does not. | `foundation.layout.Row` | `Row` | no | 4 | **7** | 6 | **The last column is what this document first recorded, and every one of its nine rows was one -low.** `docs/design/pythonx-adapter-design.md` §5.2 found this by re-counting with `javap -p`; the count +low.** `docs/archive/pythonx-adapter-design.md` §5.2 found this by re-counting with `javap -p`; the count has since been reproduced a third time, independently, against the same 1.6.11 jars. What was missed is **`$default`**. The trailing JVM parameters of `Button` are `Composer, int, int` and of `Text-fLXpl1I` are `Composer, int, int, int` — `$composer`, one or two `$changed` masks, and a @@ -381,10 +388,12 @@ because an overload set is a property of a *package* and Kotlin lets one live in `ClassNode`s, one Kotlin name, invisible to a per-class grouping, which would have emitted both under the same table key. -The dispatcher §4.4 asks for is still wanted and still cannot live here — `UpcallTable` is keyed by -name and `ExposedCallable` carries one fixed arity, so one name reaches one signature by -construction. It belongs in `pythonx` (`docs/design/pythonx-adapter-design.md` §4.1) and selects among -these names. This layer's job is to make that choice *possible*, not to make it. +The dispatcher §4.4 asks for is still wanted and still cannot live in the table — `UpcallTable` is +keyed by name and `ExposedCallable` carries one fixed arity, so one name reaches one signature by +construction. ~~It belongs in `pythonx` (`docs/archive/pythonx-adapter-design.md` §4.1) and selects +among these names.~~ **Current:** it lives in this repository's binding layer, +`python_multiplatform.binding` (`_Overloads` in `PythonxAdapter.kt`), which serves the base name and +selects among these names; the table's job is still to make that choice *possible*, not to make it. ### 3.2 Re-measured at `958c0082`: 314 @@ -428,13 +437,19 @@ entries that always fail, and declining them is correct rather than pending. §2 take at least one lambda" is the same fact seen from the other side. This is the honest remaining limit on the `Modifier` chain and it is not a metadata problem. +> **Superseded since:** a Python callable now crosses into a Kotlin `FunctionN` slot +> (`ArtifactScanner.functionSlotOrNull`, `PythonCallables.kt`; SPEC B-5, with a value-returning slot +> such as `() -> Float` still refused). `ArtifactScanner`'s KDoc records 157 `Modifier` extensions +> bound and 8 declined after that change. + The chain itself runs. `WalkedArtifactComposeModifierTest` assembles `Modifier.padding(16.dp).size(24.dp)` **from Python**, out of Compose's own jars, under Compose's own names — no `Composer` anywhere, because §2.6 already established that a `Modifier` extension is not a composable. What Python holds is an integer handle per link, and **nothing releases them**: a bare handle returned from a `CallableKind.FUNCTION` reaches Python as an `int`, which has nothing to hang a finaliser off. Three handles leak per run of that test, deliberately, because owning them is what -§4.1's proxy is for and §4.1's proxy does not exist yet. +§4.1's proxy is for and §4.1's proxy does not exist yet. *(It exists now: the binding layer's +receiver proxies, `_proxy_type` in `PythonxAdapter.kt`, SPEC U-7.)* --- @@ -457,9 +472,11 @@ Not every receiver can take a method, and the split was counted: | Kotlin collection | 29 | module-level function | | Kotlin primitive or `String` | 20 | module-level function | -Python cannot attach a method to `int`, so `Int.toDp` and its 19 siblings become -`pythonx.compose.ui.unit.to_dp(x)`. That is a real loss of fidelity and it is confined to 20 -functions. +Python cannot attach a method to `int`, so `Int.toDp` and its 19 siblings stay module-level +functions on their Kotlin-named module, taking the receiver as the first positional argument. That is +a real loss of fidelity and it is confined to 20 functions. (This file first wrote the result as +`pythonx.compose.ui.unit.to_dp(x)`; the binder renames nothing, and a snake_case spelling under +`pythonx` is pythonx-compose's to add.) ### 4.2 `Modifier` as one name for both the type and the empty modifier @@ -484,7 +501,7 @@ class Modifier(metaclass=_ModifierMeta): `m.padding(16)` resolves on the instance; `def f(m: Modifier)` still names a type. All three spellings work, and the generated `.pyi` can state all three (§4.5). -> **"All three spellings work" was not measured, and it is false.** `docs/design/pyi-generation-design.md` +> **"All three spellings work" was not measured, and it is false.** `docs/archive/pyi-generation-pythonic-stubs.md` > §4.3 ran it, in CPython 3.13 and in mypy 2.3.0, and the metaclass loses in both. > > **At run time**, `Modifier.padding` is the *instance* method, not the metaclass one. That is @@ -505,12 +522,14 @@ spellings work, and the generated `.pyi` can state all three (§4.5). > What resolves it is to stop making the stub mirror the runtime mechanism. Pick a runtime that > works (either of the two above) and declare each operation in the stub as a **class attribute whose > type is a Protocol with an overloaded `__call__`** — `padding: ClassVar[_Modifier_padding]` — which -> checks clean for all three spellings. `docs/design/pyi-generation-design.md` §4.4 adopts that and measured +> checks clean for all three spellings. `docs/archive/pyi-generation-pythonic-stubs.md` §4.4 adopts that and measured > it against mypy; PyCharm is listed there as unverified. The *observation* this section is built on > is untouched — `Modifier$Companion` > implements `androidx.compose.ui.Modifier`, so one name really can serve as both the type and the > empty modifier, and the three spellings really are the requirement. Only the mechanism proposed -> for reaching them is retired. +> for reaching them is retired. *(2026-10-03: the binding layer reaches both run-time spellings with a +> hybrid descriptor, `_Hybrid` in `PythonxAdapter.kt`; a `Modifier` class that is also the empty +> modifier is a Pythonic layer and belongs to pythonx-compose.)* **Rejected: binding the module-level name `Modifier` to the companion instance.** It is simpler at runtime and reads identically in the common case, but it destroys `Modifier` as an annotation — @@ -557,12 +576,15 @@ Adopted: 1. **Every value-class parameter accepts its proxy** — `Modifier.padding(dp(16))` always works. 2. **Raw primitives are accepted only for an explicit allowlist**, seeded with `Dp` and extended by - hand with evidence. `Dp` covers most of the ergonomic benefit on its own. + hand with evidence. `Dp` covers most of the ergonomic benefit on its own. *(Current: the binder's + allowlist starts empty and the seeding is pythonx-compose's, through `allow_raw_primitive`.)* 3. **Everything not on the allowlist rejects a raw number with an error that names the constructor - to call.** `Modifier.padding` accepting `16` while `Text(font_size=16)` raises "expected + to call.** `Modifier.padding` accepting `16` while `Text(fontSize=16)` raises "expected TextUnit; write sp(16)" is an inconsistency, and it is the honest one: the two are different because `Dp` and `TextUnit` are genuinely different, and hiding that produces a UI that silently - renders with no font size. + renders with no font size. *(Current: the binding layer refuses with "a raw number is not accepted + for TextUnit: it is a value class and not on the raw-primitive allowlist …"; naming the + constructor to call is left to the Pythonic layer, which owns `sp`.)* 4. **`Color` is on the reject list despite passing the constructor test**, and the allowlist mechanism is manual precisely so that this exception has somewhere to live. @@ -572,43 +594,15 @@ getting it wrong is a silent visual defect, not an exception. ### 4.5 `.pyi` -Generation belongs to the Gradle plugin (`docs/design/ecosystem.md` §5b), and it reads the same metadata -as the binder, so the stub and the binding cannot drift. Extension-as-method appears as an ordinary -method, and the metaclass of §4.2 is expressible: - -> **The stub below is the one that was measured to fail.** mypy resolves `Modifier.padding` to the -> instance method and rejects every class-object call; see §4.2's note and -> `docs/design/pyi-generation-design.md` §4.3. It is kept here because the three bullets that follow it are -> about `@overload`, `Dp | float` and defaults, and all three survive unchanged into the shape that -> does work — `padding: ClassVar[_Modifier_padding]` where `_Modifier_padding` is a Protocol whose -> `__call__` carries exactly these overloads (`docs/design/pyi-generation-design.md` §4.4). What changes is -> where the overloads are written, not which overloads there are. - -```python -class _ModifierMeta(type): - @overload - def padding(cls, all: Dp | float) -> Modifier: ... - @overload - def padding(cls, horizontal: Dp | float = ..., vertical: Dp | float = ...) -> Modifier: ... - @overload - def padding(cls, padding_values: PaddingValues) -> Modifier: ... - -class Modifier(metaclass=_ModifierMeta): - @overload - def padding(self, all: Dp | float) -> Modifier: ... - ... -``` - -Three things follow from the measurements: - -- **`@overload` is mandatory, not optional.** 33 of 130 `Modifier` names carry more than one - overload, and they are the load-bearing ones (§3.1). A generator that emits one signature per - name will mistype `padding`, `size`, `background`, `border` and `clickable`. -- **`Dp | float` is how the allowlist shows up in the stub**, and a type on the reject list is - stubbed as the proxy alone. The stub is then the documentation for §4.4's asymmetry. -- **Defaults must be carried.** 58% of `Modifier` extension parameters declare one; a stub with - everything required would be wrong about most of the API. Metadata's `declaresDefaultValue` is - the source; the *value* is not in metadata, so stubs emit `= ...`. +Generation belongs to the Gradle plugin (`AGENTS.md` §12.5), and it reads the same metadata walk as +the binder, so the stub and the binding cannot drift. **What it emits is the Kotlin surface, not a +Pythonic one** (SPEC B-7, `docs/design/pyi-generation-design.md`): one `def` per table key under the +Kotlin module path, Kotlin parameter names, `= ...` for a Kotlin default, the extension receiver as a +positional-only first parameter, and boundary-type annotations (`Dp` → `float`, `Modifier` → `int`) +with the Kotlin signature in the docstring. Extension-as-method, `@overload` sets over a base name and +`Dp | float` are what a Pythonic stub would say; that stub belongs to pythonx-compose. The section +this replaces — the metaclass stub and the three measured requirements for a Pythonic stub — is in +[`docs/archive/kotlin-extensions-in-python-pyi.md`](../archive/kotlin-extensions-in-python-pyi.md). ### 4.6 `@Composable` needs no new machinery here @@ -631,12 +625,33 @@ indistinguishable from an ordinary trailing `int` unless you already know the Ko Two things this paragraph did not yet know. **`$default` is the third synthetic**, and Compose carries it as a *declared* trailing parameter rather than emitting a `Button$default` bridge — so "call the synthetic to omit an argument" is not available for a composable at all -(`docs/design/pythonx-adapter-design.md` §4.5). And **the generated Kotlin cannot pass the composer -explicitly**: `docs/design/pythonx-adapter-design.md` §5.3 finds that Kotlin source can neither call a +(`docs/archive/pythonx-adapter-design.md` §4.5). And **the generated Kotlin cannot pass the composer +explicitly**: `docs/archive/pythonx-adapter-design.md` §5.3 finds that Kotlin source can neither call a `@Composable` from a non-`@Composable` lambda nor name `$composer`/`$changed`/`$default`, which are not parameters of the Kotlin declaration. That is a property of this generator, not of Compose, and it is why composables remain blocked while the `Modifier` chain of §3.2 runs. +**Update — composables are callable (SPEC B-6), and not through generated Kotlin.** What settled it +(first recorded in `docs/archive/pythonx-adapter-design.md` §5.6): + +- `@Composable` is `@Retention(BINARY)`: it appears in `RuntimeInvisibleAnnotations`, so the walker + has to read both annotation lists to recognise a composable. +- The trailing JVM shape is derivable from the Kotlin arity *n*: index *n* is the `Composer`, then + `ceil(n/10)` `$changed` ints, then `ceil(n/31)` `$default` ints exactly when the declaration + defaults something (`ComposableBindingTest.everyComposableJvmSignatureIsKotlinParamsThenComposerThenInts`). +- In the `$default` mask, bit *i* is parameter *i*; `$changed` is passed as `0`, the conservative + value. Shifting the mask by one bit made the render fail, so the encoding is load-bearing. +- A Kotlin file facade has no Kotlin name (`unresolved reference 'TextKt'`), so the call site is + emitted as **bytecode** — one `INVOKESTATIC` per composable in `ComposableThunks.kt`, name and + descriptor copied from the walked `MethodNode` at build time; no runtime lookup. +- The binding layer (`python_multiplatform.binding`) computes the `$default` mask from which + arguments the Python call wrote and threads the composer from a stack that one hand-written + `@Composable` entry pushes (`PythonComposition` in `ksp-fixtures/compose`). + +Proven by `ksp-fixtures/compose`'s render tests (`ComposableRenderTest`, `M3ProofRenderTest`, +`CallbackDrivenRenderTest`) on desktop only; Android, iOS and wasm are planned (SPEC N-2). Extension +composables and recomposition skipping remain open. + --- ## 5. What the 2024 attempt paid, for comparison @@ -659,8 +674,9 @@ be made unambiguous for Compose no matter how carefully it is written. ## 6. Open -Two of the five below have closed since this list was written, and one has moved rather than closed. -They are kept with their status rather than deleted, since each is the reason a piece of work +When this list was written, two of the five below had closed and one had moved; by 2026-10-03 the +overload dispatcher and the allowlist have closed too, and a sixth item (handle ownership) was +added. They are kept with their status rather than deleted, since each is the reason a piece of work happened. - **Scoped modifiers.** *Still open.* 74 public member extensions on `Modifier` (`RowScope.weight`, @@ -675,30 +691,33 @@ happened. - **Overload dispatch.** *Moved, not closed.* The "drop ambiguous names" rule that cost 33 of 130 `Modifier` names is gone; §3.1 records the rule that replaced it and why nothing arbitrates. What remains open is the half this layer cannot host: a Python-side dispatcher that inspects argument - count and type, selecting among `padding__Dp` / `padding__Dp_Dp` / `padding__PaddingValues`. It - lives in `pythonx` (`docs/design/pythonx-adapter-design.md` §4.1) and still interacts with §4.4 exactly as + count and type, selecting among `padding__Dp` / `padding__Dp_Dp` / `padding__PaddingValues`. + *Closed since:* it lives in this repository's binding layer (`_Overloads` in `PythonxAdapter.kt`, + serving the base name; SPEC U-8), not in `pythonx`, and it interacts with §4.4 exactly as described — if the raw `16` is what selects the `Dp` overload, then raw-primitive acceptance is - also overload resolution. + also overload resolution (a bare handle is not allowed to vote in a dispatch). - **The type gate.** *Closed*, by `f455ef41`. `Shape`, `Brush` and `PaddingValues` bind now, and the paragraph above misnamed what was missing: the boundary *type* (`TypeTag.OBJECT`) and its whole runtime (`HandleTable`, `marshalResult`, `toKotlinObject`) already existed; the walker lacked a Kotlin type name to cast to, which `@Metadata`'s classifier supplies. See §3's note on the type gate. What is left of the 45 declined `Modifier` extensions is **43 with a function-typed - parameter** (§3.2) — that is the lambda half of "66 take a lambda", and it is a real boundary - limit, not a missing type. + parameter** (§3.2) — that is the lambda half of "66 take a lambda". *(Since closed in turn: + function-typed slots bind, SPEC B-5; see §3.2's note.)* - **Handle ownership.** *New, opened by closing the type gate.* Every object crossing to Python is a strong root in `HandleTable`, and an entry bound by this walker returns it as a bare `int` with nothing to hang a finaliser off. A chained `Modifier.padding(...).size(...)` leaks one handle per intermediate link until §4.1's proxy exists to own them. Known, bounded, and true of every - `OBJECT`-returning KSP entry already — but it is now on the Compose path. + `OBJECT`-returning KSP entry already — but it is now on the Compose path. *(The proxy exists now in + the binding layer, and SPEC M-4 asserts that owned results are released when Python drops them; + the raw module still hands out bare handles.)* - **Whether the allowlist of §4.4 should be data or code**, and where a downstream consumer adds to - it for a library this repository has never seen. *Still open; a candidate answer exists.* - `docs/design/pyi-generation-design.md` §5.3 proposes it fall out of the package mapping — the same - manifest the Python package already owns. Nothing in the plugin implements an allowlist yet: - `grep -rn allowlist` over the Kotlin sources finds nothing, so §4.4's rules are decided and not - built. + it for a library this repository has never seen. *Closed:* it is run-time data in the binding + layer — `_VALUE_CLASS_ALLOWLIST`, empty in this repository, extended by the Pythonic package + through `allow_raw_primitive` (`PythonxAdapter.kt`). The manifest candidate + (`docs/archive/pyi-generation-pythonic-stubs.md` §5.3) was implemented and removed with the + Pythonic stubs; it belongs to pythonx-compose. - **klib.** *Still open, and untouched.* Everything here is JVM, and §3.2's 314 is still JVM only. `docs/design/ecosystem.md` §5b records that `LibraryAbiReader` exposes extension receivers and unerased types from klibs, which suggests the same design carries to iOS and androidNative, but no Compose - klib has been read yet — `docs/design/pyi-generation-design.md` §2.3 still lists the klib producer as - "not implemented". + klib has been read yet. *(Since: `KlibScanner` produces declarations from klibs through + `LibraryAbiReader`, but none is bound at run time yet — SPEC B-2/N-3.)* diff --git a/docs/design/marshalling-design.md b/docs/design/marshalling-design.md index 20e10437f..4f48d587d 100644 --- a/docs/design/marshalling-design.md +++ b/docs/design/marshalling-design.md @@ -38,40 +38,37 @@ differs by 2x between the same two devices. It is 3.3x better than composing on Attribute names, module names and method names are repeated literals, so encoding them again on every call is pure waste. CPython interns its own strings for the same reason. -### Status: adopted on Android, not on desktop +### Status: adopted on desktop, Android and wasm -The first row is labelled "was current" because it no longer describes Android. `internedUtf8` -and `encodeScratchUtf8` are implemented on both JVM platforms, and Android's call path uses them -— `EmbedAPI.android.kt`'s `PyObject_GetAttrString` goes through `internedUtf8`, so the live cost -is the bottom row, not the top one. Quoting 2238 ns as Android's current `getAttr` cost is wrong, -and it has been quoted that way. +The first row of the table above is labelled "was current" because it no longer describes any +platform. `internedUtf8`, `encodeScratchUtf8` and `freeUtf8` are `expect`s in +`jvmMain/.../ShapeDowncalls.kt` with an `actual` on desktop and Android, and the wasmJs target has +its own equivalent in `Wasm.kt` (same four-slot scratch). Android's call path uses them — +`EmbedAPI.android.kt`'s `PyObject_GetAttrString` goes through `internedUtf8`, so the live cost is +the bottom row, not the top one. Quoting 2238 ns as Android's `getAttr` cost is wrong, and it has +been quoted that way. -**Desktop has the primitives and does not use them.** `bindings.kt` has 59 `withUtf8` call sites -and zero `internedUtf8` ones, so every desktop call still allocates and frees a C string. That -went unnoticed because desktop was already at ~150 ns per string — Panama allocates off-heap -without crossing a boundary, which is the whole reason Android needed interning to catch up -rather than the other way round. Wiring it on desktop would take the repeated names down to a -map lookup plus a 2.65 ns crossing. +Desktop was wired later than Android (it was already at ~150 ns per string, since Panama allocates +off-heap without crossing a boundary). At the time of writing `desktopMain/.../bindings.kt` has 31 +`internedUtf8` and 27 `encodeScratchUtf8` call sites and no `withUtf8` call sites; `withUtf8` +survives only as a helper in `Panama.kt`. I did not find a desktop before/after measurement +recorded for the switch; the Android table above is the measured evidence. -This is cheaper to adopt than it looks and cheaper than composition, which ROADMAP §6 closed on -desktop: the primitives already exist and only the call sites change. - -**Do not swap all 59 blindly.** The two primitives exist for different lifetimes: +Do not mix the two primitives up. They exist for different lifetimes: | | for | lifetime | |---|---|---| | `internedUtf8` | repeated identifiers — attribute, module, method names | kept, bounded at 4096 entries | | `encodeScratchUtf8` | arbitrary content — `exec` source, user strings | thread-local scratch, freed on the next call | -Interning an `exec` source string would blow the cache. Android's actual interns only where the -argument is a name, which is the rule to follow. +Interning an `exec` source string would blow the cache; intern only where the argument is a name. ## The design Centre the abstraction on **interning**, not on a general allocator. - // jvmMain — shared by Android and desktop - internal expect fun internedUtf8(s: String): Long + // jvmMain — shared by Android and desktop (jvmMain/.../ShapeDowncalls.kt) + @PublishedApi internal expect fun internedUtf8(s: String): Long internal expect fun encodeScratchUtf8(s: String): Long internal expect fun freeUtf8(address: Long) diff --git a/docs/design/object-lifetime.md b/docs/design/object-lifetime.md index 7eae94973..a33b44e14 100644 --- a/docs/design/object-lifetime.md +++ b/docs/design/object-lifetime.md @@ -18,13 +18,18 @@ one too few leaks, and releasing one too many frees an object still in use — t somewhere unrelated, which is how the interpreter corruption during the lifetime work first appeared, as a segfault inside `Py_Finalize` far from its cause. -~~The wrapper's own release is currently **explicit only**. GC-driven release is implemented but -blocked: `Python3.initialize()` still holds the GIL, so a cleaner thread calling -`PyGILState_Ensure` blocks forever.~~ **Unblocked.** ROADMAP §1 closed: `Python3.initialize()` now -parks its thread state with `PyEval_SaveThread()`, so a cleaner thread's `PyGILState_Ensure` -attaches instead of blocking. ROADMAP §4 closed on top of that — GC-driven release is verified on -both collectors (desktop `java.lang.ref.Cleaner`, iOS `createCleaner`), and Android below API 33 -takes the `PhantomReference` fallback. See ROADMAP §1 and §4. +A wrapper's release is **GC-driven**: no `close()` is needed, and an explicit release is idempotent and +runs exactly once (SPEC M-1). `Python3.initialize()` parks its thread state with `PyEval_SaveThread()`, +so a cleaner thread's `PyGILState_Ensure` attaches instead of blocking on the GIL the initialising +thread would otherwise keep. The release path is verified on desktop (`java.lang.ref.Cleaner`), iOS +(`createCleaner`) and wasm, and Android below API 33 takes the `PhantomReference` fallback (SPEC M-2). +History: ROADMAP §1 and §4. + +On a **free-threaded** build, a decrement from a thread that does not own the object is queued to the +owner and only drained at an eval-loop checkpoint, which a pure C-API embedder never reaches; the +library therefore takes a periodic checkpoint itself (`Python3.drainPendingReleases`, +`autoDrainInterval`, on by default only for free-threaded builds). See ROADMAP §9, +`docs/investigations/gc-scheduling-investigation.md` and [`threading-and-abi.md`](threading-and-abi.md). ## Python holding Kotlin — the handle table @@ -129,8 +134,8 @@ fields are `PyObject`-typed. A traverse entry is one more generated function: // generated fun traverse_MyClass(obj: MyClass, visit: (Long) -> Unit) { - obj.someField?.let { visit(it.pointer.raw()) } - obj.otherField?.let { visit(it.pointer.raw()) } + obj.someField?.let { visit(it.pointer.toRawValue()) } + obj.otherField?.let { visit(it.pointer.toRawValue()) } } No reflection — which is the whole reason the table exists. A reflection-based binding would @@ -139,32 +144,31 @@ cycles" instead of solving it. We are not in that position. ### What is still hard -~~**`tp_traverse` runs during collection.** CPython requires it not to allocate and not to run -arbitrary code, and ours has to call into the JVM. The generated traverse must be restricted to -field reads and `visit` calls — no allocation, no user code, no locking beyond what is already -held.~~ **Built and tested this way.** `ProxyTypeFactory`'s `tp_traverse` (every target) calls only -the generated `traverse` function and `visit`; `CycleCollectionTest`/`WasmCycleCollectionTest`/ -`RefHolderCycleCollectionTest` exercise it through a real `gc.collect()`. - -~~**`tp_clear` has to mutate Kotlin state.** Breaking the cycle means nulling `K`'s `PyObject` -fields from inside a collection. A `val` field cannot be cleared at all, so either exposed -reference-holding fields must be `var`, or clearing goes through a generated accessor that -knows how.~~ **Solved a different way than this section predicted — no field mutation needed.** -`ProxyTypeFactory.tp_clear` (`desktopMain/.../ProxyTypeFactory.kt:96-108`, mirrored on every other -target) does not touch `K`'s fields at all; it calls `HandleTable.release()` on `PK`'s handle and -nothing else, deliberately not `close()`-ing the held `PyObject`s (the code comment there explains -why: CPython's own cyclic collector, not this callback, is what reclaims the other side, and -calling `close()` here would double-free). Breaking `PK -> [handle] -> K`'s strong root is enough -to let the JVM collect `K` once CPython's collector also breaks `P`'s incoming edge — no `var`, no -generated clearing accessor, and no Kotlin-side field ever needed to be nulled. +**`tp_traverse` runs during collection**, and CPython requires it not to allocate or run arbitrary +code, while ours has to call into the JVM. The generated traverse is therefore restricted to field +reads and `visit` calls. `ProxyTypeFactory`'s `tp_traverse` (every target) calls only the generated +`traverse` function and `visit`; `CycleCollectionTest` / `WasmCycleCollectionTest` / +`RefHolderCycleCollectionTest` exercise it through a real `gc.collect()` (SPEC M-3). + +**`tp_clear` needs no Kotlin field mutation.** The first design assumed breaking the cycle meant +nulling `K`'s `PyObject` fields (so they would have to be `var`, or cleared through a generated +accessor). That is not what was built. `ProxyTypeFactory.tp_clear` (`desktopMain/.../ProxyTypeFactory.kt`, +mirrored on every other target) never touches `K`'s fields; it calls `HandleTable.release()` on `PK`'s +handle and nothing else, deliberately not `close()`-ing the held `PyObject`s: CPython's own cyclic +collector, not this callback, reclaims the other side, and `close()` here would double-free. Dropping +the strong root `PK -> [handle] -> K` is enough to let the JVM collect `K` once CPython's collector +also breaks `P`'s incoming edge. `tp_dealloc` is overridden separately for the non-cyclic route; the +two are safe to run one after the other (`HandleTable.release` is idempotent). **Cycles that close on the Kotlin side.** `K1 -> P -> K2 -> K1` needs the JVM's collector to see through `P`, and there is no traverse hook on that side. Worse, the handle map holds `K2` strongly, so it is a root regardless. -The likely answer is to make the handle map hold **weakly**, with the strong reference supplied -by the Python proxy for as long as it lives — so "Python stopped using it" becomes "the JVM may -collect it". That inverts who owns the strength and has not been worked through. +**Still open.** The handle table holds strongly (`HandleTable.kt` notes it "needs to hold weakly" for +this case; ROADMAP §14b records it as untouched). The likely answer is to make the map hold weakly, +with the strong reference supplied by the Python proxy for as long as it lives — so "Python stopped +using it" becomes "the JVM may collect it". That inverts who owns the strength and has not been +worked through. ### Status of an older claim diff --git a/docs/design/pyi-generation-design.md b/docs/design/pyi-generation-design.md index 35ca018d4..1ffede955 100644 --- a/docs/design/pyi-generation-design.md +++ b/docs/design/pyi-generation-design.md @@ -1,18 +1,33 @@ # Generating `.pyi` — the reference implementation, the input, and what a stub can actually say -`pythonx` adapts Kotlin generically rather than wrapping function by function (`docs/design/ecosystem.md` -§5b): a module `__getattr__` finds the corresponding binding, adapts it once and caches it. Nothing -an editor can see is ever enumerated. `.pyi` is what pays that back — the stubs carry the fully -enumerated Pythonic surface the adapter will produce, so an IDE sees everything while the runtime -enumerates nothing. Generation belongs to the Gradle plugin, the way PyREPL did it; that is settled -and this file does not revisit it. - -What this file does is answer the three questions that were open under that decision: **what the -generator reads**, **what it can honestly write**, and **where the result goes**. +> **Revised 2026-10-03.** This file now describes only the stub product this repository ships: `.pyi` +> under the **Kotlin** module paths, with Kotlin names (SPEC B-7). The design of a second, Pythonic +> product — `pythonx.*` stubs, snake_case, `Dp | float`, `@overload` sets, the `Modifier` +> callback-Protocol shape and the `pythonx-map.toml` manifest — was removed from the plugin in +> `238119b7` and moved to +> [`docs/archive/pyi-generation-pythonic-stubs.md`](../archive/pyi-generation-pythonic-stubs.md); +> that product belongs to pythonx-compose. Section numbers are kept so that the code's KDoc +> references still resolve; sections that moved say so where they used to be. + +The binder serves Kotlin-named modules (`androidx.compose.foundation.layout`, `junit.runner.Version`, +a consumer's own package) that exist only as `sys.modules` entries made at run time +(`PythonProxySource`, and lazily `python_multiplatform.binding`). Nothing an editor can see is ever +enumerated. `.pyi` is what pays that back: the stubs carry the fully enumerated surface **under the +Kotlin names**, so an IDE sees every bound declaration while the runtime enumerates nothing. Making +that surface Pythonic is pythonx-compose's job, which reads `inspect.signature` and +`python_multiplatform.describe` at run time (`KotlinSurface.kt`) and ships its own stubs. Generation +belongs to the Gradle plugin, the way PyREPL did it (`AGENTS.md` §12.5); that is settled and this file +does not revisit it. + +What this file answers: **what the generator reads**, **what it writes**, and **where the result +goes**. The generator is `PythonStubsTask` + `renderKotlinFqnStubs` (`PyiRendering.kt`, in +`python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/stubs/`); its tests are +`PyiRenderingTest`, `KotlinNamesOnlyStubTest` and `ksp-fixtures/artifact`'s `WalkedArtifactStubTest`. Everything under "Observed" was read from working copies or run as a command on 2026-08-14, with the -path or the command given. Everything under "Judged" is a decision, and the alternatives it was -chosen over are named. Where a claim could not be checked it says so rather than being smoothed over. +path or the command given, unless dated otherwise. Everything under "Judged" is a decision, and the +alternatives it was chosen over are named. "Current" statements were checked against the code on +2026-10-03 by reading it; no Gradle build was run for this revision. --- @@ -28,14 +43,15 @@ chosen over are named. Where a claim could not be checked it says so rather than | the KSP producer | `python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/{FragmentScanner,Model,TypeShape,KsAdapters}.kt` | §2 | | the runtime proxy renderer | `python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/PythonProxySource.kt` | §2.2, §6 | | plugin wiring | `python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt` lines 145–344 | §6 | -| the Python package being stubbed | `/Volumes/macMini/thisisthepy/pythonx-compose/pythonx/**` | §5 | -| Compose desktop jars | `/Volumes/macMini/thisisthepy/pythonx-compose/pythonx/compose/lite/release/app/*.jar` | §5 | +| the Python package being stubbed | `/Volumes/macMini/thisisthepy/pythonx-compose/pythonx/**` | §5 (now archived) | +| Compose desktop jars | `/Volumes/macMini/thisisthepy/pythonx-compose/pythonx/compose/lite/release/app/*.jar` | §5 (now archived) | ### Run mypy **2.3.0** in a throwaway venv at `/Volumes/macMini/tmp/pyicheck` (nothing was installed into the repository, and no repository file other than this one was touched). Eight checks, reported in -§4.3, §4.4 and §6.3. CPython 3.13 for the two runtime attribute-lookup checks in §4.3. +§4.3, §4.4 and §6.1 — §4.3, §4.4 and most of §6.1 are now in the archive file named at the top. +CPython 3.13 for the two runtime attribute-lookup checks in §4.3. ### Not read, and therefore not claimed @@ -158,7 +174,7 @@ this repository's `KotlinMetadata.kt` acted on, two years later. | PyREPL | keep? | why | |---|---|---| | resolvable copy of `implementation`/`api` per source set | **superseded** | `PythonArtifactBindingsTask` already takes `configuration.incoming.artifacts`, which carries coordinates as well as files | -| per-package `__init__.pyi` | **keep** | it is what makes `from androidx.compose.material3 import Text` resolve, and §6.3 confirms a directory of `.pyi` with no `.py` and no `__init__.py` resolves | +| per-package `__init__.pyi` | **keep** | it is what makes `from androidx.compose.material3 import Text` resolve, and §6.1 confirms a directory of `.pyi` with no `.py` and no `__init__.py` resolves | | `...` bodies | **keep** | a stub never has to compile or run | | `def f(self, *args, **kwargs)` | **reject** | it is the whole of what makes these stubs useless for anything but name completion. Parameters are the point (§3) | | JNI-ish type map (`jint`, `jlong`) | **reject** | not Python types; a checker infers nothing from them | @@ -172,7 +188,10 @@ this repository's `KotlinMetadata.kt` acted on, two years later. ## 2. The input — judged -### 2.1 Observed: what the two producers keep today +### 2.1 Observed (2026-08-14): what the two producers kept then + +> Current: the loss described here is repaired for the stub path by `DeclarationModel` (§2.2). The +> table is kept because it is the reason the model exists. Both producers exist and both already read the right *sources*. Neither keeps enough of what it read. @@ -227,6 +246,17 @@ So: **one declaration model, produced by each scanner, consumed by two renderers ASM + kotlin-metadata-jvm ─┼─► DeclarationModel ─┬─► binding renderer (exists) LibraryAbiReader (klib) ─┘ └─► .pyi renderer (this document) +**Current (2026-10-03):** implemented as `DeclarationModel` / `DeclaredParameter` / +`KotlinTypeModel` / `ValueClassModel` in +`python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/model/StubModel.kt`. +It is produced by `ArtifactScanner.scanDeclarations` (jars) and `KlibScanner` (klibs, through the +isolated worker), in the same walk that produces the bindings. **KSP does not produce it**, so the +consumer's own KSP-bound declarations get no stub. Two differences from the plan below: the model +carries each declaration's **boundary tags** (`receiverBoundaryTag`, `DeclaredParameter.boundaryTag`, +`returnBoundaryTag`) and its table key (`bindingName`) or `declineReason`, because the stub annotates +what crosses, not what Kotlin declares (§3.1); and of the annotations it keeps only `isComposable` +(no `@Deprecated`). + `DeclarationModel` carries, per declaration: Kotlin qualified name; owner; whether it is a top-level function, a member, a constructor, a property; the extension receiver's Kotlin type or `null`; an ordered parameter list of `(name, KotlinTypeModel, declaresDefault)`; the return `KotlinTypeModel`; @@ -241,7 +271,8 @@ Three properties this buys, each of which is a defect in the current shape: `padding(a0: float)` while the binding calls `Modifier.padding(Dp(...))`. 2. **Overloads survive to where they are needed.** The binder drops an ambiguous name because it cannot dispatch (`docs/roadmap/ROADMAP.md` §16c: `Assert.assertEquals` has eight). A stub *can* state all of - them (§3.6, measured). Those are different decisions and today they are the same line of code. + them (§3.6). Those are different decisions and today they are the same line of code. *(Since + then the binder stopped dropping them: each member gets its own `name__` key, §3.6.)* 3. **What is declined stays visible.** A model entry can be marked "not bound, reason X" and still be stubbed — or deliberately not stubbed, which is the honest choice when the runtime cannot call it. Today a declined declaration returns `null` and vanishes. @@ -262,43 +293,48 @@ Three properties this buys, each of which is a defect in the current shape: | KSP | `KSFunctionDeclaration` gives names, defaults (`KSValueParameter.hasDefault`), nullability, type arguments, annotations | nothing new read — only *kept*. `KsAdapters.renderWithArguments` already walks type arguments | | artefact walker | `KmFunction` gives `valueParameters[].name`, `.declaresDefaultValue`, `KmType.isNullable`, `.arguments`, `receiverParameterType`; `ValueClassInfo` gives the rest | nothing new read — only *kept*. `functionsOf` currently throws away `KmValueParameter` and keeps `.type` | | Java classes in a walked jar (no `@Metadata`) | JVM descriptors only | **no Kotlin types, no parameter names, no nullability.** §3.2 | -| klib | not implemented | `LibraryAbiReader`, per `docs/design/ecosystem.md` §5b — unverified here | +| klib | **current:** `KlibScanner` builds `DeclarationModel` through `LibraryAbiReader` | what it cannot fill it declines on, and a declined declaration is not stubbed (`PythonStubsTask`); no klib declaration is bound at run time yet (SPEC B-2), so in practice no klib stub is emitted | That the first two need **no new reading** is the strongest argument for this shape: the information is being fetched and then dropped, one function before it would be used. ---- - ## 3. Kotlin → Python type mapping -### 3.1 The table - -`source` names where the fact comes from: `Km` = `kotlin-metadata-jvm`, `KS` = KSP, `desc` = JVM -descriptor, `list` = the hand-maintained allowlist of §3.4. - -| Kotlin | `.pyi` | source | note | -|---|---|---|---| -| `Boolean` | `bool` | Km/KS | | -| `Byte` `Short` `Int` `Long` | `int` | Km/KS | the boundary carries every one as `Long`; Python has one integer type, so the narrowing is invisible and correct | -| `Float` `Double` | `float` | Km/KS | | -| `String` | `str` | Km/KS | | -| `ByteArray` | `bytes` | Km/KS | | -| `Unit` | `None` | Km/KS | return position only | -| `Nothing` | `typing.Never` | Km/KS | a function returning it never returns; `Never` is the exact Python spelling | -| `T?` | `T \| None` | `KmType.isNullable`, `KSType.isMarkedNullable` | **not bindable today** — `resolveKotlinType` declines nullable outright. §3.3 | -| `List` `MutableList` | `list[T]` | Km/KS type args | not bindable today (the type gate) | -| `Set` | `set[T]` | | | -| `Map` | `dict[K, V]` | | | -| `Collection` `Iterable` | `Iterable[T]` | | read-only shape; `Sequence` would over-promise indexing | -| `Array` | `list[T]` | | judged: the marshaller's Python view is a list | -| `IntArray` `LongArray` `…` | `list[int]` etc. | desc/Km | `ByteArray` is the exception above | -| `() -> Unit` | `Callable[[], None]` | Km classifier `kotlin/Function0` | §3.5 | -| `(A) -> B` | `Callable[[A], B]` | `kotlin/FunctionN` type args | | -| `suspend (…) -> T` | — | `KmFunction.isSuspend`, `KSType.isSuspendFunctionType` | declined by both producers; must not be stubbed | -| value class on the allowlist | `Dp \| float` | Km + `list` | §3.4 | -| value class off the allowlist | `TextUnit` | Km | §3.4 | -| any other class/interface | its own stub name | Km/KS | requires that class to have a stub in the same namespace, which for a `TypeTag.OBJECT` handle is a promise the runtime does not yet keep — §7 | -| a generic type *parameter* (`fun f(x: T)`) | — | | `BindingPolicy` rejects these before the binder sees them; stubbing what cannot be called would be a lie. §7 | +### 3.1 The table — boundary types, with the Kotlin signature in the docstring + +**Current.** A stub describes the module the runtime actually publishes, and that module accepts and +returns what `UpcallTrampoline` marshals. So every annotation is the parameter's or return's +**boundary tag** (`python.multiplatform.reflection.TypeTag`), mapped by `boundaryAnnotation` in +`PyiRendering.kt`, and the declared Kotlin signature goes into the docstring, where a reader wants it +and a checker cannot act on it: + + def padding__Dp(receiver: int, /, all: float) -> int: + """Kotlin: androidx.compose.ui.Modifier.padding(all: androidx.compose.ui.unit.Dp): androidx.compose.ui.Modifier""" + ... + +(the exact text `WalkedArtifactStubTest.theKotlinFqnStubsDescribeTheModulesTheRuntimePublishes` +asserts against the real Compose jars). + +| boundary tag | `.pyi` | Kotlin declarations that cross as it | +|---|---|---| +| `BOOLEAN` | `bool` | `Boolean` | +| `INT` | `int` | `Byte`, `Short`, `Int`, `Long` (the boundary carries every one as `Long`; Python has one integer type) | +| `FLOAT` | `float` | `Float`, `Double` | +| `STRING` | `str` | `String`, `String?` | +| `BYTES` | `bytes` | `ByteArray`, `ByteArray?` | +| `UNIT` | `None` | `Unit`, return position only | +| `OBJECT` | `int` | any object: it crosses as a `HandleTable` integer (`UpcallTrampoline.marshalResult`), so `int` is the truth about the raw module. Function-typed slots also cross as `OBJECT` (§3.5) | +| none of the above | `object` | defensive default in `boundaryAnnotation`; a declined declaration is not stubbed at all | + +A value class crosses as its underlying primitive's tag when its constructor (parameter side) or its +property (return side) is public — `Dp` is `FLOAT` and is annotated `float` — and otherwise falls +through to an object handle (§3.4). + +**Not emitted:** `list[T]`, `dict[K, V]`, `Callable[...]`, `T | None`, `typing.Never`, or a stub name +for an object type. Each of those would describe the declared Kotlin type rather than what the raw +module accepts, and saying `-> Modifier` where Python receives an `int` is the lie §7's first item +warns about. The declared-Kotlin-type table that the removed Pythonic product implemented +(`PythonTypes.kt`) is in the archive file, §3.1. ### 3.2 Java declarations have no types to map @@ -318,125 +354,90 @@ because Java can return null and nothing in the bytecode says otherwise. This is a real asymmetry between a Kotlin jar and a Java jar and it should be visible in the output, not hidden. -### 3.3 Nullability - -`String?` → `str | None` is mechanical once the fact is carried (§2.2). The thing to record is that -**nothing is nullable today**: `resolveKotlinType` returns `null` for `type.isNullable`, so a -declaration with one nullable parameter is not bound at all. The stub generator therefore has -nothing to be nullable about until the binder's own gate moves, and the mapping row above is -provision, not description. - -`| None` is preferred over `Optional[...]`: it is the 3.10+ spelling, and a `.pyi` may use it -regardless of the runtime's target version because a stub is never executed. `from __future__ import -annotations` is not needed in a `.pyi` for the same reason. - -### 3.4 Value classes — how the allowlist shows up - -`docs/design/kotlin-extensions-in-python.md` §4.4 decided this and measured why a machine rule fails: the -rule "coerce iff the value class has a public constructor taking exactly its underlying type" -selects 36 of 111 and **admits `Color`**, whose public `ULong` constructor stores while the factory -a Kotlin author actually writes (`Color(Int)`) shifts by 32. Whether a wrapper packs is a semantic -fact with no bytecode witness. +> **Correction (2026-10-03):** the sentence above about `| None` on Java reference returns describes +> the plan, not the code. The current stub annotates a Java return by its boundary tag like any other +> (`def id() -> str:` for `junit.runner.Version.id`, asserted by `PyiRenderingTest` and +> `WalkedArtifactStubTest`); no `| None` is added. Positional-only `__a` parameters for a Java +> declaration without `-parameters` are implemented as described (`parameterNamesKnown == false`, +> `PyiRenderingTest.aJavaDeclarationGetsAnonymousPositionalParameters`), under the overload-suffixed +> table key: `def assertEquals__Long_Long(__a0: int, __a1: int, /) -> None:` (§3.6). -In a stub the distinction is one union member: - -| Kotlin parameter | `.pyi` | why | -|---|---|---| -| `Dp` (on the allowlist) | `Dp \| float` | public constructor, wraps a plain `Float`, and it *is* the identity on that float. 62% of value-class parameter occurrences on `Modifier` | -| `TextUnit` (rejected) | `TextUnit` | packed `Long`; raw `16` decodes as `Unspecified` — silently no value at all | -| `Color` (rejected despite passing the constructor test) | `Color` | `Color(int)` is `ULong(value) shl 32`; raw `0xFFFF0000` is transparent black through the constructor and red through the factory | -| every `packedValue` class (`Offset`, `Size`, `DpSize`, `IntSize`, `IntOffset`, `DpOffset`, `TransformOrigin`, `CornerRadius`, `Constraints`) | the proxy alone | a user-meaningful number is not the stored number, and none of these failures raises | +### 3.3 Nullability -**Measured, §4.4's asymmetry is expressible and enforced.** In the mypy run of §4.4, -`Text("hi", font_size=16)` produced +**Current.** The binder's gate moved since this section was first written: a nullable **object** +binds (the handle is already nullable at the boundary; `UpcallTrampoline.toKotlin` maps Python +`None` to `null`), and `String?` and `ByteArray?` bind; a nullable numeric or `Boolean` primitive and +a nullable value class stay declined (`resolveKotlinBoundary` and `nullablePrimitiveBoundaryTypeOf` +in `KotlinMetadata.kt`). The stub does **not** mark nullability: a `String?` parameter is annotated +`str`, and the `?` appears only in the docstring's Kotlin signature. Marking it `str | None` is +mechanical once wanted — `KotlinTypeModel.isNullable` carries the fact — and is listed in §7. + +### 3.4 Value classes + +**Current.** A value-class parameter is annotated with its underlying boundary type when the binder +binds it as that primitive (`Dp` → `float`, because `Dp`'s constructor is public and it wraps a +`Float`). A value class whose wrapper cannot be opened from outside its module (`TextUnit`, every +`packedValue` class) falls through to the object handle and is annotated `int`. `Color`'s public +constructor takes `ULong`, which `kotlinPrimitiveBoundaryTypeOf` has no entry for. + +The raw-primitive **allowlist** (`docs/design/kotlin-extensions-in-python.md` §4.4: `Dp` yes, `TextUnit` +and `Color` no) is not a stub concern here. It is enforced at run time by the binding layer, where it +starts **empty** and is filled by pythonx-compose through `allow_raw_primitive` +(`PythonxAdapter.kt`). The stub describes the raw module, which accepts the underlying primitive for +any value class it binds that way. How the allowlist shows up in a *Pythonic* stub (`Dp | float` +versus the proxy alone, with the mypy evidence) is archive §3.4. - error: Argument "font_size" to "Text" has incompatible type "int"; expected "TextUnit" +### 3.5 Function types, and Compose's `content` -while `Modifier.padding(16)` in the same file checked clean. So the stub is not merely documentation -of the asymmetry — a checker acts on it, and the user finds out at edit time rather than seeing a -label render with no font size. +**Current.** A function-typed parameter crosses as `OBJECT` (`ArtifactScanner.functionSlotOrNull`): +the runtime turns a Python callable into a Kotlin `FunctionN` and hands the raw boundary a handle. The +stub therefore annotates it `int`, with the declared `kotlin.FunctionN<...>` in the docstring; no +`Callable[...]` is emitted. -What a stub cannot carry is §4.4's *third* clause, the error message that names the constructor to -call (`"expected TextUnit; write sp(16)"`). mypy's own message names the type but not the remedy. -Judged: accept it. The runtime raises the full message; the stub gets the user to the runtime less -often. +Two things the current product does that the original design asked for: -**Where the allowlist lives** is `docs/design/kotlin-extensions-in-python.md` §6's open question, and §5.3 -here proposes an answer that falls out of the package mapping: the same manifest. +- **The synthetic parameters are not stubbed.** Parameters come from `KmFunction` metadata, so + `$composer`, `$changed` and `$default` never appear (`docs/design/kotlin-extensions-in-python.md` + §4.6: metadata arity is the correct source). This follows from `DeclarationModel`'s construction; + no stub test asserts it for a composable. +- **A required parameter after a defaulted one is not marked optional.** Python rejects a required + parameter after one with a default, and Compose's trailing `content` is exactly that, so a + default is marked `= ...` only when no required parameter follows it + (`KotlinNamesOnlyStubTest.aDefaultBeforeARequiredParameterIsNotMarkedSoTheStubStaysSyntacticallyValid`, + `WalkedArtifactStubTest.noGeneratedDefPutsARequiredParameterAfterADefaultedOne`). -### 3.5 Function types, and Compose's `content` +What it does not do: make `content` keyword-only. The run-time signature does +(`inspect.signature` on a binder-made function makes "required after optional" keyword-only, +`KotlinSurface.kt`), so stub and run-time signature differ there; §7. -`() -> Unit` → `Callable[[], None]`, mechanically, from `kotlin/Function0`'s type arguments. 66 of -the 177 chainable `Modifier` extensions take at least one lambda, and every `@Composable` container -takes `content`. - -Two things the stub should do and one it should not: - -- **`content` is keyword-only in Python** even though it is the last positional parameter in Kotlin. - Compose's trailing-lambda syntax has no Python equivalent, and `Column(modifier, lambda: ...)` - reads badly. Emitting `*, content: Callable[[], None]` forces `content=`. Checked in §4.4's run: - `Column(modifier=Modifier.size(4), content=lambda: Text("x"))` resolves clean. -- **`@Composable` on a lambda parameter is not expressible and should not be faked.** A - `@Composable () -> Unit` and a `() -> Unit` are the same Python `Callable[[], None]`. Kotlin's - restriction — a composable lambda may only be invoked in a composable context — has no Python - counterpart, and inventing a distinct alias would give a checker a rule it cannot enforce. - `docs/design/ecosystem.md` §5b already decided the composer is threaded as an ordinary value. -- **Do not stub the synthetic parameters.** `$composer` and `$changed` are in the JVM descriptor and - are not in `KmFunction`. `docs/design/kotlin-extensions-in-python.md` §4.6: metadata arity is the correct - source, and any code deriving arity from the descriptor is wrong for all 500 composables. +**Suspend functions are not stubbed** (`renderKotlinFqnStubs` filters `isSuspend`), although the +runtime makes them awaitable (SPEC U-5). §7. ### 3.6 Overloads -Python has `@overload`, and `docs/design/kotlin-extensions-in-python.md` §4.5 already calls it mandatory: -33 of `Modifier`'s 130 names carry more than one, and they are the load-bearing ones -(`padding`, `size`, `background`, `border`, `clickable`, …). - -**Measured** (`/Volumes/macMini/tmp/pyicheck/ovstub.pyi`, all four real `Modifier.padding` -overloads from §3 of that document, as a callback protocol): - -```python -class _padding(Protocol): - @overload - def __call__(self, paddingValues: PaddingValues) -> Modifier: ... - @overload - def __call__(self, all: Dp | float) -> Modifier: ... - @overload - def __call__(self, horizontal: Dp | float = ..., vertical: Dp | float = ...) -> Modifier: ... - @overload - def __call__(self, start: Dp | float = ..., top: Dp | float = ..., - end: Dp | float = ..., bottom: Dp | float = ...) -> Modifier: ... -``` - -All four coexist. `p(8)`, `p(horizontal=8, vertical=4)`, `p(start=1, bottom=2)` and -`p(PaddingValues())` each resolve to `Modifier`; `p("bad")` is rejected with all four variants -listed. mypy issued **no** overlap diagnostic. - -That last fact is the one to be careful about. mypy reports overlapping overloads only when their -*return types* are incompatible, and every `Modifier` extension returns `Modifier`. So variants 2 and -3 genuinely overlap — `p(8)` matches `all` and also matches `horizontal` positionally — and the -checker silently takes the **first** match. - -**Judged: overload order is part of the generated output and must be deterministic and -arity-ascending.** In the run above, arity-ascending put `all: Dp` before `horizontal, vertical`, and -`p(8)` selected `all` — the right answer, but only because of the order. Two further consequences: - -- The stub's order must be **the same order the Python-side dispatcher resolves in**. If the runtime - dispatcher (`docs/design/kotlin-extensions-in-python.md` §6, unsolved) picks differently, the stub lies - about which Kotlin function runs. The two orders must come from one place in the generator. -- **Defaults are emitted as `= ...`.** 254 of 435 `Modifier` extension parameters declare one and - metadata carries `declaresDefaultValue` but not the *value*. `= ...` is the correct stub spelling - for "has a default I cannot name" and is what makes the keyword-only overloads above work. - -Kotlin parameter names go through the same snake_case conversion as everything else -(`paddingValues` → `padding_values`), which `pythonx-compose` already does by hand: `text.py` maps -`font_size` → `fontSize`, `letter_spacing` → `letterSpacing`, and 14 more. +**Current.** The binder no longer drops an ambiguous name: `ArtifactScanner.disambiguateOverloads` +gives every member of an overload set its own table key, `name__` (`padding__Dp`, +`padding__Dp_Dp`, `padding__Dp_Dp_Dp_Dp`, `padding__PaddingValues`; a group of one keeps the bare +name; `docs/design/kotlin-extensions-in-python.md` §3.1). The stub emits **one `def` per table key** +and nothing else — `WalkedArtifactStubTest.everyStubbedFunctionIsATableKeyAndEveryTableKeyIsStubbed` +asserts the two sets are equal. The base name of an overload set (`padding`), which the binding layer +dispatches at run time (`_Overloads` in `PythonxAdapter.kt`, `(*args, **kwargs)` signature per +`KotlinSurface.kt`), has no stub; no `@overload` is emitted. + +Parameters are keyword-capable under their **Kotlin** names (`all`, `fraction`, `alignmentLine`); +nothing is converted to snake_case (`KotlinNamesOnlyStubTest.kotlinNamesAreNeverSnakeCased`). A name +that is not a Python identifier or is a Python keyword, or an unknown name, forces positional-only up +to and including that parameter, spelled `__a` +(`KotlinNamesOnlyStubTest.aParameterNamedLikeAPythonKeywordForcesAPositionalOnlyPrefix`). A Kotlin +default is `= ...`, because metadata carries `declaresDefaultValue` but not the value. + +The mypy measurements of four `padding` overloads as `@overload`s, the overlap behaviour (mypy takes +the first match silently) and the judged rule "deterministic, arity-ascending, same order as the +dispatcher" belong to a Pythonic stub and are in archive §3.6. --- -## 4. Extension functions — and the metaclass proposal does not survive contact - -This is the section the Compose surface turns on: 604 public top-level extension functions over 136 -receivers, and 177 chainable ones on `Modifier` alone. +## 4. Extension functions ### 4.1 Placement (unchanged from `docs/design/kotlin-extensions-in-python.md` §4.1) @@ -448,149 +449,19 @@ receivers, and 177 chainable ones on `Modifier` alone. | Kotlin primitive or `String` | 20 | module-level function — Python cannot attach a method to `int` | Nothing measured here disturbs that. Chaining is free because each of the 177 returns `Modifier`, and -§4.4's run confirms `Modifier.size(4).padding(2)` type-checks as `Modifier`. - -### 4.2 The proposal under test - -`docs/design/kotlin-extensions-in-python.md` §4.2 and §4.5 propose that `Modifier` be one class whose class -object also behaves like an instance, via a metaclass, so that `Modifier.padding(16)`, -`m.padding(16)` and `def f(m: Modifier)` all work: - -```python -class _ModifierMeta(type): - @overload - def padding(cls, all: Dp | float) -> Modifier: ... - ... - -class Modifier(metaclass=_ModifierMeta): - @overload - def padding(self, all: Dp | float) -> Modifier: ... - ... -``` - -### 4.3 Observed: it does not work, at run time or in a checker - -**Run time** (CPython 3.13, `/Volumes/macMini/tmp/pyicheck`): - -```python -class Meta(type): - def padding(cls, v): return ("META", v) -class Modifier(metaclass=Meta): - def padding(self, v): return ("INSTANCE", self, v) - -Modifier.padding # -> -Modifier.padding(16) # -> TypeError: Modifier.padding() missing 1 required positional argument: 'v' -``` - -This is `type.__getattribute__`'s documented order: a metaclass attribute wins **only if it is a data -descriptor**; otherwise the class's own MRO is searched first, and a plain function on the metaclass -is a non-data descriptor. So `Modifier.padding` is the *unbound instance method*, and `16` is being -bound to `self`. It raises here only because the arities differ by one; it would bind silently for -any pair that happened to match. - -**mypy 2.3.0**, on exactly the stub of §4.2: +the mypy run in archive §4.4 confirms `Modifier.size(4).padding(2)` type-checks as `Modifier` in a Pythonic stub. - reveal_type(Modifier.padding) - note: Revealed type is "Overload(def (self: Modifier, all: Dp | float) -> Modifier, ...)" +> **Current:** this placement is what a Pythonic layer does with extensions, and the counts are a +> property of Compose. The stub product here places **every** extension function as a module-level +> `def` in the module of its declaring package, with the receiver as the first positional-only +> parameter named `receiver` (`def fillMaxWidth(receiver: int, /, fraction: float = ...) -> int:`, +> `WalkedArtifactStubTest`). The method form on the receiver's proxy (SPEC U-7) exists only at run +> time in the binding layer and is not stubbed. - Modifier.padding(16) - error: No overload variant of "padding" of "Modifier" matches argument type "int" - note: Revealed type is "Any" +### 4.2–4.4 Moved -The checker reproduces the runtime rule: the metaclass declaration is invisible, `Modifier.padding` -is the instance method, and every class-object spelling fails. Ten errors in a twenty-line file, and -the only lines that passed were the instance ones. - -**The fix that works at run time does not fix the checker.** A `property` on the metaclass *is* a -data descriptor and does win: - -```python -class Meta2(type): - @property - def padding(cls): return cls.EMPTY.padding -Modifier2.padding(16) # -> ('INSTANCE', , 16) correct -``` - -as does a hybrid descriptor in the class body with no metaclass at all: - -```python -class hybrid: - def __init__(self, fn): self.fn = fn - def __get__(self, obj, owner): return self.fn.__get__(obj if obj is not None else owner.EMPTY, owner) -Modifier3.padding(16) # -> ('BOUND-TO', , 16) correct -Modifier3().padding(16) # -> correct -``` - -But declaring the metaclass side as `@property -> _PaddingCall` in the stub changes nothing for -mypy — it still resolves the class's own method and produces the same twelve errors. **mypy does not -model metaclass data-descriptor precedence.** - -### 4.4 Judged: callback-Protocol attributes, and the stub need not mirror the runtime mechanism - -The shape that satisfies all three spellings is to declare each operation on the class as an -**attribute whose type is a Protocol with an overloaded `__call__`**: - -```python -class _Modifier_padding(Protocol): - @overload - def __call__(self, all: Dp | float) -> Modifier: ... - @overload - def __call__(self, *, horizontal: Dp | float = ..., vertical: Dp | float = ...) -> Modifier: ... - -class Modifier: - padding: ClassVar[_Modifier_padding] - size: ClassVar[_Modifier_size] - background: ClassVar[_Modifier_background] -``` - -**Measured** — the same twenty-line user file that produced ten errors against §4.2 produces exactly -the two intended ones: - -| expression | revealed / result | -|---|---| -| `Modifier.padding` | `_Modifier_padding` | -| `Modifier.padding(16)` | `Modifier` | -| `Modifier.padding(dp(16))` | `Modifier` | -| `Modifier.padding(horizontal=8, vertical=4)` | `Modifier` | -| `Modifier.padding(16).background(Color(0xFFFF0000))` | `Modifier` | -| `Modifier.size(4).padding(2)` | `Modifier` | -| `m.padding(16)` where `m: Modifier` | `Modifier` | -| `takes_modifier(Modifier.padding(16))` where `def takes_modifier(m: Modifier)` | clean | -| `Text("hi", font_size=16)` | **error** — `int` is not `TextUnit` (intended, §3.4) | -| `Modifier.padding("nope")` | **error**, with all overload variants listed (intended) | - -`ClassVar` is not load-bearing for inference — a plain `padding: _Modifier_padding` gives byte-identical -results — but it is the honest annotation, and it stops a checker accepting `m.padding = something`. - -The point that makes this legitimate rather than a trick: **a `.pyi` completely replaces the `.py` -for a checker, so the stub's mechanism and the runtime's mechanism do not have to agree — only the -types do.** The runtime uses the hybrid descriptor of §4.3, which binds `self` correctly for both -spellings; the stub says "a callable attribute", which is what a checker can act on. The one -inaccuracy is that a checker believes `Modifier.padding` and `m.padding` are the same object, and -nothing observable depends on that. - -**Rejected:** - -| alternative | why not | -|---|---| -| the metaclass of §4.2 as written | measured not to work, in both mypy and CPython (§4.3) | -| metaclass whose members are `@property -> Protocol` | works at run time, does not work in mypy — the class's own member still wins. Would give a *correct runtime* with a *wrong IDE*, which is the worst of the two | -| bind the module-level name `Modifier` to the companion instance | already rejected in §4.2 of the other document and still right: it destroys `Modifier` as an annotation, and the stubs exist so that annotation works | -| drop the `Modifier.padding(...)` spelling, require `Modifier().padding(...)` | changes the API away from Kotlin's for a reason that is an implementation detail of Python attribute lookup | -| emit both the metaclass **and** the Protocol attributes | the class attribute shadows the metaclass in both engines, so the metaclass is dead text that a reader will believe | - -**Cost, stated:** one Protocol class per `(receiver, name)` pair. For `Modifier` that is 130 extra -class definitions in `pythonx/compose/ui/__init__.pyi`; across the 507 distinct `(receiver, name)` -pairs in corpus A, ~507 if every receiver is stubbed. They are generated, never read by a human, and -`.pyi` files are not executed — but a 500-class stub file is a real indexing cost for an IDE and it -has not been measured. - -**Not verified: PyCharm.** Callback protocols are standard typing and PyCharm supports `Protocol`, -but whether its inference resolves `Modifier.padding(16)` through a `ClassVar[Protocol]` was not -tested. **This is the single most load-bearing unverified claim in this document** — §4.4 is chosen -over §4.2 on mypy evidence alone, and if PyCharm follows the runtime MRO rule the way mypy does then -§4.4 works there too, while if it has its own metaclass handling the conclusion could differ. -Verify before implementing. +The `Modifier` metaclass proposal, its failure in CPython and mypy, and the callback-Protocol +attribute shape that replaced it are a Pythonic stub design: archive §4.2–§4.4. ### 4.5 Member extensions are still out @@ -604,79 +475,25 @@ that says `Modifier.weight(1.0)` checks is a stub that promises a call the runti ## 5. Package mapping -### 5.1 Observed: the shape of `pythonx-compose` - -`/Volumes/macMini/thisisthepy/pythonx-compose/pythonx/` contains **only** `compose/`, and there is -**no `pythonx/__init__.py`** — `pythonx` is an implicit namespace package, so several distributions -can contribute subpackages under it. `pythonx/compose/` has `__init__.py`, `layout/`, `lite/`, -`material3/`, `native/`, `runtime/`, `test/`, `ui/`, `ui/unit/`, `wrapper/`. +There is none. A stub lives at the Kotlin module path the runtime publishes onto — +`androidx/compose/foundation/layout/__init__.pyi`, `junit/runner/Version/__init__.pyi` for a static on +a class — and nothing is emitted under `pythonx/` +(`KotlinNamesOnlyStubTest.theStubLivesUnderTheKotlinModulePathAndNothingLivesUnderPythonx`, +`WalkedArtifactStubTest.theStubsUseKotlinNamesAndNothingIsExportedUnderPythonx`). The `androidx.` → +`pythonx.` default rule, the observation that `pythonx.compose.layout` wraps +`androidx.compose.foundation.layout`, and the `pythonx-map.toml` manifest are in archive §5. -### 5.2 Observed: it is not a rule +### 5.3 The Kotlin-FQN stubs — the one product -`unzip -l` over the Compose Multiplatform desktop jars vendored at -`pythonx/compose/lite/release/app/` shows that the `org.jetbrains.compose.*` artefacts contain -`androidx.compose.*` **packages**: +Of the two products the original §5.3 named, only the first remains: - foundation-layout-desktop-1.6.11-*.jar -> androidx/compose/foundation/layout - ui-unit-desktop-1.6.11-*.jar -> androidx/compose/ui/unit - material-desktop-1.6.11-*.jar -> androidx/compose/material - -so the Kotlin namespace is `androidx.compose.*` on desktop and Android alike, and one mapping covers -both. Against the actual directories: - -| `pythonx` module | Kotlin package | mechanical? | +| product | namespace | content | |---|---|---| -| `pythonx.compose.material3` | `androidx.compose.material3` | yes | -| `pythonx.compose.runtime` | `androidx.compose.runtime` | yes | -| `pythonx.compose.ui` | `androidx.compose.ui` | yes | -| `pythonx.compose.ui.unit` | `androidx.compose.ui.unit` | yes | -| `pythonx.compose.layout` | `androidx.compose.foundation.layout` | **no — `foundation.` is dropped** | -| `pythonx.compose.wrapper`, `.native`, `.lite`, `.test` | none | no counterpart | - -The `layout` entry is not inferred. `pythonx/compose/layout/arrangement.py` line 7 reads -`jclass("androidx.compose.foundation.layout.Arrangement")`, and `pythonx/compose/layout/__init__.py` -re-exports `Arrangement` from it. - -### 5.3 Judged: a rule plus a manifest the Python package owns - -**`androidx.` → `pythonx.` is the default rule; deviations come from a manifest, and the manifest -belongs to the Python package, not to the plugin.** - -The reason is ownership, not convenience. Which Kotlin package a `pythonx` module wraps is -`pythonx-compose`'s design decision — the plugin has no basis on which to invent -`pythonx.compose.layout` for `androidx.compose.foundation.layout`, and hard-coding Compose's -particular renames into a general-purpose Gradle plugin would make every other library's mapping -unreachable. - -So: a data file inside the Python distribution — e.g. `pythonx/compose/pythonx-map.toml` — declaring - - [modules] - "pythonx.compose.layout" = "androidx.compose.foundation.layout" - "pythonx.compose.ui.unit" = "androidx.compose.ui.unit" +| Kotlin-FQN stubs | `androidx.compose.material3`, `junit.runner`, … | 1:1 with the table keys `PythonProxySource` publishes into `sys.modules`; Kotlin names, boundary-type annotations, the Kotlin signature in each docstring, no extension-as-method | - [value-classes] - raw-primitive-allowed = ["androidx.compose.ui.unit.Dp"] - -The plugin reads it from the resolved Python package and emits `pythonx.*` stubs accordingly; with -no manifest it emits only the Kotlin-FQN stubs. This also gives -`docs/design/kotlin-extensions-in-python.md` §6's open question ("whether the allowlist should be data or -code, and where a downstream consumer adds to it") a location: the same manifest, in the same -package, next to the code whose surface it describes. - -**Consequence: there are two stub products, from one model.** - -| product | namespace | content | mechanical? | -|---|---|---|---| -| Kotlin-FQN stubs | `androidx.compose.material3`, `junit.runner`, … | 1:1 with the modules `PythonProxySource` injects into `sys.modules`; Kotlin names, no snake_case, no extension-as-method | yes | -| Pythonic stubs | `pythonx.compose.material3`, … | snake_case, extension-as-method (§4.4), value-class allowlist (§3.4), keyword-only `content` (§3.5) | needs the manifest | - -The first is the direct descendant of PyREPL's output and is the only thing an IDE could ever see for -those modules, since they exist only as runtime `sys.modules` entries and never as files. The second -is what a user actually imports. - -**Not verified:** no such manifest exists in `pythonx-compose` today, and this proposal has not been -agreed with that repository. It is a judgement about where the information has to live, not a -description of anything. +It is the direct descendant of PyREPL's output and the only thing an IDE can see for those modules, +since they exist only as runtime `sys.modules` entries. The Pythonic stubs a user of `pythonx` +imports against are pythonx-compose's product. --- @@ -684,22 +501,10 @@ description of anything. ### 6.1 Observed: what makes stubs take effect -Five mypy runs in `/Volumes/macMini/tmp/pyicheck`, each reported above or below: - -1. **A `.pyi` beside a `.py` wins.** The fixture's `pythonx/compose/ui/__init__.py` contains nothing - but `def __getattr__(name): raise AttributeError(name)` — the shape the real adapter will have — - and every name in §4.4's table resolved from `__init__.pyi`. This is the deployment that matters: - the dynamic adapter ships, the stub ships beside it, and the checker never sees the dynamism. -2. **`py.typed` is mandatory once the package is installed.** With `pythonx/` copied into the venv's - `site-packages` and `pythonx/compose/py.typed` present, everything resolved. Deleting **only** - `py.typed`: +Of the mypy runs in `/Volumes/macMini/tmp/pyicheck`, the two that bear on the Kotlin-FQN product +(measurements 1 and 2, about `.pyi` beside a `pythonx` `.py` and `py.typed` placement, are in archive +§6): - error: Skipping analyzing "pythonx.compose.ui": module is installed, but missing library - stubs or py.typed marker [import-untyped] - note: Revealed type is "Any" (every line) - - The marker goes in the top-level *regular* package of the distribution — here `pythonx/compose/`, - because `pythonx` is a namespace package (§5.1). That placement was the one tested and it works. 3. **A stub-only tree with no `.py` and no `__init__.py` resolves.** A directory containing only `androidx/compose/material3/__init__.pyi` on `MYPYPATH` gave `Revealed type is "def (text: str, color: int =, fontSize: int =)"` and rejected `Text(3)`. So @@ -727,12 +532,12 @@ So: - **Register the directory wherever the environment has a mechanism**: chaquopy `sourceSets.srcDirs` when chaquopy is applied; a `.pth` file in the venv for a desktop run; the interpreter path list for PyCharm. This is per-environment and none of it was verified. -- **For a published `pythonx` wheel the stubs belong inside the wheel** — `.pyi` beside `.py` plus - `py.typed`, both verified in §6.1 — built by `pypackpack`'s bundle stage, not regenerated per - consumer build. The plugin's per-build generation covers the consumer's own Kotlin and the - third-party jars *its* build resolves; a shipped `pythonx-compose` covers Compose once. -Two audiences, two destinations, one generator. +**Current:** implemented as written for the first two bullets — `PythonStubsTask` writes +`build/generated/pythonStubs//`, deleting it first, and `generatePythonStubs` is a +dependency of `prepareKotlinIdeaImport` (`PythonBindingsPlugin.configureStubGeneration`). No +environment registration is implemented. A `pythonx` wheel's own stubs are pythonx-compose's (archive +§6). ### 6.3 The commonMain/platform overlap @@ -746,30 +551,30 @@ what made the bug invisible. has no `JClass` and says so. The stubs must reproduce that per source set rather than emitting a union that promises every platform's surface everywhere. +**Current:** not implemented — `PythonStubsTask` is registered per source set and nothing detects or +merges an overlap. + --- ## 7. Open -- **PyCharm.** §4.4 rests entirely on mypy. Verify `ClassVar[Protocol]` resolution, `@overload` - ordering, and `.pyi`-beside-`.py` precedence in PyCharm before implementing. If PyCharm does model - metaclass data descriptors, §4.2's shape becomes viable again there and the two engines would want - different stubs — which would be a genuinely new problem. -- **The 500-class stub cost.** ~507 callback Protocols for corpus A's `(receiver, name)` pairs, and - 130 for `Modifier` alone in one file. Not measured for IDE indexing time or for mypy's own runtime. -- **Whether any real Compose overload set collapses.** §3.6 shows four `padding` overloads coexisting, - but two Kotlin overloads can map to identical Python signatures once `Dp | float` widening is - applied. The generator must detect that and drop or qualify, and how often it happens across the - 33 multi-overload `Modifier` names is unknown — it needs the scan to be run. -- **Stub order vs dispatcher order.** §3.6: the `@overload` order and the Python-side dispatcher's - resolution order must come from one place. The dispatcher does not exist - (`docs/design/kotlin-extensions-in-python.md` §6). - **What to stub for a `TypeTag.OBJECT` return.** `PythonProxySource`'s KDoc records that such a - value crosses as a bare handle integer, not as an instance of the class rendered for it. A stub - saying `-> Modifier` would then be wrong for exactly the chaining case §4.1 depends on. Needs the - handle-to-proxy wrapping that section lists as not yet done. + value crosses as a bare handle integer, not as an instance of the class rendered for it, which is + why the stub says `-> int` (§3.1). A stub saying `-> Modifier` needs the handle-to-proxy wrapping + at the raw module, which is not done (SPEC B-7: "handle-returning stubs are not wrapped"). +- **Nullability** (§3.3) is carried by the model and not rendered. +- **Suspend functions** (§3.5) are bound and awaitable but not stubbed. +- **Overload base names** (§3.6) are dispatched at run time and not stubbed. +- **`content` keyword-only** (§3.5): the run-time signature makes it keyword-only, the stub does not. +- **The consumer's own KSP-bound declarations** are not stubbed: KSP does not produce + `DeclarationModel` (§2.2), although §2.3 records it would need to read nothing new. - **Generic declarations.** `BindingPolicy` rejects them, so they are not stubbed; if the binder ever accepts them, `TypeVar` is the mapping and nothing here covers it. -- **klib.** §0. The `pythonx.*` surface on iOS and androidNative has no producer yet. -- **The manifest of §5.3** does not exist and has not been agreed with `pythonx-compose`. +- **klib.** The scanner produces declarations (§2.3), but none is bound at run time yet (SPEC B-2), + so none is stubbed. - **Member extensions** (§4.5) — 74 on `Modifier` — remain unstubbable until the scope-passing shape is chosen. +- **IDE pick-up** (§6.2): registering `build/generated/pythonStubs//` with an interpreter + or IDE is per-environment and none of it is implemented or verified; PyCharm was never tested + (§0). +- **commonMain/platform overlap** (§6.3) is not implemented. diff --git a/docs/design/threading-and-abi.md b/docs/design/threading-and-abi.md index 6ba3a7b63..da806838e 100644 --- a/docs/design/threading-and-abi.md +++ b/docs/design/threading-and-abi.md @@ -1,25 +1,23 @@ # Threading model and Stable ABI -> **Note (2026-10):** the conclusion below, "free-threading starts at 3.15t", is superseded. It dates from -> 2026-08-10. `docs/roadmap/ROADMAP.md` §9 (2026-08-12) corrects it: 3.14t works on desktop with -> `-PpythonFreeThreaded=true` (236 tests, 0 failures); the default stays `pythonFreeThreaded=false`; only -> desktop has free-threaded prebuilts; `Py_LIMITED_API` is not defined, so `abi3t` is not a blocker. The -> reasoning is kept as history. - ## Decision -Parallelism will come from **free-threaded CPython**, not from per-interpreter GIL sub-interpreters. +Parallelism comes from **free-threaded CPython**, not from per-interpreter GIL sub-interpreters +(AGENTS.md §12.8). -The library ships **two flavours**, and the developer picks one by choosing which interpreter build -they depend on: +The library supports **two flavours**, chosen at build time: -| Interpreter | Threading behaviour | -|---|---| -| `python3.X` (GIL) | Kotlin threads each attach a thread state and serialise on the GIL | -| `python3.Xt` (free-threaded) | Each Kotlin thread talks to its own free-running Python thread | +| Interpreter | Threading behaviour | Status | +|---|---|---| +| `python3.X` (GIL) | Kotlin threads each attach a thread state and serialise on the GIL | default (`pythonFreeThreaded=false` in `gradle.properties`) | +| `python3.Xt` (free-threaded) | Each Kotlin thread talks to its own free-running Python thread | **3.14t works on desktop**, opt-in with `-PpythonFreeThreaded=true` | -**Free-threading was originally planned from 3.15t onwards only** (superseded: see the note above; 3.14t -works on desktop, opt-in). The reasoning for that gate is set out below. +The interpreter version is pinned by `pythonVersion` in `gradle.properties` (3.14.7 at the time of +writing). Free-threading is **not** gated on 3.15t: the whole desktop suite runs on 3.14.7t +(236 tests, 0 failures, 1 skipped — ROADMAP §9; SPEC T-2). Only desktop has free-threaded +prebuilts; Android and iOS have none, so the opt-in is desktop-only. How the earlier "from 3.15t" +conclusion was reached, and why it did not hold, is in +[`../archive/threading-and-abi-3.15t-gate.md`](../archive/threading-and-abi-3.15t-gate.md). ## Why not per-interpreter GIL @@ -55,28 +53,25 @@ extension to implement multi-phase init and declare `Py_mod_multiple_interpreter do not simply fail to import. That would cost exactly the Python libraries this project exists to share, in return for parallelism. -## Why 3.15t and not 3.14t (historical; superseded by ROADMAP §9) - -Free-threaded builds **do not support the Limited API or the Stable ABI** in 3.13 or 3.14. All ~330 -bindings here target the Stable ABI, so a free-threaded 3.14 build cannot be used without abandoning -it and recompiling per Python version — which would defeat the version-parameterised acquisition -pipeline. +## Stable ABI on free-threaded builds -[PEP 803](https://peps.python.org/pep-0803/) — **Final**, targeting 3.15 — introduces `abi3t`, a -Stable ABI variant for free-threaded builds, with `abi3t` / `abi3.abi3t` wheel tags and forward -compatibility across all later versions. 3.15 is therefore the first release where free-threading -and the Stable ABI coexist. +CPython's Limited API / Stable ABI (`abi3`) is not offered by free-threaded builds in 3.13 and 3.14. +That does not block this project: the ~330 bindings are symbols resolved dynamically at run time, and `Py_LIMITED_API` is not defined anywhere in the build, so no +compile-time ABI contract is in play (SPEC T-2). What matters is that the symbols exist in the +free-threaded `libpython3.14t`, and the suite passing on it shows they do. The cost of the +free-threaded name changes is in the loader: such an install ships `libpython3.14t.dylib`, +`lib/python3.14t/` and `bin/python3.14t` only (`Versions.abiFlags` carries the `t`; ROADMAP §9). -### What abi3t requires of us +[PEP 803](https://peps.python.org/pep-0803/) (`abi3t`, targeting 3.15) would make a Stable ABI +available on free-threaded builds for extension *modules*. What it asks of us if we ever adopt it: | Requirement | Our status | |---|---| -| `PyObject` / `PyVarObject` become incomplete types; no field access | **Compatible.** Verified: `PyObject` appears only as `CPointer` and is never dereferenced. The only `.pointed` uses in the tree are commented-out `JNIEnv` code in `JniExport.kt`. | +| `PyObject` / `PyVarObject` become incomplete types; no field access | **Compatible.** `PyObject` appears only as an opaque pointer and is never dereferenced. | | Extensions may not embed `PyObject` in their own structs | Compatible — nothing does. | -| `PyModExport` hook (PEP 793) instead of static `PyModuleDef` | Not yet relevant. We embed CPython rather than building an extension module. **It becomes relevant for the upcall work**, where Kotlin classes are exposed to Python. | -| No backwards compatibility with 3.14 or earlier | Was accepted at the time; no longer applies, 3.14t is used on desktop (ROADMAP §9). | +| `PyModExport` hook (PEP 793) instead of static `PyModuleDef` | Not relevant: we embed CPython rather than building an extension module. It would matter if Kotlin classes were ever exposed as a CPython extension module. | -## PEP 809 may collapse the two flavours into one +## PEP 809 may collapse the two flavours into one (forward-looking) [PEP 809](https://peps.python.org/pep-0809/) — **Draft**, also targeting 3.15 — would replace `abi3` with time-bound versioned ABIs (`abi2026`), each frozen for at least ten years with at least five @@ -95,44 +90,38 @@ If PEP 809 lands, the plan simplifies: Design for two artefacts, but keep the split shallow enough to collapse later. -## Prebuilt availability, as of the 20260807 python-build-standalone release - -| Platform | GIL | Free-threaded | -|---|---|---| -| macOS arm64 / x86_64 | ✅ | ✅ (incl. `3.15.0rc1`) | -| Linux x86_64 | ✅ | ✅ (incl. `3.15.0rc1`) | -| Windows x86_64 | ✅ | ✅ (incl. `3.15.0rc1`) | -| Android (python.org) | ✅ | ❌ none published | -| iOS (Python-Apple-support) | ✅ | ❌ none published | - -Desktop free-threaded builds exist today, including 3.15 release candidates. **Mobile has none**, so -enabling free-threading before mobile artefacts appear would split the threading model across -platforms — and with it the object-lifetime and thread-state design layered on top. - -Switching the default (originally) required all three of the following; ROADMAP §9 shows 1 is not needed for 3.14t on desktop, while 2 still holds: - -1. CPython 3.15 final, for `abi3t` -2. Free-threaded mobile artefacts from python.org and Python-Apple-support -3. Adequate free-threaded wheel coverage for the C extensions users care about - -`gradle.properties` already carries `pythonFreeThreaded`, defaulting to `false`, so the switch is a -configuration change once those hold. - -## What has to be built either way - -The binding layer currently performs **no thread-state management at all**. There are no -`PyGILState_Ensure` / `PyGILState_Release` bindings, no calls to them, and `Python3.withPython` only -checks an initialisation flag. Calling into Python from any thread other than the initialising one is -therefore broken today, independently of GIL versus free-threading. - -Free-threading does not remove this requirement: `PyGILState_Ensure` still exists there and still -attaches a thread state. What disappears is contention on a global lock, not the need for a thread -state. The work is nearly identical for both flavours, so building it now also prepares the -free-threaded path. - -Two further consequences of free-threading, not yet addressed: - -- Kotlin-side caches become concurrently accessed. `PyType.getInstance` currently memoises into a - plain `mutableMapOf`, which is a data race the moment two Kotlin threads run Python in parallel. -- `PyGILState_Ensure` per call is expensive. Thread state should be cached per thread and acquired at - entry and exit boundaries, not around every FFI crossing. +## Prebuilt availability + +Desktop (macOS arm64/x86_64, Linux x86_64, Windows x86_64) has GIL and free-threaded builds on the +python-build-standalone releases; the build uses the `-freethreaded` archive when +`pythonFreeThreaded=true`. **Android (python.org) and iOS (Python-Apple-support) publish no +free-threaded build**, so free-threading stays a desktop-only opt-in, and enabling it by default +would split the threading model — and the object-lifetime and thread-state design layered on top — +across platforms. Switching the default would additionally need adequate free-threaded wheel +coverage for the C extensions users care about. Table as of the 20260807 release, kept in the +archive note above. + +## Thread-state management + +Every C API call holds an attached thread state (AGENTS.md §16, `commonMain/README.md`): +`withGIL { }` (`python/multiplatform/ffi/GILScope.kt`) calls `PyGILState_Ensure`/`Release` at the +outermost scope with a per-thread nesting counter, and `withoutGIL { }` wraps +`PyEval_SaveThread`/`RestoreThread` for long non-Python work. `Python3.withPython` goes through +`withGIL`. This is needed on **both** flavours: free-threading removes contention on a global lock, +not the requirement that a thread be attached before it touches any object. + +Entry points reached from C (upcalls) take their own unconditional `PyGILState_Ensure` instead of +trusting the nesting counter (`UpcallTrampoline.attached`; reason in `commonMain/README.md`). + +Consequences specific to free-threading, all recorded in ROADMAP §9 and +`docs/design/object-lifetime.md`: + +- Deallocation of an object decref'd by a non-owning thread is deferred to the owner's eval-loop + checkpoint; a pure C-API embedder never reaches one, so the library takes a periodic checkpoint + itself (`maybeReachEvalCheckpoint`, `Python3.drainPendingReleases`; `Python3.autoDrainInterval` defaults on for free-threaded builds and off on the GIL build). +- Kotlin-side caches become concurrently accessed. `PyType.getInstance` still memoises into a plain + `mutableMapOf` (`python/multiplatform/ffi/PyType.kt`), which relies on the caller holding the GIL; + under free-threading that is a data race once two Kotlin threads create types in parallel. The + desktop suite passing on 3.14t does not exercise that case. +- `PyGILState_Ensure` per call is expensive; the nesting counter keeps inner scopes cheap, and on a + free-threaded build widening the outermost scope costs other threads nothing. diff --git a/docs/design/typedpython.md b/docs/design/typedpython.md index fb126c89e..5ccb4044d 100644 --- a/docs/design/typedpython.md +++ b/docs/design/typedpython.md @@ -1,5 +1,11 @@ # TypedPython — 설계 제안 +> **참고 (2026-10-03):** §4.4 가 전제하는 스텁 매니페스트(`pythonx-map.toml`, 그리고 그것을 읽어 +> `pythonx.*` 스텁을 내던 `StubManifest`/`renderPythonicStubs`)는 이 저장소의 스텁 생성기에서 +> 제거되었고(`238119b7`), **pythonx-compose 저장소로 옮겨 갔다.** 이 저장소의 `PythonStubsTask` 는 +> 이제 Kotlin 모듈 경로·Kotlin 이름의 스텁만 낸다(SPEC B-7, +> [`pyi-generation-design.md`](pyi-generation-design.md)). 아래 제안 본문은 수정하지 않았다. + > **이 문서는 설계 제안이지 사양이 아니다.** 사양은 `docs/SPEC.md`, 의도는 `docs/INTENT.md` 에 있고, > 이 문서의 내용을 그 둘에 반영할지는 사용자가 정한다. 아래의 이름·표기·단계 구분은 전부 논의용이며 > 구현하면서 달라질 수 있다. diff --git a/docs/design/upcall-async-design.md b/docs/design/upcall-async-design.md deleted file mode 100644 index 089441f20..000000000 --- a/docs/design/upcall-async-design.md +++ /dev/null @@ -1,1218 +0,0 @@ -# 비동기 업콜 설계 — Python 이 Kotlin 의 `suspend` 를 부를 수 있는가 - -`docs/design/upcall-design.md` 가 만든 업콜 경로는 다섯 플랫폼에 다 붙었지만, **노출할 수 있는 것은 -일반 함수뿐이다.** 이 문서는 왜 그런지, 무엇을 대신 할 수 있는지, 후보 중 무엇을 골랐는지, -그리고 지금 무엇까지 구현되어 있는지를 정리한다. - -이 문서에서 **측정** 이라고 적힌 것은 이 저장소에서 실제로 돌려 본 것이다. **추론** 이라고 적힌 -것은 코드와 문서를 읽어 얻은 것이고, 기기가 없어 확인하지 못했다. 둘을 섞지 않는다. - ---- - -## 1. 막는 것은 구현이 아니라 C 의 호출 규약이다 - - CPython 이 C 함수 포인터를 부른다 ml_meth: PyObject *(PyObject *self, PyObject *args) - → 그 C 스택 프레임은 PyObject * 를 동기적으로 반환해야 한다 - → 중간에 suspend 하면 반환할 것이 없다 - -`runBlocking` 은 해결이 아니다. 그 스레드를 다시 붙잡는 것이므로 얻는 것이 없고, **Android 에서는 -특히 나쁘다** — 업콜이 도착한 스레드는 CPython 이 만든 pthread 이고(`docs/design/upcall-design.md` -"Android's boundary runs the other way round"), 그것을 막으면 답을 기다리는 인터프리터 쪽이 함께 -멈춘다. 즉 자기가 기다리는 대상을 자기가 막는 형태가 된다. - -그러므로 남는 선택지는 하나뿐이다: **다른 것을 반환하고, 값은 나중에 전달한다.** - ---- - -## 2. KSP 가 `suspend` 를 어떻게 다뤘는가 — 측정 (§8 에서 바뀐다) - -> **이 절은 과거형이다.** §8 이 이 동작을 바꿨다. 여기 남겨 두는 이유는 무엇이 바뀌었는지를 -> 대조할 기준이 필요하기 때문이다. `GeneratedSuspendTest` 는 더 이상 이 표를 고정하지 않는다. - -`BindingPolicy.isExposedFunctionShape` 한 줄이 전부였다. - - if (Modifier.SUSPEND in function.modifiers) return false - -**조용히 건너뛴다.** 깨지지도, 잘못된 코드를 만들지도 않는다. 로그도 없다. - -`ksp-fixtures/library/.../Suspending.kt` 를 넣고 생성된 조각을 직접 읽어 확인했다 -(`Fragment_io_github_thisisthepy_ksp_fixtures_library.kt`). 각 형태마다 같은 스코프에 비-suspend -대조군을 두었으므로, "스코프 전체가 빠져서 통과"하는 경우와 구분된다. - -| 선언 | 결과 | 대조군 | -|---|---|---| -| 최상위 `suspend fun` | 항목 없음 | `blockingTopLevel` 있음 | -| 클래스 멤버 `suspend fun` | 항목 없음. **클래스는 그대로 노출되고 생성자도 남는다** | `fetchBlocking` 있음 | -| companion 의 `suspend fun` | 항목 없음 | `createBlocking` 있음 | -| interface 의 `suspend fun` | 항목 없음 | `describe` 있음 | -| `object` 의 `suspend fun` | 항목 없음 | `pingBlocking` 있음 | -| `ReflectedClass.memberNames` | 빠진 이름은 여기에도 없다 | — | - -사용자 입장에서 보이는 것은 Python 쪽 `AttributeError` 하나뿐이고, **그 함수가 보였지만 거부되었다는 -신호는 어디에도 없다.** 이것이 이 동작의 실제 비용이다. - -`ksp-fixtures/app/src/desktopTest/.../GeneratedSuspendTest.kt` 가 이 표를 고정한다. 비동기 규약이 -생기면 이 테스트들이 깨지도록 되어 있다 — 침묵이 소리 없이 다른 침묵으로 바뀌지 않게 하는 것이 -목적이다. - -### 2.1 그런데 `suspend` **타입** 은 걸러지지 않는다 — 측정 - -정책이 읽는 것은 *선언* 의 modifier 다. `val h: suspend (Long) -> Long` 에는 그 modifier 가 없다. -그래서 다음 셋은 전부 노출된다: - -| 선언 | 생성된 것 | -|---|---| -| `val suspendingHandler: suspend (Long) -> Long` | `STATIC_GETTER`, `TypeTag.OBJECT` | -| `fun makeHandler(): suspend (Long) -> Long` | `FUNCTION`, 반환 `TypeTag.OBJECT` | -| `fun runsHandler(h: suspend (Long) -> Long)` | 파라미터 `TypeTag.OBJECT`, 캐스트는 아래 | - -파라미터 캐스트로 생성된 것은 소스에 선언이 존재하지 않는 분류자다: - - args[0] as kotlin.coroutines.SuspendFunction1 - -**그런데 이것은 컴파일된다** — `:ksp-fixtures:library:compileKotlinDesktop` 과 -`compileKotlinAndroidNativeArm64` 둘 다 `e:` 없이 통과한다. 컴파일만이 아니라 런타임 캐스트도 -통과한다(`GeneratedSuspendTest.theGeneratedCastToASuspendingFunctionTypeHoldsAtRuntime`). - -다만 **이것은 우회로가 아니다.** 건너간 것은 불투명한 핸들이고, 그 핸들을 부를 항목은 테이블 -어디에도 없다. Python 은 그것을 들고 있다가 Kotlin 에 돌려주는 것 말고 할 수 있는 일이 없다. - ---- - -## 3. 후보 셋 - -Kotlin 쪽에서 셋이 공유하는 절반은 같다: **동기 프레임 안에서 코루틴을 시작하고, 결과를 나중에 -주소를 매길 수 있는 곳에 세워 둔다.** 다른 것은 그 결과가 Python 에 도달하는 방법뿐이다. - -### (A) 핸들 + 폴링 - -업콜이 즉시 작업 핸들을 반환하고, Python 이 `is_done(h)` / `result(h)` 로 묻는다. - -**무엇이 막는가** - -1. **wasm 에서는 반드시 데드락한다 — 추론.** wasmJs 에는 스레드가 없다. Kotlin 코루틴의 재개는 JS - 마이크로태스크 큐를 통해서만 일어나고, CPython 인터프리터 자체가 같은 JS 스레드 위에서 돈다. - `while not is_done(h): pass` 는 JS 이벤트 루프에 제어를 돌려주지 않으므로, 재개가 큐에 들어간 - 채 영원히 실행되지 않는다. 이것은 폴링이 느리다는 문제가 아니라 **폴링이 진행을 막는다** 는 - 문제다. 기기 없이 확인할 수 없어 추론으로 표시한다. -2. **KSP 가 `Deferred` 를 노출할 수 없다 — 측정.** `BindingPolicy.hasRenderableSignature` 는 - 타입 파라미터가 있는 선언을 전부 거부한다(`Box` 로 이미 고정되어 있다). 그러므로 - `Deferred.await`, `Job.isCompleted` 같은 것을 그대로 테이블에 올릴 수 없고, 제네릭이 아닌 전용 - 타입을 손으로 써야 한다. 막는 것은 아니지만 설계 비용이다. -3. `kotlinx.coroutines` 는 `python-multiplatform` 의 의존성이 아니다 — 측정 - (`gradle/libs.versions.toml` 에 좌표는 있으나 `sample` 의 swing 아티팩트에서만 쓰인다). -4. JVM·Native 에서는 동작하지만 코어 하나를 태운다. GIL 을 쥔 채 도는 폴링 루프는 스위치 간격마다 - 놓았다 다시 잡으므로, 다른 Python 스레드까지 함께 느려진다. - -### (B) 콜백 (다운콜) - -Python 이 콜러블을 넘기고, 완료 시 Kotlin 이 그것을 부른다. - -**무엇이 막는가 — 그리고 막지 않는 것** - -- **완료 스레드가 GIL 을 쥘 수 있는가: 쥘 수 있다 — 측정.** - `AsyncCompletionProbeTest.aKotlinCreatedThreadCanResolveAnAsyncioFutureTheInterpreterIsWaitingOn` - 이 이것을 직접 보인다. **Kotlin 이 만든** 스레드(CPython 이 만든 것이 아니므로 스레드 상태를 - 받은 적이 없다)가 `withGIL` 로 GIL 을 얻고 Python 을 호출한다. 그것도 인터프리터의 메인 - 스레드가 `loop.run_until_complete` 안에 들어가 있는 동안에 — 테스트는 그 스레드가 - `loop.is_running()` 이 `True` 인 것을 **직접 관측한 뒤에만** 통과하도록 되어 있다. 처음 판은 - 이 조건 없이 4 ms 만에 통과했고, 그것은 경합 없는 경로를 통과한 것이라 아무 의미가 없었다. -- **Android 에서 ART attach 가 필요한가: 필요 없다 — 추론.** `409da6fc` 가 다룬 것은 **업콜** - 방향이다. 거기서 스레드는 CPython 이 만든 맨 pthread 라 ART 가 본 적이 없었다. 여기서는 완료 - 스레드가 Kotlin/JVM 이 만든 스레드이므로 ART 가 이미 알고 있고, 필요한 것은 - `PyGILState_Ensure` 뿐이다. 즉 방향이 반대라 그 문제가 재발하지 않는다. 에뮬레이터를 쓰지 - 않았으므로 추론이다. -- **진짜 어려운 점은 GIL 이 아니라 재진입이다.** 콜백은 완료 스레드 위에서 *즉시* 실행되므로, - Python 사용자 코드가 메인 스레드가 하던 일과 동시에 임의의 지점에서 돈다. 이것은 Python - `threading.Thread` 가 이미 하는 일과 같으므로 새로운 위험은 아니지만, **동기화 책임이 전부 - 사용자에게 간다.** 그리고 코루틴이 동기적으로 완료되면(§5 의 빠른 경로) 콜백은 업콜 프레임 - *안에서* 실행된다 — GIL 을 이미 쥔 채로, C 가 위에 있는 상태에서. `PyGILState_Ensure` 는 - 재진입 가능하므로 안전하지만, 규약이 그것을 명시해야 한다. -- Kotlin 이 Python 콜러블의 참조를 오래 들고 있어야 한다. 그 참조를 언제 놓는가는 이미 풀린 - 문제다 — `PyObject` 의 지연 해제 큐와 eval 체크포인트(`GILScope.kt`, - `Python3.drainPendingReleases`)가 그 일을 한다. 새 메커니즘이 아니다. - -### (C) `asyncio` 연동 - -Kotlin 이 `Future` 를 만들어 Python 이벤트 루프에 넘기고, 완료 시 -`loop.call_soon_threadsafe(fut.set_result, v)` 로 해소한다. - -**과제가 지목한 질문 — `call_soon_threadsafe` 가 이 ABI 부분집합에 있는가: 있다 — 측정.** - -특별한 진입점이 필요하지 않다. 필요한 것은 전부 이미 `EmbedAPI.kt` 에 있는 일반 호출뿐이다: - -| 필요한 것 | `EmbedAPI` | -|---|---| -| `import asyncio` | `PyImport_ImportModule` ✔ | -| `loop.call_soon_threadsafe`, `fut.set_result`, ... | `PyObject_GetAttrString` ✔ | -| 그것들을 호출 | `PyObject_Call` / `PyObject_CallObject` / `PyObject_CallNoArgs` ✔ | - -`AsyncCompletionProbeTest.everyApiThisNeedsIsAlreadyBoundSoAnAsyncioConventionAddsNoNewBinding` -이 이 경로를 그대로 밟아 `new_event_loop` / `create_future` / `call_soon_threadsafe` / -`run_until_complete` / `set_result` / `set_exception` / `done` / `result` / `cancel` 이 전부 도달 -가능함을 확인한다. **(C) 는 새 바인딩을 하나도 추가하지 않는다.** - -참고로 다른 경로였을 `Py_AddPendingCall` 은 Stable ABI 에는 있지만(`test_stable_abi_ctypes.py` 에 -등재) **`EmbedAPI` 에는 바인딩되어 있지 않다** — 측정. 그리고 그것은 C 함수 포인터를 받으므로 -`(long) -> int` 트램폴린 shape 이 하나 더 필요하다. 이미 `Panama` 의 어휘 안에 있는 shape 이지만, -(C) 가 그것 없이 되므로 필요하지 않다. - -**무엇이 막는가** - -- 루프가 **돌고 있어야** 한다. Python 메인 스레드가 `asyncio.run()` 안에 있지 않으면 - `call_soon_threadsafe` 는 큐에 넣기만 하고 아무도 꺼내지 않는다. 닫힌 루프면 예외를 던진다. - 즉 (C) 는 **애플리케이션이 async 로 짜여 있을 것을 요구한다.** -- wasm: 위 요구가 바로 그 플랫폼의 조건이다(§4). - ---- - -## 4. wasm 이 실제로 결정한다 - -세 후보 모두 JVM·Native 에서는 동작한다. 갈리는 것은 wasm 이고, 그 이유는 하나다: **wasmJs 에는 -스레드가 없고, Python 과 Kotlin 이 같은 JS 스레드를 공유한다.** - -그래서 wasm 에서는 셋 다 같은 전제를 요구한다 — **Python 이 호스트 이벤트 루프에 제어를 돌려줘야 -한다.** 그 전제 위에서: - -| 후보 | wasm | -|---|---| -| (A) 폴링 | **성립하지 않는다.** 폴링 루프가 제어를 돌려주지 않으므로 전제를 스스로 깬다 | -| (B) 콜백 | 성립한다. Python 이 돌아간 뒤 마이크로태스크에서 콜백이 실행된다 | -| (C) asyncio | 성립한다. 단 루프가 JS 에 양보해야 하므로 재래식 selector 루프로는 부족하다 | - -이것은 전부 추론이다. 이 워크스페이스에서 wasm 은 컴파일 검증만 하고 있고 -(`docs/platforms/wasm-design.md` §"Still open"), 실행 경로를 만들지 않았다. - ---- - -## 5. 고른 것: (C), 그리고 (B) 는 그 구현 수단이다 - -**(C) 를 목표 규약으로 하고, (B) 를 그 내부 메커니즘으로 쓴다. (A) 는 버린다.** - -근거는 셋이다. - -1. **(A) 는 wasm 에서 성립하지 않는다.** 나머지 넷에서 동작하는 규약을 만들고 wasm 만 예외로 - 두는 것은, 이 저장소가 다섯 타깃을 하나의 규약으로 유지해 온 방식과 맞지 않는다. -2. **(C) 는 새 바인딩을 요구하지 않는다 — 측정.** 이것은 예상이 아니라 확인이다. 반면 (A) 는 - 제네릭이 아닌 전용 타입과 그 멤버들을 테이블에 새로 올려야 한다. -3. **(C) 는 기계적으로 (B) 다.** `call_soon_threadsafe` 를 부르는 것은 완료 스레드가 Python - 콜러블을 부르는 다운콜이고, 그것이 정확히 (B) 다. 즉 둘 중 하나를 고르는 문제가 아니라, - (B) 를 만들면 (C) 가 그 위에 얹히는 관계다. 사용자에게 노출하는 표면만 (C) 로 정한다. - `await kotlin_fn(x)` 는 Python 사용자가 이미 아는 문법이고, 콜백은 아니다. - -**빠른 경로는 규약보다 앞에 온다.** `suspend` 는 시그니처이지 suspend 하겠다는 약속이 아니다. -실제 suspension 지점에 닿지 않는 `suspend fun` 은 **`start` 가 반환하기 전에 이미 완료된다** -(`PendingCallTest.aBlockThatNeverSuspendsIsAlreadyCompleteBeforeStartReturns`, 측정). 그런 호출은 -`Future` 도 이벤트 루프도 없이 실제 값을 그대로 반환하면 된다. 규약은 그 경로를 먼저 태우고, -정말로 suspend 한 경우에만 `Future` 를 만든다. - -### 뒤집어지면 이 결론도 뒤집힌다 - -- wasm 실행 경로가 열렸는데 asyncio 루프를 JS 에 양보시킬 방법이 없다면, (C) 는 wasm 에서 - (B) 로 내려앉는다 — 그때는 (B) 를 공개 표면으로 올리는 편이 낫다. -- Android 에서 완료 스레드가 실제로 ART attach 를 요구하는 것으로 밝혀지면, §3(B) 의 "방향이 - 반대라 재발하지 않는다"가 무너지고 `409da6fc` 의 pthread 키 destructor 를 다운콜 쪽에도 - 복제해야 한다. - ---- - -## 6. 구현된 것과 구현하지 않은 것 - -### 구현: `PendingCall` — 셋이 공유하는 절반 - -`python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/PendingCall.kt` - -동기 프레임 안에서 코루틴을 시작하고, 결과를 세워 두고, 하나의 리스너에게 전달한다. -`PendingCallTest`(commonTest, 9개) 가 고정한다. - -- **`kotlinx.coroutines` 의존성이 없다.** `startCoroutine` 과 `Continuation` 은 `kotlin-stdlib` - common 이다. 노출되는 `suspend fun` 은 *사용자의* 것이고, 그것이 쓰는 디스패처는 이미 사용자 - 클래스패스에 있다. 시작하는 데는 stdlib 만 있으면 된다. -- **`EmptyCoroutineContext`.** 디스패처가 없으므로 본문은 실제 suspension 까지 호출 스레드에서 - 돈다. §5 의 빠른 경로가 여기서 나온다. 대가도 여기서 나온다 — 첫 suspension 이전의 작업은 - **C 프레임 안에서, GIL 을 쥔 채** 돈다. -- **아무것도 밖으로 던지지 않는다.** 본문의 예외도 리스너의 예외도 필드에 세워 둔다. 위에 C 가 - 있는 프레임이므로 `UpcallTrampoline` 과 같은 규칙이다. -- **새 핸들 기계가 없다.** `PendingCall` 은 평범한 Kotlin 객체이므로 `HandleTable` 이 이미 - 세대 태그가 붙은 핸들을 주고, `UpcallTrampoline.releaseObject` 가 이미 Python 쪽 해제 수단을 - 준다. -- **동기화하지 않는다.** 경계의 나머지와 같은 규약을 지키는 조건에서다: 모든 변경은 GIL 을 쥔 - 스레드에서 일어난다. 이것은 추가 부담이 아니다 — Python 에 도달하려는 완료는 어차피 GIL 을 - 잡아야 하므로, 그 스코프가 이미 이 필드들을 덮는다. - -### 전달 절반 — §8 에서 구현됐다 - -이 절이 "아무것도 붙어 있지 않다"고 적었던 `onCompleted` 자리에 이제 (C) 가 붙어 있다. §8 을 보라. - ---- - -## 7. 다음에 확인해야 할 것 (순서대로) - -1. **Android 에뮬레이터에서 (B) 의 완료 스레드.** Kotlin 스레드가 `PyGILState_Ensure` 로 들어가 - Python 콜러블을 부를 때 ART attach 가 필요한가. §3(B) 의 추론이 맞는지가 이것 하나에 달려 - 있다. `UpcallThreadAttachTest` 옆에 두면 된다. **§8 이 이것을 바꾸지 못했다** — 완료 경로는 - 전부 `commonMain` 이라 다섯 타깃이 같은 소스를 컴파일하지만, 실행해 본 것은 desktop 하나다. -2. **동기 완료 비율.** §5 의 빠른 경로가 실제로 얼마나 자주 타는가. `PendingCall.isDone` 을 - `start` 직후에 세는 것으로 측정 가능하고, 이 값이 높으면 규약의 복잡도 대부분을 드문 경로로 - 미룰 수 있다. §8.4 가 이 질문을 하나 더 늘렸다 — 빠른 경로는 "suspend 하지 않은 호출"보다 - **넓다.** -3. ~~**wasm 실행 경로.** §4 가 전부 추론이다.~~ **§9.5 가 측정했다** — wasmJs 에는 실행 경로가 - 있었고(`wasmJsNodeTest`, 273개), (C) 는 `import asyncio` 가 trap 하는 지점에서 끝난다. - 남은 추론은 "루프가 도는 상태에서의 교착" 하나뿐이다. -4. **`suspend` 가 걸러졌음을 사용자에게 알리는 것.** §2 의 침묵은 §8 이 덮지 못하는 형태 - (확장 수신자를 가진 `suspend fun`, 제네릭 `suspend fun`) 에 **그대로 남아 있다.** KSP 경고가 - 맞는 자리다. -5. ~~**취소.** §8 의 범위에서 빠졌다.~~ **§9.1–9.3 이 답했고, 이른 통지는 §10 이 만들었다.** - 남은 것은 §10.5 의 두 가지 — 다섯 타깃 중 desktop 외의 실행 확인, 그리고 루프가 죽어 done 콜백이 - 돌지 못한 경우의 회수. - ---- - -## 8. 전달 절반: 구현된 것 — 측정 - -§5 가 고른 것을 그대로 만들었다. 표면은 (C), 메커니즘은 (B), 빠른 경로가 규약보다 앞에 온다. - - result = await kotlin_async_fn(x) - -`AsyncUpcallDeliveryTest`(desktopTest, 7개) 가 이것이 실제로 도는 것을 고정한다. - -### 8.1 KSP: 버리는 대신 다른 본문을 만든다 - -`BindingPolicy` 에서 `Modifier.SUSPEND` 거부 한 줄을 지웠다. 그 자리에 `isSuspending` 이 있고, -`FragmentScanner.functionBody` 가 그것을 읽어 본문 모양만 바꾼다: - - // 기존 - { args -> fixture.library.blockingTopLevel(args[0] as Long) } - // suspend - { args -> python.multiplatform.ffi.upcall.PendingCall.start { fixture.library.suspendingTopLevel(args[0] as Long) } } - -`PendingCall.start` 가 받는 것은 `suspend () -> Any?` 이므로 저 중괄호가 suspend 람다이고, -**`kotlinx.coroutines` 없이 stdlib 만으로 컴파일된다.** 생성된 조각을 직접 읽어 확인했다 -(`Fragment_io_github_thisisthepy_ksp_fixtures_library.kt`, 5개 항목). - -바뀌지 않은 것이 바뀐 것만큼 중요하다. `kind` 는 그대로다 — suspending 멤버는 여전히 `METHOD` 이고 -`args[0]` 이 수신자다. `returnType` 도 그대로 **선언된 반환 타입**이다(`PendingCall` 이 아니라). -그래야 빠른 경로가 동기 항목과 똑같이 마샬링한다. - -### 8.2 `CallableKind` 가 아니라 플래그인 이유 - -`ExposedCallable.isSuspend: Boolean` 을 새로 뒀다. `CallableKind` 에 `SUSPEND_*` 변종을 추가하는 -쪽은 택하지 않았다. 두 축이 직교하기 때문이다 — `kind` 가 답하는 것은 *인자가 어디 있는가*, -`isSuspend` 가 답하는 것은 *무엇이 돌아오는가* 이고, 하나로 접으면 모든 `hasReceiver` 판단을 -변종마다 한 번씩 다시 써야 한다. suspending 멤버 함수 하나만으로도 그 값이 드러난다. - -### 8.3 런타임 경로 - -`AsyncUpcall.deliver` 가 `UpcallTrampoline.invoke` 의 `entry.isSuspend` 분기에 붙어 있다. - -| 경우 | 반환 | 비용 | -|---|---|---| -| 이미 완료 · 성공 | 실값 (`marshalResult`) | `asyncio` import 조차 없다 | -| 이미 완료 · 실패 | `NULL` + 에러 지시자 | 동기 실패와 완전히 같은 경로 | -| 진짜 suspend | `asyncio.Future` (새 참조) | `get_running_loop` + `create_future` | - -완료 시: - - withGIL { loop.call_soon_threadsafe(future.set_result | set_exception, ...) } - -- **완료 스레드가 자기 GIL 스코프를 취한다.** `withGIL` 이고 `UpcallTrampoline.attached` 가 아니다 — - 여기는 C 진입점이 아니므로 그 스레드의 중첩 깊이는 자기 자신의 정직한 기록이다. CPython 이 본 적 - 없는 완료 스레드는 깊이 0 이라 진짜 `PyGILState_Ensure` 를 타고, 나중 업콜 안에서 오는 완료는 - 그 업콜의 `attached` 가 세운 깊이 아래에서 올바르게 건너뛴다. -- **`Future` 참조는 둘이 나눠 가진다.** Kotlin 래퍼가 `create_future` 가 준 것을 갖고, Python 은 - 증가분 하나를 받는다(트램폴린의 "결과는 새 참조" 규약 그대로). 래퍼가 살아 있는 이유는 완료 - 람다가 그것을 캡처하고, 람다는 `PendingCall` 이, `PendingCall` 은 중단된 continuation 을 쥔 - 쪽이 잡고 있기 때문이다. 완료가 발사되면 `PendingCall` 이 리스너를 놓고 둘 다 회수된다. -- **새 바인딩이 하나도 없다.** §3(C) 가 예고한 대로다. - -### 8.4 빠른 경로는 "suspend 하지 않은 호출"보다 넓다 — 측정 - -테스트를 처음 쓸 때 관측한 것이고, 예상하지 못했다. 완료 스레드가 업콜 프레임이 아직 돌고 있는 -동안 continuation 을 재개하면 — `CFUNCTYPE` 이 업콜 자체를 위해 GIL 을 놓는 순간이 그 틈이다 — -`deliver` 가 `isDone` 을 볼 때 이미 참이다. **그러면 `Future` 는 만들어지지 않고 빠른 경로가 -탄다.** 틀린 동작이 아니다(코루틴은 정말로 끝났다). 다만 §7.2 의 "동기 완료 비율" 은 정적인 -성질이 아니라 **경합의 함수**라는 뜻이고, 그 경로를 일부러 밟으려는 테스트는 `Future` 가 -건너간 것을 직접 관측해야 한다. `AsyncUpcallDeliveryTest` 는 `type(r).__name__` 을 Python 쪽에서 -기록해 그것을 단언한다 — 값만 보면 두 경로가 구별되지 않는다. - -### 8.5 플랫폼 - -| 타깃 | 컴파일 | 실행 | -|---|---|---| -| desktop | ✔ | ✔ — `await` · 예외 · 빠른 경로 전부 관측 | -| androidNativeArm64 | ✔ | 미확인 (기기 없음) — §11.5 도 그대로 | -| iosSimulatorArm64 | ✔ | ~~미확인~~ **✔ — §11 이 채웠다.** 전달·이른 취소 통지·조용한 드롭 셋 다 통과 | -| wasmJs | ✔ | **성립하지 않을 것이다 — 추론.** 아래 (§9.5 가 뒤집었다: 교착이 아니라 `import asyncio` 자체가 트랩) | -| android (JVM) | **미확인** — 이 워크스페이스에 Android SDK 가 없어 태스크가 구성조차 안 된다 | - -전달 코드는 전부 `commonMain` 이고 `expect`/`actual` 이 하나도 없다. 즉 컴파일된 다섯 소스가 -같은 소스다. 그것이 보장하는 것은 타입이 맞는다는 것뿐이고, GIL 취득 동작이 각 타깃에서 같다는 -보장은 아니다. - -**wasm 은 §4 가 예고한 그 지점에서 막힌다.** wasmJs 에는 스레드가 없으므로 완료는 JS -마이크로태스크로만 온다. 그런데 Python 이벤트 루프는 같은 스레드에서 `run_until_complete` 안에 -있고, 재래식 selector 루프는 JS 에 제어를 돌려주지 않는다. 그러면 Kotlin 의 재개가 루프 뒤에 -줄을 서고, 루프는 그 재개를 기다린다 — (A) 를 기각한 것과 같은 형태의 교착이다. 이것은 -**추론이다.** 이 워크스페이스에 wasm 실행 경로가 없어(`docs/platforms/wasm-design.md` "Still open") -확인할 수 없었고, 확인하지 않은 것을 확인했다고 적지 않는다. - -### 8.6 이번 범위에서 뺀 것 — **§9 가 앞의 둘을 채웠다** - -- **취소.** ~~`Future.cancel` 이 도달 가능하다는 것만 §3(C) 에서 확인돼 있고~~ **§9.1–9.3 이 답했다.** - 아래 원문은 §9 가 무엇을 바꿨는지 대조하기 위해 남긴다. - - > `Future.cancel` 이 도달 가능하다는 것만 §3(C) 에서 확인돼 있고, 그것을 Kotlin - > 코루틴의 취소로 옮기는 경로는 만들지 않았다. 그러므로 지금 Python 쪽에서 `Future` 를 - > 취소해도 **Kotlin 쪽 작업은 계속 돈다.** 그 뒤 완료가 도착했을 때 무슨 일이 일어나는지는 - > **테스트하지 않았고 여기서 단정하지 않는다.** - -- **Python 쪽 프록시 생성.** ~~Python 모듈 생성 자체가 아직 없으므로 생성되지는 않는다.~~ - **§9.4 가 만들었다.** -- **실행 중인 루프가 없을 때.** 실패한다(`RuntimeError: no running event loop`), 그리고 그때 - **코루틴은 이미 시작된 뒤다** — suspend 할지 여부는 시작해 봐야 알기 때문이다. 그 재개는 - 버려진다. 이것은 (C) 가 "애플리케이션이 async 로 짜여 있을 것"을 요구한다는 §3(C) 의 대가를 - 구체적으로 치르는 지점이고, 숨기지 않았다 - (`aSuspendingEntryThatSuspendsWithNoRunningLoopFailsInsteadOfReturningSomethingUnusable`). -- **`suspend` 타입 누수.** §2.1 은 그대로다. `val h: suspend (Long) -> Long` 은 여전히 `OBJECT` - 핸들로 건너가고, 그것을 부를 항목은 테이블에 없다. - ---- - -## 9. 취소와 프록시 생성 — 측정 - -§8.6 이 미룬 둘을 채웠다. 그리고 그 과정에서 **§4·§8.5 의 wasm 판단이 틀렸다는 것**이 나왔다 — -틀린 방향이 아니라, 예측한 것보다 훨씬 이른 지점에서 훨씬 나쁘게 깨진다(§9.5). - -### 9.1 고치기 전에 무슨 일이 났는가 — 측정 - -`AsyncUpcallCancellationTest` 를 먼저 쓰고, 고치기 전에 돌렸다. 관측된 것: - -| 관측 대상 | 결과 | -|---|---| -| `await` 가 받은 것 | `CancelledError` — 정상 | -| 루프의 `call_exception_handler` | **`InvalidStateError: invalid state` 1건** | -| 완료 스레드의 에러 지시자 | **깨끗하다** | -| 그 다음의 무관한 업콜 | **정상 동작** | - -즉 **오염은 없었다.** 이 저장소가 두 번 당한 "에러 지시자가 남아 무관한 다음 호출을 죽인다" 는 -일어나지 않았고, 그것을 추측이 아니라 두 가지 방법으로 확인했다 — 완료 뒤 `PyErr_Occurred()` 를 -직접 읽었고, 그 다음 파이썬에서 별개의 항목을 호출해 정상 결과를 받았다. - -일어난 것은 하나뿐이다: **루프 콜백 안에서 터진 예외**. `call_soon_threadsafe` 는 예약만 하므로 -`set_result` 는 나중에 루프 스레드에서 실행되고, 취소된 `Future` 에 대해 `InvalidStateError` 를 -던진다. asyncio 는 그것을 `call_exception_handler` 로 보낸다 — 기본 핸들러면 stderr 로그다. -**애플리케이션이 잘못한 것이 없고 대응할 수도 없는 실패 보고**이며, 값은 일부러 버린 값이다. - -### 9.2 안전하게 만든 방법 — 그리고 왜 검사 하나로는 부족한가 - -둘을 넣었고, **보장인 것은 둘째뿐이다.** - -1. 예약 전에 `Future.done()` 을 본다. 싸고 흔한 경우를 걷어낸다. **보장이 아니다** — 이 검사는 - 완료 스레드에서 일어나고 콜백은 나중에 루프 스레드에서 실행되므로, 그 사이에 취소가 끼면 - 검사를 통과한 뒤에 실패한다. -2. 예약하는 것을 `set_result` 가 아니라 `_pm_settle` 로 바꿨다. 이 함수가 **콜백 안에서 다시** - `done()` 을 본다. 순서와 무관하게 성립하는 것은 이쪽이다. - -둘 다 필요하다는 것을 단언이 아니라 실험으로 확인했다. `_pm_settle` 의 `done()` 가드만 지우고 -돌리면 **경합 테스트 하나만 빨개진다**(`InvalidStateError: invalid state`) — 나머지 취소 테스트 -둘은 1번 검사가 잡으므로 초록으로 남는다. 즉 1번은 2번을 대체하지 못한다. - -그 경합 테스트는 순서를 우연에 맡기지 않는다. 파이썬 코루틴이 `await` 하지 않고 스핀하는 동안 -Kotlin 이 완료·예약하게 만든다 — 루프의 ready 큐는 코루틴이 루프를 쥐고 있어 비워지지 않고, -CPython 은 스위치 간격마다 GIL 을 놓으므로 완료 스레드는 진행한다. **예약이 이미 큐에 들어간 뒤에** -취소가 일어나므로, Kotlin 쪽 검사는 이미 "안 끝났다" 라고 답한 상태다. - -### 9.3 Kotlin 쪽 취소 — 되는 것과 안 되는 것 - -**`PendingCall` 은 코루틴을 강제로 취소할 수 없다. 구현이 빠진 것이 아니라 구조적으로 불가능하다.** - -`start` 가 `startCoroutine` 에 넘기는 `Continuation` 은 코루틴의 **완료(completion)** 이고, 본문 -전체가 끝났을 때 딱 한 번 재개된다. 본문이 지금 멈춰 있는 **중단 지점의 continuation 은 완전히 -다른 객체**이고, 그것을 쥔 쪽은 중단을 만든 사람 — 사용자의 `suspendCoroutine`, 그들의 디스패처, -또는 이 라이브러리가 의존하지 않는 `kotlinx.coroutines` 다. `PendingCall` 은 그것을 본 적이 없고, -stdlib 에는 그것에 닿는 수단이 없다. continuation 을 두 번 재개하는 것은 정의되지 않은 동작이라 -뒷문도 없다. 강제 취소는 `Job` 트리가 하는 일이고, 그것을 만드는 것이 `kotlinx.coroutines` 다. - -**되는 것: 협조적 취소.** 코루틴의 `CoroutineContext` 를 `EmptyCoroutineContext` 에서 -`PendingCall` 자신으로 바꿨다(`PendingCall` 이 `CoroutineContext.Element` 다). 그러면 본문이 -`coroutineContext[PendingCall]` 로 자기 호출에 닿고, `ensureActive()` 가 그 한 줄 형태다. - - suspend fun slowSum(n: Long): Long { - var total = 0L - for (i in 0 until n) { ensureActive(); total += step(i) } - return total - } - -이것은 `kotlinx.coroutines` 와 **같은 계약**이다. 거기서도 취소는 협조적이고, 검사하지 않는 본문은 -거기서도 취소되지 않는다. 다른 점은 거기서는 라이브러리의 suspend 함수들이 대신 검사해 준다는 -것뿐이고, 여기에는 그런 함수가 없다. - -디스패처를 추가한 것이 아니라는 점이 중요하다. `ContinuationInterceptor` 가 아닌 element 는 본문이 -어디서 도는지 바꾸지 못하므로, **§5 의 빠른 경로는 그대로다.** - -~~**되지 않는 것: 이른 통지.**~~ **§10 이 만들었다.** 아래 원문은 무엇이 바뀌었는지 대조하기 위해 남긴다. - -> **되지 않는 것: 이른 통지.** `cancel()` 을 파이썬이 취소하는 그 순간에 부를 방법이 없다. Kotlin 이 -> Python 의 취소를 알 수 있는 지점은 완료 시점뿐이고, 그때는 이미 늦다. 그러려면 Python→Kotlin 호출이 -> 하나 더 필요하다 — 구체적으로는 (a) `_pm_release` 와 같은 `(long) -> int` 스텁 하나(모양이 이미 -> 있으므로 새 스텁 *형태* 는 아니다), (b) `Future` 에 실어 보낼 `PendingCall` 핸들, (c) 다섯 타깃의 -> 바인딩, (d) 그 핸들의 수명 관리. **이번 범위에서 하지 않았고, 되는 척하지 않는다.** - -협조하지 않는 본문이 끝까지 도는 것은 §10 이후에도 그대로다. 바뀐 것은 **협조하는 본문이 언제 -멈추는가** 뿐이다. - -### 9.4 파이썬 프록시 생성 — 런타임 `exec`, 빌드 타임 리소스가 아니라 - -`PythonProxySource.render` 가 만든다. 두 후보 중 **런타임 렌더 + `exec`** 를 골랐다. - -1. **`.py` 를 놓을 자리가 없다.** Kotlin/Native 에는 `getResourceAsStream` 이 없고, iOS·androidNative· - wasm 의 `sys.path` 는 `src/nativeInterop/cinterop/lib/...` 의 플랫폼별 CPython 트리를 가리킨다. - 생성 파일 하나를 거기에 넣고 첫 import 전에 `sys.path` 에 올리는 것은 이 저장소가 무엇에 대해서도 - 풀지 않은 플랫폼별 패키징 문제다. `Python3.exec` 는 다섯 타깃에서 이미 된다. -2. **KSP 는 무엇이 설치될지 모른다.** fragment 는 Kotlin 모듈 단위이고, 테이블은 런타임에 - `UpcallTable.install` 이 조립한다. 애플리케이션이 부분집합만 설치할 수 있다(이 저장소의 테스트가 - 전부 그렇게 한다). 빌드 타임 파이썬은 존재하지 않을 수도 있는 테이블을 기술하게 되고, 어차피 - 런타임에 klib 들을 가로질러 재집계해야 한다 — 그것이 바로 이것이고, 정보만 더 적다. -3. **`ExposedCallable` 이 이미 다 들고 있다.** `isSuspend`·`arity`·`kind` 가 정확히 입력이고, KSP 가 - 이미 싣고 있다. KSP 에서 파이썬까지 뽑으면 그 지식이 **서로 어긋날 수 있는 두 생성기**로 복제된다. - -빌드 타임 안(案)에서 실제로 가치 있던 부분은 지켰다: `render` 는 **엔트리 → 소스 텍스트의 순수 -함수**라서 인터프리터 없이 `commonTest`(`PythonProxySourceTest`)가 고정하고, 생성물은 동작이 아니라 -읽을 수 있는 산출물이다. CPython 이 필요한 것은 `install` 뿐이고 그것은 `exec` 밖에 하지 않는다. - -생성되는 모양: - - async def _pm_f_0(a0): - _pm_r = _pm_invoke(_pm_h_0, (a0,)) - if hasattr(_pm_r, '__await__'): - return await _pm_r - return _pm_r - -`_await_kotlin` 이 손으로 적어두었던 그 모양이다 — **awaitable 일 때만 await**. 이름은 -`sys.modules` 주입으로 Kotlin 패키지 트리에 실어서 `from demo.calc import doubleLater` 가 되게 했다; -import 훅이 필요 없는 이유는 CPython 이 finder 를 보기 전에 `sys.modules` 를 먼저 보기 때문이다. - -**렌더하지 않는 것:** `CallableKind.FUNCTION` 만 만든다. `METHOD`/`GETTER`/`SETTER` 는 수신자가 -프록시 *인스턴스* 에서 와야 하고, `STATIC_GETTER`/`STATIC_SETTER` 는 호출이 아니라 속성이며, -`CONSTRUCTOR` 는 프록시 인스턴스를 돌려줘야 한다. 셋 다 §7 의 `PyType_FromSpec` 타입의 몫이다. - -빠른 경로가 실제로 공짜인 것도 확인했다. 생성된 프록시는 두 경로를 호출자에게서 감추므로 -`type(r).__name__` 을 볼 수 없다 — 대신 루프의 `create_future` 를 세었고, 이것이 더 강한 진술이다: -빠른 경로에서 `Future` 는 **반환되지 않은 것이 아니라 만들어지지도 않는다**(`created == 0`). - -### 9.5 wasm — §4·§8.5 의 추론을 측정이 뒤집었다 - -> **§15 가 이 절의 진단을 뒤집었다.** 아래의 *관측*(무엇이 죽었는가)은 맞다. *원인*은 틀렸다 — -> `RuntimeError: unreachable` 은 3차 증상이었고, 진짜 원인은 Emscripten 의 **런타임** JSPI 감지와 -> `main` 을 부르지 않는 우리 부트 코드가 어긋난 것이다. `cpython.mjs` 에서 고쳤고, 지금은 -> `import selectors` 도 `import asyncio` 도 통과한다. 아래의 "첫 걸음에서 없다" 는 **틀렸다.** - -§4 와 §8.5 는 wasmJs 에서 (C) 가 **교착**으로 실패할 것이라고 추론했다. 그게 아니다. 훨씬 이르다. - - import json OK import math OK import select OK - import socket OK import contextvars OK - import selectors RuntimeError: unreachable <- trap - import asyncio{,.events,.base_events} trap (selectors 를 거쳐서) - -**`import asyncio` 가 wasm 인스턴스를 trap 시켜 Node 프로세스를 죽인다.** 파이썬 예외가 아니므로 -Kotlin 에서도 Python 에서도 잡을 수 없다. 그러므로 wasmJs 에서 (C) 는 느리거나 교착하는 것이 아니라 -**첫 걸음에서 없다**. §8.5 가 따진 교착은 거기까지 가지 못해서 도달 불가능하다. - -이것이 어떻게 발견됐는지도 적어 둔다: "루프 없이 suspend 하면 명확히 실패하는가" 를 `commonTest` 에 -넣었더니 wasm 스위트 전체가 프로세스째 죽었다. 그래서 그 케이스는 `commonTest` 에 **둘 수 없다**. -`AsyncUpcallPortabilityTest` 에는 asyncio 를 건드리기 전에 반환하는 두 경로만 남겼고 — 빠른 경로와 -suspend 이전 실패 — 그 둘은 다섯 타깃에서 실제로 돈다. wasm 애플리케이션이 의지할 수 있는 것도 -정확히 그 둘이다. - -**뒤집지 않은 것:** 루프가 도는 상태에서 진짜로 suspend 했을 때의 교착은 여전히 추론이다. 그 실험은 -프로세스를 죽이거나 매달리므로 하지 않았고, 하지 않은 것을 했다고 적지 않는다. - ---- - -## 10. 이른 통지 — 측정 - -§9.3 이 "되지 않는다" 고 적은 것을, §9.3 이 지목한 그 모양 그대로 만들었다. 결과부터: -**파이썬이 `Future.cancel()` 하면 Kotlin 코루틴은 완료를 기다리지 않고 다음 `ensureActive()` 에서 -멈춘다.** - -`AsyncUpcallEarlyCancellationTest`(desktopTest, 4개) 가 이것을 고정한다. - -### 10.1 고치기 전에 빨갰다 — 측정 - -먼저 테스트를 쓰고 구현 전에 돌렸다. 실패 메시지가 그대로 옛 동작의 서술이다: - - Kotlin did not learn about the cancellation until the call completed - -- the flag was clear and isDone was false - when Python had already cancelled and yielded to its loop - -**단언의 핵심은 `isCancelled` 가 아니라 그것을 *언제* 읽었는가다.** 옛 동작도 결국 같은 플래그를 -세웠다 — 완료 시점에, `AsyncUpcall.resolve` 의 `done()` 검사가. 그러므로 나중에 읽는 테스트는 -고치기 전에도 초록이다. 이 테스트는 **취소 이후 한 번도 재개되지 않은 호출** 에 대해 -`isCancelled && !isDone` 을 읽는다. 그 조합은 옛 동작에서 성립할 수 없다. - -두 번째 단언은 협조 지점이 실제로 발사되는가다: 통지 이후 **딱 한 번** 재개시키고, 그 틱에서 -`ensureActive()` 가 던져 본문이 끝나는 것(`failure is CancellationException`, `isDone`)을 본다. -고치기 전에는 본문이 다시 park 했다. - -### 10.2 만든 경로 - - Python: fut.cancel() - → Future 의 done 콜백 (call_soon 으로 예약됨) - → _pm_cancel(handle) (long) -> int - → UpcallTrampoline.cancelCall PyGILState_Ensure 무조건 - → HandleTable.resolveRaw → PendingCall.cancel() - → 본문의 다음 ensureActive() 가 던진다 - -§9.3 이 적어 둔 네 가지를 그대로 따랐고, 두 가지가 예상보다 싸게 끝났다. - -| §9.3 이 필요하다고 한 것 | 실제 | -|---|---| -| (a) `(long) -> int` 스텁 | `_pm_release` 와 **같은 모양**이라 새 스텁 형태 없음. 각 타깃에서 이미 있는 빌더를 한 번 더 부른다 | -| (b) `Future` 에 실을 핸들 | `HandleTable` 핸들. **속성으로 붙이지 않고** done 콜백의 기본 인자로 캡처했다 — `asyncio.Future` 가 임의 속성을 받는지에 의존하지 않게 된다 | -| (c) 다섯 타깃의 바인딩 | `_pm_cancel` 하나. 해제는 `_pm_release` 를 **재사용**한다 (핸들이 평범한 객체 핸들이므로) | -| (d) 핸들 수명 | §10.3 | - -바인딩이 붙은 곳: desktop(`UpcallStub.cancelCallStubAddr`), nativeMain(`_pm_cancel` `PyMethodDef`, -iOS·androidNative 공용), android(`jni_onload.def` 의 `pmp_upcall_cancel_meth` + `UpcallCallbacks.cancel`). -wasmJs 는 `_pm_release` 조차 Python 에 게시하지 않고 Kotlin 에서 직접 부르는 구조라 붙이지 않았다 — -§9.5 대로 그 타깃에서는 `import asyncio` 가 애초에 trap 한다. - -**바인딩이 없는 호스트는 깨지지 않는다.** `AsyncUpcall.armCancellationNotice` 가 `_pm_cancel` 과 -`_pm_release` 가 `__main__` 에 있는지 먼저 보고, 없으면 **핸들을 등록하지도 않는다.** 그런 타깃은 -§9 의 동작(완료 시점 관측)을 그대로 유지하고 아무것도 새지 않는다. 콜백 안에서 `NameError` 가 나는 -쪽을 고르지 않은 이유는, 그것이 `call_exception_handler` 로 가서 아무도 보지 않고 아무도 고칠 수 없는 -로그가 되기 때문이다 — §9.1 이 이미 한 번 겪은 실패 형태다. - -### 10.3 핸들 수명 — 놓는 쪽이 둘인 것이 설계다 - -`add_done_callback` 이 **취소·성공·실패 전부에서** 불린다는 점을 그대로 썼다. 그래서 통지와 해제가 -한 콜백이다: - - def _pm_done(_f, _h=_handle): - try: - if _f.cancelled(): - _pm_cancel(_h) - finally: - _pm_release(_h) - -여기에 Kotlin 쪽 해제를 하나 더 뒀다 — `AsyncUpcall.resolve` 의 `finally`. **중복이 아니라 서로 다른 -구멍을 막는다.** - -| 놓는 쪽 | 그것만이 막는 경우 | -|---|---| -| Python done 콜백 | **협조하지 않는 본문이 취소된 경우.** 코루틴이 영원히 안 끝나므로 Kotlin 완료 경로가 돌 기회가 없다 | -| Kotlin `resolve` | **콜백이 돌기 전에 루프가 멈춘 경우.** 예약만 되고 실행되지 않은 done 콜백은 아무것도 놓지 않는다 | - -둘 중 무엇이 먼저 와도 되고, 서로를 알 필요도 없다. `HandleTable` 의 해제가 세대 기반이라 **두 번째는 -no-op** 이고, 슬롯이 이미 남에게 재발급된 뒤라도 세대가 어긋나 남의 것을 놓지 않는다. 같은 이유로 -늦게 도착한 `_pm_cancel(stale)` 도 아무 일도 하지 않는다. - -**여전히 회수되지 않는 것:** `Future` 가 영원히 settle 되지 않고 코루틴도 영원히 끝나지 않는 호출. -그 호출은 continuation 자체를 이미 흘린 상태이고, 그것을 알아채는 것은 핸들 기계의 일이 아니다. - -### 10.4 누수 확인 — 양쪽 다 측정 - -`HandleTable.liveCount` 로 본다. - -| 경우 | 관측 | -|---|---| -| 진짜 suspend 한 호출이 미결인 동안 | `baseline + 1` — 핸들이 실제로 등록된다 | -| 취소된 뒤, **코루틴이 아직 안 끝난 시점** | `baseline` — Python done 콜백이 이미 놓았다 | -| 취소 → 본문 종료까지 끝난 뒤 | `baseline` | -| 정상 완료 뒤 | `baseline` | -| **빠른 경로** | `baseline` — 등록 자체가 없다 | - -두 번째 줄이 이 표에서 가장 정보량이 많다. 그 시점에는 Kotlin 쪽에서 이 호출에 대해 아무것도 실행된 -적이 없으므로, **회수한 것이 Python 쪽이라는 것이 그 하나로 확정된다.** 첫 줄이 없으면 나머지가 -"핸들을 애초에 안 만들었다" 와 구별되지 않으므로, 미결 중의 `baseline + 1` 을 함께 단언한다. - -### 10.5 빠른 경로는 건드리지 않았다 - -핸들 등록은 `deliver` 의 `isDone` 조기 반환 **뒤에** 있다. 즉 suspend 하지 않은 호출은 `HandleTable` -쓰기도, `__main__` 조회도, `add_done_callback` 도 지나지 않는다. 셋으로 확인했다: - -- `theFastPathRegistersNoHandleAtAll` — `liveCount` 불변 -- `AsyncUpcallDeliveryTest.aSuspendingEntryThatNeverSuspendsHandsBackTheRealValueAndBuildsNoFuture` - — **실행 중인 루프가 아예 없는 스레드에서** 통과한다. asyncio 에 손을 댔다면 실패했을 것이다 -- `PythonProxyInstallTest.theSameGeneratedProxyBuildsNoFutureWhenTheKotlinBodyNeverSuspends` - — `create_future` 호출 수 `0` - -느린 경로는 `__main__` 조회 두 번(`_pm_cancel`, `_pm_release`)과 `_pm_watch` 호출 하나가 늘었다. -`runningLoop()` 와 `settleFunction()` 이 이미 같은 종류의 조회를 하고 있고, 진짜로 suspend 한 호출은 -속성 조회를 세는 처지가 아니다(§8.3 과 같은 논거). - -### 10.6 확인하지 않은 것 — §11 이 iOS 를 채웠다 - -- ~~**desktop 외의 실행.**~~ **§11 이 iOS 시뮬레이터로 채웠다.** 아래 원문은 대조용으로 남긴다. - - > 통지 경로는 `expect`/`actual` 이 하나도 없는 `commonMain` 이고 바인딩만 타깃별이다. - > androidNative(cinterop 포함)와 wasmJs 는 **컴파일만** 확인했다. iOS·Android 기기에서 돌려보지 - > 않았다 — §7.1 이 (B) 의 완료 스레드에 대해 남겨 둔 질문이 여기에도 그대로 남는다. - - androidNative 는 여전히 컴파일만이다 — 실행하려면 에뮬레이터가 필요하고, 이번 회차에는 다른 - 작업이 쓰고 있어 쓰지 않았다(`AGENTS.md` 의 규정). Android(JVM/ART)·iOS 실기기는 여전히 미확인. -- **루프가 죽은 채로 남은 핸들.** Kotlin 쪽 `finally` 해제가 그것을 막도록 되어 있지만, 그 경우를 - 일부러 만들어 관측하지는 않았다. - ---- - -## 11. desktop 밖에서의 실행 — 측정 (iOS 시뮬레이터) - -§10.6 이 남긴 질문: 통지 경로(그리고 그 아래 전달 경로 전체)가 desktop 이 아닌 타깃에서 **실제로 -도는가.** 컴파일이 아니라 실행이다. - -### 11.1 iOS 에 `asyncio` 가 있는가 — 측정, 그리고 기존 문서의 유추는 틀렸다 - -`docs/design/upcall-design.md` §"Can Python call an address at all?" 는 "이 프로젝트의 iOS -`Python.framework` 는 `_ctypes` 도, `lib-dynload` 도 없다"고 적었고, §4·§8.5 는 그 문장에 기대어 -iOS 의 `asyncio` 가능성을 유추만 했다(직접 확인은 하지 않았다). 그 유추는 **성립하지 않는다** — -"`Python.framework` 가 `lib-dynload` 를 안 담고 있다"는 진술 자체는 맞지만, 그것이 가리키는 대상은 -앱이 링크하는 **xcframework 바이너리**(헤더 + 인터프리터 심볼뿐)이고, `asyncio` 가 실제로 찾는 -곳은 **별도로 풀리는 BeeWare stdlib 아카이브**(`extractIosSimulatorStdlib` 가 -`build/python-stdlib/ios-simulator` 에 푸는 것, `PYTHONHOME` 이 가리키는 자리)다. 둘은 다른 것이고, -후자를 직접 열어 확인했다: - - build/python-stdlib/ios-simulator/lib/python3.14/asyncio/ (순수 파이썬, 전부 있음) - build/python-stdlib/ios-simulator/lib/python3.14/lib-dynload/ - _asyncio.cpython-314-iphonesimulator.so - _socket.cpython-314-iphonesimulator.so - select.cpython-314-iphonesimulator.so - -파일이 있다는 것과 실제로 동작한다는 것은 다른 진술이므로, `AsyncioAvailabilityProbeTest` -(`iosSimulatorArm64Test`, 자기 파일 하나로 격리 — wasmJs 의 트랩과 같은 모양이 여기서도 날 경우 -번지는 범위를 파일 하나로 막기 위해서다)로 직접 돌렸다: - -```python -import asyncio -async def _pmp_probe(): return 41 + 1 -_pmp_result = asyncio.new_event_loop().run_until_complete(_pmp_probe()) -``` - -**통과한다.** import 도, `run_until_complete` 도 트랩 없이 정상 동작한다 — wasmJs 와 달리 iOS 는 -이 지점에서 막히지 않는다. `_ctypes` 유추가 `asyncio` 에는 적용되지 않는다는 것이 이제 유추가 -아니라 측정이다. - -### 11.2 그런데 데스크톱의 세 테스트는 "그대로 옮길 수 있는" 것이 아니었다 - -`AsyncUpcallDeliveryTest`·`AsyncUpcallCancellationTest`·`AsyncUpcallEarlyCancellationTest` 를 -검토한 결과, 셋 다 `commonTest`/`nativeTest` 로 옮길 수 없었다. 이유는 `asyncio` 가 아니라 -**완료/관찰 스레드를 만드는 방법**이다 — 셋 다 `java.lang.Thread` 와 -`java.util.concurrent.{LinkedBlockingQueue,TimeUnit}` 을 쓰고, 이것은 JVM 전용이라 Kotlin/Native -어디에도 없다. `asyncio` 만 봤다면 "옮길 수 있다"고 잘못 판단했을 것이다 — 이 저장소가 이미 가진 -포터블한 부분(`AsyncUpcallPortabilityTest`, `commonTest`)이 정확히 asyncio 를 건드리기 전에 -끝나는 두 경로만 남긴 이유이기도 하다: 그 두 경로는 스레드가 아예 필요 없다. - -그래서 데스크톱 파일을 옮기는 대신, 같은 주장을 **새로 만든 iOS/androidNative 쪽 완료 스레드**로 -다시 검증했다. - -### 11.3 `NativeThread` — JVM 스레드의 자리를 대신하는 것, 그리고 왜 `Worker` 가 아닌가 - -`nativeTest/.../NativeThread.kt`: `pthread_create` 를 직접 부르는 얇은 래퍼 -(`StableRef` + `staticCFunction` 트램폴린). `kotlin.native.concurrent.Worker` 를 쓰지 않은 이유는 -이 테스트들이 답해야 하는 질문 자체가 "Kotlin/Native 런타임이 **스스로 붙인 적 없는** 스레드에서 -완료가 와도 되는가"이기 때문이다 — `Worker` 는 Kotlin/Native 자신의 스레드 기계이므로 그 질문을 -우회한다. 순수 `pthread_create` 는 `CycleCollectionTest.testDeallocOnAThreadCPythonCreated` 가 이미 -같은 모양으로 측정해 둔 것과 정확히 같은 처지의 스레드다 — 거기서는 CPython 이 만든 pthread 가 -`tp_dealloc` 슬롯을 불렀고, 여기서는 이 테스트가 만든 pthread 가 `PendingCall` 의 리스너를 부른다. - -### 11.4 측정한 것 — 셋, 전부 iOS 시뮬레이터에서 통과 - -| 테스트 (`nativeTest`) | 데스크톱 대응 | 무엇을 다시 확인했나 | 결과 | -|---|---|---|---| -| `AsyncUpcallNativeDeliveryTest.pythonAwaitsASuspendedKotlinCallAndTheValueArrivesFromAThreadTheRuntimeNeverAttachedItself` | `AsyncUpcallDeliveryTest.pythonAwaitsASuspendedKotlinCallAndTheValueArrivesFromAKotlinThread` | (C)/(B) 의 실제 전달: 루프가 도는 동안 낯선 스레드가 GIL 을 얻어 `call_soon_threadsafe` 로 `Future` 를 해소 | **통과** | -| `AsyncUpcallNativeDeliveryTest.theFastPathStillTakesNoAsyncioOnThisTargetEither` | `theSameAwaitExpressionTakesTheShortcutWhenTheKotlinBodyNeverSuspends` | 빠른 경로가 이 타깃에서도 `asyncio` 를 안 타는가 | **통과** | -| `AsyncUpcallNativeEarlyCancellationTest.cancellingTheFutureReachesEnsureActiveWhileTheKotlinCallIsStillSuspendedOnAThreadTheRuntimeNeverAttachedItself` | `AsyncUpcallEarlyCancellationTest.cancellingTheFutureReachesEnsureActiveWhileTheKotlinCallIsStillSuspended` | §10 의 이른 통지: `_pm_cancel` → `ensureActive()` 가 **완료 전에** 던지는가 | **통과** | -| `AsyncUpcallNativeCancellationTest.aCompletionLandingOnACancelledFutureIsDroppedInsteadOfRaisingInsideTheLoop` | `AsyncUpcallCancellationTest.aCompletionLandingOnACancelledFutureIsDroppedInsteadOfRaisingInsideTheLoop` | §9.2 의 조용한 드롭: 비협조 본문이 취소된 `Future` 에 완료를 배달해도 루프가 다치지 않는가, 에러 지시자가 새지 않는가 | **통과** | - -세 번 반복 실행해 타이밍에 의한 우연이 아님을 확인했다(§ "검증" 참고). `_pm_cancel` 은 -`UpcallEntry.publish` 가 `_pm_resolve`/`_pm_bind`/`_pm_release` 와 함께 통째로 까는 바인딩이라 -(§10.2 표가 이미 "nativeMain, iOS·androidNative 공용"이라고 적어 둔 대로) 새 배선은 필요 없었다 — -`bindUpcallOrNull` 의 native `actual`(`UpcallRawEntryPointTest.kt`)이 이미 부르고 있었다. - -### 11.5 이번에도 확인하지 않은 것 - -- **androidNative 의 실행.** `compileTestKotlinAndroidNativeArm64` 로 새 테스트 세 파일이 그 - 타깃에서도 컴파일된다는 것만 확인했다. 실행하려면 에뮬레이터가 필요하고, 이번 회차에는 다른 - 작업이 쓰고 있어 비워 두지 않았다 — `_pm_cancel` 이 androidNative 에도 같은 방식으로 깔려 있으므로 - (§10.2) **성립할 것으로 기대하지만, 이것은 추론이지 측정이 아니다.** -- **Android(JVM/ART), iOS 실기기.** §8.5 가 이미 적어 둔 공백 그대로다 — 이 워크스페이스에 Android - SDK 가 없고, 실기기를 쓰지 않았다. -- **루프가 죽은 채로 남은 핸들.** §10.6 이 남긴 그대로. - -## 12. 생성된 프록시가 desktop 밖에서 설치조차 안 됐다 — 측정 (iOS 시뮬레이터) - -§9.4 가 `PythonProxySource` 를 만들고 §11 이 async 경로를 iOS 에서 측정했지만, **`install()` 자체는 -`desktopTest` 밖에서 한 번도 불린 적이 없었다.** 그래서 생성기와 부트스트랩이 서로 다른 것을 가정한 -채로 계속 갔다. 이번에 `PythonProxyInstallTest` 를 `commonTest` 로 올려 iOS 시뮬레이터에서 돌리자 -14개가 전부 빨갛게 나왔다. - -### 12.1 관측한 것 — 두 가지가 독립적으로 어긋나 있었다 - -| # | iOS 에서 실제로 본 것 | 원인 | -|---|---|---| -| 1 | `PyException: the raw upcall entry points are not bound: define _pm_resolve(name_bytes) -> handle and _pm_invoke(handle, args_tuple) -> result` — 14/14 | `UpcallEntry.publish` 가 `_pm_invoke` 를 **일부러** 안 깔았다. 생성기의 가드는 그것 없이는 설치를 거부하고, 생성된 모든 호출이 `_pm_invoke(handle, args)` 다 | -| 2 | `PyException: bad argument type for built-in operation` — (1)을 고친 뒤 11/11 | 생성기의 지원 코드가 `_pm_resolve(name.encode('utf-8'))` 로 **bytes** 를 넘기는데, `pmResolveMethod` 가 `PyUnicode_AsUTF8` 로 읽어 `PyErr_BadArgument` 를 세우고 NULL 을 돌려줬다 | - -(2)는 (1)의 뒤에 가려져 있어서, (1)을 고친 다음 `PyBytes_AsString` 폴백을 잠깐 되돌려 **따로 -확인했다.** 커밋 메시지의 서술이 아니라 두 번 다 시뮬레이터에서 본 문자열이다. - -세 번째로, 생성기의 지원 코드가 `_pm_bind` 를 **name → handle 로 재정의**하고 있었다. 세 부트스트랩이 -같은 이름을 handle → callable 로 깔기 때문에, `exec` 가 지나가면서 네이티브 쪽을 파괴한다. desktop 은 -`_pm_bind` 를 아예 안 깔아서 덮어쓸 것이 없었고, 그래서 조용했다. - -### 12.2 어느 쪽을 맞췄나, 그리고 왜 - -**부트스트랩 쪽을 생성기에 맞췄다.** `UpcallEntry.publish` 가 이제 `_pm_invoke` 도 깐다 -(`METH_VARARGS`, self 없이 `args = (handle, args_tuple)`). - -원래 안 깔던 이유는 주석에 적혀 있었다 — "handle 을 별도 인자로 파이썬 손에 쥐여주지 않기 위해". -**그 이유는 옆 파일과 대조하면 성립하지 않는다:** - -- 아무것도 그것을 강제하지 않았다. `_pm_bind` 가 **파이썬에서 온 생 정수**를 받아 그 handle 위의 - callable 을 돌려준다. `_pm_release`/`_pm_cancel` 도 생 handle 을 받는다. `_pm_bind(h)(*a)` 와 - `_pm_invoke(h, a)` 는 같은 트램폴린, 같은 인자, 같은 거절 경로다. 새로 열리는 권한이 없다. -- 유일한 소비자가 그것을 필요로 한다. 생성기가 만드는 모든 호출이 `_pm_invoke` 이고, 가드가 그 - 이름 없이는 설치를 거부한다. 안 깔아서 지켜진 것은 handle 이 아니라, **desktop 을 뺀 모든 타깃에서 - 프록시가 아예 안 도는 상태**였다. -- 반대 방향(생성기가 `_pm_bind` 로 `_pm_invoke` 를 합성)은 **호출 1건당 `PyCMethod_New` 할당**이고, - 런타임에 갈라지는 부트스트랩 규약을 둘로 유지하게 된다. - -인자 타입은 **`str` 과 `bytes` 를 둘 다 받게** 했다(`PyUnicode_AsUTF8` 실패 → `PyErr_Clear` → -`PyBytes_AsString`). 생성기는 계속 bytes 를 보낸다 — desktop 의 `ctypes.CFUNCTYPE(c_long, c_char_p)` -가 `str` 을 **거절**하므로, 한 번 적어서 전부에서 도는 철자는 bytes 뿐이다. 비용은 이름 1건당 실패한 -`PyUnicode_AsUTF8` 하나이고, 이름은 설치 때 한 번만 푼다. - -생성기의 name → handle 헬퍼는 `_pm_lookup` 으로 이름을 바꿨다. `PythonProxySourceTest` 에 -`support` 가 `_pm_bind`/`_pm_resolve`/`_pm_invoke`/`_pm_release`/`_pm_cancel` 중 어느 것도 -`def` 하지 않는다는 회귀 가드를 넣었다. - -### 12.3 테스트를 어디로 옮겼나 - -| 위치 | 내용 | 도는 곳 | -|---|---|---| -| `commonTest/PythonProxyInstallTest` | 프록시 계약 11개 + fixture(`ProxyFragment`, `ProxyCounter`, `ProxyLoopHarness`) | desktop · iOS · wasmJs · androidNative(컴파일) · Android(계측) | -| `desktopTest/PythonProxyDeliveryTest` | 스레드가 필요한 전달 3개 (`java.lang.Thread`) | desktop | -| `nativeTest/PythonProxyNativeDeliveryTest` | 같은 3개를 `pthread_create` 로 (**새로 추가**) | iOS · androidNative | - -부트스트랩이 없는 타깃은 **건너뛰지 않는다.** `publishesProxyEntryPoints`(`expect val`, 타깃별 -상수)가 false 인 곳에서는 생성 모듈 자신의 가드가 뜨는 것을 단언한다. 런타임에 `globals()` 를 -들여다보지 않는 이유가 이것이다 — 그러면 발행을 그만둔 타깃이 조용히 반대 분기로 넘어가 초록으로 -남고, 그게 바로 이번에 고친 실패 양식이다. - -측정: iOS 시뮬레이터 318개 0 실패(이전 303), desktop 342개 0 실패(스킵 1), wasmJs 328개 0 실패, -`compileTestKotlinAndroidNativeArm64` 와 `:sample:compileKotlinIosSimulatorArm64` 통과. - -### 12.4 못 맞춘 두 타깃과, 각각 무엇이 필요한가 - -> §13 에서 **Android(ART) 는 맞췄다.** 아래 Android 항목은 그때의 진단이고, **절반만 맞았다.** -> §14 에서 **wasmJs 도 맞췄다.** 아래 wasmJs 항목의 결론은 **틀렸다** — 새 `@WasmExport` 는 -> 필요하지 않았고, 이미 있는 하나로 다섯 개를 전부 깔 수 있었다. - -- **wasmJs — 안 된다.** 파이썬으로 건너가는 것이 *바인딩된 callable* 하나뿐이다(`UpcallEntry.bind` - 가 유일한 `@WasmExport` `pmp_invoke` 위에 만든다). 이름 해석은 Kotlin 안에서 끝난다. - `_pm_resolve`/`_pm_invoke` 를 깔려면 `@WasmExport` 가 더 필요하고, `@WasmExport` 는 `.wasm` 을 - 만드는 컴파일에서만 유효하므로 **라이브러리가 아니라 임베딩하는 애플리케이션이** - (`wasmJsTest/UpcallExports.kt` 모양으로) 선언해야 한다. 게다가 이 빌드는 `import asyncio` 가 - trap 되므로 suspend 프록시는 그다음에도 첫 `await` 에서 죽는다. 억지로 맞추지 않았다. - → **§14 가 가운데 문장을 뒤집었다.** `@WasmExport` 제약 자체는 맞고, asyncio trap 도 맞다. - 틀린 것은 "그러므로 새 export 가 필요하다" 는 결론 하나다. -- **Android(ART) — C 한 조각이 빠졌다.** §11.4 이후로도 "경계 shim 이 없다"고 적혀 있었지만 그것은 - 틀렸다. `androidMain` 의 `UpcallEntry.publish` 가 `jni_onload.def` 의 C shim 을 통해 - `_pm_resolve`/`_pm_bind`/`_pm_release`/`_pm_cancel` 을 실제로 깐다(그래서 - `UpcallThreadAttachTest` 가 `threading.Thread` 에서 업콜을 돌린다). 빠진 것은 `_pm_invoke` 하나뿐: - 기존 `pmp_upcall_invoke_meth` 옆에 `args[0]` 에서 handle, `args[1]` 에서 인자 튜플을 읽는 - `pmp_upcall_invoke_free_meth` 를 두고 `pmp_upcall_publish` 에 항목 하나를 더하면 된다. Kotlin 은 - 빠진 것이 없다 — `UpcallCallbacks.invoke(long, long)` 이 이미 그 모양이고 method ID 도 있다. - **에뮬레이터 없이 돌릴 수 없어서 넣지 않았다**; 테스트 쪽 `publishesProxyEntryPoints` 가 false 로 - 같은 메모를 들고 있으므로, 구현되는 날 "이 타깃은 거절한다"는 단언이 실패해서 알려준다. - -## 13. ART 에서 프록시가 돈다 — 그리고 §12.4 의 진단은 절반만 맞았다 - -에뮬레이터에서 돌려보고 맞췄다. 필요한 것은 **C 함수 하나가 아니라 C 변경 두 개**였다. - -### 13.1 기록된 절반: `_pm_invoke` - -§12.4 가 적은 그대로다. `jni_onload.def` 에 `pmp_upcall_invoke_free_meth(self, args)` 를 두고 -(`args[0]` 이 handle, `args[1]` 이 인자 튜플), `g_pmInvokeFreeDef` 를 `upcall_publish` 에 더했다. -Kotlin 은 정말로 빠진 것이 없었다 — `UpcallCallbacks.invoke(long, long)` 과 그 method ID 를 그대로 -썼다. `PyMethodDef` 설치는 개수를 세는 배열이 아니라 `if (!pmp_install_method(...)) return 0;` 의 -연쇄이므로 어긋날 리터럴이 애초에 없다(`RegisterNatives` 의 개수는 이전부터 `sizeof` 계산이다). - -### 13.2 기록되지 않은 절반: `_pm_resolve` 가 `bytes` 를 안 받았다 - -`_pm_invoke` 만 넣고 돌린 결과가 이것이다 — **가드는 통과하고 그다음 줄에서 죽었다.** - -| | pmp_api26 | pmp_api36 | -|---|---|---| -| `_pm_invoke` 만 추가 | 11개 전부 실패: `TypeError: bad argument type for built-in operation` | 같음 | -| `_pm_resolve` 까지 고침 | 11개 전부 통과 | 같음 | - -`pmp_upcall_resolve_meth` 는 인자를 `PyUnicode_AsUTF8` 로만 읽었는데, `PythonProxySource` 의 -`_pm_lookup` 은 `bytes` 를 보낸다(desktop 이 `ctypes.CFUNCTYPE(c_long, c_char_p)` 로 resolver 에 -닿고 `c_char_p` 가 `str` 을 거부하므로, 모든 호스트에서 통하는 철자는 `bytes` 뿐이다). nativeMain 의 -`pmResolveMethod` 는 **바로 그 이유로 이미 두 철자를 다 받게 고쳐져 있었고**, ART shim 만 안 고쳐져 -있었다. 아무도 묻지 않았기 때문이다 — 이 shim 의 유일한 호출자였던 -`androidInstrumentedTest/bindUpcallOrNull` 이 `str` 을 넘긴다. - -`support` 의 주석은 "다른 부트스트랩은 전부 두 철자를 받는다"고 단언하고 있었다. 그것은 **검증된 -적 없는 문장**이었고, ART 에서 거짓이었다. - -**소스를 읽어서는 찾을 수 없는 종류의 결함이다.** 두 번째 절반은 런타임에만 존재하고, 첫 번째 -절반을 고쳐야 비로소 도달한다. §12.3 이 `PythonProxyInstallTest` 를 `commonTest` 로 옮긴 이유가 -이것 그대로다. - -### 13.3 순서 - -`publishesProxyEntryPoints` 의 ART 값은 **11개가 실제로 통과하는 것을 본 뒤에** true 로 바꿨다. -그 전 단계에서는 두 번 다 빨간색이었고, 두 실패 메시지가 서로 다른 것을 가리켰다(먼저 -`TypeError`, 고친 뒤 "Expected an exception to be thrown, but was completed successfully"). - -### 13.4 측정 - -| | 결과 | -|---|---| -| ART 계측 (`connectedDebugAndroidTest`) | **336개 0 실패** × pmp_api26 · pmp_api36 (그중 `PythonProxyInstallTest` 11개가 이제 진짜 경로) | -| androidNative (`androidNativeArm64Test`) | 316개 0 실패 × 2대 | -| desktop (`desktopTest`) | 342개 0 실패, 스킵 1 | - -### 13.5 sample - -`ProxyDemo.android.kt` 는 "unavailable on Android" 를 반환하고 있었고, `UpcallDemo.android.kt` 의 -§3 은 "the boundary shim is desktop-only today" 라고 적혀 있었다. 둘 다 이제 거짓이므로 고쳤다. -두 에뮬레이터에서 앱을 띄워 확인한 화면: - -| 섹션 | pmp_api26 · pmp_api36 | -|---|---| -| 3 — 업콜 | `handle 4294967327 -> 0 · with args -> presses x3 = 0` (Kotlin 이 아니라 **파이썬이** `_pm_resolve`/`_pm_invoke` 로 호출) | -| 5 — 클래스 프록시 | `installed over PyMethodDef via JNI: 466 lines, 2 proxy classes`, `Greeter('Kotlin').greet(2) -> hello Kotlin! hello Kotlin!`, `g.greetings = 99 -> AttributeError (private set held)` | -| 6 — companion | `Greeter.forget() -> 100, then built=0`, `Greeter('x').built -> AttributeError (companion is class-only)` | -| 7 — fast path | `await g.greetNow(1) -> hello fast path!`, `Futures created -> 0` | -| 7 — really suspends | `await g.greetLater(2) -> hello slow path! hello slow path!`, `Futures created -> 1`, `raised -> None`, `completer thread -> clean` | - -마지막 줄이 iOS 가 아직 못 하는 것이다: `awaitSuspendingDemo` 는 파킹된 continuation 을 재개할 -Kotlin 스레드가 필요한데, `androidMain` 은 Kotlin/JVM 이라 `java.lang.Thread` 로 desktop 과 같은 -모양을 쓸 수 있다. 재개는 CPython 도 ART 도 모르는 스레드에서 오므로, 전달뿐 아니라 `pmp_attach` -까지 함께 돈다. - -## 14. wasm 에서도 프록시가 돈다 — §12.4 의 "새 export 가 필요하다" 는 틀렸다 - -§12.4 는 wasmJs 를 유일한 미구현 타깃으로 남기며 이렇게 적었다: `_pm_resolve`/`_pm_invoke` 를 깔려면 -**각자 새 `@WasmExport`** 가 필요하고, `@WasmExport` 는 `.wasm` 을 만드는 컴파일에서만 유효하므로 -라이브러리가 아니라 임베딩 앱이 선언해야 한다 — 그러므로 라이브러리는 할 수 있는 것이 없다. - -**전제는 전부 맞는데 결론이 틀렸다.** 물어보지 않은 질문이 하나 남아 있었다: *이미 하나가 -export 되어 있다면, 그 하나로 다 할 수 없나?* - -### 14.1 이미 있던 하나가 누구 것이었나 - -먼저 확인한 것은 그 하나가 **누구의 컴파일에서 나오는가**다. §12.4 의 진단은 "임베딩 앱이 선언해야 -한다" 인데, 지금 그것을 선언하고 있는 것은: - - python-multiplatform/src/wasmJsTest/kotlin/python/native/ffi/UpcallExports.kt - - @kotlin.wasm.WasmExport("pmp_invoke") - fun pmpInvoke(selfPtr: Int, argsPtr: Int): Int = UpcallEntry.invokeMethod(selfPtr, argsPtr) - -**테스트 컴파일이다.** 그리고 그 파일의 주석이 이미 정확히 그렇게 적고 있다 — "이것은 테스트 -스캐폴딩이 아니라, 이 라이브러리를 임베딩하는 애플리케이션이 스스로 써야 하는 파일이며, 스위트가 -애플리케이션이 지나가는 경로를 그대로 지나가도록 여기에 재현해 둔 것" 이다. 즉 §12.4 의 진단은 -**이미 충족되어 있었다.** 임베딩 앱이 선언해야 하는 것은 맞고, 이미 선언되어 있었다. - -배선은 이렇다. `wasmJsMain/UpcallEntry` 가 `bindings.pmpRegisterUpcall("pmp_invoke")` 로 그 export 를 -CPython 의 `__indirect_function_table` 에 `Table.set` 하고 (반환값이 곧 C 함수 포인터), 그것을 -`ml_meth` 으로 하는 `PyMethodDef` 를 하나 `malloc` 해서, `PyCFunction_NewEx(def, self, NULL)` 로 -callable 을 만든다. 파이썬이 부르는 함수 포인터는 그 하나뿐이다. - -### 14.2 디스패처는 성립한다 — 그리고 op 인자조차 필요 없다 - -과제는 "`pmp_invoke(op, a, b)` 처럼 op 코드로 갈라라" 였는데, 실제로는 **op 인자를 더할 필요가 -없었다.** `PyCFunction` 은 함수 포인터*와* `self` 를 함께 들고 다니고 `PyCFunction_NewEx` 는 호출마다 -새 객체를 만든다. 그러므로 **export 하나가 이미 서로 다른 파이썬 callable 을 얼마든지 뒷받침한다** — -`UpcallEntry.bind` 가 `CallableHandle` 로 계속 해오던 것이 정확히 그것이다. - -핸들이 들어갈 자리에 핸들이 될 수 없는 값을 넣으면 같은 포인터가 디스패처가 된다: - -| `self` | 파이썬에 깔리는 이름 | 모양 | -|---|---|---| -| `-2` | `_pm_resolve` | `(name: str \| bytes) -> handle`, 없으면 `-1` | -| `-3` | `_pm_invoke` | `(handle, args_tuple) -> result` | -| `-4` | `_pm_bind` | `(handle) -> callable` | -| `-5` | `_pm_release` | `(handle) -> int` | -| `-6` | `_pm_cancel` | `(handle) -> int` | -| 그 외 | (바인딩된 callable) | `UpcallTrampoline.invoke(self, args)` | - -**왜 음수가 안전한가:** `CallableHandle.isValid` 가 `raw >= 0` 이고, `ObjectReference` 는 -`(generation shl 32) or slot` 이며 generation 은 1 이상이다. 살아 있는 핸들은 어느 쪽이든 음수가 될 -수 없다. `-1` 은 일부러 op 로 쓰지 않았다 — `CallableHandle.NONE` 이고, 트램폴린 자신의 거절 경로에 -계속 도달해야 한다. - -그래서 **새 `@WasmExport` 는 0 개**이고, 임베딩 앱이 쓰는 파일은 이전과 글자 하나 다르지 않다. -`verifyWasmAbiSignatures` 도 그대로다 (`317 @WasmImport` + 글루 3 — 새 바인딩이 없다). - -생성기 쪽 얇은 shim 은 필요 없었다. `PythonProxySource` 가 요구하는 두 이름이 그대로 깔리므로 -`support` 절의 `_pm_lookup` 이 이미 있는 그대로 동작한다. 다만 `_pm_resolve` 는 **`str` 과 `bytes` 를 -둘 다** 받게 썼다 — §13.2 에서 ART 가 정확히 그 자리에서 죽었기 때문이다. - -### 14.3 순서 — 고치기 전에 빨간 것을 봤다 - -`publishesProxyEntryPoints` 의 wasmJs 값은 10개가 실제로 통과하는 것을 본 뒤에 true 로 바꿨다. -그리고 그 통과가 진짜인지 확인하려고 `bindUpcallOrNull` 에서 `UpcallEntry.publish` 호출만 빼고 다시 -돌렸다: - - PythonProxyInstallTest 11/11 FAILED - PyException: the raw upcall entry points are not bound: define _pm_resolve(name_bytes) - -> handle and _pm_invoke(handle, args_tuple) -> result before installing proxies - -생성 모듈 자신의 가드다. 통과가 부트스트랩에 실제로 달려 있다는 뜻이고, 빼면 §12.1 이 iOS 에서 본 -것과 같은 문자열이 wasm 에서 나온다. - -### 14.4 asyncio 는 여전히 안 된다 — 그래서 상수가 둘이다 - -> **§15 가 이 절의 전제를 뒤집었다.** `import asyncio` 는 이제 **통과한다.** 상수를 `false` 로 -> 두는 결론 자체는 유지되지만 **이유가 바뀌었다** — trap 이 아니라, 이 상수가 게이트하는 테스트가 -> 만드는 *기본* 이벤트 루프를 아직 만들 수 없기 때문이다(`socketpair` → `require('ws')`, §15.7). -> 아래 문단의 "trap 시킨다" 는 더 이상 사실이 아니다. - -§9.5 는 그대로다: `import asyncio` 가 이 wasm 빌드에서 **인스턴스를 trap** 시킨다. 파이썬 예외가 -아니라 프로세스가 죽는 것이므로 Kotlin 에서도 파이썬에서도 잡을 수 없고, 부트스트랩이 생겼다고 -달라지지 않는다. - -그러므로 `publishesProxyEntryPoints` 하나로 두 가지를 말하게 두지 않고 **`proxyBootstrapSupportsAsyncio` -를 따로 뒀다.** 하나로 뭉치면 "프록시가 아예 없다" 와 "await 할 수 있는 프록시가 없다" 가 같은 -문장이 되고, wasm 에서 실제로 성립하는 10개의 계약을 조용히 단언하지 않게 된다. - -이 상수가 거짓인 곳에서는 **거절을 단언하지 않는다.** 다른 미구현 타깃과 다른 점이 이것이다 — -거기서는 생성 모듈의 가드가 뜨는 것을 볼 수 있지만, 여기서 관측되는 것은 오류가 아니라 프로세스 -사망이라 단언할 대상이 없다. 정직한 모양은 그 한 줄을 돌리지 않는 것이고, 상수가 그 사실을 적는 -자리다. - -### 14.5 측정 - -| | 결과 | -|---|---| -| wasmJs (`wasmJsNodeTest`) | **340개 0 실패 0 스킵** — 기준선과 동일. `PythonProxyInstallTest` 11개 중 10개가 이제 진짜 경로, 1개(asyncio)는 §14.4 | -| desktop (`desktopTest`) | 360개 0 실패, 스킵 1 — 회귀 없음 | -| `verifyWasmAbiSignatures` | 통과, 317 + 글루 3 (변화 없음) | -| `compileTestKotlinAndroidNativeArm64`, `compileTestKotlinIosSimulatorArm64` | 통과 | - -`GeneratedProxyCostTest` 가 wasm 에서 처음으로 숫자를 낸다 (경계 자체가 ~300 ns 인 타깃이다): - -``` -Net of the harness floor (62.48 ns), and what the proxy layer adds - module function raw 301.91 ns proxy 318.98 ns 1.05x (+17.06 ns) - constructor raw 719.75 ns proxy 1070.98 ns 1.48x (+351.23 ns) - instance method raw 339.22 ns proxy 357.47 ns 1.05x (+18.24 ns) - property read raw 290.94 ns proxy 308.41 ns 1.06x (+17.47 ns) - property write raw 360.42 ns proxy 438.77 ns 1.21x (+78.34 ns) - static read raw 228.82 ns proxy 300.94 ns 1.31x (+72.12 ns) - module read raw 228.82 ns proxy 324.39 ns 1.41x (+95.57 ns) - -Resolution: two identical raw rows differ by -73.09 ns -HandleTable roots before the timed loops: 1, after: 1 - ---- Generated proxy install cost --- - generated source 338 lines, 14052 chars - PythonProxySource.render() 186.63 us - PythonProxySource.install() 2342.14 us (...of which Python3.exec 2155.50 us) -``` - -**해상도 주의:** 이 타깃의 동일한 두 raw 행이 73 ns 차이가 난다. 위의 static/module 행들의 +72…+96 ns -는 그 해상도와 같은 크기이므로 **숫자로 읽으면 안 된다.** 읽을 수 있는 것은 생성자의 +351 ns 뿐이고, -그것은 desktop·시뮬레이터와 같은 이유다 (`type.__call__` + `__init__` 프레임 + 속성 저장 + `__del__`). - -`HandleTable roots before/after: 1 → 1` 이 별도로 말하는 것이 하나 있다: 만 번의 `Counter(10)` 이 -전부 `__del__` → `_pm_release` 로 돌아왔다. 디스패처가 release 를 invoke 경로로 잘못 보냈다면 조용히 -전부 샜을 자리다. - -### 14.6 남은 것 - -- **async 프록시.** ~~§9.5 의 trap 이 풀리지 않는 한 불가능하다. 이 빌드의 `selectors` 가 - `RuntimeError: unreachable` 로 죽는 것이 원인이고, 그것은 이 문서가 아니라 CPython wasm 빌드 - 쪽 문제다.~~ **§15 가 이 항목을 뒤집었다.** trap 은 풀렸고, 원인은 CPython wasm 빌드가 아니라 - 우리 부트 코드였다. 남은 것은 trap 이 아니라 **기본 이벤트 루프**다 (§15.7). -- **`sample` 의 wasm 경로.** §13.5 가 Android 에 대해 한 것을 wasm 에 대해서는 하지 않았다. - ---- - -## 15. `selectors` trap 의 원인 — 측정, 그리고 §9.5·§14.4·§14.6 의 진단은 틀렸다 - -§9.5 는 `import selectors` 가 `RuntimeError: unreachable` 로 죽는 것을 관측하고 거기서 멈췄다. -§14.6 은 그것을 "CPython wasm 빌드 쪽 문제" 로 넘겼다. **둘 다 틀렸다.** 원인은 `selectors` 도, -`asyncio` 도, wasm 도, CPython 빌드도 아니다. **우리 쪽 부트 코드다.** 그리고 고쳤다. - -### 15.1 관측한 것 - -`RuntimeError: unreachable` 은 원인이 아니라 3차 증상이었다. 실제 순서는 이렇다: - - select.poll().poll(0) - -> SuspendError: trying to suspend without WebAssembly.promising <- 1차, 진짜 원인 - -> SuspendError 가 CPython 의 C 프레임을 Py_END_ALLOW_THREADS 없이 되감는다 - -> 바깥 withGIL{} 의 PyGILState_Release 가 - Fatal Python error: PyGILState_Release: thread state ... must be current when releasing - -> abort() -> wasm `unreachable` 오피코드 <- 3차, 보고된 것 - -격리된 프로브를 gradle 로 돌려 1차와 2차를 직접 읽었다. `RuntimeError: unreachable` 만 보고 -"파이썬 예외가 아니다" 라고 적었을 때 놓친 것이 그 위의 두 줄이다. - -### 15.2 왜 `select` 는 되고 `selectors` 는 안 됐나 — §9.5 가 지목한 그 대비의 답 - -**모듈을 import 하는 것과 그 모듈의 함수를 호출하는 것의 차이다.** `import select` 는 아무것도 -호출하지 않는다. `selectors.py` 는 import 시점에 호출한다: - - 582: selector_obj.poll(0) # _can_use(method) 안 - 600: elif _can_use('poll'): # DefaultSelector 선택 - -이 빌드의 `select` 에는 `kqueue`/`epoll`/`devpoll` 이 없으므로(`hasattr` 은 전부 False — 안전하게 -넘어간다) 실제로 **호출까지 가는 첫 후보가 `poll`** 이다. 그래서 `selectors` 가 첫 희생자였고, -`asyncio` 는 `selectors` 를 거쳐서만 죽었다. §9.5 가 "막는 것은 asyncio 가 아니라 그 한 모듈" 이라고 -좁힌 것은 맞았지만, 한 칸 더 들어가면 **모듈이 아니라 그 한 syscall** 이다. - -### 15.3 원인 — Emscripten 은 JSPI 를 빌드 타임이 아니라 **런타임에** 결정한다 - -CPython 의 링크 라인에 `-sJSPI` 는 **없다.** 그런데도 JSPI 를 쓴다. 글루가 이렇게 되어 있다: - - function __block_for_int(p) { return p } - if (WebAssembly.Suspending) { __block_for_int = new WebAssembly.Suspending(__block_for_int) } - function __maybe_poll_async(...) { if (!WebAssembly.promising) { return null } ... } - -그리고 짝이 되는 `WebAssembly.promising` 래퍼를 **`main` 하나에만** 씌운다. - - if (name === "main") { const main = WebAssembly.promising(orig.sym); ... } - -**우리는 `main` 을 부르지 않는다.** 이 파일 맨 위에 적혀 있는 그대로, 부트업은 `Python3.initialize()` -가 `Py_Initialize` 를 다른 C API 와 똑같이 직접 wasm 호출로 발행하는 것이다. 그러므로 JSPI 가 있는 -런타임에서는 블로킹 syscall 이 suspending import 가 되는데 **스택 어디에도 promising 프레임이 없다.** -첫 번째 블로킹 syscall 이 프로세스를 가져간다. - -즉 이것은 "wasm 에서 asyncio 가 안 된다" 가 아니라 **`main` 을 쓰지 않는 임베딩이 Emscripten 의 -JSPI 경로와 맞지 않는다** 는 문제였다. 우리가 `main` 을 쓰지 않는 것은 의도한 설계다. - -### 15.4 노드 버전이 갈랐다 — 같은 wasm 바이너리, 반대 결과 - -런타임 감지이므로 **바이너리가 아니라 실행 환경이 결정한다.** 같은 `python.wasm` 으로 측정했다: - -| | `select.poll().poll(0)` | `import selectors` | -|---|---|---| -| Node 22 (JSPI 없음) | `[]` 반환 | OK — `PollSelector` | -| Node 26 (JSPI 기본 켜짐) | `SuspendError` | 프로세스 사망 | - -gradle 의 wasm 테스트 러너는 Node **26** 에서 돈다. §9.5 를 측정한 것도 그 러너다. Node 26 에는 -JSPI 를 끄는 V8 플래그가 없다 — 실험 기능이 아니라 출시된 기능이라 `--experimental-wasm-jspi` 가 -아예 없다. - -### 15.5 고친 것 — `cpython.mjs` 에서 두 줄 - - delete WebAssembly.promising; - delete WebAssembly.Suspending; - -Emscripten 팩토리를 부르기 **전에** 지운다. 그러면 Node 26 이 Node 22 의 동기 경로로 돌아간다. -이 라이브러리가 원하는 것이 정확히 그쪽이다: 모든 호출이 Kotlin 에서 **동기 wasm 호출**로 도착하므로, -블로킹 syscall 은 실제로 **블로킹해야** 한다. 비동기 경로는 쓸 데가 없다. - -ES import 는 호이스팅되므로 이 시점에 `python.mjs` 는 이미 평가되어 있다. 상관없다 — JSPI 래핑은 -모듈 평가가 아니라 **팩토리 호출 안에서** 일어난다. 측정으로 확인했다. - -**stdlib 을 건드리지 않는다는 것이 이 선택의 핵심이다.** `sys.modules['selectors']` 에 shim 을 -넣는 방안도 후보였지만 채택하지 않았다: - -- shim 은 import 만 통과시킨다. selector 루프는 **매 이터레이션마다** `poll`/`select` 를 부르므로 - 첫 이터레이션에서 똑같이 죽는다. 증상을 한 칸 뒤로 미룰 뿐이다. -- 사용자가 진짜 `selectors` 를 원할 때 조용히 가짜를 준다. 명시적 opt-in 을 붙여야 하는 것은 이 쪽이다. -- 고칠 지점이 아니다. 문제는 stdlib 이 아니라 **우리 부트 코드**였다. - -지금 `selectors` 는 **진짜 모듈**이고 `DefaultSelector` 도 진짜 `PollSelector` 로 해결된다. -그러므로 opt-in 도 필요 없다 — 끄고 켤 동작 차이가 없고, 되던 것이 늘어날 뿐이다. - -### 15.6 측정 — 어디까지 가나 - -`WasmSelectorsImportTest`(wasmJsTest, 4개) 가 고정한다. 고치기 전에 빨간 것을 봤다: 프로브를 -gradle 로 돌려 §15.1 의 fatal error 를 직접 읽었고, 그 실행은 테스트 실패가 아니라 -`Test running process exited unexpectedly` 였다. - -| | 결과 | -|---|---| -| `select.poll().poll(0)`, `select.select(...)` | **통과** — suspend 하지 않고 반환한다 | -| `import selectors` → `DefaultSelector is PollSelector` | **통과** — 진짜 모듈, 진짜 디스패치 | -| `import asyncio`, `asyncio.events`, `asyncio.base_events` | **통과** | -| selector 를 안 쓰는 루프로 코루틴 1개 완주 (`await Future`, `asyncio.sleep(0)`) | **통과** — 84 | - -즉 **async 업콜이 필요로 하는 표면은 이 타깃에서 성립한다**: 도는 루프, 진짜 `asyncio.Future`, -`create_future`, `call_soon`. - -### 15.7 남은 것 — 기본 루프는 여전히 못 만든다 (다른 원인) - -`asyncio.run()` 은 아직 안 된다. **JSPI 와 무관한 별개의 벽이다:** - - BaseSelectorEventLoop.__init__ -> _make_self_pipe -> socket.socketpair() - -> Emscripten ___syscall_listen -> require('ws') -> MODULE_NOT_FOUND - -Emscripten 은 소켓을 Node 의 `ws` 패키지로 흉내내는데 그것이 설치되어 있지 않다. 이것은 §15.5 -이전에도 있었고 지금도 있다 — Node 22 에서 측정했을 때도 똑같이 났다. 그래서 §15.6 의 마지막 줄이 -**커스텀 루프**다: Pyodide 가 같은 문제를 푸는 방식이고, selector 도 self-pipe 도 열지 않으므로 -두 벽을 모두 비켜간다. - -`proxyBootstrapSupportsAsyncio` 는 **`false` 로 둔다.** `import asyncio` 는 이제 되지만 그 상수가 -게이트하는 테스트는 *기본* 이벤트 루프를 만들고 `create_future` 를 세므로, 그 경로는 아직 없다. -상수를 뒤집으려면 라이브러리가 커스텀 루프를 제공할지부터 정해야 하고 — 그것은 §7 의 프록시 설계가 -아니라 새 결정이다. 여기서 하지 않는다. **§9.5 의 "wasm 에서 (C) 는 첫 걸음에서 없다" 는 이제 -틀렸다. 첫 걸음은 있다. 없는 것은 기본 루프다.** - -### 15.8 `ws` 를 실제로 넣어 봤다 — (A) 는 패키지 문제가 아니라 구조적으로 막혀 있다 - -§15.7 이 연 질문("`ws` 를 넣으면 그것만으로 되는가")에 실제로 답했다. `wasmJsTest` 에 -`npm("ws", "8.18.0")` 을 임시로 추가하고, 격리된 프로브 파일(`WasmAsyncioRunProbeTest`, 커밋하지 -않음)에서 `asyncio.run()` 을 호출해 관측했다. - -**한 걸음 나아갔다.** `MODULE_NOT_FOUND` 로 프로세스가 죽던 것이, `ws` 를 넣자 진짜 파이썬 -예외로 바뀌었다: - - OSError: [Errno 28] Invalid argument - at socket.py:633, _fallback_socketpair -> lsock.accept() - -**그리고 거기서 막힌다 — 근본적으로.** CPython 이 임베드하는 Emscripten 글루(`python.mjs`)를 -직접 읽어 원인을 확인했다. `SOCKFS.websocket_sock_ops` 는 소켓을 실제 `ws` 연결로 흉내내는데, -`listen()` 은 `require("ws").Server` 로 진짜 `WebSocketServer` 를 열고, `connect()` 는 진짜 -`WebSocket` 클라이언트를 연다 — 루프백이라도 실제 TCP 핸드셰이크와 WebSocket 업그레이드가 오간다. -`accept()` 의 구현은 이렇다: - - accept(listensock){ - if (!listensock.server || !listensock.pending.length) { throw new FS.ErrnoError(28) } - ... - } - -즉 **핸드셰이크가 끝나 `pending` 에 연결이 들어와 있어야 `accept()` 가 성공한다.** 그런데 -`_fallback_socketpair()` 는 `bind() -> listen() -> connect() -> accept()` 를 전부 한 C 함수 -안에서, 하나의 동기 wasm 호출로 순서대로 부른다. §15.3 이 이미 적어 둔 그대로 이 임베딩은 `main` 을 -부르지 않으므로 이 호출들 사이에 **Node 이벤트 루프가 틱을 돌 기회가 없다** — `connect()` 가 시작한 -WebSocket 핸드셰이크는 다음 매크로태스크가 되어야 진행되는데, 그 다음 매크로태스크가 올 시점에는 -이미 `accept()` 가 (그리고 그 바깥의 동기 wasm 호출 전체가) 끝나 있다. `pending` 은 언제나 비어 -있고, `accept()` 는 언제나 EINVAL 이다. - -**이것은 `ws` 가 없어서 나는 에러가 아니다.** `ws` 를 설치하면 1차 장벽(`require` 실패)은 없어지고 -2차 장벽(핸드셰이크와 동기 호출의 경쟁)이 드러날 뿐이며, 2차 장벽은 **어떤 npm 패키지로도 못 -넘는다** — `socketpair()` 를 웹소켓으로 흉내내는 구현 자체가 비동기이고, 이 라이브러리의 부트업 -모델(모든 호출이 Kotlin 에서 동기 wasm 호출로 도착) 은 정확히 §15.3 이 JSPI 를 꺼야 했던 이유와 -같은 이유로 그 비동기성을 담을 곳이 없다. JSPI 를 다시 켜도 소용없다 — 켜져 있었다면 `main` 을 -부르지 않는 한 애초에 `SuspendError` 로 죽었을 자리다(§15.3). 그러므로 (A) 는 "아직 안 해서 안 되는 -것"이 아니라 **이 임베딩 모델과 구조적으로 상충한다.** - -**브라우저에서는 더 나쁘다.** 여기서 확인한 것은 Node 전용 경로다 — `require("ws")` 는 -`ENVIRONMENT_IS_NODE` 분기에서만 있고, 브라우저 분기는 전역 `WebSocket` 을 그대로 쓴다. 즉 -브라우저에서 `listen()` 자체가 없다(`if(!ENVIRONMENT_IS_NODE){throw new FS.ErrnoError(138)}`) -- -`_fallback_socketpair` 가 루프백 서버를 열 수조차 없다. 설령 (A) 가 Node 에서 성립했더라도 그것은 -**Node 테스트 환경 전용 해법**이었을 것이고, 브라우저 배포에는 애초에 적용되지 않았다. 이번 측정으로 -그 구분조차 낼 필요가 없어졌다 — Node 에서도 막히기 때문이다. - -측정 후 `ws` 의존성과 프로브 파일은 되돌렸다(`kotlinWasmUpgradeYarnLock` 으로 lockfile 도 원복). -얻는 것 없이 npm 의존성만 늘리는 선택이기 때문이다. - -**결론 — (C).** 세 선택지 중 남는 것은 (B) 커스텀 루프를 라이브러리가 제공하는 것과, (C) 아무것도 -하지 않고 임베더에게 맡기는 것 둘뿐이다. (B) 는 §15.6 의 `_PmpMiniLoop` 로 *메커니즘은* 이미 -증명되어 있다 -- selector 도 self-pipe 도 열지 않는 루프는 코루틴을 완주시킨다. 그러나 `asyncio.run()` -이 기대하는 것은 그 최소 루프가 아니라 `Runner`/`BaseEventLoop` 계약 전체(`call_later`, -`shutdown_asyncgens`, `shutdown_default_executor`, 예외 처리 프로토콜, `KeyboardInterrupt` 취소 등)이고, -그것을 CPython 버전에 맞춰 계속 맞는 상태로 유지하는 것은 "라이브러리가 파이썬 런타임을 흉내내는" -바로 그 비용이다. 이번 조사가 확정한 것은 asyncio.run() 을 막는 벽의 **정체**(비동기 소켓 흉내가 -동기 부트업과 상충)이지, 그 벽을 대신할 만큼 신뢰할 수 있는 대체 루프의 존재가 아니다. 그러므로 -`proxyBootstrapSupportsAsyncio` 는 **여기서는 뒤집지 않는다.** 커스텀 루프를 제공할지는 여전히 별도 -결정이고, §15.6 이 이미 증명한 최소 메커니즘 위에서 언제든 다시 집어들 수 있다. diff --git a/docs/design/upcall-design.md b/docs/design/upcall-design.md deleted file mode 100644 index db542912d..000000000 --- a/docs/design/upcall-design.md +++ /dev/null @@ -1,1018 +0,0 @@ -# 업콜 설계 — Python → Kotlin - -Python 코드가 Kotlin 함수·클래스를 가져다 실행하고, 인스턴스화하고, 상속하는 방향. 반대 방향은 -[`downcall-design.md`](downcall-design.md). - -무엇을 노출할지는 [`binding-policy.md`](binding-policy.md). - -**현재 미구현이다.** 이 문서는 확정된 설계와, 아직 열려 있는 지점을 구분해 기록한다. - -## 런타임 리플렉션을 쓰지 않는다 - -이유가 두 겹이고, 두 번째가 더 근본적이다. - -1. **GraalVM Native Image** 는 closed-world 라 런타임 리플렉션이 등록 없이는 동작하지 않는다. -2. **Kotlin/Native 에는 리플렉션이 사실상 없다.** `::class.simpleName` 정도이고 멤버 열거가 안 된다. - iOS 와 androidNative 는 GraalVM 과 무관하게 애초에 리플렉션 경로가 없다. - -멀티플랫폼 라이브러리인 이상 **빌드 타임에 생성한 테이블 말고는 선택지가 없다.** - -## 구조 - -``` -Python 사용자 코드 obj.baz(1, 2) - ↓ -프록시 타입 우리가 PyType_FromSpec 으로 만든 힙 타입. - 인스턴스 데이터에 핸들을 담고 있다. - ↓ -업콜 진입점 (플랫폼당 1개) invoke(handle, args) - ↓ -Kotlin 함수 테이블 핸들 → 이미 해석된 호출 대상 -``` - -Python 사용자 코드에는 문자열도 정수도 드러나지 않는다. - -## 핸들 해석은 Kotlin 이, 캐시는 프록시 객체가 - -ObjC 가 빠른 이유는 테이블이 있어서가 아니라, **셀렉터가 인터닝된 포인터라 매번 문자열을 찾지 않기 -때문**이다. `objc_msgSend` 는 클래스 메서드 캐시에서 몇 개 명령어로 IMP 를 찾아 점프한다. - -같은 성질을 이렇게 얻는다: - -| 시점 | 하는 일 | -|---|---| -| 최초 1회 | Python 이 이름(`"Bar.baz"`)을 넘기고, **Kotlin 이 테이블에서 해석**해 핸들을 돌려준다 | -| 이후 매 호출 | 프록시 객체가 자기 인스턴스 데이터에 담아둔 핸들을 그대로 넘긴다 | - -핵심은 **호출마다 문자열을 넘기면 어떤 테이블 설계로도 ObjC 수준에 도달할 수 없다**는 것이다: - -| 방식 | 호출당 비용 | -|---|---| -| 매번 문자열 전달 | `PyUnicode` → UTF-8 변환(할당) + FFI 경계 + 해시 + 문자열 비교 → 수백 ns | -| 최초만 해석, 이후 핸들 | FFI 경계 + 테이블 조회 → 10~20ns 목표 | - -테이블 항목은 `KFunction.call` 이 아니어야 한다. `KFunction.call` 은 호출마다 인자 개수와 타입을 -다시 검사하고 박싱해 50~100ns 가 든다. JVM 에서는 `MethodHandle`, Kotlin/Native 에서는 생성된 함수 -참조를 쓴다. 후자는 컴파일 타임에 직접 호출로 낮아진다. - -## 왜 트램폴린이 적어도 되는가 - -CPython 콜백 규약은 **전부 관련 객체를 인자로 넘긴다.** 헤더에서 확인했다: - -| 슬롯 | 시그니처 | 식별 정보 | -|---|---|---| -| `PyCFunction` | `(PyObject *self, PyObject *args)` | `self` | -| `destructor` | `(PyObject *self)` | `self` | -| `getter` / `setter` | `(PyObject *self, void *closure)` | **명시적 closure 인자** | -| `initproc` | `(PyObject *self, args, kwargs)` | `self` | -| `newfunc` | `(PyTypeObject *type, args, kwargs)` | `type` | - -즉 **라우팅을 함수 포인터가 아니라 데이터로 한다.** 소수의 공유 트램폴린이 무제한의 Kotlin 대상을 -서비스할 수 있고, **런타임 코드 생성(LLVM JIT, libffi 클로저)이 필요 없다.** JPype·PyO3·JEP 가 전부 -이 방식이다. - -업콜 shape 는 4종으로 수렴하며, 그 4종은 다운콜의 14종에 **이미 포함**된다. 업콜 때문에 추가되는 -트램폴린이 없다. - -런타임 코드 생성이 실제로 필요해지는 경우는 하나뿐이다 — **컨텍스트 인자가 전혀 없는 순수 C 콜백** -(`qsort` 비교자 등)에 Kotlin 함수 포인터를 넘겨야 할 때. CPython 객체 모델에는 해당이 없다. - -## 상속과 인스턴스화 - -`PyType_FromSpec` 으로 힙 타입을 만들면: - -- Python 이 그 타입을 상속하면 `tp_new`/`tp_init` 은 우리 공유 트램폴린을 그대로 물려받는다 -- 오버라이드한 메서드는 그냥 Python 함수다 -- Kotlin 이 가상 메서드를 부를 때는 `PyObject_CallMethod` 가 일반 MRO 조회를 한다 - -**새 네이티브 스텁이 하나도 추가되지 않는다.** - -## 플랫폼별 진입점 - -진입점 뒤는 전부 같은 테이블이다. - -| 플랫폼 | 진입 방법 | -|---|---| -| Desktop | FFM `Linker.upcallStub` | -| Android | `PyMethodDef` whose `ml_meth` is a C shim in `jni_onload.def`, calling Kotlin/JVM through JNI | -| iOS / androidNative | `PyMethodDef` + `PyCMethod_New` (`@CName` 심볼은 androidNative 에서만 유효) | - -원래 여기에는 "iOS 는 Python 과 Kotlin/Native 가 같은 바이너리이므로 `ctypes.CDLL(None)` 으로 `@CName` -심볼에 바로 닿는다"고 적혀 있었다. **측정해 보니 iOS 에서는 양쪽 절반이 모두 성립하지 않는다.** -[아래 표](#what-each-platform-still-owes)에 측정값이 있다. 실제 경로는 -`nativeMain` 의 `python.native.ffi.UpcallEntry` 이며, C 글루가 필요 없다는 결론 자체는 유지된다 — -`PyMethodDef` 를 Kotlin/Native 가 직접 채우기 때문이다. - ---- - -## The trampoline that carries arguments - -Written for desktop, settled for everyone. `python.multiplatform.ffi.upcall.UpcallTrampoline` is -`commonMain` and depends on nothing platform-specific; what each platform still owes is only the -few lines that give Python the *address* of an entry point. - -Before this, the table carried arity and a per-argument `TypeTag` and nothing read them. Both -desktop stubs were `(long) -> long`, so the only entries Python could reach were zero-argument ones -returning a `Long` — Python could call Kotlin but could not pass it anything (ROADMAP §13). - -### The shape count is one, and it was already implied - -Counted the way the downcall side was counted, and it converges harder: **argument passing needs -exactly one new C shape.** - - long pm_invoke(long callableHandle, PyObject *args) -> PyObject * // (long, long) -> long - -Arity and types travel inside the tuple and inside the table entry, never in the C signature. A -stub specialised per signature would need one per `(arity, tag-vector)` — unbounded, and -impossible to pre-generate for a closed world. This is the same argument as -[왜 트램폴린이 적어도 되는가](#왜-트램폴린이-적어도-되는가), applied one level down. - -The remaining CPython slots were already in the vocabulary, which is why the total is one and not -five: - -| Slot | C signature | Carrier shape | Status | -|---|---|---|---| -| `PyCFunction` | `PyObject *(PyObject *self, PyObject *args)` | `(long, long) -> long` | **added** | -| `getter` | `PyObject *(PyObject *self, void *closure)` | `(long, long) -> long` | same one | -| `initproc` / `setter` / `traverse` | `int(long, long, long)` | `(long, long, long) -> int` | already there | -| `inquiry` (`tp_clear`) | `int(PyObject *self)` | `(long) -> int` | already there | -| `destructor` | `void(PyObject *self)` | `(long) -> void` | already there | -| name → handle | `long(const char *)` | `(long) -> long` | already there | - -`newfunc` (`PyObject *(type, args, kwargs)`) is the only one still unaccounted for, and only if -`tp_new` is ever bound directly rather than reached through `PyCFunction`. - -The `PyCFunction` correspondence is deliberate: when the generated proxy type lands, `self` takes -the callable handle's place in the same stub and **no new shape appears then either**. - -### What crosses, per tag - -The `args: Array` an `ExposedCallable` receives has exactly one representation per `TypeTag`, -which is what `python-multiplatform-ksp`'s `TypeShape.castExpression` already narrows from: - -| `TypeTag` | Python side | Kotlin side | -|---|---|---| -| `INT` | `int` | `Long` (a declared `Int`/`Short`/`Byte` is narrowed by generated code) | -| `FLOAT` | `float` | `Double` | -| `BOOLEAN` | `bool` | `Boolean` — a distinct Python type, so not folded into `INT` | -| `STRING` | `str` | `String` | -| `BYTES` | `bytes` | `ByteArray` | -| `UNIT` | `None` | `Unit` | -| `OBJECT` | `int` handle, **or** any Python object | the Kotlin instance from `HandleTable`, **or** a `PyObject` | -| — | `None` | `null`, whatever the tag | - -`OBJECT` is the one tag that is two things, and it is resolved by what Python actually sent rather -than by the tag: Python cannot hold a Kotlin reference on any target, so a Kotlin object crosses as -an `ObjectReference` integer; anything else is a Python object and reaches a parameter declared as -`PyObject`. - -`BYTES` is correct and slow — an item at a time through `PyObject_GetItem`, because -`PyBytes_AsString` is bound here as a NUL-terminated UTF-8 *string* read and destroys exactly the -payloads `ByteArray` exists for. `PyBytes_AsStringAndSize` is in no platform's `EmbedAPI` yet; when -it is, both directions collapse to one call. - -### Two conventions that are not negotiable - -- **Arguments are borrowed.** `PyTuple_GetItem` lends, so a `PyObject` built over one takes - `borrowed = true`. `borrowed = false` gives back a reference nobody ever took — one per call — - and the free lands somewhere unrelated. That is the bug that crashed this repo twice. -- **The result is a new reference.** Python takes ownership. Measured end to end through each - platform's real entry point (`UpcallEntryTest.theReferenceReturnedToPythonIsTakenOverExactlyOnce`, - `commonTest`): a hundred calls move the returned object's refcount by zero. - -And one that is not about references: **nothing may be thrown out of a trampoline.** The return -path is C; a Kotlin exception crossing a Panama upcall stub terminates the VM and on Kotlin/Native -terminates the process. Every failure leaves as `NULL` with the error indicator set. - -### The GIL is not the caller's to promise - -`withGIL` skips `PyGILState_Ensure` when the thread's nesting depth is already non-zero. That is -right for Kotlin-side code and **wrong for an entry point from C**, because C may have dropped the -GIL inside a scope that is still open. `ctypes.CFUNCTYPE` does precisely that — it releases the GIL -around the foreign call, unlike `PYFUNCTYPE` — so the first version of this trampoline segfaulted -in `_PyThreadState_GET` (`PyErr_Occurred+0x1c`) on a thread whose own counter said it held the GIL. - -A trampoline therefore takes its own `PyGILState_Ensure`/`Release` pair unconditionally. Every -platform's entry point needs this, not just desktop's. - -### What each platform still owes - -Only the address-publishing step; the marshalling is shared. - -| Platform | What is needed | Cost | -|---|---|---| -| **Desktop** | done — `Panama.createUpcallStubII_L`, `UpcallStub.invokeWithArgsStubAddr` | — | -| **iOS / androidNative** | done — `python.native.ffi.UpcallEntry` (`nativeMain`), a `PyMethodDef` whose `ml_meth` is a `staticCFunction` over `UpcallTrampoline.invoke` and whose `self` carries the handle. `UpcallEntryTest` (`nativeTest`) runs it on the iOS simulator and, since `androidNativeArm64Test` exists, on `pmp_api26` and `pmp_api36` as well | cheapest of the three | -| **Android** | done — `python.native.ffi.UpcallEntry` / `UpcallCallbacks` (`androidMain`) plus the `pmp_upcall_*` shims in `artMain/cinterop/jni_onload.def`. `UpcallEntryTest` (`androidInstrumentedTest`) runs it on `pmp_api26` and `pmp_api36` | one shim per shape, plus a JNI upcall per call | -| **wasm** | `@WasmExport` on the entry point plus `Table.set` to publish it, measured at 3.1 ns/call in `wasm-experiment` | already proven | - -Nothing in that list touches `UpcallTrampoline`; each is the platform's existing upcall mechanism -pointed at it. - -The "Cost" column above is per-platform prose, not a comparison. For one measured next to the others -— desktop, iOS, wasmJs and Android in the same units, from the same test — see "One upcall, across -all five platforms" below. In particular the wasm row's **3.1 ns is the bare `call_indirect` -mechanism with no arguments** and is not what an upcall carrying an argument tuple costs. - -### Android's boundary runs the other way round, and `RegisterNatives` is not it - -The Android row above used to read "a JNI `static jlong` native method registered with -`RegisterNatives`, and a C shim of `PyCFunction` shape that calls it". `RegisterNatives` binds a -JVM `external fun` to a C function — it is the **downcall** direction, and no arrangement of it -lets C call Kotlin. What Python needs here is the opposite: a `PyMethodDef`'s `ml_meth` has to be a -real C function pointer, and Kotlin/JVM on ART can produce none. - -So on Android the entry points are C functions in `artMain/cinterop/jni_onload.def` -(`pmp_upcall_invoke_meth` and the `_pm_resolve` / `_pm_bind` / `_pm_release` trio), and *they* call -Kotlin, with `CallStaticLongMethod` against `python/native/ffi/UpcallCallbacks` — a class looked up -once in `JNI_OnLoad` and held as a global ref. `RegisterNatives` still appears, but only for the -one cold `upcallPublish(long)` that installs the bootstrap into a namespace dict. - -That is not a new mechanism. It is exactly the inversion the cycle-collecting proxy type already -makes for `tp_traverse`/`tp_clear` (`ProxyCallbacks`), down to reusing `pmp_attach` — so the third -consequence falls out with it: **a Python worker thread is a bare pthread ART has never seen**. -Every `threading.Thread` is one, so `GetEnv` fails and the shim must -`AttachCurrentThreadAsDaemon`. `UpcallEntryTest.anUpcallArrivesOnAThreadCPythonCreatedRatherThanFailingToFindTheJvm` -runs the upcall inside a `threading.Thread` and asserts the thread it landed on is not the -instrumentation thread, so a pass cannot come from `GetEnv` having succeeded after all. - -#### The attach is paid once per thread, and the reason it used to be once per call was wrong - -This paragraph used to end "and detach again on the way out. Skipping the detach is not an option — -ART aborts the process when a thread it knows about exits without detaching." The first run of -`UpcallOverheadTest` priced that decision, and it was the most expensive thing on the Android upcall -path by an order of magnitude: - -| per upcall | API 26 | API 36 | -|---|---|---| -| instrumentation thread (ART already knows it) | 1307–1695 ns | 5307–6693 ns | -| Python worker thread, steady state | 59712–63293 ns | 19820–31219 ns | -| **the attach/detach pair** | **58405–61598 ns** | **14514–24526 ns** | -| upcall / downcall of the same shape, worker | 44.59–47.93x | 11.69–13.23x | - -(Two runs per device, in each table below; where they differ the range is given, because a single -emulator reading is not a measurement.) - -The justification does not survive contact with ART. `Thread::ThreadExitCallback` -(`runtime/thread.cc`, same shape in `android-8.0.0_r1` and in `main`) logs a warning on its first -invocation and only reaches `LOG(FATAL)` on a second one — and the only thing that arms a second -invocation is a `pthread_setspecific` in an `#else` branch that Android does not compile, because -on Android it restores `__get_tls()[TLS_SLOT_ART_THREAD_SELF]` instead. Bionic clears a key's value -before running its destructor and never re-reads it, so the callback fires at most once and the -abort is unreachable. - -Read, then run: `UpcallThreadAttachTest.aThreadThatExitsWithoutDetachingLeaksItsPeerRatherThanAbortingArt` -attaches a bare pthread as a daemon, lets it exit without detaching, and continues — on both -`pmp_api26` and `pmp_api36`, with no `Native thread exiting without having called -DetachCurrentThread` line in logcat at all. - -**The detach is still mandatory, for a different reason.** The same test then asks whether the peer -became reclaimable, and it did not: an attachment that is never given back leaves a -`java.lang.Thread` in the thread list as a GC root, with a recorded stack the pthread has already -unmapped. So the detach moved from call exit to *thread death* — a `pthread_key_create` destructor -(`pmp_thread_exit_detach`), which is the case ART's own warning text points at. It defers to its -second invocation rather than detaching on its first, so it cannot run before ART's own exit -callback whatever order bionic visits the keys in. - -Re-measured, same test, same devices: - -| per upcall | API 26 | API 36 | -|---|---|---| -| instrumentation thread | 1280–1327 ns | 2982–5086 ns | -| Python worker thread, steady state | **1209–1329 ns** | **2301–3086 ns** | -| worker minus instrumentation thread | −118 to +49 ns | −681 to −2000 ns | -| upcall / downcall of the same shape, worker | 44.59–47.93x → **0.99–1.09x** | 11.69–13.23x → **2.00–2.34x** | - -The attach is no longer visible in the steady state at all: on both devices the difference between -a Python worker and a thread ART already knows now straddles zero, i.e. it is inside the run-to-run -spread rather than being a cost. On API 26 an upcall from a Python worker is the same price as a -downcall of the same shape. - -> **Both blocks of absolute figures above and below are superseded: `UpcallOverheadTest` now warms -> 100 000 calls, not 3 000, and every number in this section was taken at 3 000.** They are kept as -> the record of how the defect was found, not as current costs, and the rows below are quoted from -> `409da6fc` and its five-suite re-run rather than re-measured here. -> -> **Re-run unchanged over five full suites per emulator, this table's absolute figures have fallen -> a long way and its API 26 conclusion has weakened.** At the time, `UpcallOverheadTest` still warmed -> 3 000 calls, and since `09bf2397` put `commonTest` on the device it ran *after* -> `UpcallBoundaryCostTest` had driven several hundred thousand upcalls through the same ART process, -> so ART's JIT was warm before it started. That is the observation the warmup fix was made from: the -> two rows below differ by a factor of four with **the same warmup and the same emulator**, and the -> only variable was how much ran first. -> -> | re-run, 5 suites each | API 26 | API 36 | -> |---|---|---| -> | instrumentation thread | 1085–1273 ns (was 1280–1327) | 1032–1229 ns (was 2982–5086) | -> | Python worker, steady state | 1249–1334 ns (was 1209–1329) | 982–1219 ns (was 2301–3086) | -> | worker minus instrumentation | **+43 to +249 ns** (was −118 to +49) | **−50 to +12 ns** (was −681 to −2000) | -> | first upcall on a fresh worker | 88 250–242 375 ns | 41 208–611 416 ns | -> -> **On API 36 the claim survives and sharpens** — the gap is −50 to +12 ns, tight around zero. -> **On API 26 it no longer straddles zero**: five runs out of five put the worker above the -> instrumentation thread, by 43–249 ns. That is small against a ~1250 ns call and it does not restore -> the per-call attach this section disproved — the old 44.59–47.93x is long gone — but the honest -> statement for API 26 is now "a worker costs a little more", not "the difference is inside the -> spread". The first upcall on a fresh worker remains enormous and enormously variable, which is -> where the attach actually shows up. Read these rows against each other, not against the boundary -> table's, and see that table's warmup footnote. -> -> **What survives the warmup change and what does not.** The *differences* and *ratios* here are -> within-run comparisons — worker against instrumentation thread, upcall against downcall, both -> measured in the same process at the same warmup — so raising the warmup moves both sides together -> and they are the quotable quantities, exactly as they were for wasm in `7e9c6b8c`. The absolute -> nanosecond columns are not: they were read three tiers early and are expected to fall when this -> section is re-measured at 100 000. The structural assertion the test actually enforces — one ART -> thread per Python worker, not one per call — is a count and is unaffected by either. - -**The first upcall on each worker still pays the attach in full** — 56–142 µs, larger than the -per-call charge it replaced — and that is the honest shape of the cost now: once per thread, not -once per call. A worker that upcalls once is no better off than before; a worker that upcalls in a -loop is 20–50x better off. - -#### What holding the attachments costs, and why it is not a new risk - -Keeping a thread attached makes ART's collector responsible for it: the peer is a GC root, the -thread carries a JNI local reference table, and every `SuspendAll` walks the thread list. So "what -if there are hundreds of Python workers?" is a fair question to ask of this change. - -It is largely the wrong question, because **peak simultaneous attachments do not change.** Under -the per-call scheme every worker that is upcalling at a given instant is attached at that instant; -what the destructor changes is how long each attachment is *resident*, not how many can exist at -once. The ceiling either way is the number of live Python threads, which the application already -pays for in pthread stacks. - -What does change is that an attachment now outlives the call, so the release has to be reliable. -`UpcallThreadAttachTest.manyConcurrentWorkersAreEachAttachedOnceAndAllReleased` holds 32 Python -workers on a `threading.Barrier` so that all 32 are attached simultaneously, checks each one sees a -single ART thread across the barrier and that no two share one, and then requires every peer to -become reclaimable once the workers finish. Green on both devices. - -The instrument for that matters as much as the result. It reads GC reachability of the peer, not -`ThreadGroup.enumerate`, because the first version did use `enumerate` and it is **unusable on API -26**: it does not report the peer of an attached native thread even while that thread is running and -upcalling, so every release assertion passed while measuring nothing. A positive control — assert -the peer is *un*reclaimable while its worker is deliberately parked mid-upcall — is what caught it, -and it is kept in the test for that reason. - -The GIL rule is unchanged and is what makes this safe: CPython holds the GIL when it calls a -`PyCFunction`, so the C shims take nothing, and `UpcallTrampoline` still takes its own -`PyGILState_Ensure`/`Release` pair unconditionally on the Kotlin side. `PyGILState_Ensure` is bound -as **ordinary JNI** on Android (`PyGILState_EnsureN`), which is what keeps a thread blocked on the -GIL from stalling the JVM collector — a `@FastNative` or `@CriticalNative` binding of it would -deadlock the two runtimes against each other the moment upcalls exist. `androidMain/README.md` -step 3 says this in the abstract; this is the path that makes it concrete. - -Cost, which the row's "one shim per shape" understated: one JNI upcall per call, plus an attach the -*first* time Python calls from a thread it created. Both are now measured — see the table above and -`UpcallOverheadTest`. Neither is on the marshalling path. - -### One upcall, across all five platforms - -Everything above prices Android, because Android had a question the others do not: whether the -attach a Python worker needs is paid once per thread or once per call. That left the upcall path -measured on exactly one target, while the downcall path has been comparable across all of them since -`overhead/BenchmarkTest` — so "is the upcall expensive?" had no answer that could be given per -platform. - -`UpcallBoundaryCostTest` (`commonTest`) is `UpcallOverheadTest` with the attach half removed: same -loop shape, the same three baselines measured in the same run (plus the two controls the next heading -introduces), and the same printed-not-asserted policy. The one thing it no longer inherits is the -warmup count — it ran 10 000 iterations after 3 000 of warmup, copied from the Android test, and -3 000 turned out to be the defect rather than a detail. It is now 10 000 after **100 000**. It sits -in `commonTest` because every seam it needs is already common — `bindUpcallOrNull` (the per-platform -binding step `UpcallEntryTest` introduced), `UpcallTrampoline`, and the `expect` C API — so one copy -runs on every target. - -> **This table is hand-cut and is a record of the round it was taken in.** The same rows, taken by -> a harness that runs all six targets from one command and writes the conditions next to the -> numbers, are in [`cost-table.md`](../investigations/cost-table.md) — re-cut with -> `./benchmarks/cost-table.sh --runs 3` on a quiet machine. Prefer that one when the two disagree: -> it records the commit, the warmup, the load average and the device identity of its own cut, and -> this one cannot, because those were transcribed by hand and only partly. - -| Platform | upcall | downcall, same shape | trampoline alone | upcall / downcall | upcall / trampoline | -|---|---|---|---|---|---| -| **desktop** (JVM 21.0.12, macOS arm64) | 542.84–576.45 ns | 260.27–265.81 ns | 170.44–247.42 ns | 2.04–2.20x | 5.54–6.46x | -| **wasmJs** (Node) | 302.41–310.72 ns | 96.53–100.95 ns | 149.50–164.88 ns | 3.07–3.14x | 2.39–2.46x | -| **iOS simulator** (iOS 26.2, arm64) | 2253.20–2312.97 ns | 1585.46–1655.40 ns | 2794.42–2843.16 ns | 1.39–1.42x | 1.14–1.15x | -| **androidNative** (`pmp_api36`, arm64) | 3348.75–3443.23 ns | 2194.26–2365.61 ns | 3957.01–4069.40 ns | 1.45–1.54x | 1.19–1.20x | -| **androidNative** (`pmp_api26`, arm64) | 3234–3310 ns † | 2470–2554 ns | 2731–3228 ns | 1.28–1.32x | 1.02–1.20x | -| **Android ART** (`pmp_api36`, arm64) | 995.61–1154.10 ns ‡ | 804.27–838.60 ns | 1100.87–1167.17 ns | 1.18–1.43x | 1.10–1.29x | -| **Android ART** (`pmp_api26`, arm64) | 1196.52–1272.28 ns ‡ | 1253.41–1295.79 ns | 1800.86–1863.97 ns | 0.92–1.01x | 1.05–1.18x | - -_For rows 1–4 and 6–7: Source is `docs/investigations/cost-table.md` (commit `958c0082b294`), `UpcallBoundaryCostTest` with warmup 100,000, min–max over 3 runs per target (except desktop, wasmJs, iosSimulatorArm64 and androidNativeArm64 cut from cut-hostless, artApi36 from cut-api36, artApi26 from cut-api26 — see that document's "Conditions" table for per-cut details)._ - -**Prior values (warmup 3,000):** desktop 510–560 ns, wasmJs 290–304 ns, iOS simulator 2266–2301 ns, androidNative pmp_api36 3228–3275 ns, androidNative pmp_api26 3234–3310 ns, ART pmp_api36 944–1063 ns, ART pmp_api26 1146–1233 ns. - -† androidNative `pmp_api26`: This row was re-measured to 100,000 warmup but does not appear in `cost-table.md` (only `androidNativeArm64` on `pmp_api36` is there). **Still awaiting re-measurement** under the same conditions as the cost-table harness. - -‡ **Android ART rows (`artApi26`, `artApi36`):** Marked "failed" in cost-table.md. Those failures are in `PhantomCleanerRegistryTest` (reference-release path), not the benchmark rows themselves. The benchmark measurements of boundary cost are valid. Failures were fixed in `c180c46f`. - -**Every row is now measured at a warmup large enough to mean anything; getting there is why the table -was re-cut twice.** `UpcallBoundaryCostTest` warmed 3 000 calls per row -and every figure it had ever published was taken before the host JIT had finished tiering up, so the -number it printed was a function of how much upcall traffic the *rest of the suite* had pushed -through the same process first. The warmup is now 100 000, chosen from a measured convergence point -rather than picked; the counts, the sweep behind them and the check below are in -[`downcall-design.md`](downcall-design.md) under "The benchmark was measuring the benchmark". - -**These desktop and wasmJs figures agree between a full suite and a suite with the boundary's biggest -other consumer removed** — the exact comparison that used to break them. Running the whole suite with -`GeneratedProxyCostTest` short-circuited, so it drives none of its ~270 000 calls, moves desktop's -upcall row from 510–560 ns to 501–530 ns and wasmJs's from 290–304 ns to 287–301 ns; both overlap. -Under the old 3 000-call warmup the same lever moved desktop from 674 ns to 537 ns and wasmJs from -322 ns to 292 ns. **13 desktop runs and 12 wasmJs runs of the full suite, 3 of each with the lever -pulled.** - -A desktop `--tests`-filtered run of this class alone reads 506–550 ns, inside the full-suite band, and -its pure-Python control rows are identical to the full suite's (8.4–8.6 ns against 8.4 ns). **wasmJs -is the exception and it is not fixable from inside this test**: filtered to this class alone it reads -321–353 ns, but its *pure-Python* rows — an empty Python loop and a Python callee, with no boundary -anywhere in them — read 23.1 ns and 83.1 ns against the full suite's 15.0 ns and 47.2 ns. CPython is -itself a wasm module there, so a short-lived Node process runs the interpreter's own bytecode 55–74% -slower and inflates numerator and denominator alike. Tripling the warmup to 300 000 does not move it, -which is what says the residual is host-lifetime-bound rather than call-count-bound. **wasmJs's ratio -columns survive the filtering (2.66–3.13x filtered against 2.91–3.14x in the suite); its absolute -columns are a full-suite figure and are only meaningful as one.** - -**The five device configurations were checked for the same effect and none of them has it, so wasmJs -remains the only exception.** Three filtered runs of this class alone against the five full-suite -runs above: - -| filtered vs full suite | upcall, filtered | upcall, full suite | empty Python loop | pure-Python callee | -|---|---|---|---|---| -| iOS simulator | 2320–2322 ns | 2266–2301 ns | 9.9–11.4 vs 9.9–10.3 ns | 36.8–38.0 vs 36.8–38.3 ns | -| androidNative `pmp_api36` | 3275–3294 ns | 3228–3275 ns | 17.21 vs 17.22 ns | 48.9–58.2 vs 48.9–74.2 ns | -| androidNative `pmp_api26` | 3245–3280 ns | 3234–3310 ns | 20.0–23.6 vs 20.0–20.6 ns | 51.7–55.3 vs 51.7–62.0 ns | -| ART `pmp_api36` | 996–1031 ns | 944–1063 ns | 17.2–19.3 vs 17.2–22.8 ns | 49.0–60.0 vs 49.2–63.3 ns | -| ART `pmp_api26` | 1118–1171 ns | 1146–1233 ns | 20.0–20.1 vs 20.0–21.5 ns | 51.7–63.4 vs 51.7–65.6 ns | - -**The control columns are the test.** On wasmJs the pure-Python rows moved 55–74% under filtering, -which is what proved the whole interpreter had slowed down; here they are unchanged everywhere, to -the last significant figure on `pmp_api36`'s empty loop (17.21 against 17.22 ns). The upcall columns -overlap their full-suite bands in four of five cases and miss by 1% on iOS (2320–2322 against -2266–2301). **So the device rows' absolute values are quotable as absolute values**, unlike wasmJs's, -and the reason is structural: on these five, CPython is native code the host is merely executing, not -a wasm module the host must itself compile. - -‡ **Each platform threw exactly one outlying run out of eleven, and both are quoted here rather than -dropped**, for the reason `pmp_api26`'s row below is kept — the min–max convention exists to show -when a host's spread is larger than the effect. Desktop's outlier is confined to the GIL-held -trampoline row: 10 of 11 runs sit at 76–91 ns and one read 186.81 ns, which is the single run behind -that ratio column's other value, 2.89x against 5.62–7.02x for the rest. wasmJs's is the opposite -shape — one run inflated *every* boundary row at once (upcall 432.55 ns, downcall 144.64 ns, -trampoline 264.44 ns) while the empty GIL scope stayed at 24.67 ns; the eleven other runs give the bands -above and three deliberate re-runs immediately afterwards read 298.27, 301.73 and 304.47 ns. **Its -ratio column was unaffected — 2.99x, inside the band** — because numerator and denominator moved -together, which is the case for preferring the ratio on that target. - -**Every device row has now been re-measured at the 100 000 warmup.** No row in this table is a -3 000-warmup reading any more, so the ¶ and † marks that carried that caveat are gone from it. The -three Kotlin/Native rows barely moved; the two ART rows moved a great deal. That split is the -finding. - -| re-measured at 100 000 | upcall, old → new | downcall, old → new | -|---|---|---| -| iOS simulator (26.2, arm64) | 2263–2502 → **2266–2301 ns** | 1599–1826 → **1599–1610 ns** | -| androidNative `pmp_api36` | 3339–3598 → **3228–3275 ns** | 2170–2222 → **2155–2229 ns** | -| androidNative `pmp_api26` | 3314–4140 → **3234–3310 ns** | 2636–5554 → **2470–2554 ns** | - -Every new band sits inside or just below its old one, and every one is narrower — `pmp_api36`'s -downcall column is the extreme case, moving from 2170–2222 to 2155–2229, i.e. not moving at all. -Compare what the same change did to the hosts with a JIT: desktop's upcall row fell 674 → 537 ns, -wasmJs's 322 → 292 ns, and ART `pmp_api36`'s 2301–3086 → 944–1063 ns. **The three Kotlin/Native -targets did not fall; all three JIT hosts did.** That is what the tier-up diagnosis predicts rather -than merely permits, and it is the check that separates the two explanations. The defect `7e9c6b8c` -fixed was real, and it is a property of the host's compiler, not of the boundary. - -**But "did not fall" is not the same as "needed no warmup", and the sweep below says so.** It would -be easy to read the flat Kotlin/Native rows as meaning 3 000 was always enough there. It was not: -from cold, androidNative's first 10 000 upcalls read ~1.3x the plateau. The old figures landed near -the plateau anyway because the rest of the suite had already driven the boundary before this test was -timed — which is the *same* dependence on what ran first, just with a smaller amplitude. The old -Kotlin/Native rows were right by luck rather than by margin, and that is why they are replaced rather -than kept. The old ART rows were not right at all. - -> **`7e9c6b8c` raised the warmup in `UpcallBoundaryCostTest` and nowhere else. That has since been -> carried to the two benchmarks that were left behind.** The observation was this: in that round's -> ART logcat, alongside this file's `warmup: 100000`, `UpcallOverheadTest` printed `warmup: 3000`, -> and its API 36 instrumentation-thread row read 1032–1229 ns where `409da6fc` recorded -> 2982–5086 ns — **at the same warmup count and on the same emulator.** Nothing about that test had -> changed; what changed is that `09bf2397` put the whole of `commonTest` on the device, so -> `UpcallBoundaryCostTest`'s own several hundred thousand upcalls ran *before* it and left ART's JIT -> warm. That is precisely the "the number is a function of what ran first" failure this section -> documents, in a test the table does not quote but other sections do. -> -> `UpcallOverheadTest` now uses 100 000 for both its Python-driven loops and its Kotlin-driven -> downcall rows — the latter were on `N / 4`, i.e. 2 500 — and `overhead/BenchmarkTest`, which runs on -> all six targets, now uses 100 000 uniformly in place of a mix of 100 and `Benchmark`'s 1 000 -> default. The value comes from the sweep immediately below: ART `pmp_api36` is the tightest path -> measured and only reaches its plateau at ~90 000–100 000. -> -> **The coroutine/proxy `warmup: 5000` is not an instance of this defect and is deliberately left -> alone.** `GeneratedProxyCostTest` warms nineteen rows before it times any, so the shared -> `_pm_invoke` path receives 19 x 5 000 before the first measurement — which is past the same knee. -> `7e9c6b8c` justified that number when it set it; it is not a leftover. - -#### Is 100 000 enough on a device? Swept, not assumed — and on one target it only just is - -`WARMUP`'s doc justifies 100 000 from a sweep run on desktop and wasmJs only (40 000 cold and 70 000 -warm on desktop, 70 000 on wasm). Devices are slower, so the number was extrapolated rather than -checked. Sweeping them the same way — 40 consecutive reps of 10 000 calls, from cold, nothing warmed -before the first — gives the calls-to-plateau below. The first-rep ratio is that rep divided by the -mean of reps 21–40. - -| target | rep 1 / plateau, Python-driven upcall | flat from | margin at 100 000 | -|---|---|---|---| -| iOS simulator | 1.01x | ~10 000 | 10x | -| androidNative `pmp_api36` | 1.27x | ~40 000 | 2.5x | -| androidNative `pmp_api26` | 1.30x | ~30 000 | 3.3x | -| ART `pmp_api26` | 1.93x | ~20 000, noisily | 5x | -| **ART `pmp_api36`** | **8.56x** | **~90 000–100 000** | **~1x — none** | - -**`pmp_api36`'s ART row is the one to act on.** Its sweep runs 9194.8, 3218.9, 2288.7, 1648.3, -1218.3, 1173.3, 1129.5, 1532.7, 1057.8, 1044.6 ns, against a plateau of 1074.2 — so it is still 9% -high at 70 000 calls and only arrives around 90 000–100 000. **The constant is not wrong there, but it -has no margin**, and a slower device, a colder emulator or a busier host would read the boundary -before it settles. Its Kotlin-driven rows need less: the downcall is flat from ~80 000 and the -GIL-held trampoline from ~50 000, so the Python-driven upcall sets the requirement, as it does -everywhere else. - -The two control rows behave on every device exactly as they did on the two hosts, which is what says -the sweep is measuring the boundary rather than the interpreter: the empty Python loop and the -pure-Python callee are flat from the very first 10 000 calls on all five device configurations -(iOS 10.5 → 10.2 ns and 37.9 → 38.0 ns; ART `pmp_api36` 17.3 → 17.3 ns and 49.7 → 50.4 ns). - -**`UpcallBoundaryCostTest`'s own number is not changed here.** Raising it is a change to a benchmark -and would invalidate the table it was measured for; this pass was a re-measurement. What the sweep -establishes is the fact needed to decide: 100 000 covers every device path measured, and covers -`pmp_api36` on ART by approximately nothing. - -**This sweep is what set the value for the two benchmarks that were still on the old warmup.** ART -`pmp_api36`'s ~90 000–100 000 is the largest requirement of the six configurations, so it is the one -that fixes the constant, and it is why `UpcallOverheadTest` — which runs only on ART, i.e. only on -the tightest host — starts at 100 000 rather than at desktop's ~70 000. The desktop sweep of -`overhead/BenchmarkTest` agrees independently: its worst row, `PyUnicode_AsUTF8 (8 chars)`, reads -557 ns at a 40 000 warmup and 78–84 ns at 70 000 and above, a 7x step in one interval, so 70 000 is -the desktop knee there too and 100 000 is that with margin. - -Ranges are min–max over five runs of the whole suite (four for wasmJs); a single reading is not a -measurement. **The two ART rows are no longer quoted from another test.** They used to be — the † -mark meant "taken from `UpcallOverheadTest` via commit `409da6fc`", which is why three of their five -columns read *not recorded*. Since `09bf2397` put `commonTest` on the device, `UpcallBoundaryCostTest` -itself runs on ART, so all five columns are now filled from the same test, the same run and the same -thread as every other row in the table. The two ART rows are two different emulators, not two runs of -one, and so are the two androidNative rows — the same two emulators, in fact, which is why the ART -and Kotlin/Native figures for one API level can be read against each other. - -**Those ART rows are measured on the instrumentation thread, and that is now the right thread to -measure.** The old † rows quoted the *Python worker thread, steady state* figure instead, because a -worker is the thread ART has never seen and has to attach. The section above is what makes the -instrumentation thread the fair choice: with the attach amortised, the two are close enough that -choosing between them does not decide the row. This round's `UpcallOverheadTest`, re-run five times -per emulator, puts the worker −50 to +12 ns from the instrumentation thread on API 36 and +43 to -+249 ns on API 26 — so on API 26 the instrumentation thread is now the *cheaper* of the two by a -little, and the ART row above is that much optimistic against a `threading.Thread` caller. The first -upcall on a fresh worker is still expensive (88 250–611 416 ns across the two devices) and is still -the number to quote for attach, not for a call. - -**What replacing them changed, and it is not small on API 36.** The old † row said 2301–3086 ns; the -same target now reads 944–1063 ns. API 26 barely moved — 1209–1329 → 1146–1233 ns. The asymmetry is -the tier-up story again, and this time on the target that most nearly settles it: **ART has a JIT, so -it belongs with desktop and wasmJs rather than with the three ahead-of-time rows, and it moved like -them.** Every host in this table with a JIT fell when the warmup was raised; no host without one did. - -※ **`pmp_api26`'s androidNative row is noisy and is left noisy — but it is a different single-run -outlier each time.** In the 3 000-warmup round, one of five runs read 5554 ns for the downcall and -4863 ns for the trampoline where the other four sat at 2636–3245 and 2843–3672, widening its ratio -columns to 0.62–1.34x and 0.71–1.20x. In the 100 000-warmup round that outlier is gone — the downcall -column tightened to 2470–2554 ns — and a new one appeared in a different row: run 5's GIL-held -trampoline read 3227.87 ns where the other four sat at 2731–2761, and that one run is the whole of -what widens the `upcall / trampoline` column from 1.16–1.18x to 1.02–1.20x. Neither is dropped, -because the min–max convention here exists precisely to show that an emulator's spread can be larger -than the effect being measured. **That the outlier moves rows between rounds is the point**: it is -the host, not the row. Read `pmp_api36`'s row for the shape and `pmp_api26`'s for how much confidence -an emulator supports. - -**The history that produced the two roman rows.** This table's original absolute figures were shown -to depend on the rest of the suite rather than on the commit: desktop's 861–1313 ns and wasmJs's -703–1075 ns both fell on re-measurement, in each case with the downcall column standing still. On -wasmJs the cause was pinned to one commit (`4472f83a`) and demonstrated — it made generated proxies -installable there, so `GeneratedProxyCostTest` went from driving zero upcalls to driving ~270 000 of -them through the same Node process *before* `UpcallBoundaryCostTest` is timed, and short-circuiting -that one test put the figure back. Desktop moved for a repository-located reason too, in a different -and non-overlapping window. - -That was diagnosed but not fixed, so the numbers stayed unusable for two more passes. It is fixed -now, and the fix is the warmup count: the same lever no longer moves either platform's figure. The -bisections, the controlled experiment, the 40-rep sweep that identified host JIT tier-up as the -mechanism and ruled CPython-side state out, and the cost of the change are all in -[`downcall-design.md`](downcall-design.md) under "Ratio consistency" and "The benchmark was measuring -the benchmark". - -**The androidNative row was empty because the target had no test *run* task**, only -`androidNativeArm64TestBinaries` — KGP registers an execution task only where it knows how to reach -a host (`KotlinNativeTest` for the build machine, `KotlinNativeSimulatorTest` for simctl), and an -Android device is neither. `commonTest` compiled for the target on every build and had never once -been executed on it. - -`:python-multiplatform:androidNativeArm64Test` (see `build.gradle.kts`) is that missing task. It -pushes `test.kexe` to `/data/local/tmp`, pushes the CPython prefix beside it because -`Py_Initialize()` aborts the process rather than failing without a standard library, runs the binary -under the Kotlin/Native runner's TeamCity logger, and parses the service messages back into JUnit XML -under `build/test-results/androidNativeArm64Test/` so this target is counted the same way as every -other. With no serial given it runs on every connected device whose ABI matches, which is where the -two rows above come from. The figures are five runs of the whole 252-test suite on each. - -**Nothing new broke when the suite finally ran there** — 252 tests, 0 failures, on both emulators, -in all five runs. That is worth stating because the precedent pointed the other way: ROADMAP §11b -attached this same suite to Android/ART for the first time and it died at the 2nd test and again at -the 12th, surfacing two real defects. androidNative shares `nativeMain` with iOS, and iOS has been -running the suite all along, so the shared code was already exercised; what had never been exercised -was `artMain` and the androidNative `cinterop` bindings, and those came up clean. - -#### The trampoline column needed a control before it meant anything - -"Upcall minus trampoline is the boundary" is only true if the two differ by the boundary alone, and -in the obvious arrangement they do not. `UpcallTrampoline` takes its own `PyGILState_Ensure`/ -`Release` pair **unconditionally** — see "The GIL is not the caller's to promise" — so driving it -from a bare Kotlin thread pays a real GIL acquisition per call, while Python drives it already -holding the GIL. The first run made that concrete: on iOS the trampoline alone came out *more* -expensive than the whole upcall through it, i.e. the boundary priced negative (0.77–0.81x). -**Re-measuring at the 100 000 warmup reproduces it, and on every device configuration rather than -just iOS**: five runs each give 0.79–0.82x on the iOS simulator, 0.82–0.85x and 0.77–0.78x on the two -androidNative emulators, and 0.86–0.95x and 0.65–0.70x on the same two under ART. The negative price -is a property of the unconditional GIL pair, not of the old warmup — and the reason it never shows on -desktop or wasmJs is visible in the table below: their GIL scope is 24–57 ns against 276–1221 ns on a -device, so the uncontrolled acquisition is a rounding error there and the dominant term here. - -So it is measured twice, differing in exactly that one thing, with an empty `Python3.withPython { }` -beside them for scale: - -| | desktop | wasmJs | iOS simulator | androidNative `pmp_api36` | androidNative `pmp_api26` | ART `pmp_api36` | ART `pmp_api26` | -|---|---|---|---|---|---|---|---| -| trampoline, caller holding nothing | 164–181 ns | 146–164 ns | 2794–2862 ns | 3789–3893 ns | 4174–4267 ns | 1093–1148 ns | 1693–1822 ns | -| trampoline, GIL already held | 76–91 ns ‡ | 122–129 ns | 1962–1990 ns | 2699–2760 ns | 2731–3228 ns ※ | 793–907 ns | 1012–1087 ns | -| `Python3.withPython { }`, empty | 50–57 ns | 24–28 ns | 720–731 ns | 932–1002 ns | 1112–1221 ns | 276–303 ns | 498–549 ns | - -The **GIL-held** row is the one the table above uses, because it is the one the Python-driven -numerator is comparable with. - -All seven columns are now at the 100 000 warmup. The re-measurement changed these -rows by more than it changed the headline: desktop's empty `withPython` scope went from 124–152 ns to -50–57 ns and wasmJs's from 79–81 ns to 24–28 ns, because a GIL round trip is the cheapest thing in -the report and therefore the thing an unwarmed loop overstates most. **This is why the old table's -"boundary priced negative" problem looked worse than it was** — one of the two rows being subtracted -was further from its steady state than the other. - -The ordering also flipped on wasmJs and the flip is real: its GIL-held trampoline (122–129 ns) is now -*more* expensive than desktop's (76–91 ns) even though its empty GIL scope is half the price -(24–28 ns against 50–57 ns), so what wasm pays for is the trampoline body rather than the GIL. - -androidNative is the second target to make the case for this control on its own: without it the -boundary prices negative there too (0.86–0.88x on `pmp_api36`, 0.68–0.86x on `pmp_api26`), for the -same reason as iOS and not because anything about the boundary differs. The gap between its two -trampoline rows — roughly a microsecond — is one uncontended GIL round trip on an emulator, and it -is charged to downcalls exactly as much as to upcalls. - -#### Where the differences actually come from - -**iOS has the highest absolute upcall of the three originally measured here and the lowest ratio, -and neither is about the boundary.** (androidNative, added later, is higher still on both counts of -absolute cost — see below; the explanation is the same one, which is the point.) -There is no runtime boundary on that target at all — a `PyMethodDef` whose `ml_meth` is a -`staticCFunction` in the same binary. What is expensive is the per-call scaffolding every C API call -shares: an empty `withPython` scope costs 725–982 ns there against 50–57 ns on desktop and 24–28 ns -on wasm, and `Py_IncRef + Py_DecRef` — two calls, two GIL scopes, no marshalling — costs 1551–2032 ns -against 110–120 ns and 44–49 ns. (iOS's own figures are the old unwarmed ones and will come down when -that target is re-measured; the gap is wide enough that the conclusion survives, but the multiple is -not quotable.) The denominator is inflated by the same thing as the numerator, -which is exactly why the ratio comes out near 1: an iOS upcall is barely more than an iOS downcall, -and both are expensive for a reason that has nothing to do with upcalls. This is where the work is -if the iOS number is to move, not in `UpcallEntry`. - -**Desktop is the opposite shape.** Its trampoline with the GIL held is 76–91 ns, and the -Python-driven figure is 510–560 ns, so roughly 420–480 ns per call is the Panama upcall stub plus the -`ctypes` shim in front of it — a real boundary, and the largest one measured here in *relative* -terms (5.62–7.02x). Both halves came down when the warmup was fixed and the *ratio went up*, because -the trampoline row had further to fall than the Python-driven one; the shape of the conclusion is -unchanged and its size is larger than previously recorded. One caveat is genuine and is not measured -away: on desktop `_pm_bound` is a -Python `lambda *a: _pm_invoke(_pm_h, a)` over a `ctypes.CFUNCTYPE`, where the other targets bind a -`PyCFunction` directly. Desktop therefore pays one extra Python call (26.7–27.2 ns, printed as the -pure-Python callee) plus ctypes' own argument conversion inside every figure in its row. **Desktop's -row is an upper bound on its boundary cost**, not a measurement of the boundary alone. - -**wasm's 3.1 ns is not this number, and was never claiming to be.** The "already proven" row in the -platform table quotes `wasm-experiment`'s `call_indirect` figure, which is the *mechanism* with no -arguments. The argument-passing path measured here is 290–304 ns, of which 122–129 ns is the -trampoline; the remaining ~170–180 ns is the Python-side callable plus the crossing with a real -argument tuple. Two orders of magnitude between the two is not a contradiction — they measure -different things — but the 3.1 ns figure must not be quoted for an upcall that carries arguments. - -**wasm's scaffolding is the cheapest and by far the most stable of the three** (24.47–27.52 ns for an -empty scope across eleven runs), which follows from there being no OS thread machinery under it. That -stability is within one suite scope, though, and does not extend across scopes: the same row reads -34–37 ns when this class is run on its own, for the process-lifetime reason the table's note gives. - -**androidNative is iOS's shape, moved onto an emulator.** It has the highest absolute upcall -measured here (3.3–4.1 µs) and, on `pmp_api36`, a ratio of 1.52–1.62x — and neither figure is about -the boundary, because on this target there is no boundary either: the same `nativeMain` -`PyMethodDef` whose `ml_meth` is a `staticCFunction` in the same binary. What it shares with iOS is -the reason both are expensive: an empty `withPython` scope costs 937–1010 ns here against 50–57 ns -on desktop, and `Py_IncRef + Py_DecRef` costs 2010–2132 ns against 110–120 ns. (As with iOS, the -androidNative side of both comparisons is an old unwarmed reading.) The per-call -scaffolding is the whole story on both native targets, and it is where the work is if either number -is to move. - -Two differences from iOS are worth keeping separate from that. Its ratio sits *above* iOS's -1.33–1.43x rather than at it, because its downcall denominator (2170–2222 ns) is cheaper relative to -its upcall than iOS's is — the numerator is inflated by scaffolding on both, the denominator less so -here. And these figures are from an emulator on Apple Silicon, not hardware: `pmp_api26`'s spread -(above) is the honest width of that, and no androidNative figure here should be quoted as a -device number. - -Nothing in the test asserts a duration. A wall-clock threshold on a shared build machine, an -emulator or a Node host is a flake generator, and this repo has had exactly that failure; what is -asserted is structural — the name bound, the loop demonstrably reached Kotlin, and every figure came -back positive rather than zero from a clock with no resolution, which is the failure mode that would -otherwise report the upcall as free. - -### What a user actually calls: the generated proxy, priced against that boundary - -Everything above measures `_pm_invoke(handle, args)` called straight from Python. **Nobody writes -that.** What a user writes is `demo.calc.ping()`, `Counter(10)`, `c.increment(5)`, `c.value`, -`Counter.created`, `demo.calc.tally`, `await g.fetch()` — and each of those goes through a layer of -generated Python (`PythonProxySource`) that the boundary figure says nothing about: a global lookup, -a bound-method dispatch, an argument tuple built one element longer to carry the receiver, a -`property` descriptor, a metaclass descriptor, or an `async def` wrapper with a `hasattr` in it. That -layer had never been measured. - -`GeneratedProxyCostTest` (`commonTest`) is `UpcallBoundaryCostTest`'s design applied one level up: -every baseline in the same run, identical warmup in identical shape, nothing asserting a duration. -Every row is a zero-argument Python function called by one shared loop, so each pays exactly one -identical call frame; the row **net of that floor** is the body. **Each proxy row is priced against -the raw `_pm_invoke` call it wraps**, measured through the same object the proxy calls through, so -the difference is the generated layer and nothing else. - -One departure from `UpcallBoundaryCostTest`: each row is the **minimum of three timed loops**, not -one. The quantity here is a *difference* of tens of nanoseconds sitting on a boundary of hundreds, -and wall-clock noise is one-sided — it lands entirely in that difference. With a single loop per -row, four consecutive runs put the instance-method delta at −17, −31, −60 and −71 ns, i.e. the proxy -came out cheaper than the boundary it wraps, every time. - -Ranges are min–max over three runs. **The last row of each table is this report's own resolution**, -and it is measured rather than assumed: two of the raw rows are the *same call* (`_pm_invoke(h, ())`, -arity zero, different handle), so whatever they differ by is what cannot be resolved. A delta below -it is not a number. - -| Surface a user writes | raw boundary | through the proxy | ratio | the proxy layer adds | -|---|---|---|---|---| -| **desktop** (JVM 21.0.12, macOS arm64) | | | | | -| `demo.calc.ping()` module function | 452–475 ns | 468–485 ns | 1.00–1.03x | +5…+17 ns | -| `c.increment(5)` instance method | 514–558 ns | 537–573 ns | 1.02–1.05x | +14…+27 ns | -| `c.value` property read | 495–527 ns | 509–542 ns | 1.02–1.03x | +14…+19 ns | -| `c.label = 'x'` property write | 522–548 ns | 569–609 ns | 1.09–1.11x | +47…+62 ns | -| `Counter.created` static read (metaclass) | 449–485 ns | 480–510 ns | 1.01–1.07x | +6…+34 ns | -| `Counter.created = 12` static write | 466–507 ns | 511–573 ns | 1.07–1.13x | +34…+66 ns | -| `demo.calc.tally` top-level read | 449–485 ns | 484–521 ns | 1.03–1.07x | +14…+36 ns | -| `demo.calc.tally = 9` top-level write | 466–507 ns | 520–553 ns | 1.08–1.11x | +44…+54 ns | -| `Counter(10)` constructor | 682–761 ns | 588–627 ns | — | *not separable, see below* | -| *resolution of this table* | | | | *−3…+9 ns* | -| **iOS simulator** (arm64) | | | | | -| `demo.calc.ping()` module function | 1801–1890 ns | 1829–1895 ns | 1.00–1.01x | +6…+28 ns | -| `c.increment(5)` instance method | 2901–2987 ns | 2895–3029 ns | 0.98–1.01x | −44…+42 ns | -| `c.value` property read | 2504–2601 ns | 2510–2620 ns | 0.99–1.00x | −3…+19 ns | -| `c.label = 'x'` property write | 2832–2940 ns | 2925–3092 ns | 1.03–1.05x | +92…+153 ns | -| `Counter.created` static read (metaclass) | 1838–1931 ns | 1906–1982 ns | 1.01–1.03x | +28…+68 ns | -| `Counter.created = 12` static write | 2180–2315 ns | 2277–2365 ns | 1.02–1.04x | +50…+103 ns | -| `demo.calc.tally` top-level read | 1838–1931 ns | 1927–1981 ns | 1.02–1.05x | +48…+99 ns | -| `demo.calc.tally = 9` top-level write | 2180–2315 ns | 2286–2363 ns | 1.02–1.04x | +48…+105 ns | -| `Counter(10)` constructor | 2591–2724 ns | 2769–2864 ns | 1.05–1.06x | +140…+177 ns | -| *resolution of this table* | | | | *+15…+79 ns* | - -**The headline is that the boundary is the whole cost.** The boundary was 861–1313 ns on desktop and -2263–2502 ns on the simulator when this table was taken, and the generated layer on top of it is tens -of nanoseconds — 1.00x to 1.13x, and on the simulator half the rows are inside the table's own -resolution. The sugar is not where the money goes, and a user who bypassed the proxies to call -`_pm_invoke` by hand would recover almost nothing. - -Those two boundary figures are the ones current when this table was measured, and the section above -has since re-measured both — desktop to 510–560 ns and the simulator to 2266–2301 ns. **The -conclusion is unchanged and the arithmetic is not**: this table's own columns were taken in the older -round at that section's older warmup, so it may be read internally (raw against proxy, which is what -the ratio column is) but its absolute figures must not be mixed with the re-measured table's. - -**The await fast path is the same story**, which is the whole claim §8.6 makes for it: - -| | desktop | iOS simulator | -|---|---|---| -| `_pm_invoke(h, (21,))` in the same coroutine loop, no await | 561–617 ns | 3025–3055 ns | -| `await` a **pure-Python** coroutine that never suspends | 67–70 ns | 84.5–84.8 ns | -| `await demo.calc.doubleNow(21)` through the generated `async def` | 618–647 ns | 3186–3239 ns | -| ratio, and what the `async def` wrapper adds | 1.03–1.10x, +24…+57 ns | 1.04–1.06x, +152…+185 ns | -| `asyncio.Future`s constructed during the timed loops | **0** | **0** | - -The zero is asserted, not observed in passing: it is the only way to tell from Python which path was -taken, because the generated `async def` deliberately makes the two indistinguishable to its caller, -and a slow-path figure would be a different measurement wearing the same label. The pure-Python row -is the ceiling on what the wrapper can cost — an `async def` that never suspends is one coroutine -frame — and both targets land under it. - -**`install()` is a startup cost, not a per-call one**, and it is small enough not to need managing. -For this repository's 11-entry, 2-class fixture table it renders 306 lines / 12.4 kB: - -| | desktop | iOS simulator | -|---|---|---| -| `PythonProxySource.render()` — Kotlin string building | 226–271 µs | 606–664 µs | -| `Python3.exec` of the result | 816–927 µs | 2481–2532 µs | -| `install()` end to end | 1.06–1.20 ms | 3.09–3.20 ms | - -Roughly a millisecond on desktop and three on the simulator, once per process. Compiling and -executing the Python is 75–80% of it on both; rendering the text is the cheap half. It scales with -the size of the table rather than with anything else, so an application exposing ten times this -surface should expect ten times this — still tens of milliseconds, against a `Py_Initialize` that -already costs more. - -#### The one row that was expensive, and why - -A **top-level `val`/`var` read cost 551–587 ns more than the boundary call underneath it** on -desktop — 2.16–2.29x, against +6…+34 ns for the metaclass descriptor doing the identical job for a -class static. That is fifty times its sibling for the same work, and it was reproducible to within -±20 ns across every run. - -The cause is where CPython looks first. `PythonProxySource` answered a module attribute with a -`__getattr__` hook on a `ModuleType` subclass, and `__getattr__` is the **fallback** hook: it is -consulted only after normal attribute lookup has already failed, and on a module "already failed" -means `module_getattro` has run to completion and *raised* — building the fully formatted -`module 'demo.calc' has no attribute 'tally'` `AttributeError`, `__spec__` inspection and all — for -the hook to catch and throw away. One raised-and-discarded exception on every read of every exposed -top-level property. - -A standalone probe of the three available shapes, net of an empty call: - -| | ns per read | -|---|---| -| plain module attribute (the floor) | 14 ns | -| PEP 562 module-dict `__getattr__` | 214 ns | -| type-level `__getattr__` hook (what was there) | 569 ns | - -The fix is not PEP 562's middle answer but the shape the class statics already used: a `property` — -a **data** descriptor, consulted *ahead* of the instance dict and in both directions — on a -`ModuleType` subclass generated **per module**. Per module because the descriptor is named after the -Kotlin declaration, and one shared type would answer `demo.calc.tally` on every other proxy module -too; that is the same price the metaclass already pays for a class's statics, and it buys the same -thing. It also removes the `__setattr__` hook outright, because a data descriptor answers the write -half as well. - -| desktop, net of the boundary | before | after | -|---|---|---| -| `demo.calc.tally` read | +551…+587 ns (2.16–2.29x) | +14…+36 ns (1.03–1.07x) | -| `demo.calc.tally = 9` write | +109…+136 ns (1.21–1.29x) | +44…+54 ns (1.08–1.11x) | - -`PythonProxyInstallTest` passes unchanged across the change — including the assertion that a -read-only top-level property *refuses* assignment, which the setter-less descriptor answers with a -message naming the Kotlin declaration exactly as the old `__setattr__` branch did. **Nothing about -the two shapes differs in behaviour**, both read as correct, and no amount of reading the generated -source says which one costs fifty times the other. The measurement is what found it, which is the -argument for having it. - -#### Two things this table cannot say - -**The constructor row is not reportable on desktop.** `Counter(10)` and its raw counterpart both -root a Kotlin object in `HandleTable` per call and nothing releases it, so each is measured against a -table that grows underneath it — the test prints the count, and it ends the run at **78 002 live -handles** from those two rows alone. On desktop the raw baseline moved -between 511 and 775 ns across runs — larger than the effect — and the delta swings either side of -zero; on the simulator it is stable at +140…+177 ns, which is what a `type.__call__` plus an -`__init__` frame plus one attribute store should cost. The desktop figure is a property of the -fixture, not of the constructor proxy. - -**Half the simulator's rows are inside its own resolution** (+15…+79 ns), which is why the -`resolution` row exists. The simulator's boundary is 2.5–3 µs and its per-row variation is ~80 ns, so -a +20 ns proxy layer is genuinely unmeasurable there. Desktop resolves to ±10 ns and is where the -small deltas can be read. - -#### wasmJs was absent from these tables, and that judgement did not survive being tested - -This section read: "**There are no generated proxies on wasmJs, and there structurally cannot be.** -Nothing crosses into Python on that target but an already-*bound* callable — `UpcallEntry.bind` -builds one over the single `@WasmExport`ed `pmp_invoke`, and resolution happens in Kotlin — so -`_pm_resolve`/`_pm_invoke` do not exist in `__main__` and `PythonProxySource.install()` refuses at -its own entry-point guard." - -Every clause of that was true except the one it turned on. The `@WasmExport` constraint is real and -unchanged; what was wrong is the assumption that a second entry point needs a second export. A -`PyCFunction` carries a **`self`** as well as a function pointer, and `PyCFunction_NewEx` mints a -fresh object per call, so one exported pointer already backs arbitrarily many distinct Python -callables — which is exactly what `UpcallEntry.bind` had been doing with a `CallableHandle` all -along. Putting an op code where the handle would sit turns the same pointer into a dispatcher, and -all five names land with no new export. `docs/design/upcall-async-design.md` §14 has the measurement. - -wasmJs now fills the sync rows of this table for real: module function +17 ns (1.05x), instance -method +18 ns (1.05x), property read +17 ns (1.06x), constructor +351 ns (1.48x), against a boundary -of ~300 ns. The one half that did not follow is async: `import asyncio` **traps the instance** on -this wasm build rather than raising, so the await row alone stays behind -`proxyBootstrapSupportsAsyncio` — the precedent `AsyncUpcallPortabilityTest` exists to record. - -Android and androidNative are absent for a different and weaker reason: both run this file (it is -`commonTest`), but doing so needs an emulator, and this pass had none available. Their rows are -missing, not zero. - -### The `@CName` + `ctypes.CDLL(None)` route, measured - -The row above used to read "`@CName` symbol, reached from Python via `ctypes.CDLL(None)`; one -binary, so the symbol is already in the global table". Both halves were checked when the entry -point was actually built, and on iOS **neither one holds**. - -*Is the `@CName` symbol in the binary?* `nm` on each artifact this project produces, for the same -three functions (`pm_upcall_invoke`, `pm_upcall_resolve`, `pm_upcall_release_object`): - -| Binary | Result | -|---|---| -| androidNative `libmultiplatform_python3.14.so` | **exported** (`T` in `nm -D`) | -| iOS `PythonMultiplatform.framework` | **absent** — a Kotlin/Native framework's export set is the Objective-C surface plus the Konan runtime, and a `@CName` alias is not in it | -| Kotlin/Native test executable, either target | **absent**, and absent from the per-file caches as well, so it is never emitted rather than dropped at link time | - -*Can Python call an address at all?* Only where `_ctypes` exists. This project's iOS -`Python.framework` exports `PyInit__abc` … `PyInit_time` and ships no `lib-dynload` at all, so -`import ctypes` raises `ModuleNotFoundError` there. Android's CPython does ship `_ctypes.so`. - -So the route is real for androidNative and unavailable on iOS in both directions at once. The -`@CName` functions are kept because they are what a C host embeds against and what `ctypes` -reaches on Android, and `UpcallEntry.invokeAddress` publishes the same addresses without -depending on the link — `UpcallEntryTest.theRawEntryPointsAreCallableCFunctionsOfTheDocumentedShape` -calls all three through the C ABI that way. - -What replaces it is not a workaround but the destination: a `PyMethodDef` is what the generated -proxy type installs anyway, and desktop's `ctypes` shim (in `UpcallEntryTest`, `commonTest`) exists -precisely to stand in for a `PyCFunction` slot. Kotlin/Native fills the struct itself, so the claim -that no C glue is needed survives — it was the reason for the claim that did not. - -## 테이블 생성 — KSP - -- 우리가 **KSP 프로세서를 아티팩트로 배포**한다 -- 사용자가 자기 빌드에 `ksp(...)` 를 추가한다 — **사용자 모듈에서도 KSP 가 돌아야 한다.** 사용자의 - Kotlin 클래스를 Python 에 노출하는 것이 목표이므로 필수다 -- 모듈마다 자기 테이블 조각을 생성한다 - -### 미해결: 모듈 조각 수집 - -조각들을 런타임에 모으는 방법이 플랫폼마다 갈린다. - -| 플랫폼 | 사정 | -|---|---| -| JVM (Desktop/Android) | `ServiceLoader` 사용 가능 | -| **Kotlin/Native (iOS)** | **`ServiceLoader` 없음.** 최상위 `object` 는 지연 초기화되므로, 아무도 참조하지 않는 조각은 **초기화 자체가 일어나지 않아 등록되지 않는다** | - -후보: - -1. **생성된 애그리게이터** — 최종 앱 모듈의 KSP 가 모든 조각을 아는 통합 객체를 만들고 명시적으로 - 참조한다. 컴파일 타임에 확정되므로 트리 쉐이킹과도 맞는다. **다만 앱 모듈에도 KSP 가 붙어야 하고, - 라이브러리는 조각만 만들고 앱이 모으는 2단계 구조가 된다.** -2. `@EagerInitialization` — Kotlin/Native 에 있으나 실험적이고 트리 쉐이킹을 무력화할 위험이 있다 -3. 사용자에게 명시적 등록 호출을 요구 — 가장 단순하지만 수동이다 - -**1번이 유력하다.** - -### 미해결: 트리 쉐이킹 - -테이블이 모든 `public` 을 참조하면 Native Image 의 데드코드 제거가 무력화되어 바이너리가 부푼다. - -이 문제는 **업콜에만 해당한다.** 다운콜은 CPython 심볼을 찾는 구조라 무관하다. - -방향으로 제시된 것: 테이블에 `KFunction<*>` 참조 대신 **람다(함수 리터럴)** 를 넣으면 도달 가능성 -분석이 개별 함수 단위로 이뤄질 수 있다. 위의 애그리게이터 방식(모듈 단위 분리)도 같은 문제를 완화한다. - -확정된 답은 아직 없다. 업콜 구현에 착수할 때 결론을 내야 한다. - -## 스레드 상태 - -업콜은 Python 스레드에서 Kotlin 으로 들어오므로, 반대 방향의 스레드 상태 문제가 있다. -[`threading-and-abi.md`](threading-and-abi.md) 를 본다. - ---- - -## Cycle collection is part of the table's job - -Settled after the fact, and it changes what the generator emits. - -A Python proxy holding a Kotlin object that holds a Python object forms a cycle neither -collector can break on its own: CPython's traversal stops at the opaque handle, and the JVM sees -the handle map as a live root. Reference counting cannot help — it never frees cycles in any -language, which is why CPython carries a cyclic collector at all. - -The fix is to make the Kotlin side's Python references visible to `tp_traverse`, so the proxy's -traverse reaches through the handle and enumerates the `PyObject`-typed fields of the Kotlin -object. KSP already walks every exposed class to build the call table and knows those field -types at that point, so a traverse function per class is one more generated entry rather than a -new mechanism. - -This is why the reflection-free, build-time table matters beyond dispatch cost: a -reflection-based binding cannot afford this, which is why mature ones document "do not create -cycles" instead of solving it. - -Full mechanism, and the three parts that remain hard — traverse running during collection, -`tp_clear` having to mutate Kotlin state, and cycles that close on the Kotlin side — are in -`docs/design/object-lifetime.md`. diff --git a/docs/design/upcall-table-design.md b/docs/design/upcall-table-design.md deleted file mode 100644 index dfe4e0d15..000000000 --- a/docs/design/upcall-table-design.md +++ /dev/null @@ -1,913 +0,0 @@ -# Upcall Table Design: Fragment Generation, Collection, and Tree Shaking - -This document resolves the two open questions from -[`upcall-design.md`](upcall-design.md) and docs/roadmap/ROADMAP.md §7: - -1. How do per-module function-table fragments find each other at runtime, - given that Kotlin/Native has no `ServiceLoader` and no useful reflection? -2. How do we avoid defeating dead-code elimination? - -## Status of claims - -| Claim | Basis | -|---|---| -| KSP in app module sees generated code from library modules | **Shipped.** `python-multiplatform-ksp/`, with `ksp-fixtures/` running its generated table | -| `getDeclarationsFromPackage` is the discovery API | **Demonstrated** | -| Two-round processing lets the app discover its own fragment too | **Demonstrated** | -| `.klib` fragment discovery works on a real Kotlin/Native target | **Demonstrated.** `ksp-fixtures/{library,app}` on `androidNativeArm64`: `AppProcessor` round 2 found `Fragment_ksp_fixture_library` from the library's compiled `.klib`, `FunctionTable` compiled, and a real test binary linked (`linkDebugTestAndroidNativeArm64`). Compiled and linked only -- not run, since `androidNativeArm64` targets a device this workspace does not execute tests against | -| A KSP-generated table passes the same runtime API a hand-written one does | **Demonstrated.** `ksp-fixtures/app`'s `GeneratedTableTest` (11 tests, desktop) runs `UpcallTableTest`'s exact scenarios -- constructor/method/getter/setter round trip through `HandleTable`, `@PythonInternal` exclusion, narrow-`Int`/`Float` boundary widening, `tp_traverse` field detection -- against a table KSP generated, not a hand-written fragment | -| Kotlin/Native LLVM DCE operates at function-level granularity | Web research; not measured directly. What *was* measured is the net effect (next row), which is consistent with it but does not isolate the mechanism | -| Lambda references are eliminated when unreachable | Web research (LLVM `GlobalDCE`); not measured | -| Binary-size cost of a full table on K/N | **Measured**, §4 below: ~1.84 KB/entry (stripped) for 200 trivial exposed functions | -| `@EagerInitialization` is deprecated and should not be used | Confirmed (deprecated, slated for removal) | -| A fragment object can be `internal` as sketched below | **Wrong.** `internal` is enforced per Kotlin module; the app module's generated code referencing a library's fragment is a different module even inside one Gradle build, and fails with "cannot access ... it is internal". Fragments must be `public`. See §1's implementation note | - ---- - -## 1 Fragment generation (per module) - -A KSP processor scans every source file in the current compilation unit. -For each eligible `public` declaration it records the qualified name, the -callable reference, arity, parameter-type tags, and return-type tag. - -The processor emits a **fragment object** into a well-known package: - -```kotlin -// generated in package: python.multiplatform.generated.fragments -// file: Fragment_com_example_mylib.kt - -package python.multiplatform.generated.fragments - -internal object Fragment_com_example_mylib : FunctionTableFragment { - override fun entries(): List = listOf( - ExposedCallable( - name = "com.example.mylib.greet", - callable = { args -> greet(args[0] as String) }, - arity = 1, - paramTypes = listOf(ParamType.STRING), - returnType = ReturnType.STRING, - ), - // ... - ) -} -``` - -### What is scanned - -Per [`binding-policy.md`](binding-policy.md): - -| Declaration | Included | Notes | -|---|---|---| -| `public` top-level functions | ✅ | | -| `public` top-level properties | ✅ | module attribute: `STATIC_GETTER`/`STATIC_SETTER` | -| `public` classes + constructors | ✅ | constructor only if concrete, non-`inner` | -| `public` member functions | ✅ | | -| `public` properties | ✅ | getter; setter only for `var` | -| `companion object` members | ✅ | under the *owner's* name, no receiver | -| `object` declarations | ✅ | the same shape; no constructor | -| `interface` | ✅ | members only; no constructor, receiver arrives as a handle | -| `enum class` | ✅ | one `STATIC_GETTER` per entry, plus `name`/`ordinal`/`valueOf` | -| nested classes / interfaces / enums / objects | ✅ | under `Outer.Inner`; the walk stops at a non-exposed owner | -| `annotation class` | ❌ | reading one back needs runtime reflection; an instance has nothing to attach to | -| `enum` `values()` / `entries` | ❌ | return collections; `ReflectedClass.enumEntryNames` carries the same data | -| generic declarations (`class Box`, `fun f`) | ❌ | `args[0] as T` / `as Box` do not compile | -| abstract / sealed class constructors | ❌ | "Cannot create an instance of an abstract class" | -| `internal` / `private` / `protected` | ❌ | `internal` is `Modifier.INTERNAL` in KSP | -| `suspend` functions | ❌ | hidden `Continuation` parameter | -| extension functions | ❌ | no receiver available from Python | -| compiler-generated (`copy`, `componentN`) | ❌ | KSP reports these two for a `data class`; not `equals`/`hashCode`/`toString` | -| `expect` declarations | ❌ | `actual` is found on the platform side | -| annotated `@PythonInternal` | ❌ | opt-out annotation | -| library's own packages (`python.multiplatform.*`) | ❌ | excluded by KSP option | - -Everything in this table is asserted against a KSP-generated fragment in -`ksp-fixtures/app/src/desktopTest/.../GeneratedDeclarationKindsTest.kt`, not against a -hand-written one. - -### What the fragment holds - -Each `ExposedCallable` carries: - -- **`name`**: the Python-visible qualified name -- **`callable`**: a lambda wrapping the target — *not* a `KFunction` reference - (see §4 on tree shaking for why) -- **`arity`**: parameter count -- **`paramTypes`**: list of type tags (`INT`, `FLOAT`, `STRING`, `BYTES`, `OBJECT`) -- **`returnType`**: return-type tag - -The lambda form `{ args -> greet(args[0] as String) }` is deliberate. On the JVM, -`KFunction.call()` boxes arguments and does per-call arity/type checking (50–100 ns -overhead per call, measured in this project's benchmarks). On Kotlin/Native, -`KFunction.call()` is not available at all. A generated lambda that directly invokes -the target function: - -- avoids boxing on both platforms -- compiles to a direct call on Kotlin/Native (the compiler inlines non-capturing lambdas) -- creates a **static, traceable reference** that the linker can reason about (see §4) - -On the JVM, the generated code can use `MethodHandle` instead of a lambda for even -lower overhead, but this is an optimisation within the same design — the fragment -shape stays the same. - -### Fragment naming convention - -The well-known package is `python.multiplatform.generated.fragments`. - -Fragment objects are named `Fragment_`, where the module name -is passed as a KSP option `python.multiplatform.moduleName`. Periods and hyphens are -replaced with underscores. - -This naming convention is not load-bearing for discovery — the aggregator finds -fragments by scanning the entire well-known package — but it prevents collisions -and makes generated code legible. - ---- - -## 2 Fragment collection (the aggregator) - -### The problem, concretely - -On Kotlin/Native: - -- There is no `ServiceLoader`. -- A top-level `object` that nobody references is never initialised (lazy init - is the default since K/N 1.7.20; `@EagerInitialization` is deprecated). -- Runtime reflection cannot enumerate class members. - -So a fragment that registers itself in its own initialiser simply never runs. -The fragments must be referenced explicitly by something that *is* reachable. - -### The solution: a build-time aggregator in the app module - -This was the suspected answer. **It is now verified experimentally.** - -A throwaway project once stood at `ksp-experiment/`. It was removed once the real processor -shipped; what it demonstrated is now carried by `ksp-fixtures/`, whose tests run against a table -KSP actually generated. The shape it established -demonstrates the complete flow: - -``` -ksp-experiment/ -├── ksp-processor/ — the KSP processor (fragment + aggregator) -├── library-module/ — a library with public functions -└── app-module/ — the application, depends on library-module -``` - -The library module's KSP generates `Fragment_library_module` into the well-known -package. The app module's KSP: - -1. **Round 1**: generates `Fragment_app_module` for the app's own functions. -2. **Round 2**: calls `resolver.getDeclarationsFromPackage("...fragments")` - and finds **both** `Fragment_app_module` (just generated) and - `Fragment_library_module` (compiled into the library's JAR on the classpath). -3. Generates a `FunctionTable` that explicitly references every fragment. - -**Verified output** (exit code 0): -``` -Entries discovered: [experiment.app.appFunctionOne, experiment.app.main, - experiment.library.libraryFunctionOne, - experiment.library.libraryFunctionTwo] -``` - -### Why this works - -KSP processes one module at a time and cannot see other modules' *source* files. -But `resolver.getDeclarationsFromPackage` queries the compilation classpath, which -includes compiled `.class` files (or `.klib` metadata on Native) from dependencies. - -When the library module builds: -1. KSP generates `Fragment_library_module.kt` in the well-known package -2. The Kotlin compiler compiles it into the library's JAR (or `.klib`) - -When the app module builds: -1. The library JAR/klib is on the compilation classpath -2. KSP's `getDeclarationsFromPackage` sees the compiled `Fragment_library_module` -3. The aggregator generates a `FunctionTable` with explicit references to all fragments - -The generated `FunctionTable` looks like: - -```kotlin -package python.multiplatform.generated - -object FunctionTable { - private val entries: Map = buildMap { - Fragment_library_module.entries().forEach { put(it.name, it) } - Fragment_app_module.entries().forEach { put(it.name, it) } - } - - fun lookup(name: String): ExposedCallable? = entries[name] -} -``` - -Because `FunctionTable` textually references every `Fragment_*` object, the -Kotlin/Native compiler sees them as reachable. No `ServiceLoader`, no -`@EagerInitialization`, no runtime reflection needed. - -### Platform uniformity - -The same aggregator design works on all four targets: - -| Platform | Fragment discovery | Why it works | -|---|---|---| -| **JVM (Desktop)** | classpath (JAR) | `getDeclarationsFromPackage` sees JARs | -| **Android** | classpath (JAR/AAR) | same mechanism | -| **GraalVM Native Image** | classpath at build time | generated table is statically reachable → no reflection config needed | -| **Kotlin/Native (iOS)** | klib metadata | `getDeclarationsFromPackage` sees `.klib` dependencies | - -On JVM/Android, `ServiceLoader` *could* work as a fallback, but using the same -aggregator everywhere eliminates a platform split and avoids runtime classpath -scanning. - -### What the user has to do - -**Library author:** -```kotlin -plugins { - id("com.google.devtools.ksp") -} -dependencies { - ksp("python.multiplatform:ksp-processor:$version") -} -ksp { - arg("python.multiplatform.role", "library") - arg("python.multiplatform.moduleName", "my_library") -} -``` - -**App developer:** -```kotlin -plugins { - id("com.google.devtools.ksp") -} -dependencies { - implementation("com.example:my-kotlin-library:1.0") - ksp("python.multiplatform:ksp-processor:$version") -} -ksp { - arg("python.multiplatform.role", "app") - arg("python.multiplatform.moduleName", "my_app") -} -``` - -Ideally the `role` distinction could be inferred (the app module is the one that -produces a final binary / framework / APK), but KSP has no built-in way to detect -this. A Gradle plugin wrapping the KSP configuration could set the role -automatically based on `plugins.hasPlugin("application")` or -`plugins.hasPlugin("com.android.application")`. - -### Reaching `FunctionTable` from shared code: `@InstallsUpcallTable` - -`FunctionTable` is emitted into the compilation that generated it, so only that target's own -source set can name it. An **intermediate** source set — `commonMain`, `iosMain`, -`androidNativeMain` — is one the generating leaves *depend on*, and it gets `Unresolved reference -'FunctionTable'`. `androidMain` is the exception, being the Android target's own source set; a -per-platform exception is exactly what shared code cannot be written against. - -Moving generation to `kspCommonMainMetadata` would fix the visibility and break the scan: a -fragment has to see the target's own declarations, and a common-only scan exposes only what is -common. So generation stays in the leaves and the *seam* is generated instead: - -```kotlin -// commonMain -@InstallsUpcallTable -expect fun installGeneratedUpcallTable() -``` - -The processor (app role, round 1) finds the annotated `expect` — a leaf compilation resolves its -whole source set closure, which is the same reason the fragment scanner sees `commonMain` -declarations — and writes one `actual` per leaf compilation, into the `expect`'s own package: - -```kotlin -@python.multiplatform.reflection.PythonInternal -actual fun installGeneratedUpcallTable() { - python.multiplatform.generated.FunctionTable.installInto() -} -``` - -`@PythonInternal` because the `actual` is an ordinary public top-level function in the user's -package, and an incremental round that fed the scanner its own previous output would otherwise -offer Python a callable that reinstalls the table underneath its caller. - -A `library`-role module carrying the annotation is an error rather than a silent skip: it -aggregates nothing, so there is no `FunctionTable` for the `actual` to call, and the alternative -diagnostic is an unimplemented `expect` reported against the user's `commonMain` with nothing -naming the cause. - -`expect`/`actual` rather than a common interface plus a runtime registry, for the reason §4 and §9 -give: the fragment reference chain has to stay static or the Kotlin/Native linker is free to drop -what nothing names. The generated `actual` is that name. - -### Discovery key: well-known package, not annotation - -The aggregator does **not** scan for annotations. It calls -`resolver.getDeclarationsFromPackage("python.multiplatform.generated.fragments")` -and filters for objects whose name starts with `Fragment_`. - -This is simpler and faster than annotation scanning (`getSymbolsWithAnnotation`), -which KSP documents as potentially expensive because it must scan the entire -classpath. - -An alternative — keying on a marker interface `FunctionTableFragment` — would also -work. The aggregator could filter by supertype instead of name prefix. Either way, -the well-known package is the primary discovery mechanism. - ---- - -## 3 What about `getDeclarationsFromPackage` being `@KspExperimental`? - -`resolver.getDeclarationsFromPackage` is annotated `@KspExperimental` in KSP 2.x. -This means it may change in future KSP releases. However: - -1. It is the *documented* API for cross-module discovery. KSP's own documentation - recommends it for this use case. -2. No alternative exists that is non-experimental and works cross-module. -3. It has been stable in practice since KSP 1.x. - -If it is removed or changed, the fallback is resource-file-based discovery: each -module's KSP writes a metadata file (e.g., JSON) to `META-INF/`, and the aggregator -reads those files from the classpath. This is more complex but equally viable. -The current design should abstract the discovery step so that switching mechanisms -does not change the fragment or table shape. - ---- - -## 4 Tree shaking - -### The problem - -If the aggregator generates a `FunctionTable` that references every fragment, and -every fragment references every `public` callable in its module, then: - -- The linker sees the entire public API as reachable -- Dead-code elimination cannot remove unused functions -- An app that calls three Kotlin functions from Python ships the entire library - -This is the opposite of pay-for-what-you-use. - -### Analysis: how bad is it, actually? - -Before optimising, it is worth understanding the cost structure: - -**What a fragment reference actually keeps alive:** - -A fragment entry like `ExposedCallable("greet", { args -> greet(args[0] as String) }, ...)` -creates a lambda that closes over nothing (it calls the function by name). On -Kotlin/Native, non-capturing lambdas compile to singleton objects. The cost per -entry is: - -| Component | Approximate cost | -|---|---| -| The lambda object (singleton) | ~16–32 bytes | -| The string literal (name) | ~40–80 bytes | -| The `ExposedCallable` wrapper | ~32 bytes | -| **The target function itself** | **varies — this is the real cost** | - -The metadata overhead per entry is ~100–150 bytes. For a library with 300 public -functions, that's ~30–45 KB of metadata. This is noise. - -The real cost is keeping the target functions alive. If `greet()` calls -`formatMessage()` which calls `translateString()` which pulls in an i18n library, -then the reference to `greet` keeps the entire dependency chain alive. - -**But:** Kotlin/Native's LLVM backend performs function-level DCE with -`-ffunction-sections` + `--gc-sections`. Each function occupies its own section. -If `greet()` is referenced but `calculateTax()` is not, only `greet()` and its -transitive dependencies survive. The fragments reference every function, but the -linker still eliminates functions that are not referenced *by other code* in the -binary. - -Wait — **that reasoning is wrong.** The fragment *does* reference every function -via the lambda. The lambda `{ args -> greet(args[0] as String) }` is a static -reference to `greet`. If the fragment is reachable (which it is, via the -aggregator), then *every lambda in it* is reachable, and therefore *every target -function* is reachable. - -### The fundamental tension - -The blacklist model ("all public unless opted out") means the fragment references -everything public. The aggregator references every fragment. Therefore the full -table keeps the entire public surface alive by construction. - -There is no way to have all three of: -1. Blacklist model (all public exposed by default) -2. Full table referenced at build time (needed for K/Native) -3. Tree shaking of unused entries - -You must give up at least one. The options: - -### Option A: Accept the cost - -**How:** Keep the current design unchanged. Every public function is in the table, -the table is reachable, nothing is eliminated. - -**Cost:** For a library with 300 public functions averaging 50 bytes of machine -code each, that's ~15 KB of code that cannot be eliminated. For a library with -larger functions or transitive dependencies, the cost grows. On Kotlin/Native -where the baseline binary is 2–5 MB (Kotlin runtime + GC), 15–100 KB may be -acceptable. - -**Advantages:** -- Simplest design -- No user-facing complexity -- Python can call any public function without configuration - -**When this is right:** When the library is small or medium-sized, or when the -app uses most of the library's surface anyway. - -**Measurement needed:** Build a release Kotlin/Native binary with and without the -full table and measure the delta. If it's under 100 KB for a realistic library, -accept it and move on. **This has not been done.** - -### Option B: Fine-grained fragments - -**How:** Instead of one fragment per module, generate one fragment per file (or -per class). The aggregator still references all of them, but if the linker can -trace through the `FunctionTable.entries` map and determine that certain -`Fragment_*` objects are never *looked up* at runtime, it could eliminate them. - -**Problem:** The linker cannot do this. `FunctionTable` calls -`Fragment_X.entries()` unconditionally during its initialisation. All fragments -are reachable regardless of which names are actually looked up at runtime. - -Making fragments lazily initialised (`by lazy { ... }`) would defer the call, -but `lazy` still captures a reference to the lambda that calls -`Fragment_X.entries()`, so the fragment is still reachable. - -**Verdict:** Fine-grained fragments alone do not help. The aggregator's explicit -reference is the root cause. - -### Option C: Opt-in at the call site (Python side) - -**How:** Instead of generating a table of everything, require the Python code -(or a configuration file) to declare which Kotlin functions it wants to use. -The KSP processor reads this configuration and generates a table containing -only those entries. - -```python -# python_bindings.txt (read by KSP) -com.example.mylib.greet -com.example.mylib.Calculator -``` - -**Advantages:** -- Perfect tree shaking: only referenced functions survive -- Binary contains exactly what is needed - -**Disadvantages:** -- Contradicts the blacklist model ("all public, minus opt-out") -- User must maintain a manifest — fragile, especially during development -- Breaks the "write Kotlin, import from Python" promise - -**Verdict:** This is a different product. The user request says the blacklist -model is not to be re-litigated. - -### Option D: Lazy table with runtime resolution (JVM only) - -**How:** On the JVM (where dead-code elimination is not an issue because classes -are loaded lazily), skip the aggregator entirely. Use `ServiceLoader` or simple -classpath scanning to discover fragments at runtime. - -On Kotlin/Native, keep the aggregator but accept the cost (Option A). - -**Advantages:** -- JVM apps pay no tree-shaking cost (not that they care — JVM doesn't tree-shake) -- Kotlin/Native pays the cost, but that's where it matters least (the library - and app are typically small on iOS) - -**Disadvantages:** -- Platform split in the collection mechanism -- `ServiceLoader` on JVM adds classpath scanning overhead at import time - -**Verdict:** Not worth the complexity. The aggregator is simpler on all platforms. - -### Option E: Two-tier table (recommended) - -**How:** Split the table into a **skeleton** (names and metadata only) and a -**body** (the callable lambdas). The skeleton is always present. The body is -loaded lazily, per fragment, on first access. - -```kotlin -// Generated skeleton — always present, small -object Fragment_mylib_Skeleton { - val names: List = listOf( - "com.example.mylib.greet", - "com.example.mylib.calculate", - ) -} - -// Generated body — loaded only when a name from this fragment is actually called -object Fragment_mylib_Body { - fun resolve(name: String): ((Array) -> Any?)? = when (name) { - "com.example.mylib.greet" -> { args -> greet(args[0] as String) } - "com.example.mylib.calculate" -> { args -> calculate(args[0] as Long, args[1] as Long) } - else -> null - } -} -``` - -The `FunctionTable` references all skeletons (keeping them alive — ~100 bytes -per entry, names and metadata only). When Python does a `lookup("com.example.mylib.greet")`, -the table identifies which fragment owns the name and calls `Fragment_mylib_Body.resolve()` -for the first time, which triggers lazy initialisation of the body object. - -**Key insight:** If no name from `Fragment_mylib` is ever looked up, the body -object is never initialised, and the lambdas inside it are never created. -**But the body class is still on the classpath and still reachable in principle.** - -On the JVM, this is fine — class loading is lazy, so `Fragment_mylib_Body` is -never loaded if never referenced at runtime. - -On Kotlin/Native, `object` initialisation is lazy (since 1.7.20), so the body's -`when` expression is never evaluated if the object is never accessed. **However,** -the body class itself is still compiled into the binary. The lambdas inside the -`when` arms are only created when that arm is executed, but the function references -they point to must still be compiled. - -So on Kotlin/Native, Option E reduces the *runtime* cost (no lambda objects for -unused fragments) but does **not** reduce the *binary size*. The compiled code for -`greet()` is still in the binary because the `when` arm statically references it. - -**Verdict:** Marginal improvement. Not worth the complexity unless runtime memory -is the concern rather than binary size. - -### Recommended approach - -**Start with Option A (accept the cost). Measure before optimising.** - -The reasoning: - -1. **The cost may be small.** The metadata overhead is ~100–150 bytes per entry. - The function code itself varies, but on Kotlin/Native with a 2–5 MB baseline - binary, even 100 KB of "extra" code is 2–5% — noticeable but not fatal. - -2. **The cost may be unavoidable.** Given the constraints (blacklist model, no - reflection, build-time table), there may be no mechanism that keeps a static - reference to a function while allowing the linker to remove it. This is a - fundamental property of static linking. - -3. **The measurement is easy.** Build a Kotlin/Native release binary with and - without the table. Diff the sizes. If the delta is under 100 KB for a - realistic library (50–200 public functions), the problem is not worth solving. - -4. **If the cost is unacceptable**, the only real mitigation is to change the - exposure model — from "all public" to "explicitly annotated" — which is a - product decision, not a technical one. The document noted in the user request - says this is not to be re-litigated, so it stays as Option A for now. - -### Measured - -`ksp-fixtures/library` got 200 synthetic top-level functions (`fun bulkN(x: Long): Long -= x + N`), and `ksp-fixtures/app`'s `androidNativeArm64` debug test binary was linked -before and after (`linkDebugTestAndroidNativeArm64`), then stripped with the NDK's -`aarch64-linux-android-strip` from the same toolchain Kotlin/Native already downloads: - -| | bytes | delta | -|---|---|---| -| baseline (unstripped) | 7,578,480 | | -| +200 entries (unstripped) | 8,343,184 | 764,704 (≈3.8 KB/entry) | -| baseline (stripped) | 3,264,056 | | -| +200 entries (stripped) | 3,632,696 | 368,640 (**≈1.84 KB/entry**) | - -Read this as a floor, not a typical case: each synthetic function is a one-line -arithmetic expression, so almost the entire 1.84 KB/entry is the `ExposedCallable` -object, its lambda, its name string, and Kotlin/Native's per-function metadata -- -not the function body. A real library's functions would add their own body size on -top of this floor. For a 200-function library that floor alone is ~360 KB against a -~3.26 MB stripped baseline (≈11%) — noticeable, in line with Option A's prediction, -and not disqualifying for a mobile/desktop target. This closes the "measurement -needed" item; whether ~11% for 200 entries is acceptable for a *real* library (whose -function bodies dominate the delta) is still a product judgement, not resettled here. - ---- - -## 5 Detailed flow: from `ksp(...)` to Python `import` - -### Build time - -``` -┌──────────────┐ KSP ┌────────────────────────────────────────┐ -│ library-mod │ ────────── │ Fragment_mylib.kt │ -│ src/ │ (role= │ package ...generated.fragments │ -│ mylib/*.kt │ library) │ object Fragment_mylib { │ -│ │ │ fun entries() = listOf(...) │ -│ │ │ } │ -└──────────────┘ └────────────────────────────────────────┘ - │ │ - │ compile │ compiled into JAR/klib - ▼ ▼ -┌──────────────┐ ┌────────────────────────────────────────┐ -│ app-module │ KSP │ FunctionTable.kt │ -│ depends on │ ────────── │ package ...generated │ -│ library-mod │ (role= │ object FunctionTable { │ -│ │ app) │ private val entries = buildMap { │ -│ │ │ Fragment_mylib.entries()... │ -│ │ │ Fragment_app.entries()... │ -│ │ │ } │ -│ │ │ fun lookup(name) = entries[name] │ -│ │ │ } │ -└──────────────┘ └────────────────────────────────────────┘ -``` - -### Runtime - -``` -Python: from kotlin import greet - │ - ├─ CPython import mechanism invokes _kotlin_bridge module init - │ - ├─ Module init calls FunctionTable (which initialises the object, - │ pulling in all fragments) - │ - ├─ Python: greet("world") - │ │ - │ ├─ tp_getattro on the module proxy - │ │ First time: FunctionTable.lookup("greet") → handle - │ │ Cached in proxy's __dict__ - │ │ - │ ├─ Invoke handle with marshalled args - │ │ JVM: MethodHandle.invokeExact(...) - │ │ K/N: direct function call (lambda inlined) - │ │ - │ └─ Marshal return value back to Python - │ - └─ Result: "Hello, world!" -``` - ---- - -## 6 KSP processor design - -The project ships a single KSP processor module. It contains one -`SymbolProcessorProvider` that delegates to two internal processors -based on a KSP option: - -| Option | Value | Processor | What it does | -|---|---|---|---| -| `python.multiplatform.role` | `library` | `FragmentProcessor` | Scan module, emit fragment | -| `python.multiplatform.role` | `app` | `AppProcessor` | Emit fragment (round 1), then discover all fragments and emit `FunctionTable` (round 2) | - -Both processors use the same fragment-generation logic. The `AppProcessor` -additionally runs the aggregation step in KSP's second processing round -(triggered by the new file generated in round 1). - -### KSP options - -| Option | Required | Description | -|---|---|---| -| `python.multiplatform.role` | Yes | `library` or `app` | -| `python.multiplatform.moduleName` | Yes | Sanitised module name for the fragment | -| `python.multiplatform.excludePackages` | No | Comma-separated packages to skip (e.g., the library's own packages) | - -### Multi-round processing - -The `AppProcessor` exploits KSP's multi-round processing: - -1. **Round 1:** Generate `Fragment_` for the app's own functions. - This produces a new file, which triggers round 2. -2. **Round 2:** Call `resolver.getDeclarationsFromPackage("...fragments")`. - This now sees both the freshly generated app fragment and all dependency - fragments (compiled into JARs/klibs on the classpath). Generate `FunctionTable`. - -This was verified in the experiment: the round-2 query correctly returns -declarations from both sources. - ---- - -## 7 The Gradle plugin - -**Built:** `python-multiplatform-gradle-plugin/`, an included build (registered in the root -`settings.gradle.kts` under `pluginManagement`) publishing the id -`io.github.thisisthepy.python.multiplatform.bindings`. A consumer writes: - -```kotlin -plugins { - id("io.github.thisisthepy.python.multiplatform.bindings") -} - -// only if the module produces the final binary without `application`/`com.android.application`: -pythonBindings { role.set("app") } -``` - -and gets: the KSP plugin applied, the processor added to every target's main compilation -(`kspDesktop`, `kspAndroidNativeArm64`, ... — never a `...Test` one, never -`kspCommonMainMetadata`, both of which would emit a second fragment under the same name), the role -inferred, and `moduleName` derived. `ksp-fixtures/{library,app}` are the worked example; their -build files lost the two `add("ksp", ...)` lines and the `ksp { arg(...) }` block each. - -Two things the original sketch below did not anticipate: - -- **`moduleName` carries the Maven group, not just the project path.** Fragment objects from every - artifact share one package, so two independent libraries both called `:core` would emit the same - object name into it. `:ksp-fixtures:app` with group `io.github.thisisthepy` derives - `io_github_thisisthepy_ksp_fixtures_app`. -- **The plugin does not compile against KSP's Gradle plugin.** `kotlin-dsl` builds against - Gradle's embedded Kotlin (2.0.20 on Gradle 8.11.1) and KSP 2.3.11 is compiled with Kotlin 2.3: - "Class 'com.google.devtools.ksp.gradle.KspExtension' was compiled with an incompatible version - of Kotlin". The dependency is `runtimeOnly` — a plugin id resolves from a classpath resource, - not from a compiled type — and the two `ksp { arg(...) }` calls go through one reflective - lookup of `arg(String, String)`. -- **Which KSP configurations count is decided by name, and Android names them differently.** AGP's - source-set names put the build type last (`kspAndroidTestDebug`), so an `endsWith("Test")` check - put the processor on the unit-test compilation, which emitted a duplicate fragment that shadowed - `main`'s inside that compilation. `Test` is matched as a camel-case word now; see ROADMAP §13 and - `ksp-fixtures/android`, the fixture that carries an Android plugin. - -The original sketch, for the record: - -The KSP options (`role`, `moduleName`, `excludePackages`) are mechanical. -A Gradle plugin (`python-multiplatform-bindings`) could: - -1. Apply the KSP plugin automatically -2. Add the processor dependency -3. Infer the role from the module type: - - `com.android.application` or `application` plugin → `app` - - everything else → `library` -4. Derive `moduleName` from the Gradle project path -5. Set `excludePackages` based on the module's own `group`/`namespace` - -This would reduce the user's configuration to: - -```kotlin -plugins { - id("python-multiplatform-bindings") -} -``` - -This is a usability concern for later. The KSP processor works without it. - ---- - -## 8 What was NOT verified - -### Kotlin/Native target with KSP - -The experiment uses JVM-only modules. KSP processes Kotlin/Multiplatform -modules identically for all targets (it operates on Kotlin IR, not -platform-specific bytecode), so the fragment generation should work the -same way on Native targets. However, this was not tested end-to-end. - -Specifically not tested: -- Whether `getDeclarationsFromPackage` works correctly when the dependency - is a `.klib` rather than a `.jar` -- Whether the generated `FunctionTable` compiles on Kotlin/Native targets -- Whether the aggregator's explicit references actually prevent the K/N - linker from stripping the fragments - -**Recommendation:** Before implementing, build a minimal KMP project -(commonMain + nativeMain) with the same two-module structure and verify -compilation on at least one Native target (e.g., `macosArm64`). - -### Transitive dependencies - -Only tested direct dependency (app → library). Whether -`getDeclarationsFromPackage` discovers fragments from transitive -dependencies (app → libA → libB, where libB has fragments) was not tested. - -It should work because transitive classes are on the compilation classpath, -but "should work" has been wrong three times in this project. - -### Published Maven artifacts - -Only tested with `project()` dependencies. When a library is published to -Maven and consumed as `implementation("group:artifact:version")`, the -generated fragment classes are in the published JAR. KSP should see them -the same way. Not tested. - -### Binary-size impact - -The actual KB cost of keeping unused functions alive on Kotlin/Native has -not been measured. This is the most important open measurement. - -### GraalVM Native Image compatibility - -The aggregator design avoids reflection entirely — `FunctionTable` has -static references to all fragments, which have static references to all -callables. Under GraalVM's closed-world analysis, this should work without -any `reflect-config.json`. Not tested. - ---- - -## 9 Alternatives considered and rejected - -### `@EagerInitialization` - -Kotlin/Native's `@EagerInitialization` forces a top-level property to -initialise at program startup. This could make fragments self-registering. - -**Rejected because:** -- Deprecated and slated for removal -- Does not solve the discovery problem (who calls `register()`?) -- Defeats tree shaking by definition (if it's eagerly initialised, it's reachable) - -### Resource files (META-INF) - -Each module's KSP writes a metadata file (e.g., JSON listing function names) -to `META-INF/python-multiplatform/`. The aggregator reads these resources -from the classpath. - -**Not needed because** `getDeclarationsFromPackage` works for this use case. -Resource files are more complex (need classpath I/O in the processor) and -less type-safe (JSON parsing vs. direct class references). - -**However**, resource files remain a viable fallback if -`getDeclarationsFromPackage` is removed or changed (it is `@KspExperimental`). -The design should keep the discovery step abstract enough to swap. - -### Manual registration - -Require the user to call `FunctionTable.register(Fragment_mylib)` in their -`main()`. Simplest possible design. - -**Rejected because** it contradicts the "add the KSP plugin and nothing else" -goal. Every new library dependency would require a new registration call. - -### Compiler plugin instead of KSP - -A Kotlin compiler plugin (IR plugin) could inject fragment registration -directly, without generating source files. More powerful but: - -- Much harder to write and maintain -- Compiler plugin API is less stable than KSP -- Not needed — KSP is sufficient for this use case - ---- - -## 10 Summary of the design - -| Aspect | Decision | -|---|---| -| Fragment generation | KSP processor per module, one fragment object per module | -| Fragment package | `python.multiplatform.generated.fragments` | -| Discovery mechanism | `resolver.getDeclarationsFromPackage` in the app module's KSP | -| Aggregation | `FunctionTable` generated in app module, explicit references to all fragments | -| Platform uniformity | Same mechanism on JVM, Android, GraalVM, and Kotlin/Native | -| Tree shaking | Accept the cost for now; measure before optimising | -| User experience | Add KSP plugin + processor dependency; set role and module name | -| Callable representation | Generated lambda (not `KFunction.call`) | -| Discovery key | Well-known package name, not annotation | - -## 11 Open questions for implementation - -Numbers 1, 2, 3, 4 and 6 are resolved; the resolution is recorded next to each so the -reasoning survives without the implementation session. 5 is still open. - -1. **~~Measure binary-size impact on Kotlin/Native.~~ Resolved.** ≈1.84 KB/entry - stripped, §4 above. Over the 100 KB guess for a 200-entry table (368,640 bytes - measured) but the guess was for a *typical* function; the measured entries were - deliberately trivial one-liners to isolate the table's own floor cost from - whatever a real function body adds on top. - -2. **~~Verify `.klib` discovery.~~ Resolved: it works.** `ksp-fixtures/{library,app}` - on `androidNativeArm64` -- round 2's `getDeclarationsFromPackage` found the - library's fragment from its compiled `.klib`, and the app's generated - `FunctionTable` compiled *and linked* into a real test binary. Not run (no - device), but the mechanism up through linking is confirmed, not inferred. - -3. **~~Decide annotation name.~~ Resolved: `@PythonInternal`,** matching - `binding-policy.md` (the "`@PythonHidden`" alternative from an earlier - conversation was not carried into that document and there is nothing to - reconcile). - -4. **~~Handle `main()`.~~ Resolved.** `BindingPolicy.isExposedTopLevelFunction` - excludes a function named `main` with zero parameters or a single `Array<...>` - parameter. - -5. **~~Incremental KSP.~~ Answered by measurement: the aggregator is not the cause, - and narrowing it changes nothing.** - - `Dependencies(false)` (no files, not aggregating) means "never regenerate this - file", which would leave a newly-added fragment undiscovered; `ALL_FILES` is - correct but reprocesses on every change. The open question was whether something - narrower would restore incrementality. It does not. - - Measured with `ksp.incremental.log=true` on `:ksp-fixtures:app`, a two-file module, - modifying one file: - - | aggregator `Dependencies` | dirty / all | - |---|---| - | `ALL_FILES` | 100.00% | - | `Dependencies(aggregating = true, )` | 100.00% | - - The source-to-output map explains it. A module's own `Fragment_` is an - *aggregating* output over every source file in the module -- correctly so, because - blacklist exposure means any file can add an entry -- so it is keyed by KSP's - `AnyChanges` wildcard and by all sources. Any change regenerates it, and KSP then - marks every source that maps to it dirty. Whatever the aggregator declares is - downstream of that. - - A classpath-only change behaves the same way: adding a function to - `:ksp-fixtures:library` dirtied both app files with `Modified` empty and - `CP changes` listing `Fragment_ksp_fixture_library`. - - `ALL_FILES` therefore stays: the narrower form buys nothing measurable, and whether - it stays correct when a dependency's fragment appears with no local source change - was never established. The only route to real incrementality is **per-file - fragments** -- one isolating output per source file -- which changes fragment - naming, `UpcallTable`'s per-module idempotency (`moduleName`) and where duplicate - names are detected. That is a design change, and it would also be the lever for - §4's tree-shaking option B. - -6. **~~KSP for KMP.~~ Resolved for the targets this project applies it to.** Per-target - configurations (`add("kspDesktop", ...)`, `add("kspAndroidNativeArm64", ...)`) work - as documented; `ksp { arg(...) }` applies shared options to all of them. A Gradle - convenience plugin to apply this automatically (§7) remains unbuilt. diff --git a/docs/design/upcall.md b/docs/design/upcall.md new file mode 100644 index 000000000..aaf9a4076 --- /dev/null +++ b/docs/design/upcall.md @@ -0,0 +1,1331 @@ +# Upcalls — Python → Kotlin + +How Python code imports Kotlin packages, constructs Kotlin classes, calls their functions (including +`suspend` ones) and reads their properties. The opposite direction is +[`downcall-design.md`](downcall-design.md); what is exposed at all is +[`binding-policy.md`](binding-policy.md); object lifetimes and cycle collection are +[`object-lifetime.md`](object-lifetime.md); thread state is [`threading-and-abi.md`](threading-and-abi.md). + +## 0. Status and how to read this document + +This is a **design record**. The behavioural contract is [`SPEC.md`](../SPEC.md) §5 (U-1 … U-8) and +the boundary is [`INTENT.md`](../INTENT.md) §2.1, §2.2, §2.6; where this document and those disagree, +they win. It replaces three earlier documents (`upcall-design.md`, `upcall-table-design.md`, +`upcall-async-design.md`); their history is in git. + +What is implemented, in one paragraph: a KSP processor generates a function table for every `public` +declaration of a module at build time (U-1); one entry point per platform marshals arguments into it +(U-2); generated Python proxies make Kotlin packages importable under their own names with classes, +properties, companions and statics (U-3, U-4); `suspend` functions are awaitable (U-5); upcalls work in +a GraalVM native image (U-6, manually verified); every function on a Kotlin-named module carries Kotlin's +own surface (U-8). It runs on desktop, Android (ART), iOS, androidNative and wasmJs, with the exceptions +listed in §8. + +Conventions used below: + +- **Measured** means it was run in this repository and the test that prints or asserts it is named. + **Inferred** means it was read from code or documentation and not run. **(unverified)** marks a + statement this consolidation could not check against code or tests. +- Every measurement carries its conditions (platform, warmup, run count, commit where recorded). + Nothing in the test suite asserts a duration: wall-clock thresholds on a shared machine, an emulator or + a Node host are flake generators. Tests assert structure (the call reached Kotlin, the figure is + positive) and print the numbers. +- The authoritative, harness-cut cost figures are [`cost-table.md`](../investigations/cost-table.md) + (`./benchmarks/cost-table.sh --runs 3`). Hand-cut tables here are records of the round they were + taken in; prefer the cost table where they disagree. + +--- + +## 1. Decisions + +### 1.1 No runtime reflection + +Two reasons, the second more fundamental: + +1. **GraalVM native image** is closed-world: runtime reflection does not work without registration. +2. **Kotlin/Native has effectively no reflection** (`::class.simpleName`, no member enumeration). iOS + and androidNative have no reflective path regardless of GraalVM. + +A multiplatform library therefore has **no option but a table generated at build time** +(AGENTS.md §12.3, INTENT §2.1). The same choice is what makes cycle collection possible (§6). + +### 1.2 Resolve a name once, then pass a handle + +Objective-C is fast not because it has a table but because a selector is an interned pointer: +`objc_msgSend` finds the IMP in the class's method cache in a few instructions without touching a string. +The equivalent here: + +| When | What happens | +|---|---| +| Once per name | Python passes a name (`"demo.Counter.increment"`); **Kotlin resolves it** in the table and returns a `CallableHandle` | +| Every call after that | The handle crosses, never the string | + +Passing a string per call costs `PyUnicode` → UTF-8 (an allocation) + the boundary + hashing + comparison — +hundreds of ns — whatever the table looks like. `UpcallTable.resolve` returns a `CallableHandle` whose raw +value packs the table **epoch** (high 32 bits) and the index (low 32), so a handle cached across a table +reinstall is rejected rather than calling a different function +(`python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/ExposedCallable.kt`, +`UpcallTable.kt`). The in-Kotlin part of the comparison is measured by +`commonTest/.../reflection/UpcallOverheadTest.kt` (`testNameResolutionVersusCachedHandle`, 200-entry +table); that test's numbers are a lower bound on the gap because the boundary is not in them. + +A table entry is a **generated lambda**, not a `KFunction`: `KFunction.call` re-checks arity and types and +boxes on every call (50–100 ns, measured in this project's benchmarks) and does not exist on +Kotlin/Native, while `{ args -> greet(args[0] as String) }` compiles to a direct call there and gives the +linker a static reference (§2.6). + +### 1.3 Route by data, so a few trampolines serve everything + +Every CPython callback convention passes the relevant object as an argument (checked against the headers): + +| Slot | Signature | What identifies the target | +|---|---|---| +| `PyCFunction` | `(PyObject *self, PyObject *args)` | `self` | +| `destructor` | `(PyObject *self)` | `self` | +| `getter` / `setter` | `(PyObject *self, void *closure)` | the explicit `closure` | +| `initproc` | `(PyObject *self, args, kwargs)` | `self` | +| `newfunc` | `(PyTypeObject *type, args, kwargs)` | `type` | + +So routing is done **with data, not function pointers**: a handful of shared trampolines serve an +unbounded number of Kotlin targets, and **no runtime code generation** (LLVM JIT, libffi closures) is +needed. JPype, PyO3 and JEP all work this way. The only case that would need runtime code generation is a +pure C callback with no context argument at all (a `qsort` comparator); the CPython object model has none. + +Argument passing needs exactly **one** carrier shape (§3.2), already inside the downcall vocabulary. + +### 1.4 One entry point per platform, one table behind all of them + +Behind the per-platform entry point every platform uses the same generated table and the same +`commonMain` trampoline (`UpcallTrampoline`); a platform contributes only the few lines that give Python +the address of an entry point (§3.7). INTENT §2.6. + +### 1.5 Exposure is a blacklist + +Every `public` declaration is registered; `@PythonInternal` opts a class or member out. A whitelist +(`@PythonAPI`) was rejected (AGENTS.md §12.7; [`binding-policy.md`](binding-policy.md)). This is the +premise the tree-shaking decision (§2.6) rests on and must not be re-litigated there. + +### 1.6 Kotlin names only + +A Kotlin package is importable under its own fully-qualified name and nothing else. Functions on a +Kotlin-named module take keyword arguments by **Kotlin parameter names**, honour Kotlin defaults for +omitted parameters, and expose their signature through `inspect.signature` and +`python_multiplatform.describe` (SPEC U-8; contract in the KDoc of +`python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/KotlinSurface.kt`). Nothing is +renamed: no snake_case, no synthetic `pythonx` module (INTENT §2.2–2.3, AGENTS.md §12.1–12.2). Making the +surface Pythonic is pythonx-compose's job, built on this metadata. See §4.3. + +--- + +## 2. Table generation + +### 2.1 Fragments, one per module + +The KSP processor (`python-multiplatform-ksp/`) is shipped as an artifact and runs in **every** module that +exposes Kotlin to Python — including the user's own modules, since exposing the user's classes is the +point. Each module emits one **fragment object** into the well-known package +`python.multiplatform.generated.fragments`: + +```kotlin +// GENERATED by python-multiplatform-ksp. Do not edit. +package python.multiplatform.generated.fragments + +object Fragment_io_github_thisisthepy_ksp_fixtures_library : python.multiplatform.reflection.FunctionTableFragment { + override val moduleName: String = "io_github_thisisthepy_ksp_fixtures_library" + override fun entries(): List = listOf( + ExposedCallable(name = "fixture.library.greet", arity = 1, paramTypes = listOf(TypeTag.STRING), + returnType = TypeTag.STRING, paramNames = listOf("name"), callable = { args -> fixture.library.greet(args[0] as String) }), + // ... + ) + override fun classes(): List = listOf(/* ... */) +} +``` + +(Shape from `SourceRendering.kt`'s `renderFragmentSource`; abbreviated.) + +- **Name**: `Fragment_`, where `moduleName` is the KSP option + `python.multiplatform.moduleName` with `.` and `-` replaced by `_` (`Constants.kt`). The Gradle plugin + derives it from the **Maven group plus the project path** (`:ksp-fixtures:app` with group + `io.github.thisisthepy` → `io_github_thisisthepy_ksp_fixtures_app`), because fragments from every + artifact share one package and two libraries both called `:core` would otherwise collide. The name is + not load-bearing for discovery (the whole package is scanned) but prevents collisions. +- **Visibility: `public`.** An `internal` fragment does not work: `internal` is enforced per Kotlin + module, and the app module's aggregator referencing a library's fragment is another module even inside + one Gradle build ("cannot access … it is internal"). +- **Each `ExposedCallable` carries** the Python-visible qualified name, `arity`, a `TypeTag` per + parameter and for the return, a `CallableKind` (where the arguments are: `FUNCTION`, `CONSTRUCTOR`, + `METHOD`, `GETTER`, `SETTER`, `STATIC_GETTER`, `STATIC_SETTER`), `isSuspend` (§5.3), the Kotlin + `paramNames`, declared type names, extension/receiver data, `paramHasDefault`, and the lambda. + `ReflectedClass` entries carry class metadata and, for classes with `PyObject`-typed properties, a + generated `tp_traverse` body (§6). +- The library's own packages (`python.native.ffi`, `python.multiplatform.ffi`, `…reflection`, `…ref`) + and the generated packages are always excluded; `python.multiplatform.excludePackages` adds more. + +### 2.2 What is scanned + +| Declaration | Exposed | Notes | +|---|---|---| +| `public` top-level functions | yes | `main()` with no parameters or one `Array<…>` is excluded (`BindingPolicy.isExposedTopLevelFunction`) | +| `public` top-level properties | yes | module attribute: `STATIC_GETTER` / `STATIC_SETTER` | +| `public` classes + constructors | yes | constructor only if concrete and non-`inner` (`isConstructible`) | +| `public` member functions | yes | | +| `public` properties | yes | getter; setter only for a `var` whose setter is itself public (`private set` / `internal set` are read-only from Python, `isExposedSetter`) | +| `companion object` members | yes | under the **owner's** name, no receiver | +| `object` declarations | yes | same shape, no constructor | +| `interface` | yes | members only; not constructible; receiver arrives as a handle | +| `enum class` | yes | one `STATIC_GETTER` per entry, plus `name`/`ordinal`/`valueOf` | +| nested classes / interfaces / enums / objects | yes | under `Outer.Inner`; the walk stops at a non-exposed owner | +| `suspend` functions | **yes** | body wrapped in `PendingCall.start { }`, `isSuspend = true` (§5.3) | +| `annotation class` | no | reading one back needs runtime reflection; an instance has nothing to attach to | +| enum `values()` / `entries` | no | return collections; `ReflectedClass.enumEntryNames` carries the same data | +| generic declarations (`class Box`, `fun f`) | no | `args[0] as T` does not compile | +| abstract / sealed class constructors | no | "Cannot create an instance of an abstract class" | +| `internal` / `private` / `protected` | no | | +| extension functions / extension properties | no (KSP) | the artifact walker does bind extensions, as methods on the receiver's proxy ([`kotlin-extensions-in-python.md`](kotlin-extensions-in-python.md); SPEC U-7 is `partial` and not asserted for KSP-generated proxies) | +| `@Composable` functions in the module's own source | no | a generated non-composable lambda cannot call one; the artifact walker binds composables instead (`BindingPolicy.isComposable` KDoc) | +| declarations whose type is a `suspend` function type, or one nested in a generic argument | no, **with a KSP warning** | `"has a type with no exposable classifier"` (`FragmentScanner.warnUnexposableType`; `GeneratedSuspendTest`) | +| compiler-generated `copy`, `componentN` | no | the subset KSP reports for a `data class` (no `equals`/`hashCode`/`toString`) | +| `expect` declarations | no | the `actual` is found on the platform side | +| `@PythonInternal` | no | opt-out | + +Asserted against KSP-generated fragments, not hand-written ones: +`ksp-fixtures/app/src/desktopTest/.../GeneratedDeclarationKindsTest.kt`, +`GeneratedSuspendTest.kt`, `GeneratedTableTest.kt` (which runs `UpcallTableTest`'s scenarios — constructor +/ method / getter / setter round trip through `HandleTable`, `@PythonInternal` exclusion, narrow +`Int`/`Float` widening, `tp_traverse` field detection — against a generated table), and +`python-multiplatform-ksp/src/test/.../SourceRenderingTest.kt`. + +### 2.3 Aggregation in the app module + +Kotlin/Native has no `ServiceLoader`; a top-level `object` that nothing references is never initialised +(lazy initialisation is the default since K/N 1.7.20); and reflection cannot enumerate. So a fragment that +registers itself in its own initialiser never runs. Fragments must be referenced explicitly by something +reachable. + +The **app-role** module's KSP therefore generates an aggregator that names every fragment: + +```kotlin +package python.multiplatform.generated + +object FunctionTable { + val fragments: List = listOf( + python.multiplatform.generated.fragments.Fragment_my_library, + python.multiplatform.generated.fragments.Fragment_my_app, + ) + fun installInto() { python.multiplatform.reflection.UpcallTable.install(fragments) } +} +``` + +(`renderAggregatorSource`.) `UpcallTable.install` clears the table (bumping the epoch) and registers each +fragment; a fragment already registered under the same `moduleName` is a no-op, and **a name claimed by two +different fragments is an error**, detected before anything is mutated (SPEC U-1). + +**Discovery** uses `resolver.getDeclarationsFromPackage("python.multiplatform.generated.fragments")`, in two +KSP rounds: + +1. **Round 1** — generate the app's own `Fragment_`. The new file triggers round 2. +2. **Round 2** — `getDeclarationsFromPackage` now sees the app's fresh fragment **and** every dependency's + fragment, which is on the compilation classpath as compiled classes (JAR) or klib metadata. Generate + `FunctionTable`. + +KSP cannot see other modules' *sources*, but this query reads the classpath, which is why it works. The key +is the well-known package, not an annotation: `getSymbolsWithAnnotation` is documented as potentially +expensive because it scans the whole classpath; the package query is cheap. (A marker-interface filter on +`FunctionTableFragment` would also work; the package is the primary mechanism.) + +`getDeclarationsFromPackage` is `@KspExperimental`. It is the documented cross-module API, nothing +non-experimental does the job, and it has been stable since KSP 1.x. Discovery sits behind the +`FragmentDiscovery` interface (`FragmentDiscovery.kt`) so a fallback — each module writing a metadata file +under `META-INF/`, read by the aggregator — can replace it without changing the fragment or table shape. + +The same mechanism serves every target: + +| Platform | Discovery source | Status | +|---|---|---| +| JVM desktop | JARs on the classpath | `ksp-fixtures/app` desktop tests | +| Android | JAR/AAR classpath | `ksp-fixtures/android` (`GeneratedAndroidTableTest`, unit and instrumented) | +| GraalVM native image | classpath at build time; the table is statically reachable, so no reflection config | verified manually, [`graal-native-image-verification.md`](../platforms/graal-native-image-verification.md) (SPEC U-6) | +| Kotlin/Native | klib metadata | `ksp-fixtures/{library,app}` on `androidNativeArm64`: round 2 found the library's fragment in its `.klib`, `FunctionTable` compiled and linked, and `ksp-fixtures/app/src/androidNativeArm64Test/.../NativeSmokeTest.kt` runs it (SPEC U-3) | + +`ServiceLoader` could work on JVM/Android, but one aggregator everywhere avoids a platform split and runtime +classpath scanning. + +### 2.4 Reaching `FunctionTable` from shared code: `@InstallsUpcallTable` + +`FunctionTable` is emitted into each leaf compilation, so only that target's own source set can name it; +an intermediate source set (`commonMain`, `iosMain`, `androidNativeMain`) gets +`Unresolved reference 'FunctionTable'`. Generating in `kspCommonMainMetadata` instead would fix visibility +and break the scan (a common-only scan sees only common declarations). So generation stays in the leaves +and the **seam** is generated: + +```kotlin +// commonMain +@InstallsUpcallTable +expect fun installGeneratedUpcallTable() +``` + +The app-role processor finds the annotated `expect` (a leaf compilation resolves its whole source-set +closure) and writes one `actual` per leaf compilation into the `expect`'s package: + +```kotlin +@python.multiplatform.reflection.PythonInternal +actual fun installGeneratedUpcallTable() { python.multiplatform.generated.FunctionTable.installInto() } +``` + +`@PythonInternal` because the `actual` is a public top-level function in the user's package and an +incremental round could otherwise expose "reinstall the table" to Python. A `library`-role module carrying +the annotation is an error (it aggregates nothing). `expect`/`actual` rather than an interface plus a +runtime registry, because the reference chain must stay static or the Kotlin/Native linker may drop what +nothing names. The seam's shape is checked in `InstallSeam.kt` (top-level, no parameters, returns `Unit`, +public or internal). + +### 2.5 The processor, its options, and the Gradle plugin + +One `SymbolProcessorProvider` (`PythonBindingProcessorProvider.kt`) with two roles: + +| `python.multiplatform.role` | Does | +|---|---| +| `library` | scan the module, emit its fragment | +| `app` | emit its fragment (round 1), discover all fragments and emit `FunctionTable` and the `@InstallsUpcallTable` actuals (round 2) | + +| KSP option | Required | Meaning | +|---|---|---| +| `python.multiplatform.role` | yes | `library` or `app` | +| `python.multiplatform.moduleName` | yes | sanitised fragment name | +| `python.multiplatform.excludePackages` | no | comma-separated packages to skip, on top of the always-excluded ones | + +**The Gradle plugin is built**: `python-multiplatform-gradle-plugin/`, id +`io.github.thisisthepy.python.multiplatform.bindings`. Applying it applies KSP, adds the processor to every +target's **main** KSP configuration (never a `…Test` one and never `kspCommonMainMetadata`, either of which +would emit a second fragment under the same name), infers the role (`application` / +`com.android.application` → `app`, else `library`; override with `pythonBindings { role.set("app") }`) and +derives `moduleName`. Notes from building it: + +- **It does not compile against KSP's Gradle plugin.** `kotlin-dsl` builds against Gradle's embedded + Kotlin (2.0.20 on Gradle 8.11.1) and KSP 2.3.11 is compiled with Kotlin 2.3 ("compiled with an + incompatible version of Kotlin"). The dependency is `runtimeOnly` and the `ksp { arg(...) }` calls go + through one reflective lookup of `arg(String, String)`. +- **Test configurations are matched as a camel-case word `Test`.** AGP puts the build type last + (`kspAndroidTestDebug`), so an `endsWith("Test")` check once put the processor on a unit-test + compilation, which emitted a duplicate fragment shadowing `main`'s. `ksp-fixtures/android` carries the + Android plugin for this reason ([ROADMAP](../roadmap/ROADMAP.md) §13). + +### 2.6 Tree shaking — decision: accept the cost (measured) + +If the aggregator references every fragment and every fragment references every `public` callable, the +linker sees the entire public API as reachable. Each lambda `{ args -> greet(args[0] as String) }` is a +static reference to `greet`; a reachable fragment makes every target reachable. Function-level DCE +(`-ffunction-sections` + `--gc-sections`) cannot help, because the references are real. Three things +cannot all hold: (1) blacklist exposure, (2) a full table referenced at build time (required on +Kotlin/Native), (3) tree shaking of unused entries. + +**Chosen: give up (3) — accept the cost**, because (1) is settled (§1.5) and (2) is forced by §2.3. The +remaining options, one line each: + +| Option | Verdict | +|---|---| +| B. Finer fragments (per file/class) | Does not help alone: `FunctionTable` still references every fragment unconditionally; `by lazy` still captures the reference. (Per-file fragments are still the lever for incremental KSP, §2.7.) | +| C. Opt-in manifest (`python_bindings.txt`) | A different product: contradicts the blacklist and the "write Kotlin, import from Python" promise | +| D. `ServiceLoader` on JVM, aggregator on Native | A platform split for no gain: the JVM does not tree-shake anyway | +| E. Two-tier table (skeleton + lazily initialised body) | Saves runtime lambda objects, **not binary size**: the `when` arms still statically reference every target on Kotlin/Native | + +**Measured.** `ksp-fixtures/library` got 200 synthetic top-level functions (`fun bulkN(x: Long): Long = x + N`); +`ksp-fixtures/app`'s `androidNativeArm64` debug test binary was linked before and after +(`linkDebugTestAndroidNativeArm64`) and stripped with the NDK's `aarch64-linux-android-strip`: + +| | bytes | delta | +|---|---|---| +| baseline (unstripped) | 7,578,480 | | +| +200 entries (unstripped) | 8,343,184 | 764,704 (≈3.8 KB/entry) | +| baseline (stripped) | 3,264,056 | | +| +200 entries (stripped) | 3,632,696 | 368,640 (**≈1.84 KB/entry**) | + +This is a **floor**: each function is a one-line expression, so almost all of the 1.84 KB is the +`ExposedCallable`, its lambda, its name string and Kotlin/Native per-function metadata. A real library adds +its function bodies on top. For 200 entries the floor is ≈11% of a 3.26 MB stripped baseline — noticeable, +not disqualifying. Whether that is acceptable for a real library is a product judgement; the only real +mitigation would be changing the exposure model, which is settled. + +### 2.7 Incremental KSP — measured, `ALL_FILES` stays + +`Dependencies(false)` would never regenerate the aggregator and miss a newly added fragment; +`Dependencies.ALL_FILES` is correct but reprocesses on every change. A narrower declaration buys nothing. +Measured with `ksp.incremental.log=true` on `:ksp-fixtures:app` (two files), modifying one: + +| aggregator `Dependencies` | dirty / all | +|---|---| +| `ALL_FILES` | 100.00% | +| `Dependencies(aggregating = true, )` | 100.00% | + +A module's own `Fragment_` is an *aggregating* output over every source file — correctly, since under +blacklist exposure any file can add an entry — so any change regenerates it and KSP marks every source that +maps to it dirty. A classpath-only change behaves the same (adding a function to `:ksp-fixtures:library` +dirtied both app files with `CP changes` listing `Fragment_ksp_fixture_library`). The only route to real +incrementality is **per-file fragments**, which changes fragment naming, `UpcallTable`'s per-module +idempotency and where duplicate names are detected — a design change, not done. + +### 2.8 Alternatives rejected + +| Alternative | Why not | +|---|---| +| `@EagerInitialization` | Deprecated and slated for removal; does not answer who calls `register()`; defeats tree shaking by definition | +| Resource files under `META-INF/` | Not needed while `getDeclarationsFromPackage` works; more complex (classpath I/O) and less type-safe. Kept as the fallback behind `FragmentDiscovery` | +| Manual `FunctionTable.register(Fragment_x)` in `main()` | Contradicts "apply the plugin and nothing else"; every new dependency needs a call | +| A compiler (IR) plugin | Harder to write and maintain, less stable API, and KSP suffices | + +### 2.9 What is still not verified for generation + +- **Transitive dependencies** (app → libA → libB with a fragment in libB): should work because transitive + classes are on the compilation classpath. (unverified) +- **Published Maven artifacts** consumed as `implementation("g:a:v")` rather than `project(...)`: should + work the same way. (unverified) + +--- + +## 3. Runtime path + +### 3.1 From install to call + +``` +Kotlin, once: UpcallBootstrap.publishToGlobals() publishes _pm_resolve / _pm_invoke / _pm_bind* / _pm_release / _pm_cancel + installGeneratedUpcallTable() -> FunctionTable.installInto() -> UpcallTable.install(fragments) + PythonProxySource.install() renders Python from UpcallTable + ClassLookup, exec's it +Python: from demo.calc import Counter sys.modules hit -- no import hook + c = Counter(10) proxy __init__ -> _pm_invoke(handle, (10,)) -> UpcallTrampoline.invoke + c.increment(5) _pm_invoke(method_handle, (c._pm_handle, 5)) +``` + +(`*` `_pm_bind` is published by the `PyMethodDef` bootstraps, not by desktop's.) Names are resolved to +handles when the generated module is executed; the calls afterwards carry only handles (§1.2). +`UpcallBootstrap` is the consumer-facing route (`commonMain/.../ffi/upcall/UpcallBootstrap.kt`); calling it +twice is safe (`UpcallBootstrapTest`). The published names live in `__main__` and do not survive +`Py_Finalize`; the addresses behind them are process-lifetime, so a caller that re-initialises must publish +again (inferred from code; not covered by a test because the suite shares one interpreter). + +### 3.2 The trampoline and its one carrier shape + +`python.multiplatform.ffi.upcall.UpcallTrampoline` (`commonMain`) depends on nothing platform-specific. +Argument passing needs exactly one C shape: + + PyObject *pm_invoke(long callableHandle, PyObject *args) // (long, long) -> long + +Arity and types travel inside the tuple and the table entry, never in the C signature. A stub specialised +per signature would need one per `(arity, tag-vector)` — unbounded and impossible to pre-generate for a +closed world. The remaining slots were already in the downcall vocabulary: + +| Slot | C signature | Carrier shape | | +|---|---|---|---| +| `PyCFunction` | `PyObject *(PyObject *self, PyObject *args)` | `(long, long) -> long` | the argument-passing shape | +| `getter` | `PyObject *(PyObject *self, void *closure)` | `(long, long) -> long` | same one | +| `initproc` / `setter` / `traverse` | `int(long, long, long)` | `(long, long, long) -> int` | pre-existing | +| `inquiry` (`tp_clear`) | `int(PyObject *self)` | `(long) -> int` | pre-existing | +| `destructor` | `void(PyObject *self)` | `(long) -> void` | pre-existing | +| name → handle | `long(const char *)` | `(long) -> long` | pre-existing | +| release / cancel | `int(long)` | `(long) -> int` | pre-existing (`_pm_release`, `_pm_cancel`) | + +`newfunc` is unaccounted for only if `tp_new` is ever bound directly; it is not — construction goes +through a `CONSTRUCTOR` entry called from Python (§4.2). + +### 3.3 What crosses, per tag + +The `Array` an entry receives has one representation per `TypeTag`, which the generated cast +(`TypeShape.castExpression`) narrows from: + +| `TypeTag` | Python | Kotlin | +|---|---|---| +| `INT` | `int` | `Long` (a declared `Int`/`Short`/`Byte` is narrowed by generated code) | +| `FLOAT` | `float` | `Double` | +| `BOOLEAN` | `bool` | `Boolean` — a distinct Python type, not folded into `INT` | +| `STRING` | `str` | `String` | +| `BYTES` | `bytes` | `ByteArray` (NUL bytes survive; SPEC U-2) | +| `UNIT` | `None` | `Unit` | +| `OBJECT` | `int` handle, **or** any Python object | the Kotlin instance from `HandleTable`, **or** a `PyObject` | +| — | `None` | `null`, whatever the tag | + +`OBJECT` is resolved by what Python actually sent: Python cannot hold a Kotlin reference, so a Kotlin +object crosses as an `ObjectReference` integer (`(generation shl 32) | slot`, generation ≥ 1); anything +else is a Python object and reaches a parameter declared `PyObject`. + +`BYTES` is correct and slow — one item at a time — because `PyBytes_AsString` is bound as a NUL-terminated +string read and `PyBytes_AsStringAndSize` is in no platform's `EmbedAPI` yet (`UpcallTrampoline.toByteArray` +KDoc). When it is, both directions collapse to one call. + +### 3.4 Conventions that are not negotiable + +- **Arguments are borrowed.** `PyTuple_GetItem` lends, so a `PyObject` built over one takes + `borrowed = true`. `borrowed = false` gives back a reference nobody took — one per call — and the free + lands somewhere unrelated. That bug crashed this repository twice. +- **The result is a new reference**; Python takes ownership. Measured end to end through each platform's + real entry point: a hundred calls move the returned object's refcount by zero + (`commonTest/.../native/ffi/UpcallEntryTest.kt`, + `theReferenceReturnedToPythonIsTakenOverExactlyOnce`). +- **Nothing is thrown out of a trampoline.** The return path is C; a Kotlin exception crossing a Panama + upcall stub terminates the VM, on Kotlin/Native the process. Every failure leaves as `NULL` with the error + indicator set; a Kotlin exception becomes a Python exception and `Unit` becomes `None` (SPEC U-2, + `UpcallTrampolineTest`). + +### 3.5 The GIL is not the caller's to promise + +`withGIL` skips `PyGILState_Ensure` when the thread's nesting depth is non-zero. That is right for Kotlin +code and **wrong for an entry from C**, because C may have released the GIL inside a scope that is still +open. `ctypes.CFUNCTYPE` does exactly that (unlike `PYFUNCTYPE`); the first version of the trampoline +segfaulted in `_PyThreadState_GET` (`PyErr_Occurred+0x1c`) on a thread whose own counter said it held the +GIL. Every trampoline entry (`invoke`, `releaseObject`, `cancelCall`) therefore takes its own +`PyGILState_Ensure`/`Release` pair **unconditionally** (`UpcallTrampoline.attached`), on every platform. + +On Android `PyGILState_Ensure` is bound as **ordinary JNI** (`PyGILState_EnsureN`): a `@FastNative` or +`@CriticalNative` binding would let a thread blocked on the GIL stall the ART collector and deadlock the two +runtimes the moment upcalls exist ([`androidMain/README.md`](../../python-multiplatform/src/androidMain/README.md)). + +### 3.6 Handles and their lifetimes + +- `CallableHandle` — table index + epoch (§1.2). `-1` is `NONE`; a valid handle is never negative. +- `HandleTable` / `ObjectReference` — a Kotlin object given to Python is rooted in `HandleTable` under a + generation-tagged handle; releases are generational, so a second release is a no-op and a stale handle + never frees a reused slot (SPEC M-4, `HandleTableTest`, `ProxyHandleLifetimeTest`, + `OwnedResultLifetimeTest`). Generated proxies give their handle back in `__del__` (§4.2). + +### 3.7 Per-platform entry points + +Only the address-publishing step differs; the marshalling is shared. + +| Platform | How Python reaches `UpcallTrampoline` | Names published | +|---|---|---| +| **Desktop** | FFM `Linker.upcallStub` (`UpcallStub.invokeWithArgsStubAddr`, `cancelCallStubAddr`, …) wrapped by `ctypes.CFUNCTYPE` in `UpcallBootstrap.desktop.kt` | `_pm_resolve`, `_pm_invoke`, `_pm_release`, `_pm_cancel` | +| **iOS / androidNative** | `python.native.ffi.UpcallEntry` (`nativeMain`): a `PyMethodDef` whose `ml_meth` is a `staticCFunction` over the trampoline, `self` carrying the handle; Kotlin/Native fills the struct, no C glue | all five | +| **Android (ART)** | C shims in `artMain/cinterop/jni_onload.def` (`pmp_upcall_invoke_meth`, `pmp_upcall_invoke_free_meth`, `pmp_upcall_resolve_meth`, …, `pmp_upcall_cancel_meth`) calling `python/native/ffi/UpcallCallbacks` via `CallStaticLongMethod` | all five | +| **wasmJs** | one `@WasmExport("pmp_invoke")` placed in CPython's `__indirect_function_table` with `Table.set`, five `PyCFunction`s over it told apart by an op code in `self` (§3.10) | all five | + +Tests: `UpcallEntryTest` (`commonTest`, runs on every target through the per-platform `bindUpcallOrNull`), +`nativeTest/.../UpcallRawEntryPointTest.kt`, `androidInstrumentedTest/.../UpcallThreadAttachTest.kt`. + +The `str`/`bytes` contract of `_pm_resolve`: the generated support code sends **bytes**, because desktop's +`ctypes.CFUNCTYPE(c_long, c_char_p)` refuses a `str`; every `PyMethodDef` bootstrap accepts both +(`PyUnicode_AsUTF8`, on failure `PyErr_Clear` then `PyBytes_AsString`). The generated module's own +name → handle helper is called `_pm_lookup` so it never overwrites a bootstrap's `_pm_bind` +(handle → callable). `PythonProxySourceTest` guards that the generated support defines none of the five +bootstrap names. How these were found: when `PythonProxyInstallTest` was first run outside desktop, iOS +failed 14/14 because `UpcallEntry.publish` did not publish `_pm_invoke`, then 11/11 on the `bytes` +argument; ART later failed 11/11 on the same `bytes` gap in its separate C shim +(`TypeError: bad argument type for built-in operation`, on `pmp_api26` and `pmp_api36`). The rule that +came out of it: the generator's assumptions about bootstraps are only proven by running +`PythonProxyInstallTest` on each target. + +### 3.8 Android: the boundary runs the other way, and threads attach once + +`RegisterNatives` binds a JVM `external fun` to a C function — the **downcall** direction. A +`PyMethodDef`'s `ml_meth` must be a real C function pointer and Kotlin/JVM on ART can produce none. So the +entry points are C functions in `jni_onload.def`, and *they* call Kotlin with `CallStaticLongMethod` against +`UpcallCallbacks`, a class looked up once in `JNI_OnLoad` and held as a global ref. `RegisterNatives` appears +only for the one cold `upcallPublish(long)`. This is the same inversion the cycle-collecting proxy type uses +for `tp_traverse`/`tp_clear` (`ProxyCallbacks`), down to reusing `pmp_attach`. + +**A Python worker thread is a bare pthread ART has never seen**, so `GetEnv` fails and the shim must +`AttachCurrentThreadAsDaemon` (SPEC C-5). +`UpcallThreadAttachTest.anUpcallArrivesOnAThreadCPythonCreatedRatherThanFailingToFindTheJvm` upcalls from a +`threading.Thread` and asserts it did not land on the instrumentation thread. + +**Attach once per thread, detach at thread death.** Attaching and detaching per call was the most expensive +thing on the Android upcall path (§7.6). The belief that forced it — "ART aborts if a thread exits without +detaching" — is false: `Thread::ThreadExitCallback` (`runtime/thread.cc`, same shape in `android-8.0.0_r1` +and `main`) warns on its first invocation and only reaches `LOG(FATAL)` on a second, which only an `#else` +branch Android does not compile could arm; bionic clears a key's value before its destructor and never +re-reads it. `UpcallThreadAttachTest.aThreadThatExitsWithoutDetachingLeaksItsPeerRatherThanAbortingArt` +lets an attached pthread exit undetached on `pmp_api26` and `pmp_api36` with no abort and no logcat warning. +**The detach is still mandatory**, for a different reason: an attachment never given back leaves its +`java.lang.Thread` in the thread list as a GC root. So the detach happens at thread death, in a +`pthread_key_create` destructor (`pmp_thread_exit_detach`) that defers to its second invocation so it cannot +run before ART's own exit callback. + +**Holding attachments is not a new risk.** Peak simultaneous attachments do not change — under the per-call +scheme every worker upcalling at an instant is attached at that instant; the ceiling is the number of live +Python threads either way. What changes is that release must be reliable: +`manyConcurrentWorkersAreEachAttachedOnceAndAllReleased` holds 32 workers on a `threading.Barrier`, checks +each sees one ART thread and no two share one, and requires every peer reclaimable afterwards (green on both +emulators). The instrument reads GC reachability of the peer, not `ThreadGroup.enumerate`, which on API 26 +does not report an attached native thread's peer at all; a positive control (peer unreclaimable while its +worker is parked mid-upcall) is kept in the test because it is what caught that. + +The C shims take no GIL — CPython holds it when calling a `PyCFunction` — and the trampoline takes its own +pair (§3.5). + +### 3.9 iOS and androidNative: the `@CName` + `ctypes.CDLL(None)` route, measured + +The original design said Python would reach `@CName` symbols via `ctypes.CDLL(None)` because both live in one +binary. On iOS **neither half holds**: + +| Binary | `pm_upcall_invoke` / `pm_upcall_resolve` / `pm_upcall_release_object` | +|---|---| +| androidNative `libmultiplatform_python3.14.so` | **exported** (`T` in `nm -D`) | +| iOS `PythonMultiplatform.framework` | **absent** — a K/N framework exports the Objective-C surface plus the Konan runtime | +| Kotlin/Native test executable, either target | **absent**, also from per-file caches — never emitted | + +and this project's iOS `Python.framework` binary has no `_ctypes`, so `import ctypes` raises +`ModuleNotFoundError` (Android's CPython ships `_ctypes.so`). The `@CName` functions +(`pm_upcall_invoke`, `pm_upcall_resolve`, `pm_upcall_release_object`, `pm_upcall_cancel_call`) are kept for a +C host and for `ctypes` on Android; `UpcallEntry.invokeAddress` publishes the same addresses without the +link (`UpcallRawEntryPointTest.theRawEntryPointsAreCallableCFunctionsOfTheDocumentedShape`). The real route is +the `PyMethodDef` of §3.7. + +### 3.10 wasmJs: one export, five names + +`@WasmExport` is valid only in the compilation that produces the `.wasm`, so the embedding application +declares `@WasmExport("pmp_invoke") fun pmpInvoke(selfPtr: Int, argsPtr: Int): Int = +UpcallEntry.invokeMethod(selfPtr, argsPtr)`; this repository reproduces that file in +`wasmJsTest/.../UpcallExports.kt` so the suite takes the application's path. `wasmJsMain`'s `UpcallEntry` +puts the export in CPython's table (`pmpRegisterUpcall`) and builds `PyCFunction_NewEx(def, self, NULL)` +objects over it. + +A `PyCFunction` carries `self` as well as the function pointer, so one export already backs any number of +distinct callables. Putting a value no handle can take in `self` turns it into a dispatcher — **no second +export, and no op argument**: + +| `self` | Name | Shape | +|---|---|---| +| `-2` | `_pm_resolve` | `(name: str \| bytes) -> handle`, `-1` if none | +| `-3` | `_pm_invoke` | `(handle, args_tuple) -> result` | +| `-4` | `_pm_bind` | `(handle) -> callable` | +| `-5` | `_pm_release` | `(handle) -> int` | +| `-6` | `_pm_cancel` | `(handle) -> int` | +| other | bound callable | `UpcallTrampoline.invoke(self, args)` | + +Negative ops are safe because `CallableHandle.isValid` is `raw >= 0` and `ObjectReference`'s generation is +≥ 1; `-1` is not an op because it is `CallableHandle.NONE` and must still reach the trampoline's own refusal. +The application's file and `verifyWasmAbiSignatures` (317 `@WasmImport` + 3 glue) are unchanged by it. +Removing the `UpcallEntry.publish` call makes `PythonProxyInstallTest` fail 11/11 with the generated module's +guard ("the raw upcall entry points are not bound …"), so the passing run depends on the bootstrap. + +The bare mechanism (`call_indirect`, no arguments) was measured at 3.1 ns in the former standalone +`wasm-experiment` (against 10.9 ns for `addFunction` around a JS closure); that experiment is now +`wasmJsTest/.../WasmUpcallRouteOverheadTest.kt`, measured from Python as in production. **3.1 ns must not be +quoted for an upcall that carries arguments** — that is ~300 ns (§7.2). + +--- + +## 4. Generated Python proxies + +### 4.1 Rendered at run time, not shipped as a `.py` + +`PythonProxySource.render` (`commonMain/.../ffi/upcall/PythonProxySource.kt`) renders Python from the +installed table; `install()` executes it. Not a KSP-emitted `.py` resource, because: + +1. **There is no resource path that works everywhere.** Kotlin/Native has no `getResourceAsStream`. + `PythonPayload` now puts a directory on `sys.path` on desktop (jar resource) and Android (APK asset), but + iOS, androidNative and wasm have no packaging step, so a shipped file would exist on two of five. +2. **KSP does not know what will be installed.** The table is assembled at run time and an application may + install a subset; build-time Python would describe a table that may not exist. +3. **`ExposedCallable` already carries everything** (`kind`, `arity`, `isSuspend`, names); a second + generator could disagree with the first. + +`render` is a pure function from entries to source text, pinned without an interpreter by +`PythonProxySourceTest`. Nothing here reintroduces reflection, so the native-image argument (§1.1) holds. + +### 4.2 Shapes + +- **Module functions** — published into `sys.modules` under the Kotlin package (`_pm_module`), so + `from demo.calc import doubleLater` works with no import hook (CPython checks `sys.modules` before any + finder). Each is wrapped by `python_multiplatform.kotlin_function` (§4.3). +- **Classes** — a plain Python `class` whose `__init__` calls the `CONSTRUCTOR` entry and stores the handle in + `self._pm_handle`; methods and properties pass it as `args[0]`. A `private set` property refuses + assignment. Interfaces and abstract classes have no constructor. +- **Handle release** — `__del__` (`tp_finalize`, run by `tp_dealloc`) calls `_pm_release`, looked up once at + class-definition time and carried as a default argument (globals are `None` during finalisation). Without + it, `GeneratedProxyCostTest`'s constructor row ended at 78 002 live handles. Alternatives measured on CPython + 3.13, construct + destruct net of a plain object: `__del__` +66 ns; per-instance holder +158 ns; + `weakref.ref` registry +260 ns; `weakref.finalize` +555 ns. The accepted weakness: a subclass that defines + `__del__` without calling `super().__del__()` leaks its handle (measured). `ProxyHandleLifetimeTest` pins + release, including through a cycle. +- **GC base** — where `ProxyTypeFactory.installGcBase()` succeeds it publishes the cycle-collecting + `PyType_FromSpec` type as `_pm_proxy_base`, and generated classes subclass it through a `PyMemberDef` named + `_pm_handle`, so `self._pm_handle = …` writes into the C slot `tp_traverse` reads (§6). Otherwise they fall + back to `_PmObject`. Every platform's `actual` now implements `installGcBase` (code read; the `expect`'s + KDoc saying Android and wasm return `false` is stale). +- **Class statics** (companion / `object` properties, enum entries) — a descriptor on a **metaclass** rendered + beside the class, so instances cannot see them (Kotlin's rule). **Companion functions** go on the metaclass + too: a `staticmethod` in the class body would be reachable through an instance, and an accessor pair would + disagree with the Kotlin declaration. +- **Top-level `val`/`var`** — a `property` (a *data* descriptor) on a `ModuleType` subclass generated **per + module** (one shared type would answer the attribute on every module). The previous shape, a + `__getattr__`/`__setattr__` pair on one shared subclass, cost ~50× more per read because `__getattr__` is + the fallback hook: `module_getattro` first raises a fully formatted `AttributeError` that the hook discards + (§7.7). +- **`suspend` functions** — `async def` wrappers that await the result only if it is awaitable: + + async def _pm_f_0(a0): + _pm_r = _pm_invoke(_pm_h_0, (a0,)) + if hasattr(_pm_r, '__await__'): + return await _pm_r + return _pm_r + +### 4.3 Kotlin-named surface + +`KotlinSurface` (`commonMain/.../ffi/upcall/KotlinSurface.kt`) installs `python_multiplatform`, the binder's +root module, shared by both installers (`PythonProxySource` and the binding layer `PythonxAdapter`, which +lives at `python_multiplatform.binding`). Its KDoc is the contract (SPEC U-8): + +- keyword arguments by **Kotlin parameter names**; an extension receiver is positional-only; +- a parameter with a Kotlin default has `default is python_multiplatform.KOTLIN_DEFAULT`; leaving it out (or + passing that object) means Kotlin's own default; +- a required parameter after a defaulted one is keyword-only; +- annotations are the declared Kotlin type names; synthetic `$composer`/`$changed`/`$default` never appear; +- an overload set answers `(*args, **kwargs)`; +- `inspect.signature(fn)` (built lazily via `__wrapped__`) and `python_multiplatform.describe(fn)` give the + metadata. The answer is the same whichever installer ran first (stamped epochs). + +No member or parameter is renamed and the binder creates no `pythonx` module; a real `pythonx` package on +disk is what `import pythonx` loads. Tests: `desktopTest/.../pythonx/KotlinNamedSurfaceTest.kt`, +`ksp-fixtures/compose/.../KotlinSignatureMetadataTest.kt`. `.pyi` stubs are generated by the Gradle plugin +under Kotlin module paths only (SPEC B-7). + +### 4.4 Inheritance + +A generated proxy is an ordinary Python class, so Python can subclass it and override methods; instances of +such subclasses keep their handle and are collected through cycles (SPEC M-3; e.g. +`desktopTest/.../ref/CycleCollectionTest.kt` `aPythonSubclassOfTheProxyTypeSurvivesBeingUntrackedTwice`). +**A Python override is not visible to Kotlin callers**: nothing dispatches a Kotlin virtual call to a Python +method (no test asserts it, and the table has no such path). The original design's "Kotlin calls the +override through `PyObject_CallMethod`, no new stubs" is therefore not implemented. + +### 4.5 Where proxies run + +`PythonProxyInstallTest` lives in `commonTest`. Each test target declares `publishesProxyEntryPoints` and +`proxyBootstrapSupportsAsyncio` as constants (`commonTest/.../ffi/upcall/ProxyBootstrap.kt` and its +`actual`s) instead of probing `globals()` at run time, so a target that stops publishing fails loudly instead +of silently taking the other branch. + +| Target | `publishesProxyEntryPoints` | `proxyBootstrapSupportsAsyncio` | +|---|---|---| +| desktop | true | true | +| iOS / androidNative (`nativeTest`) | true | true | +| Android ART | true | true | +| wasmJs | true | **false** (§5.8) | + +Threaded delivery tests are split by how they create a thread: `desktopTest/.../PythonProxyDeliveryTest.kt` +(`java.lang.Thread`) and `nativeTest/.../PythonProxyNativeDeliveryTest.kt` (`pthread_create`). + +--- + +## 5. `suspend` functions (async upcalls) + +SPEC U-5: `await g.greetNow(1)` works; a function that never suspends completes without a Future; cancelling +the Python future cancels the Kotlin coroutine. Implemented on desktop and Kotlin/Native; on ART the sample +demonstrates it (§5.8); `planned` on wasm. + +### 5.1 Why a suspend function cannot simply be called + +CPython calls a C function pointer whose frame must return a `PyObject *` synchronously; a suspension has +nothing to return. `runBlocking` is not a solution: it blocks the thread the upcall arrived on — on Android a +CPython-created pthread — so the interpreter waiting for the answer is the thing being blocked. The only +option: **return something else now and deliver the value later.** + +### 5.2 Three candidates; (C) chosen, with (B) as its mechanism + +All three share the Kotlin half — start the coroutine inside the synchronous frame and park the result where +it can be addressed later. + +| Candidate | Verdict | +|---|---| +| **(A) handle + polling** (`is_done(h)` / `result(h)`) | **Rejected.** On wasmJs Python and Kotlin share one JS thread and resumption only happens when control returns to the host, so a polling loop that never yields prevents its own progress (inferred). Also burns a core and re-takes the GIL at every switch interval, and KSP cannot expose a generic `Deferred` | +| **(B) callback** — Python passes a callable, Kotlin calls it on completion | Works: a Kotlin-created thread can take the GIL and call Python while the main thread is inside `run_until_complete` (measured, `AsyncCompletionProbeTest.aKotlinCreatedThreadCanResolveAnAsyncioFutureTheInterpreterIsWaitingOn`, which passes only after observing `loop.is_running()`). Hands all synchronisation to the user | +| **(C) asyncio** — Kotlin creates a `Future` on the running loop and resolves it with `loop.call_soon_threadsafe` | **Chosen.** Needs no new binding: `PyImport_ImportModule`, `PyObject_GetAttrString` and `PyObject_Call*` are enough (measured, `AsyncCompletionProbeTest.everyApiThisNeedsIsAlreadyBoundSoAnAsyncioConventionAddsNoNewBinding`). `await kotlin_fn(x)` is syntax Python users already know | + +(C) is mechanically (B): `call_soon_threadsafe` is a downcall from the completion thread. `Py_AddPendingCall` +(in the Stable ABI, not bound in `EmbedAPI`) was not needed. The cost of (C): the application must be written +async — the loop must be running. + +**The fast path comes before the convention.** `suspend` is a signature, not a promise to suspend: a body +that reaches no suspension point is complete before `PendingCall.start` returns +(`PendingCallTest.aBlockThatNeverSuspendsIsAlreadyCompleteBeforeStartReturns`), and its value is returned +directly with no `Future` and no event loop. + +What would reverse this choice: if a target's loop could not be made to yield to its host, (B) would be the +better public surface there; if a completion thread on Android were found to need an ART attach, the +`pmp_thread_exit_detach` destructor (§3.8) would have to be replicated on the completion side. (A Kotlin/JVM +thread is already known to ART, so no attach is expected — inferred; the ART sample in §5.8 resumes from a +`java.lang.Thread` and works.) + +### 5.3 KSP: a different body, not an exclusion + +`BindingPolicy.isSuspending` selects a body shape: + + { args -> fixture.library.blockingTopLevel(args[0] as Long) } // ordinary + { args -> python.multiplatform.ffi.upcall.PendingCall.start { fixture.library.suspendingTopLevel(args[0] as Long) } } // suspend + +The braces are a `suspend` lambda; it compiles with the stdlib only. `kind` is unchanged (a suspending +member is still `METHOD`, receiver in `args[0]`) and `returnType` is the **declared** return type, so the +fast path marshals exactly like a synchronous entry. `ExposedCallable.isSuspend` is a flag rather than +`SUSPEND_*` variants of `CallableKind` because the axes are orthogonal: `kind` says where the arguments are, +`isSuspend` what comes back. Pinned by `ksp-fixtures/app/.../GeneratedSuspendTest.kt` against +`ksp-fixtures/library/.../Suspending.kt`, which pairs every suspending declaration (top-level, member, +companion, interface, object) with a non-suspending control in the same scope. + +A `suspend` **function type** (`val h: suspend (Long) -> Long`, a parameter or return of that type, nullable, +behind a `typealias`, or nested in a generic argument) is **not exposed** and KSP **warns** +(`FragmentScanner.warnUnexposableType`); `GeneratedSuspendTest` asserts each rejection. (An earlier state +exposed it as an opaque `OBJECT` handle no entry could call. The `BindingPolicy.isSuspending` KDoc still says +that; it is stale.) + +### 5.4 `PendingCall` + +`commonMain/.../ffi/upcall/PendingCall.kt`, pinned by `PendingCallTest`: + +- **No `kotlinx.coroutines` dependency.** `startCoroutine` and `Continuation` are stdlib; the dispatcher a + user's `suspend fun` needs is already on the user's classpath. +- **The context is the `PendingCall` itself** (a `CoroutineContext.Element`), with no dispatcher, so the body + runs on the calling thread until its first real suspension — which is where the fast path comes from, and + why work before the first suspension runs **inside the C frame, holding the GIL**. +- **Nothing escapes**: body and listener failures are parked, as in the trampoline. +- **No new handle machinery**: it is an ordinary object in `HandleTable`. +- **Unsynchronised**, on the boundary's rule that every mutation happens on a GIL-holding thread; a + completion that reaches Python takes the GIL anyway. + +### 5.5 Delivery + +`AsyncUpcall.deliver` runs on `UpcallTrampoline.invoke`'s `entry.isSuspend` branch: + +| Case | Returned | Cost | +|---|---|---| +| already complete, success | the real value (`marshalResult`) | not even `import asyncio` | +| already complete, failure | `NULL` + error indicator | identical to a synchronous failure | +| really suspended | `asyncio.Future` (new reference) | `get_running_loop` + `create_future` | + +On completion: `withGIL { loop.call_soon_threadsafe(_pm_settle, future, ok, payload) }`. `withGIL`, not the +trampoline's unconditional pair, because this is not a C entry: a fresh completion thread has depth 0 and +really ensures; a completion inside a later upcall correctly nests. The Kotlin wrapper and Python each hold a +reference to the `Future`; the completion lambda keeps the wrapper alive until it fires. + +**The fast path is wider than "did not suspend".** If the completion thread resumes the continuation while the +upcall frame is still running — `CFUNCTYPE` releases the GIL for the upcall itself — `deliver` sees `isDone` +and returns the value. Correct, but the synchronous-completion ratio is a function of the race, and a test of +the slow path must observe that a `Future` actually crossed (`AsyncUpcallDeliveryTest` records +`type(r).__name__`; the proxy tests count `create_future` calls instead and assert `0` on the fast path — +`PythonProxyInstallTest.theSameGeneratedProxyBuildsNoFutureWhenTheKotlinBodyNeverSuspends`). + +**No running loop:** a call that really suspends fails with `RuntimeError: no running event loop`, and the +coroutine has already started (whether it suspends is only known by starting it); its resumption is dropped +(`AsyncUpcallDeliveryTest.aSuspendingEntryThatSuspendsWithNoRunningLoopFailsInsteadOfReturningSomethingUnusable`). +A suspending entry that never suspends works with no loop at all +(`aSuspendingEntryThatNeverSuspendsHandsBackTheRealValueAndBuildsNoFuture`). + +### 5.6 Cancellation + +**A completion landing on a cancelled `Future` is dropped, not raised.** Measured before the fix +(`AsyncUpcallCancellationTest`): `await` got `CancelledError`; the loop's `call_exception_handler` got one +`InvalidStateError: invalid state`; the completion thread's error indicator was clean and the next unrelated +upcall worked — no contamination, just an unactionable log entry. Two guards, only the second a guarantee: +`AsyncUpcall.resolve` checks `done()` before scheduling (cheap, racy), and the scheduled callback is +`_pm_settle`, which checks `done()` **again on the loop thread**. Removing only `_pm_settle`'s guard turns +exactly the race test red; the race is forced, not left to timing (the Python coroutine spins without +awaiting while Kotlin schedules, then cancels). + +**Kotlin-side cancellation is cooperative, by construction.** The continuation `startCoroutine` receives is +the coroutine's *completion*; the suspension point's continuation belongs to whoever suspended (the user's +`suspendCoroutine`, their dispatcher, `kotlinx.coroutines`). `PendingCall` never sees it, the stdlib cannot +reach it, and resuming twice is undefined. Forced cancellation is what a `Job` tree does. What works: +`ensureActive()` (top-level, reads `coroutineContext[PendingCall]`) throws `CancellationException` once the +call is cancelled — the same contract as `kotlinx.coroutines`, where a body that never checks is never +cancelled either. The element is not a `ContinuationInterceptor`, so the fast path is untouched. + + suspend fun slowSum(n: Long): Long { + var total = 0L + for (i in 0 until n) { ensureActive(); total += step(i) } + return total + } + +**Early notification.** Python's `fut.cancel()` reaches Kotlin before completion: + + fut.cancel() -> done callback (call_soon) -> _pm_cancel(handle) (long) -> int + -> UpcallTrampoline.cancelCall (unconditional PyGILState_Ensure) + -> HandleTable.resolveRaw -> PendingCall.cancel() -> next ensureActive() throws + +`_pm_cancel` has `_pm_release`'s shape (no new stub shape). The `PendingCall` handle is captured as a default +argument of the done callback rather than set as a `Future` attribute. `AsyncUpcall.armCancellationNotice` +registers the handle only if `_pm_cancel` and `_pm_release` exist in `__main__`; a host without them keeps +completion-time observation and leaks nothing (a `NameError` inside a loop callback would be another +unactionable log). `AsyncUpcallEarlyCancellationTest` was red first, with the old behaviour as its message +("Kotlin did not learn about the cancellation until the call completed"); it asserts +`isCancelled && !isDone` on a call never resumed since cancellation (the old code also set the flag, but only +at completion), then resumes once and requires `CancellationException` and `isDone`. + +**Two releasers by design.** `_pm_watch`'s done callback calls `_pm_cancel` if cancelled and always +`_pm_release`; `AsyncUpcall.resolve` releases too, in a `finally`. They cover different holes: + +| Releaser | Only it covers | +|---|---| +| Python done callback | a cancelled body that never cooperates — the Kotlin completion never runs | +| Kotlin `resolve` | the loop stopping before the scheduled done callback ran | + +Generational release makes the second a no-op and a late `_pm_cancel(stale)` harmless. Measured with +`HandleTable.liveCount` (`AsyncUpcallEarlyCancellationTest`): + +| Case | Live handles | +|---|---| +| while a really-suspended call is pending | baseline + 1 | +| after cancel, coroutine not yet finished | baseline (Python released it) | +| after cancel and body exit | baseline | +| after normal completion | baseline | +| fast path | baseline — nothing registered (`theFastPathRegistersNoHandleAtAll`) | + +Registration happens after `deliver`'s `isDone` early return, so the fast path touches neither `HandleTable`, +`__main__` nor `add_done_callback`. The slow path pays two `__main__` lookups and one `_pm_watch` call. Not +reclaimed: a call whose `Future` never settles and whose coroutine never ends — that has leaked its +continuation already. + +### 5.7 Desktop and Kotlin/Native tests + +The desktop tests use `java.lang.Thread` and `java.util.concurrent`, so they cannot move to `commonTest`; the +claims are re-verified on Kotlin/Native with `nativeTest/.../NativeThread.kt`, a thin `pthread_create` +wrapper. Not `Worker`: the question is whether a completion may come from a thread the Kotlin/Native runtime +never attached itself, which `Worker` would sidestep — the same situation +`CycleCollectionTest.testDeallocOnAThreadCPythonCreated` measures for `tp_dealloc`. + +| `nativeTest` | Desktop counterpart | Re-verifies | +|---|---|---| +| `AsyncUpcallNativeDeliveryTest` (2 tests) | `AsyncUpcallDeliveryTest` | delivery from a foreign thread while the loop runs; fast path takes no asyncio | +| `AsyncUpcallNativeEarlyCancellationTest` | `AsyncUpcallEarlyCancellationTest` | `_pm_cancel` reaches `ensureActive()` before completion | +| `AsyncUpcallNativeCancellationTest` | `AsyncUpcallCancellationTest` | a completion on a cancelled `Future` is dropped; no error indicator leaks | + +All passed on the iOS simulator, three repeated runs. They are in `nativeTest`, which +`androidNativeArm64Test` also runs, and they predate the androidNative full-suite runs recorded in §5.8 +(316 tests, 0 failures) — but no per-test androidNative result for them was recorded (unverified). +`AsyncUpcallPortabilityTest` (`commonTest`) keeps only the two paths that return before touching asyncio — fast +path and failure-before-suspend — which need no thread and run on all targets. + +### 5.8 Per platform + +**Desktop** — everything above (`AsyncUpcallDeliveryTest`, `AsyncUpcallCancellationTest`, +`AsyncUpcallEarlyCancellationTest`, `PythonProxyDeliveryTest`). + +**iOS simulator** — asyncio is available. The iOS `Python.framework` binary has no `lib-dynload`, but asyncio +comes from the separately extracted BeeWare stdlib (`build/python-stdlib/ios-simulator/lib/python3.14/` with +`asyncio/` and `lib-dynload/_asyncio…so`, `select…so`, `_socket…so`). +`iosSimulatorArm64Test/.../AsyncioAvailabilityProbeTest.kt` (isolated in its own file) runs +`asyncio.new_event_loop().run_until_complete(...)` and passes. Delivery, early cancellation and the silent drop +pass (§5.7). + +**Android (ART)** — proxies, including `async` ones, run on `pmp_api26` and `pmp_api36` +(`connectedDebugAndroidTest`: 336 tests, 0 failures on each; `PythonProxyInstallTest`'s 11 on the real path). +The sample app on both emulators showed: + +| Section | Output | +|---|---| +| 3 — upcall | `handle 4294967327 -> 0 · with args -> presses x3 = 0` (called from Python via `_pm_resolve`/`_pm_invoke`) | +| 5 — class proxy | `installed over PyMethodDef via JNI: 466 lines, 2 proxy classes`, `Greeter('Kotlin').greet(2) -> hello Kotlin! hello Kotlin!`, `g.greetings = 99 -> AttributeError (private set held)` | +| 6 — companion | `Greeter.forget() -> 100, then built=0`, `Greeter('x').built -> AttributeError (companion is class-only)` | +| 7 — fast path | `await g.greetNow(1) -> hello fast path!`, `Futures created -> 0` | +| 7 — really suspends | `await g.greetLater(2) -> hello slow path! hello slow path!`, `Futures created -> 1`, `raised -> None`, `completer thread -> clean` | + +The last row resumes the parked continuation from a `java.lang.Thread`, as on desktop. (The original record +adds that `pmp_attach` runs on that path too; a `java.lang.Thread` is already known to ART, so that is +(unverified).) The iOS sample cannot yet show that row (a Kotlin/Native completion thread needs a +`Worker` or memory-model decision in the sample; the library's `PythonProxyNativeDeliveryTest` does it with +`pthread_create`). + +**wasmJs** — the synchronous proxy surface runs (10 of `PythonProxyInstallTest`'s 11); `await` does not yet: + +1. *The trap is fixed.* `import selectors` / `import asyncio` used to kill the Node process with + `RuntimeError: unreachable`. That was the third symptom: `select.poll().poll(0)` raised + `SuspendError: trying to suspend without WebAssembly.promising`, which unwound CPython's C frames without + `Py_END_ALLOW_THREADS`, so the outer `PyGILState_Release` hit + `Fatal Python error: PyGILState_Release: thread state … must be current`, then `abort()`. Cause: Emscripten + decides JSPI **at run time** (`if (WebAssembly.Suspending) …`) and wraps only `main` in + `WebAssembly.promising`; this library never calls `main` (it issues `Py_Initialize` as an ordinary wasm + call). The same `python.wasm` returns `[]` on Node 22 (no JSPI) and dies on Node 26 (JSPI on by default, no + flag to disable it); Gradle's runner uses Node 26. `selectors` was the first victim because it *calls* + `poll(0)` at import time; `asyncio` died only through it. **Fix:** `cpython.mjs` + (`python-multiplatform/src/wasmJsMain/resources/`) deletes `WebAssembly.promising` and + `WebAssembly.Suspending` before calling the Emscripten factory, putting Node 26 on the synchronous path — + which is what this library needs, since every call arrives from Kotlin as a synchronous wasm call. A + `sys.modules['selectors']` shim was rejected: it only moves the death to the first loop iteration, hands + users a fake module, and fixes the wrong layer. `wasmJsTest/.../WasmSelectorsImportTest.kt` (4 tests): + `poll`/`select` return; `DefaultSelector is PollSelector`; `asyncio`, `asyncio.events`, + `asyncio.base_events` import; a coroutine runs to completion on a selector-free loop. +2. *The default event loop still cannot be built.* `BaseSelectorEventLoop.__init__` → + `_make_self_pipe` → `socket.socketpair()` → Emscripten `___syscall_listen` → `require('ws')` → + `MODULE_NOT_FOUND`. Installing `ws` (tried temporarily with `npm("ws", "8.18.0")` and a probe, then + reverted) only exposes the next wall: `OSError: [Errno 28]` from `lsock.accept()`, because Emscripten's + socket emulation needs a real WebSocket handshake to populate `pending` and `_fallback_socketpair()` runs + `bind/listen/connect/accept` inside one synchronous wasm call, so Node's event loop never ticks in between. + No npm package fixes that, and re-enabling JSPI brings back §1's `SuspendError`. In a browser `listen()` + does not exist at all (`ENVIRONMENT_IS_NODE` only). +3. *Decision:* `proxyBootstrapSupportsAsyncio` stays **false** on wasmJs. A selector-free loop works as a + mechanism (the `WasmSelectorsImportTest` run), but shipping one means keeping the whole + `Runner`/`BaseEventLoop` contract (`call_later`, `shutdown_asyncgens`, `shutdown_default_executor`, + exception handling, `KeyboardInterrupt`) correct across CPython versions. Whether the library ships a + custom loop is an open decision (§8). Until then a wasm application can rely on the fast path and on + failure-before-suspend (`AsyncUpcallPortabilityTest`). + +--- + +## 6. Cycle collection is part of the table's job + +A Python proxy holding a Kotlin object that holds a Python object forms a cycle neither collector can break +alone: CPython's traversal stops at the opaque handle, and the JVM sees the handle table as a live root. +Reference counting never frees cycles in any language. The fix makes the Kotlin side's Python references +visible to `tp_traverse`: the proxy's traverse reaches through the handle and enumerates the `PyObject`-typed +properties of the Kotlin object. KSP already walks every exposed class, so a traverse body per class is one +more generated field (`FragmentScanner.traverseBody`, `ClassModel.traverseBody`), not a new mechanism. A +reflection-based binding cannot afford this, which is why mature ones document "do not create cycles" +instead. Generated proxies reach the traversable type through the GC base of §4.2. + +Status: SPEC M-3 — implemented on desktop (`desktopTest/.../ref/CycleCollectionTest.kt`, +`ksp-fixtures/app/.../RefHolderCycleCollectionTest.kt`), partial elsewhere. Mechanism and the hard parts +(traverse during collection, `tp_clear` mutating Kotlin state, cycles closing on the Kotlin side) are in +[`object-lifetime.md`](object-lifetime.md). + +--- + +## 7. Measurements + +How to reproduce: the cost table is cut by `./benchmarks/cost-table.sh --runs 3` (see +[`cost-table.md`](../investigations/cost-table.md) "다시 뽑는 법"). Individual tests print their own blocks: + +| Test | Source set | Prints | +|---|---|---| +| `UpcallBoundaryCostTest` | `commonTest` (`python/native/ffi/`) | one upcall vs. downcall of the same shape vs. trampoline, with controls | +| `GeneratedProxyCostTest` | `commonTest` (`ffi/upcall/`) | proxy layer over the raw call; install cost | +| `UpcallOverheadTest` | `androidInstrumentedTest` | worker vs. instrumentation thread, first upcall on a fresh worker | +| `UpcallOverheadTest` | `commonTest` (`reflection/`) | name resolution vs. cached handle, inside Kotlin | +| `WasmUpcallRouteOverheadTest` | `wasmJsTest` | `Table.set` route vs. `addFunction` around a JS closure | + +Rules: run alone, unfiltered, after checking `uptime` (AGENTS.md §8); ranges are min–max over the stated runs; +a single reading is not a measurement. + +### 7.1 One upcall, per platform + +Current figures, from `cost-table.md` (commit `958c0082b294`, 2026-08-14, Apple M1, `UpcallBoundaryCostTest` +warmup 100 000, 10 000 iterations per loop, 3 runs per target; desktop/iOS/wasmJs/androidNative from +`cut-hostless`, ART from `cut-api36` / `cut-api26`): + +| Platform | upcall | downcall, same shape | trampoline alone | upcall / downcall | upcall / trampoline | +|---|---|---|---|---|---| +| desktop (JVM 21.0.12, macOS arm64) | 542.84–576.45 ns | 260.27–265.81 ns | 170.44–247.42 ns | 2.04–2.20x | 5.54–6.46x | +| wasmJs (Node) | 302.41–310.72 ns | 96.53–100.95 ns | 149.50–164.88 ns | 3.07–3.14x | 2.39–2.46x | +| iOS simulator (26.2, arm64) | 2253.20–2312.97 ns | 1585.46–1655.40 ns | 2794.42–2843.16 ns | 1.39–1.42x | 1.14–1.15x | +| androidNative `pmp_api36` | 3348.75–3443.23 ns | 2194.26–2365.61 ns | 3957.01–4069.40 ns | 1.45–1.54x | 1.19–1.20x | +| androidNative `pmp_api26` † | 3234–3310 ns | 2470–2554 ns | 2731–3228 ns | 1.28–1.32x | 1.02–1.20x | +| ART `pmp_api36` ‡ | 995.61–1154.10 ns | 804.27–838.60 ns | 1100.87–1167.17 ns | 1.18–1.43x | 1.10–1.29x | +| ART `pmp_api26` ‡ | 1196.52–1272.28 ns | 1253.41–1295.79 ns | 1800.86–1863.97 ns | 0.92–1.01x | 1.05–1.18x | + +† Not in the cost table (it cuts androidNative on `pmp_api36` only); hand-cut at warmup 100 000, five runs. +Its spread is the emulator's: in two rounds a different single run was the outlier in a different row (5554 ns +downcall at warmup 3 000; a 3227.87 ns GIL-held trampoline at 100 000, which alone widens `upcall / trampoline` +from 1.16–1.18x). Read `pmp_api36` for the shape and `pmp_api26` for how much confidence an emulator supports. +No androidNative figure is a hardware number. +‡ The cost table marks both ART cuts "failed"; the failures were in `PhantomCleanerRegistryTest`, not the +benchmark, and were fixed in `c180c46f`. + +Controls from the same cut: + +| Target | empty Python loop | pure-Python callee | GIL-held trampoline | +|---|---|---|---| +| desktop | 8.67–8.68 ns | 25.77–26.09 ns | 85.52–97.95 ns | +| iOS simulator | 9.84–10.17 ns | 36.80–38.32 ns | 1948.58–2020.43 ns | +| wasmJs | 14.74–15.34 ns | 47.89–49.22 ns | 123.15–129.83 ns | +| androidNative `pmp_api36` | 17.22–17.31 ns | 48.85–58.45 ns | 2790.19–2857.59 ns | +| ART `pmp_api26` | 20.01–20.08 ns | 56.10–65.66 ns | 1073.40–1142.08 ns | +| ART `pmp_api36` | 17.22–18.14 ns | 58.40–65.30 ns | 873.48–911.95 ns | + +The previous hand-cut round (warmup 100 000; five full suites per device, eleven on the hosts) read: +desktop 510–560 ns, wasmJs 290–304 ns, iOS 2266–2301 ns, androidNative `pmp_api36` 3228–3275 ns, +`pmp_api26` 3234–3310 ns, ART `pmp_api36` 944–1063 ns, ART `pmp_api26` 1146–1233 ns. (The predecessor document +labelled this list "warmup 3,000"; that was wrong — the 3 000-warmup desktop and wasmJs figures were 674 ns and +322 ns.) In that round each host had exactly one outlying run in eleven, kept rather than dropped: desktop's +GIL-held trampoline once read 186.81 ns against 76–91 ns (its ratio 2.89x against 5.62–7.02x); one wasmJs run +inflated every boundary row together (upcall 432.55, downcall 144.64, trampoline 264.44 ns) while the empty GIL +scope stayed at 24.67 ns, so its ratio (2.99x) stayed in band — the case for preferring the ratio on wasm. + +### 7.2 Warmup: why 100 000, and what the old figures were measuring + +`UpcallBoundaryCostTest` warmed 3 000 calls and every figure it published was taken before the host JIT +finished tiering up, so the number depended on how much upcall traffic **the rest of the suite** had pushed +through the process first. Diagnosis: on wasmJs, commit `4472f83a` made generated proxies installable, so +`GeneratedProxyCostTest` began driving ~270 000 upcalls before this test, and short-circuiting that one test +restored the figure; the original absolutes (desktop 861–1313 ns, wasmJs 703–1075 ns) fell on re-measurement +with the downcall column standing still. The fix (`7e9c6b8c`) is the warmup count, chosen from a measured +convergence point; bisections, the controlled experiment and the 40-rep sweep that identified host JIT tier-up +and ruled out CPython-side state are in [`downcall-design.md`](downcall-design.md) ("Ratio consistency" and +"The benchmark was measuring the benchmark"). + +**Check that it is fixed:** at warmup 100 000, short-circuiting `GeneratedProxyCostTest` moves desktop +510–560 → 501–530 ns and wasmJs 290–304 → 287–301 ns (overlapping); at 3 000 the same lever moved desktop +674 → 537 ns and wasmJs 322 → 292 ns. 13 desktop and 12 wasmJs full-suite runs, 3 of each with the lever +pulled. + +**JIT hosts fell, AOT hosts did not** — the prediction that separates the explanations: + +| re-measured at 100 000 | upcall, old → new | downcall, old → new | +|---|---|---| +| iOS simulator | 2263–2502 → 2266–2301 ns | 1599–1826 → 1599–1610 ns | +| androidNative `pmp_api36` | 3339–3598 → 3228–3275 ns | 2170–2222 → 2155–2229 ns | +| androidNative `pmp_api26` | 3314–4140 → 3234–3310 ns | 2636–5554 → 2470–2554 ns | + +against desktop 674 → 537 ns, wasmJs 322 → 292 ns and ART `pmp_api36` 2301–3086 → 944–1063 ns (API 26 barely +moved, 1209–1329 → 1146–1233 ns). ART has a JIT and moved like desktop and wasmJs. "Did not fall" is not "needed +no warmup": from cold, androidNative's first 10 000 upcalls read ~1.3× the plateau; the old Kotlin/Native rows +landed near it only because the rest of the suite had warmed the path. + +**Device sweep** — 40 consecutive reps of 10 000 calls from cold; first-rep ratio = rep 1 / mean of reps 21–40: + +| Target | rep 1 / plateau (Python-driven upcall) | flat from | margin at 100 000 | +|---|---|---|---| +| iOS simulator | 1.01x | ~10 000 | 10x | +| androidNative `pmp_api36` | 1.27x | ~40 000 | 2.5x | +| androidNative `pmp_api26` | 1.30x | ~30 000 | 3.3x | +| ART `pmp_api26` | 1.93x | ~20 000, noisily | 5x | +| **ART `pmp_api36`** | **8.56x** | **~90 000–100 000** | **~1x — none** | + +ART `pmp_api36`'s sweep: 9194.8, 3218.9, 2288.7, 1648.3, 1218.3, 1173.3, 1129.5, 1532.7, 1057.8, 1044.6 ns against +a plateau of 1074.2 — still 9% high at 70 000. Its downcall is flat from ~80 000 and its GIL-held trampoline +from ~50 000, so the Python-driven upcall sets the requirement. The controls are flat from the first rep on all +five device configurations (iOS 10.5 → 10.2 ns and 37.9 → 38.0 ns; ART `pmp_api36` 17.3 → 17.3 ns and +49.7 → 50.4 ns), so the sweep measures the boundary, not the interpreter. The host sweeps behind `WARMUP`'s doc: +40 000 cold / 70 000 warm on desktop, 70 000 on wasm; `overhead/BenchmarkTest`'s worst desktop row +(`PyUnicode_AsUTF8 (8 chars)`) reads 557 ns at 40 000 and 78–84 ns from 70 000. **100 000 is therefore the +constant everywhere**: `UpcallOverheadTest` (ART only) now uses it for its Python-driven loops and Kotlin-driven +downcall rows (previously 3 000 and 2 500), and `overhead/BenchmarkTest` uses it uniformly. With no margin on +ART `pmp_api36`, a slower device or busier host may read the boundary before it settles. The +`GeneratedProxyCostTest` warmup of 5 000 per row is deliberate: nineteen rows warm the shared `_pm_invoke` path +19 × 5 000 before the first timing. + +**Filtered runs.** A desktop `--tests` run of this class alone reads 506–550 ns, inside the full-suite band, and +its controls match (8.4–8.6 vs 8.4 ns). **wasmJs is the exception**: filtered it reads 321–353 ns and its +*pure-Python* controls read 23.1 and 83.1 ns against the suite's 15.0 and 47.2 — CPython is itself a wasm +module there, so a short-lived Node process runs the interpreter 55–74% slower. Tripling the warmup to 300 000 +does not move it (host-lifetime-bound). wasmJs's ratios survive filtering (2.66–3.13x vs 2.91–3.14x); its +absolutes are a full-suite figure only. The devices do not have the effect (three filtered runs vs five +full-suite runs): + +| filtered vs full suite | upcall, filtered | upcall, full | empty Python loop | pure-Python callee | +|---|---|---|---|---| +| iOS simulator | 2320–2322 ns | 2266–2301 ns | 9.9–11.4 vs 9.9–10.3 ns | 36.8–38.0 vs 36.8–38.3 ns | +| androidNative `pmp_api36` | 3275–3294 ns | 3228–3275 ns | 17.21 vs 17.22 ns | 48.9–58.2 vs 48.9–74.2 ns | +| androidNative `pmp_api26` | 3245–3280 ns | 3234–3310 ns | 20.0–23.6 vs 20.0–20.6 ns | 51.7–55.3 vs 51.7–62.0 ns | +| ART `pmp_api36` | 996–1031 ns | 944–1063 ns | 17.2–19.3 vs 17.2–22.8 ns | 49.0–60.0 vs 49.2–63.3 ns | +| ART `pmp_api26` | 1118–1171 ns | 1146–1233 ns | 20.0–20.1 vs 20.0–21.5 ns | 51.7–63.4 vs 51.7–65.6 ns | + +So device absolutes are quotable as absolutes: there CPython is native code the host executes, not a wasm +module the host compiles. + +**ART rows are measured on the instrumentation thread**, which is fair now that the attach is amortised (§7.6). +On API 26 the worker is 43–249 ns dearer than the instrumentation thread, so the ART row is that much optimistic +against a `threading.Thread` caller. + +**androidNative had no test run task** until `:python-multiplatform:androidNativeArm64Test` +(`python-multiplatform/build.gradle.kts`): KGP only registers execution tasks for the host and simctl. The task +pushes `test.kexe` and the CPython prefix (`Py_Initialize` aborts without a stdlib) to `/data/local/tmp`, runs +it under the TeamCity logger and converts the service messages to JUnit XML under +`build/test-results/androidNativeArm64Test/`, on every connected device with a matching ABI. First runs: 252 +tests, 0 failures, both emulators, five runs — `artMain` and the androidNative cinterop came up clean. + +### 7.3 The trampoline column needs a control + +`UpcallTrampoline` takes its GIL pair unconditionally (§3.5), so driving it from a bare Kotlin thread pays a real +GIL acquisition per call while Python drives it already holding the GIL. Uncontrolled, the boundary prices +**negative** on devices: at warmup 100 000, five runs each, upcall / trampoline was 0.79–0.82x on the iOS +simulator, 0.82–0.85x and 0.77–0.78x on androidNative `pmp_api36`/`pmp_api26`, 0.86–0.95x and 0.65–0.70x on ART +`pmp_api36`/`pmp_api26`. It is measured twice, differing in that one thing, with an empty +`Python3.withPython { }` for scale (hand-cut round, warmup 100 000): + +| | desktop | wasmJs | iOS sim | androidNative 36 | androidNative 26 | ART 36 | ART 26 | +|---|---|---|---|---|---|---|---| +| trampoline, caller holds nothing | 164–181 ns | 146–164 ns | 2794–2862 ns | 3789–3893 ns | 4174–4267 ns | 1093–1148 ns | 1693–1822 ns | +| trampoline, GIL already held | 76–91 ns | 122–129 ns | 1962–1990 ns | 2699–2760 ns | 2731–3228 ns | 793–907 ns | 1012–1087 ns | +| `Python3.withPython { }`, empty | 50–57 ns | 24–28 ns | 720–731 ns | 932–1002 ns | 1112–1221 ns | 276–303 ns | 498–549 ns | + +The **GIL-held** row is the comparable denominator (§7.1 uses it). Desktop and wasmJs never showed the effect +because their GIL scope is 24–57 ns against 276–1221 ns on a device. At the old 3 000 warmup desktop's empty +scope read 124–152 ns and wasmJs's 79–81 ns — the cheapest operation is the one an unwarmed loop overstates +most. wasmJs's GIL-held trampoline is dearer than desktop's although its GIL scope is half the price, so wasm +pays for the trampoline body, not the GIL. + +### 7.4 Where the differences come from + +- **iOS and androidNative: the scaffolding, not the boundary.** There is no runtime boundary on these targets — + a `staticCFunction` in the same binary. What is expensive is per-call scaffolding every C API call shares: an + empty `withPython` scope is 720–1002 ns there against 50–57 ns on desktop and 24–28 ns on wasm, and the same + inflation hits the downcall denominator, which is why the ratio sits near 1.4. (In the round that measured + `Py_IncRef + Py_DecRef` — two calls, two GIL scopes — it cost 1551–2032 ns on iOS and 2010–2132 ns on + androidNative against 110–120 ns on desktop and 44–49 ns on wasm; those device figures were unwarmed, so the + multiple is not quotable, but the gap is wide enough that the conclusion survives.) Work to move either number + belongs in the scaffolding, not in `UpcallEntry`. +- **Desktop: a real boundary.** GIL-held trampoline 76–91 ns against a 510–560 ns upcall (hand-cut round): ~420–480 + ns is the Panama upcall stub plus the `ctypes` shim, the largest relative boundary measured (5.62–7.02x in that + round, 5.54–6.46x in the cost table). Desktop's `_pm_invoke` is a `ctypes.CFUNCTYPE` (and in this benchmark a + Python `lambda *a: _pm_invoke(_pm_h, a)`), where other targets bind a `PyCFunction`, so desktop's row is an + **upper bound** on its boundary — it includes ctypes' argument conversion and, in the benchmark, one extra + Python call (26.7–27.2 ns). +- **wasmJs: ~300 ns with arguments**, of which 122–129 ns is the trampoline; the rest is the Python-side callable + and the crossing with a real tuple. Its scaffolding is the cheapest and most stable (24.47–27.52 ns for an + empty scope across eleven runs), as there is no OS thread machinery; that row reads 34–37 ns when the class is + run alone, for the reason in §7.2. +- **ART: one JNI upcall per call**, plus the per-thread attach the first time a CPython-created thread calls + (§7.6). Neither is on the marshalling path. + +### 7.5 What a user calls: the generated proxy over the boundary + +`GeneratedProxyCostTest` (`commonTest`) prices each proxy surface against the raw `_pm_invoke` call it wraps, +measured through the same object, in the same run, each row a zero-argument Python function called by one +shared loop (the floor is subtracted). Each row is the **minimum of three timed loops**: the quantity is a +difference of tens of ns on a boundary of hundreds and noise is one-sided — with one loop per row, four runs put +the instance-method delta at −17, −31, −60 and −71 ns. The last row of each table is the report's own +resolution: two raw rows are the same call (`_pm_invoke(h, ())`, different handle), so their difference is what +cannot be resolved. Three runs; taken in the round before §7.2's warmup fix, so read internally (raw against +proxy), not against §7.1's absolutes. + +| Surface a user writes | raw boundary | through the proxy | ratio | proxy layer adds | +|---|---|---|---|---| +| **desktop** (JVM 21.0.12, macOS arm64) | | | | | +| `demo.calc.ping()` module function | 452–475 ns | 468–485 ns | 1.00–1.03x | +5…+17 ns | +| `c.increment(5)` instance method | 514–558 ns | 537–573 ns | 1.02–1.05x | +14…+27 ns | +| `c.value` property read | 495–527 ns | 509–542 ns | 1.02–1.03x | +14…+19 ns | +| `c.label = 'x'` property write | 522–548 ns | 569–609 ns | 1.09–1.11x | +47…+62 ns | +| `Counter.created` static read (metaclass) | 449–485 ns | 480–510 ns | 1.01–1.07x | +6…+34 ns | +| `Counter.created = 12` static write | 466–507 ns | 511–573 ns | 1.07–1.13x | +34…+66 ns | +| `demo.calc.tally` top-level read | 449–485 ns | 484–521 ns | 1.03–1.07x | +14…+36 ns | +| `demo.calc.tally = 9` top-level write | 466–507 ns | 520–553 ns | 1.08–1.11x | +44…+54 ns | +| `Counter(10)` constructor | 682–761 ns | 588–627 ns | — | not separable (below) | +| *resolution* | | | | *−3…+9 ns* | +| **iOS simulator** (arm64) | | | | | +| `demo.calc.ping()` module function | 1801–1890 ns | 1829–1895 ns | 1.00–1.01x | +6…+28 ns | +| `c.increment(5)` instance method | 2901–2987 ns | 2895–3029 ns | 0.98–1.01x | −44…+42 ns | +| `c.value` property read | 2504–2601 ns | 2510–2620 ns | 0.99–1.00x | −3…+19 ns | +| `c.label = 'x'` property write | 2832–2940 ns | 2925–3092 ns | 1.03–1.05x | +92…+153 ns | +| `Counter.created` static read (metaclass) | 1838–1931 ns | 1906–1982 ns | 1.01–1.03x | +28…+68 ns | +| `Counter.created = 12` static write | 2180–2315 ns | 2277–2365 ns | 1.02–1.04x | +50…+103 ns | +| `demo.calc.tally` top-level read | 1838–1931 ns | 1927–1981 ns | 1.02–1.05x | +48…+99 ns | +| `demo.calc.tally = 9` top-level write | 2180–2315 ns | 2286–2363 ns | 1.02–1.04x | +48…+105 ns | +| `Counter(10)` constructor | 2591–2724 ns | 2769–2864 ns | 1.05–1.06x | +140…+177 ns | +| *resolution* | | | | *+15…+79 ns* | + +**The boundary is the whole cost**: the generated layer adds tens of ns (1.00–1.13x); bypassing proxies to call +`_pm_invoke` by hand would recover almost nothing. The desktop constructor row is not reportable: when it was +taken nothing released constructed objects, so both rows ran against a `HandleTable` growing to 78 002 live +handles and the raw baseline moved between 511 and 775 ns (since fixed by `__del__`, §4.2). On the simulator it +is a stable +140…+177 ns (`type.__call__` + `__init__` frame + attribute store). Half the simulator rows are +inside its resolution; desktop resolves to ±10 ns. + +**wasmJs** (`wasmJsNodeTest`, one run as printed; harness floor 62.48 ns): + +``` + module function raw 301.91 ns proxy 318.98 ns 1.05x (+17.06 ns) + constructor raw 719.75 ns proxy 1070.98 ns 1.48x (+351.23 ns) + instance method raw 339.22 ns proxy 357.47 ns 1.05x (+18.24 ns) + property read raw 290.94 ns proxy 308.41 ns 1.06x (+17.47 ns) + property write raw 360.42 ns proxy 438.77 ns 1.21x (+78.34 ns) + static read raw 228.82 ns proxy 300.94 ns 1.31x (+72.12 ns) + module read raw 228.82 ns proxy 324.39 ns 1.41x (+95.57 ns) +Resolution: two identical raw rows differ by -73.09 ns +HandleTable roots before the timed loops: 1, after: 1 +``` + +The static and module rows (+72…+96 ns) are the size of the resolution (73 ns) and are not numbers; the +constructor's +351 ns is (`type.__call__` + `__init__` + attribute store + `__del__`). Roots 1 → 1 shows ten +thousand `Counter(10)`s all released through `__del__` → `_pm_release` (a mis-dispatched release would have +leaked silently). Android and androidNative also run this `commonTest` file; no proxy-cost rows were recorded +for them (missing, not zero). + +**The top-level property fix.** A top-level `val` read used to cost +551…+587 ns over its boundary on desktop +(2.16–2.29x) against +6…+34 ns for the metaclass doing the same job for a class static. A standalone probe, net of +an empty call: plain module attribute 14 ns; PEP 562 module `__getattr__` 214 ns; type-level `__getattr__` hook +(the old shape) 569 ns. After moving to a per-module data descriptor: + +| desktop, net of the boundary | before | after | +|---|---|---| +| `demo.calc.tally` read | +551…+587 ns (2.16–2.29x) | +14…+36 ns (1.03–1.07x) | +| `demo.calc.tally = 9` write | +109…+136 ns (1.21–1.29x) | +44…+54 ns (1.08–1.11x) | + +`PythonProxyInstallTest` passed unchanged across it, including the read-only refusal; nothing in the source +says which shape is fifty times dearer — the measurement found it. + +**The await fast path costs one coroutine frame:** + +| | desktop | iOS simulator | +|---|---|---| +| `_pm_invoke(h, (21,))` in the same coroutine loop, no await | 561–617 ns | 3025–3055 ns | +| `await` a pure-Python coroutine that never suspends | 67–70 ns | 84.5–84.8 ns | +| `await demo.calc.doubleNow(21)` through the generated `async def` | 618–647 ns | 3186–3239 ns | +| ratio, and what the wrapper adds | 1.03–1.10x, +24…+57 ns | 1.04–1.06x, +152…+185 ns | +| `asyncio.Future`s constructed during the timed loops | **0** (asserted) | **0** (asserted) | + +**Install is a startup cost.** For the 11-entry, 2-class fixture (306 lines / 12.4 kB): + +| | desktop | iOS simulator | wasmJs (338 lines, 14 052 chars) | +|---|---|---|---| +| `PythonProxySource.render()` | 226–271 µs | 606–664 µs | 186.63 µs | +| `Python3.exec` of the result | 816–927 µs | 2481–2532 µs | 2155.50 µs | +| `install()` end to end | 1.06–1.20 ms | 3.09–3.20 ms | 2342.14 µs | + +Compiling and executing the Python is 75–80% of it; it scales with table size (ten times the surface ≈ tens of +ms, less than `Py_Initialize`). + +### 7.6 Android: the attach, priced + +All figures below were taken by `UpcallOverheadTest` (`androidInstrumentedTest`) at its old 3 000-call warmup, +two runs per emulator unless stated; they are kept as the record of how the per-call attach was found and +removed. **The within-run differences and ratios are quotable; the absolute ns are not** (they were read before +tier-up and are expected to fall at 100 000). + +Per-call attach/detach (before): + +| per upcall | API 26 | API 36 | +|---|---|---| +| instrumentation thread (ART already knows it) | 1307–1695 ns | 5307–6693 ns | +| Python worker thread, steady state | 59712–63293 ns | 19820–31219 ns | +| **the attach/detach pair** | **58405–61598 ns** | **14514–24526 ns** | +| upcall / downcall of the same shape, worker | 44.59–47.93x | 11.69–13.23x | + +Attach once per thread (after, commit `409da6fc`): + +| per upcall | API 26 | API 36 | +|---|---|---| +| instrumentation thread | 1280–1327 ns | 2982–5086 ns | +| Python worker thread, steady state | **1209–1329 ns** | **2301–3086 ns** | +| worker minus instrumentation thread | −118 to +49 ns | −681 to −2000 ns | +| upcall / downcall of the same shape, worker | **0.99–1.09x** | **2.00–2.34x** | + +Re-run unchanged over five full suites per emulator after `09bf2397` put `commonTest` on the device (so +`UpcallBoundaryCostTest`'s several hundred thousand upcalls ran first and left the JIT warm — the observation the +warmup fix came from: the API 36 instrumentation row differs fourfold at the same warmup and emulator): + +| re-run, 5 suites each | API 26 | API 36 | +|---|---|---| +| instrumentation thread | 1085–1273 ns | 1032–1229 ns | +| Python worker, steady state | 1249–1334 ns | 982–1219 ns | +| worker minus instrumentation | **+43 to +249 ns** | **−50 to +12 ns** | +| first upcall on a fresh worker | 88 250–242 375 ns | 41 208–611 416 ns | + +On API 36 the claim sharpens (the gap is tight around zero). On API 26 a worker costs a little more (+43–249 ns +against a ~1250 ns call) — not a return of the per-call attach. **The first upcall on each worker still pays the +attach in full** (56–142 µs in the first round, 41–611 µs in the re-run): once per thread, not once per call. A +worker that upcalls once gains nothing; one that upcalls in a loop is 20–50× better off. The structural assertion +the test enforces — one ART thread per Python worker — is a count and unaffected by warmup. + +--- + +## 8. Open items + +Only what is really open; each with what proves the done part. + +1. **Async proxies on wasmJs.** Synchronous proxies run (`PythonProxyInstallTest` 10/11 on `wasmJsNodeTest`); + `import asyncio` works (`WasmSelectorsImportTest`); the default event loop cannot be built (§5.8). Open + decision: whether the library ships a selector-free loop. SPEC U-5 `planned` on wasm. +2. **Kotlin-side forced cancellation.** Structurally impossible without `kotlinx.coroutines`' `Job` tree; only + cooperative `ensureActive()` exists (§5.6). Not planned unless that dependency is taken. +3. **A handle left by a dead loop.** The Kotlin `finally` release is meant to cover a loop that stopped before + its done callback ran; that case was never constructed and observed (unverified). +4. **Silent exclusions without a KSP warning.** A `suspend` function *type* warns (§5.3); a `suspend fun` with + an extension receiver or type parameters is still dropped silently by the shape checks + (`BindingPolicy.isExposedFunctionShape`). The warning belongs there too. +5. **Extension functions through KSP.** KSP excludes them; the artifact walker binds them as receiver methods + (SPEC U-7 `partial`). +6. **Kotlin calling a Python override.** Not implemented (§4.4). +7. **Per-target gaps outside desktop.** U-3/U-4 are asserted on desktop, with smoke coverage elsewhere + (`NativeSmokeTest`, `GeneratedAndroidTableTest`); the native async tests have no recorded per-test + androidNative result (§5.7); no proxy-cost rows for Android/androidNative (§7.5); the iOS sample cannot show + the really-suspending demo (§5.8). +8. **`BYTES` marshalling** is per item until `PyBytes_AsStringAndSize` is bound (§3.3). +9. **Incremental KSP** needs per-file fragments (§2.7). **Transitive and published-artifact discovery** are + unverified (§2.9). +10. **Native-image verification is manual** (SPEC U-6, N-5); automation is planned. +11. **Proxy-cost re-measurement.** §7.5's tables predate the 100 000 warmup; §7.6's absolutes predate it too. diff --git a/docs/guide/status.html b/docs/guide/status.html index af0cc05c8..c171fcf1c 100644 --- a/docs/guide/status.html +++ b/docs/guide/status.html @@ -54,7 +54,7 @@

What works today지금 되 Compose from Python on Android, iOS, wasmAndroid, iOS, wasm 에서 Python 으로 Composeplanned계획 -

The full, item-by-item contract with the test that backs each item is docs/SPEC.md on the develop branch; the progress log is the roadmap.항목별 계약과 각 항목을 받치는 테스트는 develop 브랜치의 docs/SPEC.md 에, 진척 기록은 로드맵 에 있다.

+

The full, item-by-item contract with the test that backs each item is docs/SPEC.md on the develop branch; the work still left is listed in the roadmap.항목별 계약과 각 항목을 받치는 테스트는 develop 브랜치의 docs/SPEC.md 에, 남은 작업은 로드맵 에 있다.

FAQ

Is it published on Maven Central?Maven Central 에 배포되어 있나?

diff --git a/docs/investigations/android-unregistered-surface.md b/docs/investigations/android-unregistered-surface.md index 3cbd6571f..26ce36956 100644 --- a/docs/investigations/android-unregistered-surface.md +++ b/docs/investigations/android-unregistered-surface.md @@ -1,267 +1,59 @@ -# Reachability Analysis: Android Unregistered Surface +# Android unregistered surface -> **Status: superseded. Every count and every risk ranking below is historical.** -> -> The unregistered surface is empty as of the pass recorded in ROADMAP §2: 363 of 367 `external fun` -> are bound through `RegisterNatives`, and the four that are not are deliberate — `ffiAllocUtf8`, -> `ffiFreeUtf8` and `ffiReadUtf8` are hand-written JNI exports with the correct prologue, and -> `echoCriticalNamed` exists in order to be name-linked, so that a benchmark can measure that path. -> -> The claim here worth carrying forward is that the *ordering* was right and the *counts* were not. -> When the last 157 were registered, a recount found **zero** of them reachable from -> `commonMain`/`commonTest` — the earlier passes had already taken the whole reachable surface, so -> the "71 reachable" figure below had long since been consumed. The call-graph traces are kept -> because they are still the record of which public API path reaches which C function. -> -> Everything from here down is retained unedited. +## Conclusion -> **Earlier status: the string half of this report is closed; the counts below are stale.** -> -> Recounted from the two files: 187 registered, 178 unregistered (was 145 / 224). Declarations -> still carrying a `jstring` across the boundary went from 52 to 6, and none of those six is on -> the broken path except `ffiSymbolRaw`, which has no caller anywhere in `src/`. The -> "Critical Risk: Functions taking `String`" section below is therefore resolved in full — not -> just the four functions it happens to list. What remains is the pointer-only surface, and the -> reachability reasoning for that still holds. -> -> See ROADMAP §2 for the current numbers and `docs/design/marshalling-design.md` for the per-argument -> intern/scratch rule the migration followed. +**The unregistered surface is empty.** Every `external fun` in `androidMain/.../bindings.kt` has a +`JNINativeMethod` entry in `artMain/cinterop/jni_onload.def`, except four that are unregistered on +purpose: -This report analyzes the reachability of the 304 unregistered functions in the Android JNI surface (`bindings.kt`). An unregistered function relies on name-based `@CName` linking which on Android leads to arguments arriving shifted, Kotlin `String`s being passed incorrectly, and `java.lang.Long` boxing issues. Hitting any of these functions is a latent `SIGSEGV` or similar crash. +| declaration | why it is not in the table | +|---|---| +| `ffiAllocUtf8`, `ffiFreeUtf8`, `ffiReadUtf8` | hand-written JNI exports (`artMain/.../JNIOnLoadExporter.kt`) with the correct `JNIEnv*, jclass` prologue | +| `echoCriticalNamed` | exists to be name-linked, so a benchmark can measure that path | -We traced the call graph backwards from the public API wrappers in `commonMain` (specifically under `ffi/`) through `EmbedAPI.kt` to the actual Android bindings. +Why it matters: an unregistered `external fun` does not fail cleanly. It falls back to the +`@CName` export in `nativeMain`, which has no `JNIEnv*`/`jclass` prologue, so the arguments arrive +shifted by two registers and the process dies on a truncated pointer, a `jstring` handed to a +Kotlin/Native `String` parameter, or a boxed `java.lang.Long`. -## 1. Reachable and Unregistered +**Status: acted on, and enforced by a test.** +`python-multiplatform/src/desktopTest/.../native/ffi/JniCallConventionClassificationTest.kt` +`everyDeclarationIsRegisteredOrIsOneOfTheFourNamedExceptions` parses `bindings.kt` and +`jni_onload.def` on every desktop build and fails if a declaration has no entry or the exception +set above goes stale. The rule is written in `androidMain/README.md` ("Every `external fun` here +needs a table entry — no exceptions by reachability"). The migration this report recommended +(strings first, then core object model, then exceptions, then the rest) is complete. -Out of the unregistered functions, exactly **71 are reachable** from the current public API. +## Numbers (all historical) -The list below shows the reachable functions, ordered by how likely an ordinary user is to hit them. Functions that take strings are placed at the top because they are guaranteed to crash the process due to `jstring` vs Kotlin/Native `String` mismatch. Functions used by core objects (`PyObject`, `PyDict`, `PyList`, etc.) follow. +| | at the time of this report | later | +|---|---|---| +| unregistered `external fun` | 304 of 375 | 178 of 365 (187 registered) after the string half | +| `jstring`-carrying declarations | 52 | 6, none on a live path | +| reported "reachable from `commonMain`" | 71 | **0** when the last 157 were registered | -### Critical Risk: Functions taking `String` -These functions take a string argument and are highly likely to crash immediately if reached. +The ordering recommended (reachable, string-taking first) was right; the counts were not. By the +time the last 157 were registered, earlier passes had already taken the whole reachable surface, +so "71 reachable" had been consumed long before. Current counts are not recorded here because they +change with every binding; the test above re-derives them. The ROADMAP §2 pass recorded +363 of 367 registered, with the four exceptions above. -- **`PyImport_AddModule`** - - `ffi/PyTypeChecks.kt` -> `EmbedAPI.PyImport_AddModule()` -> `bindings.PyImport_AddModule()` -- **`PyObject_DelAttrString`** - - `ffi/PyObject.kt` -> `EmbedAPI.PyObject_DelAttrString()` -> `bindings.PyObject_DelAttrString()` -- **`PyObject_SetAttrString`** - - `ffi/PyObject.kt` -> `EmbedAPI.PyObject_SetAttrString()` -> `bindings.PyObject_SetAttrString()` -- **`PyErr_SetString`** - - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyErr_SetString()` -> `bindings.PyErr_SetString()` +## How to reproduce -### High Risk: Core Object Model (Collections and Types) -These functions manipulate common Python objects (Dicts, Lists, Tuples, Sets, etc.) and are very likely to be called during normal usage. +```bash +./gradlew :python-multiplatform:desktopTest --tests 'python.native.ffi.JniCallConventionClassificationTest' \ + --console=plain > .tmp/jni-classification.log 2>&1; echo "EXIT=$?" +``` -- **`PyDict_Clear`** - - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyDict_Clear()` -> `bindings.PyDict_Clear()` -- **`PyDict_Contains`** - - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyDict_Contains()` -> `bindings.PyDict_Contains()` -- **`PyDict_DelItem`** - - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyDict_DelItem()` -> `bindings.PyDict_DelItem()` -- **`PyDict_GetItem`** - - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyDict_GetItem()` -> `bindings.PyDict_GetItem()` -- **`PyDict_Items`** - - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyDict_Items()` -> `bindings.PyDict_Items()` -- **`PyDict_Size`** - - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyDict_Size()` -> `bindings.PyDict_Size()` -- **`PyDict_Values`** - - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyDict_Values()` -> `bindings.PyDict_Values()` -- **`PyList_Append`** - - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PyList_Append()` -> `bindings.PyList_Append()` -- **`PyList_AsTuple`** - - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyList_AsTuple()` -> `bindings.PyList_AsTuple()` -- **`PyList_Insert`** - - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PyList_Insert()` -> `bindings.PyList_Insert()` -- **`PyList_Reverse`** - - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PyList_Reverse()` -> `bindings.PyList_Reverse()` -- **`PyList_Sort`** - - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PyList_Sort()` -> `bindings.PyList_Sort()` -- **`PyTuple_GetItem`** - - `ffi/PyType.kt` -> `EmbedAPI.PyTuple_GetItem()` -> `bindings.PyTuple_GetItem()` - - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PyTuple_GetItem()` -> `bindings.PyTuple_GetItem()` - - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyTuple_GetItem()` -> `bindings.PyTuple_GetItem()` - - `ffi/types/collections/PyTuple.kt` -> `EmbedAPI.PyTuple_GetItem()` -> `bindings.PyTuple_GetItem()` -- **`PyTuple_GetSlice`** - - `ffi/types/collections/PyTuple.kt` -> `EmbedAPI.PyTuple_GetSlice()` -> `bindings.PyTuple_GetSlice()` -- **`PyTuple_Size`** - - `ffi/PyType.kt` -> `EmbedAPI.PyTuple_Size()` -> `bindings.PyTuple_Size()` - - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PyTuple_Size()` -> `bindings.PyTuple_Size()` - - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PyTuple_Size()` -> `bindings.PyTuple_Size()` - - `ffi/types/collections/PyTuple.kt` -> `EmbedAPI.PyTuple_Size()` -> `bindings.PyTuple_Size()` -- **`PySequence_Contains`** - - `ffi/types/collections/PyDict.kt` -> `EmbedAPI.PySequence_Contains()` -> `bindings.PySequence_Contains()` - - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PySequence_Contains()` -> `bindings.PySequence_Contains()` - - `ffi/types/collections/PyTuple.kt` -> `EmbedAPI.PySequence_Contains()` -> `bindings.PySequence_Contains()` -- **`PySequence_List`** - - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PySequence_List()` -> `bindings.PySequence_List()` -- **`PySequence_Tuple`** - - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PySequence_Tuple()` -> `bindings.PySequence_Tuple()` -- **`PySet_Add`** - - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PySet_Add()` -> `bindings.PySet_Add()` -- **`PySet_Clear`** - - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PySet_Clear()` -> `bindings.PySet_Clear()` -- **`PySet_Contains`** - - `ffi/types/collections/PyFrozenSet.kt` -> `EmbedAPI.PySet_Contains()` -> `bindings.PySet_Contains()` - - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PySet_Contains()` -> `bindings.PySet_Contains()` -- **`PySet_Discard`** - - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PySet_Discard()` -> `bindings.PySet_Discard()` -- **`PySet_New`** - - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PySet_New()` -> `bindings.PySet_New()` -- **`PySet_Pop`** - - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PySet_Pop()` -> `bindings.PySet_Pop()` -- **`PySet_Size`** - - `ffi/types/collections/PyFrozenSet.kt` -> `EmbedAPI.PySet_Size()` -> `bindings.PySet_Size()` - - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PySet_Size()` -> `bindings.PySet_Size()` -- **`PyFrozenSet_New`** - - `ffi/types/collections/PyFrozenSet.kt` -> `EmbedAPI.PyFrozenSet_New()` -> `bindings.PyFrozenSet_New()` -- **`PyObject_Call`** - - `ffi/PyObject.kt` -> `EmbedAPI.PyObject_Call()` -> `bindings.PyObject_Call()` -- **`PyObject_DelItem`** - - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PyObject_DelItem()` -> `bindings.PyObject_DelItem()` -- **`PyObject_GetItem`** - - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PyObject_GetItem()` -> `bindings.PyObject_GetItem()` -- **`PyObject_GetIter`** - - `ffi/PyType.kt` -> `EmbedAPI.PyObject_GetIter()` -> `bindings.PyObject_GetIter()` - - `ffi/types/collections/PyTuple.kt` -> `EmbedAPI.PyObject_GetIter()` -> `bindings.PyObject_GetIter()` -- **`PyObject_IsInstance`** - - `ffi/PyType.kt` -> `EmbedAPI.PyObject_IsInstance()` -> `bindings.PyObject_IsInstance()` - - `ffi/PyTypeChecks.kt` -> `EmbedAPI.PyObject_IsInstance()` -> `bindings.PyObject_IsInstance()` -- **`PyObject_Repr`** - - `ffi/PyObject.kt` -> `EmbedAPI.PyObject_Repr()` -> `bindings.PyObject_Repr()` -- **`PyObject_RichCompare`** - - `ffi/PyObject.kt` -> `EmbedAPI.PyObject_RichCompare()` -> `bindings.PyObject_RichCompare()` -- **`PyObject_RichCompareBool`** - - `ffi/types/basic/PyFloat.kt` -> `EmbedAPI.PyObject_RichCompareBool()` -> `bindings.PyObject_RichCompareBool()` - - `ffi/types/basic/PyInt.kt` -> `EmbedAPI.PyObject_RichCompareBool()` -> `bindings.PyObject_RichCompareBool()` - - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PyObject_RichCompareBool()` -> `bindings.PyObject_RichCompareBool()` -- **`PyObject_SetItem`** - - `ffi/types/collections/PyList.kt` -> `EmbedAPI.PyObject_SetItem()` -> `bindings.PyObject_SetItem()` -- **`PyObject_Type`** - - `ffi/PyObject.kt` -> `EmbedAPI.PyObject_Type()` -> `bindings.PyObject_Type()` - - `ffi/PyType.kt` -> `EmbedAPI.PyObject_Type()` -> `bindings.PyObject_Type()` - - `ffi/PyTypeChecks.kt` -> `EmbedAPI.PyObject_Type()` -> `bindings.PyObject_Type()` - - `ffi/conversion/Context.kt` -> `EmbedAPI.PyObject_Type()` -> `bindings.PyObject_Type()` - - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyObject_Type()` -> `bindings.PyObject_Type()` - - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PyObject_Type()` -> `bindings.PyObject_Type()` -- **`PyModule_GetDict`** - - `ffi/types/modules/PyModule.kt` -> `EmbedAPI.PyModule_GetDict()` -> `bindings.PyModule_GetDict()` -- **`PyModule_GetFilenameObject`** - - `ffi/types/modules/PyModule.kt` -> `EmbedAPI.PyModule_GetFilenameObject()` -> `bindings.PyModule_GetFilenameObject()` -- **`PyModule_GetName`** - - `ffi/types/modules/PyModule.kt` -> `EmbedAPI.PyModule_GetName()` -> `bindings.PyModule_GetName()` -- **`PyType_GetName`** - - `ffi/PyType.kt` -> `EmbedAPI.PyType_GetName()` -> `bindings.PyType_GetName()` - - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyType_GetName()` -> `bindings.PyType_GetName()` -- **`PyType_IsSubtype`** - - `ffi/PyType.kt` -> `EmbedAPI.PyType_IsSubtype()` -> `bindings.PyType_IsSubtype()` +(Desktop is enough: the test is a source-level parse, not a device run. Device-side registration is +exercised by `androidInstrumentedTest/.../native/ffi/JniWiringTest.kt`.) -### Medium Risk: Exception and Error Handling -These functions are less likely to be called during standard execution, but are critical when an error happens. A crash here masks the actual problem. +## What stays useful -- **`PyErr_GetRaisedException`** - - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyErr_GetRaisedException()` -> `bindings.PyErr_GetRaisedException()` -- **`PyErr_SetRaisedException`** - - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyErr_SetRaisedException()` -> `bindings.PyErr_SetRaisedException()` -- **`PyException_GetCause`** - - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyException_GetCause()` -> `bindings.PyException_GetCause()` -- **`PyException_GetContext`** - - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyException_GetContext()` -> `bindings.PyException_GetContext()` -- **`PyException_GetTraceback`** - - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyException_GetTraceback()` -> `bindings.PyException_GetTraceback()` +The call-graph traces (public `commonMain` API → `EmbedAPI.` → `bindings.`) are the record +of which API path reaches which C function; they are in the archived original, with the +per-function reachability lists: +[`docs/archive/investigations/android-unregistered-surface.md`](../archive/investigations/android-unregistered-surface.md). -### Medium Risk: Mathematics and Basic Types -- **`PyNumber_Add`** - - `ffi/types/basic/PyFloat.kt` -> `EmbedAPI.PyNumber_Add()` -> `bindings.PyNumber_Add()` - - `ffi/types/basic/PyInt.kt` -> `EmbedAPI.PyNumber_Add()` -> `bindings.PyNumber_Add()` -- **`PyNumber_And`** - - `ffi/types/collections/PyFrozenSet.kt` -> `EmbedAPI.PyNumber_And()` -> `bindings.PyNumber_And()` - - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PyNumber_And()` -> `bindings.PyNumber_And()` -- **`PyNumber_Multiply`** - - `ffi/types/basic/PyFloat.kt` -> `EmbedAPI.PyNumber_Multiply()` -> `bindings.PyNumber_Multiply()` - - `ffi/types/basic/PyInt.kt` -> `EmbedAPI.PyNumber_Multiply()` -> `bindings.PyNumber_Multiply()` -- **`PyNumber_Or`** - - `ffi/types/collections/PyFrozenSet.kt` -> `EmbedAPI.PyNumber_Or()` -> `bindings.PyNumber_Or()` - - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PyNumber_Or()` -> `bindings.PyNumber_Or()` -- **`PyNumber_Subtract`** - - `ffi/types/basic/PyFloat.kt` -> `EmbedAPI.PyNumber_Subtract()` -> `bindings.PyNumber_Subtract()` - - `ffi/types/basic/PyInt.kt` -> `EmbedAPI.PyNumber_Subtract()` -> `bindings.PyNumber_Subtract()` - - `ffi/types/collections/PyFrozenSet.kt` -> `EmbedAPI.PyNumber_Subtract()` -> `bindings.PyNumber_Subtract()` - - `ffi/types/collections/PySet.kt` -> `EmbedAPI.PyNumber_Subtract()` -> `bindings.PyNumber_Subtract()` -- **`PyFloat_AsDouble`** - - `ffi/types/basic/PyComplex.kt` -> `EmbedAPI.PyFloat_AsDouble()` -> `bindings.PyFloat_AsDouble()` - - `ffi/types/basic/PyFloat.kt` -> `EmbedAPI.PyFloat_AsDouble()` -> `bindings.PyFloat_AsDouble()` - - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PyFloat_AsDouble()` -> `bindings.PyFloat_AsDouble()` -- **`PyFloat_FromDouble`** - - `ffi/types/basic/PyFloat.kt` -> `EmbedAPI.PyFloat_FromDouble()` -> `bindings.PyFloat_FromDouble()` -- **`PyLong_AsInt`** - - `ffi/exceptions/PyTraceback.kt` -> `EmbedAPI.PyLong_AsInt()` -> `bindings.PyLong_AsInt()` -- **`PyLong_AsLongLong`** - - `ffi/PyObject.kt` -> `EmbedAPI.PyLong_AsLongLong()` -> `bindings.PyLong_AsLongLong()` - - `ffi/types/basic/PyInt.kt` -> `EmbedAPI.PyLong_AsLongLong()` -> `bindings.PyLong_AsLongLong()` - - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PyLong_AsLongLong()` -> `bindings.PyLong_AsLongLong()` -- **`PyBool_FromLong`** - - `ffi/types/basic/PyBool.kt` -> `EmbedAPI.PyBool_FromLong()` -> `bindings.PyBool_FromLong()` -- **`PyUnicode_Concat`** - - `ffi/types/basic/PyString.kt` -> `EmbedAPI.PyUnicode_Concat()` -> `bindings.PyUnicode_Concat()` -- **`PyUnicode_Contains`** - - `ffi/types/basic/PyString.kt` -> `EmbedAPI.PyUnicode_Contains()` -> `bindings.PyUnicode_Contains()` - -### Miscellaneous Utility Functions -- **`PyCallable_Check`** - - `ffi/PyObject.kt` -> `EmbedAPI.PyCallable_Check()` -> `bindings.PyCallable_Check()` -- **`PyEval_GetBuiltins`** - - `ffi/PyTypeChecks.kt` -> `EmbedAPI.PyEval_GetBuiltins()` -> `bindings.PyEval_GetBuiltins()` - - `ffi/types/basic/PyComplex.kt` -> `EmbedAPI.PyEval_GetBuiltins()` -> `bindings.PyEval_GetBuiltins()` - - `ffi/types/basic/PyNone.kt` -> `EmbedAPI.PyEval_GetBuiltins()` -> `bindings.PyEval_GetBuiltins()` -- **`PyIter_Next`** - - `ffi/types/iteration/PyIterator.kt` -> `EmbedAPI.PyIter_Next()` -> `bindings.PyIter_Next()` -- **`Py_GetBuildInfo`** - - `ffi/Python3.kt` -> `EmbedAPI.Py_GetBuildInfo()` -> `bindings.Py_GetBuildInfo()` -- **`Py_GetCompiler`** - - `ffi/Python3.kt` -> `EmbedAPI.Py_GetCompiler()` -> `bindings.Py_GetCompiler()` -- **`Py_GetCopyright`** - - `ffi/Python3.kt` -> `EmbedAPI.Py_GetCopyright()` -> `bindings.Py_GetCopyright()` -- **`Py_GetPlatform`** - - `ffi/Python3.kt` -> `EmbedAPI.Py_GetPlatform()` -> `bindings.Py_GetPlatform()` -- **`Py_RunMain`** - - `ffi/Python3.kt` -> `EmbedAPI.Py_RunMain()` -> `bindings.Py_RunMain()` -- **`ffiReadUtf8`** (Helper used for marshalling) - - `ffi/PyObject.kt` -> `EmbedAPI.PyUnicode_AsUTF8()` -> `bindings.ffiReadUtf8()` - - `ffi/PyType.kt` -> `EmbedAPI.PyUnicode_AsUTF8()` -> `bindings.ffiReadUtf8()` - - `ffi/Python3.kt` -> `EmbedAPI.Py_GetVersion()` -> `bindings.ffiReadUtf8()` - - `ffi/exceptions/PyException.kt` -> `EmbedAPI.PyUnicode_AsUTF8()` -> `bindings.ffiReadUtf8()` - - `ffi/exceptions/PyTraceback.kt` -> `EmbedAPI.PyUnicode_AsUTF8()` -> `bindings.ffiReadUtf8()` - - `ffi/types/basic/PyString.kt` -> `EmbedAPI.PyUnicode_AsUTF8()` -> `bindings.ffiReadUtf8()` - - `ffi/types/collections/CollectionSupport.kt` -> `EmbedAPI.PyUnicode_AsUTF8()` -> `bindings.ffiReadUtf8()` - ---- - -## 2. Unreachable Today - -There are **233** functions present in `bindings.kt` as unregistered `external fun`s that are not reached by any code in `commonMain` today (totalling 304 unregistered out of 375 overall, though exact counts from parsing show 71 reachable and 222 unreachable). - -While they should eventually be migrated to ensure the bindings layer is safe to use in its entirety, they **do not pose an immediate crash risk** for current users of the library. - -## 3. Cannot Determine - -Every path from the public `commonMain` API layer down to the underlying bindings layer could be definitively traced statically. There are **0** functions whose reachability could not be determined. The analysis explicitly covered every wrapper and `expect`/`actual` binding chain. - -## 4. Suggested Migration Order - -1. **String-taking reachable functions (The immediate crashers)** - - Because they process strings incorrectly across JNI natively, calling any of these guarantees a crash. - - `PyImport_AddModule`, `PyErr_SetString`, `PyObject_DelAttrString`, `PyObject_SetAttrString` - -2. **Core Object Model (Primitives & Collections)** - - Heavily used functions that could lead to crashes due to missing `JNIEnv` or boxing issues (`java.lang.Long` vs `jlong`). - - `PyDict_Clear`, `PyDict_Contains`, `PyDict_DelItem`, `PyDict_GetItem`, `PyDict_Items`, `PyDict_Size`, `PyDict_Values`, `PyList_Append`, `PyList_AsTuple`, `PyList_Insert`, `PyList_Reverse`, `PyList_Sort`, `PyModule_GetDict`, `PyModule_GetFilenameObject`, `PyObject_Call`, `PyObject_DelItem`, `PyObject_GetItem`, `PyObject_GetIter`, `PyObject_IsInstance`, `PyObject_Repr`, `PyObject_RichCompare`, `PyObject_RichCompareBool`, `PyObject_SetItem`, `PyObject_Type`, `PySequence_List`, `PySequence_Tuple`, `PyTuple_GetItem`, `PyTuple_GetSlice`, `PyTuple_Size`, `PyType_GetName`, `PyType_IsSubtype`, etc. - -3. **Exception and Error handling** - - Less likely to be hit in a happy path, but critical for clean failure handling. - - `PyErr_GetRaisedException`, `PyErr_SetRaisedException`, `PyException_GetCause`, `PyException_GetContext`, `PyException_GetTraceback`. - -4. **Remaining reachable functions** - - Other utility functions (e.g. maths and module utilities). - -5. **Unreachable functions** - - Deferred until the public surface is expanded to use them. +Related: [`jni-call-convention-audit.md`](jni-call-convention-audit.md) (which convention a +registered function may use). diff --git a/docs/investigations/cost-table.md b/docs/investigations/cost-table.md index 2cc8c4877..ed9b30d15 100644 --- a/docs/investigations/cost-table.md +++ b/docs/investigations/cost-table.md @@ -2,7 +2,15 @@ Kotlin ↔ Python 경계의 비용을, **여섯 타깃 전부에 대해 한 번에** 뽑아 놓은 표. -`docs/design/upcall-design.md` 와 `docs/design/downcall-design.md` 는 각자의 조사 과정에서 나온 표를 갖고 있고, +> **결론과 상태 (2026-10-03 확인).** 업콜 1건은 같은 모양의 다운콜의 약 0.9~3.1 배다 — 호스트 JVM 에서 +> 약 543–576 ns (다운콜 260–266), iOS 시뮬레이터 약 2.3 µs, androidNative 약 3.4 µs, ART API 26/36 약 1.0–1.3 µs, +> wasmJs 약 300 ns. 모든 숫자는 아래 마커 사이의 표(커밋 `958c0082b294`, 2026-08-14, Apple M1, 타깃당 3회의 +> min–max, 워밍업 100,000)에서 온 것이고, 그 표의 숫자는 이 문장보다 우선한다. 하네스(`benchmarks/cost-table.sh`, +> `benchmarks/cost_table.py`)는 이 문서의 마커 사이만 다시 쓴다 — 마커와 그 바깥의 "다시 뽑는 법" 절은 하네스가 +> 기대하는 형식이므로 손으로 고치지 않는다. 이 표는 **측정을 기록할 뿐 임계값을 단언하지 않는다**(SPEC X-1). +> 표는 2026-08-14 컷이므로 그 뒤의 커밋을 반영하지 않는다. 다시 뽑으려면 아래 "다시 뽑는 법". + +`docs/design/upcall.md` 와 `docs/design/downcall-design.md` 는 각자의 조사 과정에서 나온 표를 갖고 있고, 그 표들은 **조사가 끝난 시점의 기록**이다. 이 문서는 다르다 — 여기 있는 표는 사람이 옮겨 적은 것이 아니라 `benchmarks/cost_table.py` 가 실행 결과에서 직접 렌더링한 것이고, 아래 "다시 뽑는 법" 한 줄이면 통째로 갱신된다. @@ -68,10 +76,10 @@ python3 benchmarks/cost_table.py render --check # 문서가 최신 JSO 정상이다. 그럴 때는 각각 뜨고 렌더링할 때 합친다. ```bash -./benchmarks/cost-table.sh collect --targets desktop,iosSimulatorArm64,wasmJs --out /tmp/host.json -./benchmarks/cost-table.sh collect --targets artApi36,androidNativeArm64 --out /tmp/a36.json -./benchmarks/cost-table.sh collect --targets artApi26 --out /tmp/a26.json -python3 benchmarks/cost_table.py render --in /tmp/host.json,/tmp/a36.json,/tmp/a26.json +./benchmarks/cost-table.sh collect --targets desktop,iosSimulatorArm64,wasmJs --out benchmarks/results/host.json +./benchmarks/cost-table.sh collect --targets artApi36,androidNativeArm64 --out benchmarks/results/a36.json +./benchmarks/cost-table.sh collect --targets artApi26 --out benchmarks/results/a26.json +python3 benchmarks/cost_table.py render --in benchmarks/results/host.json,benchmarks/results/a36.json,benchmarks/results/a26.json ``` **커밋이 다른 cut 을 합치는 것은 거부된다.** 서로 다른 트리에서 나온 행을 한 표에 놓는 것이 @@ -92,7 +100,7 @@ python3 benchmarks/cost_table.py render --in /tmp/host.json,/tmp/a36.json,/tmp/a | `artApi26` | `:python-multiplatform:connectedDebugAndroidTest` | `build/outputs/androidTest-results/connected/debug/<기기>/logcat-*.txt` | **에뮬레이터** (`pmp_api26`) | | `artApi36` | `:python-multiplatform:connectedDebugAndroidTest` | 〃 | **에뮬레이터** (`pmp_api36`) | -기본 여섯 외에 `androidNativeArm64Api26` 이 하나 더 선언되어 있다 — `upcall-design.md` 의 표가 +기본 여섯 외에 `androidNativeArm64Api26` 이 하나 더 선언되어 있다 — `docs/design/upcall.md` 의 표가 androidNative 를 API 레벨별로 나눠 기록하기 때문이고, `--targets` 로 이름을 대면 돈다. **ART 만 결과를 읽는 경로가 다르다.** AGP 가 쓰는 JUnit XML 에는 `` 이 없다 — @@ -320,7 +328,7 @@ These also come through `Benchmark.printReport`, and they are **not comparable w ## The React Native comparison's Android column is unquotable, and not for the reason assumed The cross-runtime comparison this table is read against -(`/Volumes/macMini/rn-benchmark/COMPARISON.md`) has an Android column that cannot +(`benchmarks/rn-benchmark/COMPARISON.md`) has an Android column that cannot be quoted. Two solo runs on a quiet machine, one at the documented warmup and one at four times it, both ended with the benchmark refusing its own numbers: diff --git a/docs/investigations/gc-scheduling-investigation.md b/docs/investigations/gc-scheduling-investigation.md index f43d32d52..31b9b373a 100644 --- a/docs/investigations/gc-scheduling-investigation.md +++ b/docs/investigations/gc-scheduling-investigation.md @@ -1,514 +1,259 @@ # Why a pure C API embedder never collects cycles, and never merges a refcount -Two things this document is careful about, because it corrects a previous document that was not: - -- **CPython file:line citations are now verified against the tagged source.** The v3.14.7 tarball - (the version this build pins — `gradle.properties: pythonVersion=3.14.7`) was fetched to - `/Volumes/macMini/cpython-src` for §8d and every line number in this document was re-checked - against it. Two in §1 were wrong by ~100 lines and are corrected below; the rest were right. - Nothing is vendored into this repo. Where a claim matters, §3, §4 and §8 still back it with a - measurement taken here rather than with a citation. -- **Every number in §3, §5 and §8 was measured on this machine**, on `desktopTest`, macOS arm64, - CPython 3.14.7, both the default and the `-PpythonFreeThreaded=true` builds. -- **§8 corrects §7.** A measurement probe in `GCSchedulingMeasurementTest` was leaking the list - returned by `gc.get_objects()`, which holds the whole tracked heap, and that leak is what - produced the earlier free-threaded conclusion. The correction was itself measured; the - hypothesis it replaced was measured and rejected rather than merely doubted. +Conditions for every number below unless stated: `desktopTest`, macOS arm64 (Apple M1), CPython +3.14.7 (`gradle.properties: pythonVersion=3.14.7`), both the default (GIL) build and the opt-in +free-threaded build (`-PpythonFreeThreaded=true`, 3.14t; desktop only, default is `false`). CPython +file:line citations were checked against the v3.14.7 tarball (fetched outside the repo, not +vendored). Printed by `GCSchedulingMeasurementTest`, `EvalCheckpointTest`, `FreeThreadedGCGateTest`, +`CycleCollectionTest` (all under `python-multiplatform/src/{desktopTest,commonTest}/.../ref/`). ## 0. Summary -Two symptoms that looked unrelated share one cause, and a third that looked like the same thing is -not. - -- **The cyclic collector never runs.** An embedder that only ever calls `PyObject_Call` and friends - never triggers a generational collection, however much it allocates. True on both builds. (§1) -- **Free-threaded, a completed `Py_DecRef` can leave the object alive.** The count is correct and - the decrement happened, but `tp_dealloc` has not run and the memory is not back. (§2) - - These two are the same thing: CPython schedules both jobs by setting a bit on the *eval breaker*, - and the only code that reads those bits is `_Py_HandlePending`, which the evaluation loop calls - and nothing else does. No bytecode frame, no checkpoint; no checkpoint, no collection and no - merge. -- **`CycleCollectionTest`'s free-threaded failure is *not* the same thing**, and an eval-loop - checkpoint provably does not fix it. It is deferred reference counting on the heap type, and what - materialises a deferred reference is a collection, not a checkpoint. (§3) -- **The checkpoint fixes §1 on the build with the global lock, and on the free-threaded build it - often does not.** It fires exactly as designed there and still reclaims nothing. This is *not* - caused by the FFI wrapper's per-call GIL scopes — that hypothesis was tested by holding the - checkpoint count fixed and widening the scope, and rejected. (§8) - - The cause is that free-threaded, `_Py_RunGC` re-asks `gc_should_collect` *after* the checkpoint - has read the scheduled bit, and that check gates generation 0 on the growth of the **whole - process's** memory footprint — the JVM's. Python-side allocation cannot move it, so the gate - shuts one-way as soon as the JVM's footprint settles. Confirmed by opening and closing the gate - on demand, including by faulting in 400 MB of ballast and watching Python cycles get collected - with no threshold changed. (§8d) +Conclusions (current): + +- **The cyclic collector never runs for an embedder that executes no Python frame.** Allocation + only sets a bit on the eval breaker; the only reader is `_Py_HandlePending`, called from the eval + loop. True on both builds. (§1) +- **Free-threaded, a completed `Py_DecRef` can leave the object alive** (biased reference counting: + a non-owner thread's final decrement is queued to the owner, drained only by the eval loop). (§2) +- **`CycleCollectionTest`'s free-threaded failure was a different thing:** the probe read + `ob_tid` instead of a refcount (header layout differs), and heap types are deferred-ref-counted, + so the type's count is observable only after a collection. An eval-loop checkpoint provably does + not fix it; two `PyGC_Collect()` calls in the test do. (§3) +- **The checkpoint fixes §1 on the GIL build; free-threaded it often reclaims nothing**, because + `_Py_RunGC` re-asks `gc_should_collect`, which gates generation 0 on growth of the **whole + process's** memory footprint (the JVM's). Python-side allocation cannot move it. Not caused by + per-call `PyGILState_Ensure/Release` scopes (tested, rejected). (§8, §8d) +- **Supported way to collect cycles in an embedder that runs no bytecode, identical on both + builds: `PyGC_Collect()`**, cost proportional to the heap (§5). `drainPendingReleases()` is the + cheap checkpoint that merges refcounts (and collects on the GIL build). + +**Status: acted on.** +- `Python3.drainPendingReleases()` and `Python3.autoDrainInterval` (default 32 on free-threaded + builds, 0 otherwise): `python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt` + (~lines 737-802; checkpoint function `__pmp_eval_checkpoint__`); rules in + `python-multiplatform/src/commonMain/README.md`. +- Tests: `commonTest/.../ref/EvalCheckpointTest.kt`, `desktopTest/.../ref/GCSchedulingMeasurementTest.kt`, + `FreeThreadedGCGateTest.kt`, `CycleCollectionTest.kt`. SPEC M-3, T-2. +- Free-threaded default behaviour (no bound on reclamation at default thresholds) is a documented + consequence, not fixed. ## 1. `_Py_ScheduleGC` only sets a bit -Allocation-driven collection has not been a direct call since 3.12. `_Py_ScheduleGC` sets -`_PY_GC_SCHEDULED_BIT` on the eval breaker and returns: +Allocation-driven collection has not been a direct call since 3.12. -| build | function | file:line (v3.14.7, verified) | +| build | function | file:line (v3.14.7) | |---|---|---| -| global lock | `_Py_ScheduleGC` | `Python/gc.c:1846` | +| GIL | `_Py_ScheduleGC` | `Python/gc.c:1846` | | free-threaded | `_Py_ScheduleGC` | `Python/gc_free_threading.c:2795` | -The bit's only reader is `_Py_HandlePending` (`Python/ceval_gil.c:1357`), which calls `_Py_RunGC` -when it is set (`Python/ceval_gil.c:1397-1399`). `_Py_HandlePending` in turn has one caller family: the -`_CHECK_PERIODIC` / `_CHECK_PERIODIC_IF_NOT_YIELD_FROM` uops in `Python/bytecodes.c`, which open -every Python-level frame (`RESUME`) and close every call instruction. There is no public entry -point to it. +The bit's only reader is `_Py_HandlePending` (`Python/ceval_gil.c:1357`; runs `_Py_RunGC` at +`:1397-1399`), reached from `_CHECK_PERIODIC` uops that open every Python frame (`RESUME`) and +close every call instruction. No public entry point. So allocation schedules a collection that only +the evaluation loop will perform; `gc.collect()`/`PyGC_Collect` is the only thing that has ever +collected anything in this library's tests. -So **allocation schedules a collection that only the evaluation loop will ever perform.** An -embedder that executes no Python-level frame accumulates the scheduled bit forever and its cycles -are never broken. This is not a free-threading property; it is the same on both builds, and it is -why `gc.collect()` (or the C `PyGC_Collect`) has been the only thing that has ever collected -anything in this library's tests. - -### `Py_MakePendingCalls` is not a substitute - -It is the obvious Stable ABI candidate and it does not work. It forwards to -`_PyEval_MakePendingCalls` (`Python/ceval_gil.c:1034`), which handles `handle_signals` and -`make_pending_calls` and returns. It never looks at `_PY_GC_SCHEDULED_BIT`, and on a free-threaded -build it never looks at the merge bit either. - -This one is checked here rather than taken on trust: -`EvalCheckpointTest.testDrainPendingReleasesReclaimsWhatTheCleanerGaveBack` calls -`Py_MakePendingCalls`, asserts it returned 0, and asserts the reference count did not move. +`Py_MakePendingCalls` is **not** a substitute: it forwards to `_PyEval_MakePendingCalls` +(`ceval_gil.c:1034`), which never looks at the GC bit nor the merge bit. Checked by +`EvalCheckpointTest.testDrainPendingReleasesReclaimsWhatTheCleanerGaveBack` (calls it, asserts 0 +return and that the refcount did not move). ## 2. Free-threading: biased reference counting, and the queue only the eval loop drains -`Py_DECREF` on a free-threaded build is not one operation. It branches on whether the decrementing -thread is the object's owner, recorded in `ob_tid`: - -- **Owner.** `ob_ref_local` is decremented non-atomically. If it reaches zero and the shared count - is zero too, `tp_dealloc` runs immediately, on this thread. -- **Not the owner.** The decrement goes to `ob_ref_shared` atomically. If that count was already at - zero — i.e. this decrement is the one that takes the object to zero — `tp_dealloc` *cannot* run - here, because the object belongs to another thread. It is pushed onto that thread's queue by - `_Py_brc_queue_object` (`Objects/object.c:411`, reached from the inline decref in - `Include/refcount.h:363`). - -The queue is drained by `_Py_brc_merge_refcounts`, whose only caller is, again, `_Py_HandlePending` -(`Python/ceval_gil.c:1388`), behind `_PY_EVAL_EXPLICIT_MERGE_BIT`. - -**And once a collection has run, a surviving object can become a non-owner case for everybody.** -On the way out, `gc_restore_tid` (`Python/gc_free_threading.c:325`) puts each survivor's owning -thread id back — *except* when its shared refcount has been merged, in which case `ob_tid` is left -at 0 (`:330`) and no thread matches any more, so every further decrement of that object takes the -shared path. (The inherited citation for this was `:205`, which is not the mechanism; merging -itself is at `:264`. The behaviour is as described, but it applies to merged survivors rather than -to all of them.) - -That is the mechanism behind the ROADMAP §9 measurement: 1000 wrappers released by the JVM cleaner, -2003 `Py_DecRef` calls demonstrably made, and `sys.getrefcount` frozen at 1002 through 50 forced -JVM GCs, 500 further C API round trips and two seconds of wall clock — then one trivial bytecode -frame released all thousand at once. - -## 3. `CycleCollectionTest`: two defects, and the one that is *not* a checkpoint problem - -`CycleCollectionTest.testHandleReleasedWhenProxyDiesWithoutCycle` was the last free-threaded -failure. It has two independent causes, and the guard assertion — "the type's count must rise by -exactly `rounds` while the instances are alive, otherwise the probe is not reading a refcount" — -caught both, exactly as it was written to. - -### 3a. The probe read the wrong bytes - -`ob_refcnt` is at offset 0 only on a build with the global lock. Free-threaded CPython lays the -header out differently: - -```c -/* Include/object.h, Py_GIL_DISABLED */ -struct _object { - uintptr_t ob_tid; /* +0, 8 bytes -- owning thread id, NOT a refcount */ - uint16_t ob_flags; /* +8 */ - PyMutex ob_mutex; /* +10 */ - uint8_t ob_gc_bits; /* +11 */ - uint32_t ob_ref_local; /* +12, 4 bytes */ - Py_ssize_t ob_ref_shared; /* +16, 8 bytes, shifted by _Py_REF_SHARED_SHIFT (2), - low 2 bits are state flags */ - PyTypeObject *ob_type; /* +24 */ -}; -``` - -`Py_REFCNT` there is `ob_ref_local + (ob_ref_shared >> 2)`. Reading eight bytes at offset 0 yields -`ob_tid` — which is why the failure reported a ten-digit number that was *identical* before and -after creating 100 instances: measured here, `6171668704` both times. The layout above was -confirmed against this build by dumping the header (see 3b); it is not taken from documentation. - -The probe now reads the right fields on each build. +On a free-threaded build `Py_DECREF` branches on whether the thread owns the object (`ob_tid`): +owner decrements `ob_ref_local` and may `tp_dealloc` at once; a non-owner decrements +`ob_ref_shared` atomically, and if that takes it to zero the object is queued to its owner by +`_Py_brc_queue_object` (`Objects/object.c:411`). The queue is drained by +`_Py_brc_merge_refcounts`, whose only caller is `_Py_HandlePending` (`ceval_gil.c:1388`, behind +`_PY_EVAL_EXPLICIT_MERGE_BIT`). After a collection, merged survivors keep `ob_tid == 0` +(`gc_restore_tid`, `gc_free_threading.c:325/:330`), so every later decrement takes the shared path. -### 3b. Correcting the correction: heap types **are** deferred-reference-counted +Measurement (ROADMAP §9): 1000 wrappers released by the JVM cleaner, 2003 `Py_DecRef` calls made, +`sys.getrefcount` frozen at 1002 through 50 forced JVM GCs, 500 further C API round trips and two +seconds of wall clock; one trivial bytecode frame then released all thousand at once. -This section exists because two successive diagnoses of this failure were wrong in opposite -directions, and only a direct measurement settles it. +## 3. `CycleCollectionTest`: two defects -ROADMAP §9 originally said: +Two successive diagnoses were wrong in opposite directions; a direct header dump settled it. -> Fixing the offset would not save the test. Its premise is that "each live instance of a heap type -> holds one reference to that type", and free-threaded CPython gives heap types **deferred -> reference counting** [...] The invariant simply does not exist there, so `tp_dealloc`'s -> obligation to release the type reference cannot be checked this way at all. +**3a. The probe read the wrong bytes.** `ob_refcnt` is at offset 0 only on the GIL build. +Free-threaded `struct _object` is `ob_tid` +0 (8 B), `ob_flags` +8, `ob_mutex` +10, `ob_gc_bits` ++11, `ob_ref_local` +12 (4 B), `ob_ref_shared` +16 (8 B, shifted 2, low 2 bits flags), `ob_type` ++24; `Py_REFCNT = ob_ref_local + (ob_ref_shared >> 2)`. The failure printed an identical ten-digit +number (`6171668704`) before and after 100 instances: that was `ob_tid`. -A later source trace proposed the opposite: that this was wrong, that `_PyGC_BITS_DEFERRED` is set -only on modules, top-level functions, descriptors and immortal objects, that nothing marks a -`PyType_FromSpec` heap type or its instances as deferred, and that the real cause was queued BRC -decrements that an eval-loop checkpoint would flush. +**3b. Heap types are deferred-reference-counted.** Header of the proxy type, free-threaded: -**Measured, both of those are partly wrong, and the practical conclusion belongs to the first.** -Dumping the proxy type's header on the free-threaded build, before and after 100 instantiations: - -| | `ob_tid` (+0) | `ob_gc_bits` (+11) | `ob_ref_local` (+12) | `ob_ref_shared` (+16) | +| | `ob_tid` | `ob_gc_bits` | `ob_ref_local` | `ob_ref_shared` | |---|---|---|---|---| | fresh type | `0x16d9b70e0` | `0x41` | 2 | `0x3ffffffffffffffd` | -| 100 instances alive | `0x16d9b70e0` | `0x41` | 2 | `0x3ffffffffffffffd` | -| after `drainPendingReleases()` | `0x16d9b70e0` | `0x41` | 2 | `0x3ffffffffffffffd` | -| after `PyGC_Collect()` | `0x16d9b70e0` | `0x41` | 2 | `0x400000000000018d` | +| 100 instances alive | same | `0x41` | 2 | `0x3ffffffffffffffd` | +| after `drainPendingReleases()` | same | `0x41` | 2 | `0x3ffffffffffffffd` | +| after `PyGC_Collect()` | same | `0x41` | 2 | `0x400000000000018d` | | after 100 `Py_DecRef` → 100 `tp_dealloc` | `0` | `0x41` | 0 | `0x4000000000000007` | -Reading that off: - -- `ob_gc_bits == 0x41` is `_PyGC_BITS_TRACKED | _PyGC_BITS_DEFERRED`. The heap type **is** marked - deferred. -- `ob_ref_shared == 0x3ffffffffffffffd` is `(0x0fffffffffffffff << 2) | 1`: a shared count of - `2^60 - 1` = `PY_SSIZE_T_MAX / 8`, which is the deferred sentinel, with the - `_Py_REF_MAYBE_WEAKREF` flag. Two independent signals agreeing. -- **Creating 100 instances changed nothing.** `PyType_GenericAlloc` increments the type through - `_Py_INCREF_TYPE`, which is a no-op for a deferred type on its owning thread. The header is - bit-identical. -- **An eval-loop checkpoint changed nothing either.** This is the decisive one: the proposed fix - does not work, and cannot, because there is no queued decrement to merge. The increments were - never made in the first place. -- **A collection is what makes deferred references real.** `ob_ref_shared` rose by exactly - `100 << 2`. This is `gc_free_threading.c` converting deferred references into counted ones as it - walks. -- **And then `tp_dealloc` gives back exactly 100.** Accounting for the flag bits moving from - `_Py_REF_MAYBE_WEAKREF` (1) to `_Py_REF_MERGED` (3) and `ob_ref_local` merging to zero, the total - goes from `1152921504606847077` back to `1152921504606846977` — the pre-instantiation value, - exactly. - -So the original ROADMAP claim was right about the *mechanism* (heap types are deferred) and wrong -about the *consequence* (that the invariant does not exist and cannot be checked). The invariant -holds on both builds. Free-threaded it is merely not **observable** until a collection has -materialised the deferred references. The later trace was right that instances are ordinary -reference-counted objects — but the test reads the *type's* count, not an instance's, so that -correction did not touch the failing assertion. - -`CycleCollectionTest` therefore takes a `PyGC_Collect()` on either side of its instantiation loop, -free-threaded only. The assertions are unchanged: the same "rises by exactly 100" and "returns to -exactly where it started" are asserted on both builds, and both now hold. - -### What this means for `abi3t` - -The conclusion drawn from the wrong premise — "so `abi3t` making `PyObject` an incomplete type -would sink this test" — does not follow either. The test does need to know the header layout, which -is a real cost of a direct probe and the one place in this repo that reads inside a `PyObject`. But -that is a property of the probe, not of the invariant, and it would be equally solvable by counting -through `sys.getrefcount` at the price of the argument's own temporary reference. The library -proper stays clean: `ProxyTypeFactory` writes only into memory it obtained from -`PyObject_GetTypeData`. +`0x41` = tracked | deferred. `ob_ref_shared` = `2^60 - 1` sentinel plus `_Py_REF_MAYBE_WEAKREF`. +Creating instances changes nothing (`_Py_INCREF_TYPE` is a no-op for a deferred type on its owning +thread); a checkpoint changes nothing (no queued decrement exists); a collection materialises the +references (`ob_ref_shared` rose by exactly `100 << 2`) and `tp_dealloc` gives back exactly 100. +So the invariant ("each live instance holds one reference to its type") holds on both builds; +free-threaded it is observable only after a collection. `CycleCollectionTest` takes a +`PyGC_Collect()` on either side of its instantiation loop, free-threaded only. + +`abi3t` is not a blocker: the probe's header-layout knowledge is a property of the probe (the one +place in this repo reading inside a `PyObject`), not of the invariant; `ProxyTypeFactory` writes +only into memory from `PyObject_GetTypeData`. ## 4. What this repo does about §1 and §2 -`Python3.drainPendingReleases()` reaches a checkpoint the cheapest way there is: it calls a cached, -already-compiled, zero-argument Python function whose body is `pass`, through -`PyObject_CallNoArgs`. That is one `RESUME`, hence one `_CHECK_PERIODIC`, hence one -`_Py_HandlePending` — one BRC merge, one QSBR sweep, and one scheduled collection if one was -scheduled. - -It is deliberately **not** `PyRun_SimpleString("pass")` or `exec("pass")`: - -- `PyRun_SimpleString` runs `PyErr_Print()` on failure, which clears the error indicator. This repo - has twice had unrelated tests die because a helper wiped an indicator its caller had not read - yet; `Python3.exec` is built on `PyRun_String` for exactly that reason. -- `exec("pass")` recompiles a module on every call — 23× the cached call, §5. - -`withGIL` takes one automatically at its outermost entry, behind two gates: at most one per -`Python3.autoDrainInterval` outermost scopes, and skipped entirely unless `ReleaseCounter.released` -has moved since the last one. It defaults to 32 on free-threaded builds and 0 (off) otherwise. -`reachEvalCheckpointHoldingGIL` additionally declines while `PyErr_Occurred()` is non-null, for the -same error-indicator reason. - -It is not taken from the cleaner. The queue that needs merging belongs to the thread that *owns* -the object; a cleaner owns nothing, so a checkpoint there would run Python on a cleaner thread (the -ROADMAP §1 deadlock) in order to drain an empty queue. -`EvalCheckpointTest.testCleanerActivityAloneTakesNoCheckpoint` pins the suppression. - -`PyGC_Collect` is the one Stable ABI function that reclaims on behalf of *another* thread, because -`gc_collect_internal` stops the world and walks every thread state, calling -`_PyObject_MergePerThreadRefcounts` and `merge_queued_objects` for each. It costs a heap walk -rather than a queue pop — §5 — and, as §3b shows, it is also the only thing that materialises a +`Python3.drainPendingReleases()` reaches a checkpoint by calling a cached, compiled, zero-argument +Python function (body `pass`) through `PyObject_CallNoArgs`: one `RESUME` → one `_CHECK_PERIODIC` → +one `_Py_HandlePending` (BRC merge, QSBR sweep, scheduled collection if any). Not +`PyRun_SimpleString("pass")` (its `PyErr_Print()` clears the error indicator) and not `exec("pass")` +(recompiles each time, 23× the cached call). + +`withGIL` takes one automatically at its outermost entry behind two gates: at most one per +`Python3.autoDrainInterval` outermost scopes, and only if `ReleaseCounter.released` has moved. +`reachEvalCheckpointHoldingGIL` declines while `PyErr_Occurred()` is non-null. It is not taken from +the cleaner thread (the queue belongs to the owning thread; running Python there is the ROADMAP §1 +deadlock) — pinned by `EvalCheckpointTest.testCleanerActivityAloneTakesNoCheckpoint`. +`PyGC_Collect` is the one Stable ABI call that reclaims on behalf of other threads (it stops the +world and merges each thread state), at heap-walk cost, and the only thing that materialises a deferred reference. ## 5. Measurements -`EvalCheckpointTest.testCheckpointCostAgainstTheAlternatives`, free-threaded desktop build, macOS -arm64. The test asserts the *ordering*, not the absolute numbers: the ordering is a property of the -design and would break the moment the checkpoint started compiling or allocating. +`EvalCheckpointTest.testCheckpointCostAgainstTheAlternatives`, free-threaded desktop, macOS arm64. +The test asserts the *ordering*, not the numbers. | | ns/op | | |---|---:|---| -| `withGIL { }` — attach and detach, nothing else | 170.51 | the floor | -| `Python3.drainPendingReleases()` | 302.70 | the checkpoint, ~132 ns over the floor | -| `withGIL { Py_MakePendingCalls() }` | 197.34 | cheap, and does not merge | -| `Python3.exec("pass")` | 7 037.04 | 23× the checkpoint — it recompiles | -| `withGIL { PyGC_Collect() }` | 224 625.00 | 742× the checkpoint — heap walk, but drains every thread | - -At the default `autoDrainInterval = 32` the automatic checkpoint amortises to about 4 ns on a -~170 ns outermost scope, and a workload that drops no wrappers skips it on a field compare. +| `withGIL { }` — attach and detach | 170.51 | floor | +| `Python3.drainPendingReleases()` | 302.70 | ~132 ns over floor | +| `withGIL { Py_MakePendingCalls() }` | 197.34 | cheap, does not merge | +| `Python3.exec("pass")` | 7 037.04 | 23× the checkpoint | +| `withGIL { PyGC_Collect() }` | 224 625.00 | 742× the checkpoint | -`PyGC_Collect` being ~740× the checkpoint is also the cost `CycleCollectionTest` pays for the two -collections §3b requires: ~0.45 ms for the whole test, free-threaded only. +At `autoDrainInterval = 32` the automatic checkpoint amortises to ~4 ns on a ~170 ns outermost +scope. `CycleCollectionTest`'s two collections cost ~0.45 ms, free-threaded only. ## 6. What is still true, and what to watch -- The scheduled-GC half of §1 affects **both** builds. Nothing in this repo currently depends on - allocation-driven collection, and `autoDrainInterval` defaults to 0 with the global lock, so the - default configuration still never runs a generational collection on its own. That is a choice, - not an oversight, and reversing it is one assignment. -- `EvalCheckpointTest` asserts that `Py_MakePendingCalls` does *not* merge. If CPython ever changes - its mind, that test fails and says so. -- The header layout in §3a is a private detail read in exactly one place, and that read is guarded - by an assertion that fails loudly if the offsets stop being right. It already has, twice. -- The deferred sentinel gives roughly `2^60` of headroom, so nothing here is at risk of the count - actually reaching zero. But note the asymmetry `ProxyTypeFactory` lives with: its hand-written - `tp_dealloc` releases the type through the public `Py_DecRef`, which is deferred-unaware, while - the matching increment went through the deferred-aware `_Py_INCREF_TYPE` and did not happen. That - balances out once a collection has materialised the references — measured above — but it is - balanced by the collector, not by the two calls. - -## 7. GC Accumulation and Reentrancy Measurement - -Measurements taken to verify the consequences of §1 and the safety of the checkpoint confirm: - -- **Cyclic Garbage Accumulation (GIL Build)**: A workload creating 10,000 cyclic object groups entirely through the C API (never running a Python bytecode evaluation loop) with `autoDrainInterval = 0` accumulates **20,000+** cyclic garbage objects indefinitely. Because no evaluation loop runs, `_Py_ScheduleGC`'s scheduled bit is never checked by `_CHECK_PERIODIC`, and cyclic GC never occurs. -- **`autoDrainInterval` Effectiveness (GIL build)**: When running the exact same C API workload but with `autoDrainInterval = 32`, `python-multiplatform` evaluates a dummy function (`__pmp_eval_checkpoint__`) to force the evaluation loop to run periodically. This allows `_CHECK_PERIODIC` to see the scheduled bit, trigger `_Py_RunGC()`, and reclaim the cyclic garbage (leaving ~1,970 of ~20,000). **This result is specific to the build with the global lock.** Free-threaded, the same checkpoints frequently reclaim nothing at all — §8. -- **`__del__` Reentrancy Risk**: Executing `PyGC_Collect()` or processing a checkpoint can invoke `__del__` methods. A scenario where `__del__` directly calls back into Kotlin (via Panama upcalls) was executed (`GCSchedulingMeasurementTest.testReentrancyDuringCheckpoint`). The reentrancy is safe on the GIL build: it does not deadlock, it does not loop infinitely, and it successfully executes the Kotlin upcall while the garbage collection is in progress. - -## 8. Free-threaded: the checkpoint fires and reclaims nothing, and it is not the scope shape - -Everything in this section was measured here, on `desktopTest`, macOS arm64, CPython 3.14.7, both -builds. The workload is `GCSchedulingMeasurementTest.buildAndDropCycles`: 10,000 unreachable -`list` cycles, 20,000 objects, built entirely through the C API. - -### 8a. The probe was holding the evidence - -The measurement chain used to be written inline as - -```kotlin -gc.getAttr("get_objects").invoke().getAttr("__len__").invoke().toString().toInt() -``` - -which closes none of its temporaries. `gc.get_objects()` returns a list holding **a strong -reference to every tracked object in the interpreter**, so a leaked one roots the very garbage the -next measurement is trying to see. - -That leak is where §7's earlier free-threaded conclusion came from — the claim that "a trailing -explicit `gc.collect()` finds nothing further to reclaim, so the residue is not simply garbage -waiting for the next checkpoint," and the inference from it that free-threaded CPython was failing -to *reclaim* something. With the temporaries released, **an explicit `gc.collect()` reclaims the -residue essentially completely, on both builds, in every configuration tried**: 60+ repetitions, -post-collection residue between −13 and +2 objects out of ~20,000. - -So the residue is ordinary collectable cyclic garbage. What differs free-threaded is what makes the -collector *run*, not what it can take. `GCSchedulingMeasurementTest` now asserts this on both -builds (`assertResidueIsOrdinaryCollectableGarbage`), which is the opposite of what the previous -text asserted about it. - -### 8b. Rejected: "per-call `PyGILState_Ensure`/`Release` destroys the scheduled GC bit" - -The standing hypothesis was that because free-threading keeps `eval_breaker` per thread state, an -FFI wrapper that attaches and detaches around every C API call destroys and recreates the thread -state, discarding the scheduled GC bit with it — and that widening the scope would therefore fix -collection. - -**Measured and rejected.** Holding the checkpoint count fixed at 2,000 explicit -`Python3.drainPendingReleases()` calls (`autoDrainInterval = 0`, one drain every 5 rounds) and -varying *only* the enclosing scope, free-threaded, residue out of 20,000: - -| enclosing scope | reps | residue | -|---|---:|---| -| none — one `withGIL` per C API call | 5 | 20,000 ×5 | -| one per round (~12 C API calls) | 5 | 20,000 ×5 | -| one per 100 rounds | 10 | 20,000 ×10 | -| one for the whole 10,000-round workload | 10 | 11,562 / 1,280 / 1,280, then 20,000 ×7 | - -Widening the scope to 100 rounds — which gives the thread state a long life and hundreds of -allocations before it is destroyed — changes nothing whatsoever. The hypothesis predicts the -opposite, so it is wrong. - -The single-scope row does not rescue it either: **it is bimodal within one JVM.** Those ten -repetitions are consecutive and identical; it reclaimed on the first three and then never again. -The configuration latches, which is a property of accumulated interpreter state, not of the scope -shape the call happens to be made under. - -With the automatic checkpoint instead (`autoDrainInterval = 32`), free-threaded: - -| enclosing scope | checkpoints taken | residue | -|---|---:|---| -| none — one `withGIL` per C API call | 1,875 | 20,000 ×5; also 7,427 and 11,523 in two single-repetition JVMs | -| one per round | 312–313 | 20,000 ×5 | -| one per 100 rounds | 3 | 20,000 ×5 | -| one for the whole workload | **0** | 20,000 ×3 | - -The last row is a property of the gate, not of the collector: a single outermost scope decrements -`checkpointCountdown` once, so at interval 32 no automatic checkpoint is ever reached. An embedder -that wraps a long batch in one `withPython` therefore gets *fewer* automatic checkpoints, not more. - -### 8c. The contrast that is actually there - -The same sweep on the build with the global lock is stable and near-complete in **every** shape: - -| configuration | checkpoints | residue | -|---|---:|---| -| per C API call, `autoDrainInterval = 32` | 1,875 | 1,964–1,972 | -| per C API call, 2,000 explicit drains | 2,000 | 1,920 ×5 | -| one per 100 rounds, 2,000 explicit drains | 2,000 | 1,920 ×5 | -| whole workload in one scope, 2,000 explicit drains | 2,000 | 1,920–1,962 | -| one per round, `autoDrainInterval = 32` | 312–313 | 1,556–1,616 | -| one per 100 rounds, `autoDrainInterval = 32` | **3** | 1,600–3,200 | - -Three checkpoints reclaim ~90% on the GIL build. Two thousand reclaim nothing free-threaded. That -gap — not the scope shape, which is irrelevant on both builds — is the whole finding. +- The scheduled-GC half of §1 affects both builds. Nothing in this repo depends on + allocation-driven collection, and `autoDrainInterval` defaults to 0 on the GIL build, so by + default no generational collection runs on its own. A choice, not an oversight; reversing it is + one assignment. +- `EvalCheckpointTest` asserts `Py_MakePendingCalls` does *not* merge; it fails if CPython changes. +- The header layout in §3a is read in exactly one place, guarded by an assertion that has already + fired twice. +- `ProxyTypeFactory`'s hand-written `tp_dealloc` releases the type through the deferred-unaware + `Py_DecRef`, while the matching increment went through deferred-aware `_Py_INCREF_TYPE` and did + not happen; balanced by the collector, not by the two calls (measured in §3b). + +## 7. GC accumulation and re-entrancy (GIL build) + +- 10,000 cyclic groups built purely through the C API with `autoDrainInterval = 0`: 20,000+ + objects accumulate indefinitely (no bytecode runs, the bit is never read). +- Same workload with `autoDrainInterval = 32`: checkpoints let `_Py_RunGC` run; ~1,970 of ~20,000 + remain. **Specific to the GIL build** (free-threaded: §8). +- `__del__` calling back into Kotlin during `PyGC_Collect()` or a checkpoint + (`GCSchedulingMeasurementTest.testReentrancyDuringCheckpoint`): safe on the GIL build, no + deadlock, no loop, the upcall runs. + +## 8. Free-threaded: the checkpoint fires and reclaims nothing + +Workload: `GCSchedulingMeasurementTest.buildAndDropCycles`, 10,000 unreachable `list` cycles +(20,000 objects) built through the C API. + +**8a. The probe held the evidence.** Measuring with `gc.get_objects()` unclosed leaked a list that +strongly references every tracked object, rooting the garbage being measured; that produced the +earlier "free-threaded does not reclaim" conclusion. With temporaries released, an explicit +`gc.collect()` reclaims the residue essentially completely on both builds (60+ repetitions, +residue −13..+2 of ~20,000). The residue is ordinary collectable garbage; what differs is what +makes the collector *run*. Do not restore the inline `gc.get_objects()` chain. + +**8b. Rejected: per-call `PyGILState_Ensure/Release` destroys the scheduled bit.** 2,000 fixed +explicit drains, only the enclosing scope varied, free-threaded, residue of 20,000: no scope 20,000 +×5; per round 20,000 ×5; per 100 rounds 20,000 ×10; one scope for the whole workload bimodal within +one JVM (11,562 / 1,280 / 1,280, then 20,000 ×7). Widening the scope changes nothing. With +`autoDrainInterval = 32`: 1,875 / 312–313 / 3 / **0** checkpoints for the four scope shapes (one +outermost scope decrements the countdown once, so a long batch in one `withPython` gets *fewer* +automatic checkpoints), residue 20,000 throughout. + +**8c. The contrast.** On the GIL build every shape is stable and near-complete: residue 1,920–1,972 +for per-call/explicit-drain shapes, 1,556–1,616 per round at interval 32; **3 checkpoints at +interval 32 per-100-rounds still reclaim ~90%** (residue 1,600–3,200). Three checkpoints reclaim +~90% on the GIL build; two thousand reclaim nothing free-threaded. That gap, not scope shape, is +the finding. ### 8d. Answered: the checkpoint is fine, the collector asks a second question -The question §8c leaves open is why a checkpoint that demonstrably runs -(`CheckpointCounter.reached` climbs by an identical, deterministic amount every run) triggers no -collection free-threaded, and why the same configuration latches from "reclaims" to "never -reclaims" inside one JVM. - -CPython v3.14.7 was fetched to `/Volumes/macMini/cpython-src` (not vendored here) and read. **The -checkpoint is not where this goes wrong.** The checkpoint reads the scheduled bit exactly as §1 -describes, on both builds. What differs is that free-threaded, `_Py_RunGC` then asks a *second* -question, and it is that one that answers no. - -#### The asymmetry, in four lines of source +Source-confirmed asymmetry (v3.14.7): -| | global lock (`Python/gc.c`) | free-threaded (`Python/gc_free_threading.c`) | +| | GIL (`gc.c`) | free-threaded (`gc_free_threading.c`) | |---|---|---| -| schedules on | `generations[0].count > threshold`, and nothing else (`gc.c:1866`) | `gc_should_collect` (`:2153`) | -| re-checked when the checkpoint runs it | `gc_select_generation` — generation 0 on count alone (`gc.c:1258`) | `gc_should_collect` **again** (`:2328`) | -| process-memory gate | **none**; `gc.c` has no `last_mem` field and never calls `get_process_mem_usage` | `gc_should_collect_mem_usage` (`:2080`) | -| `long_lived_total / 4` | oldest generation only (`gc.c:1300`) | **every** generation-0 decision (`:2131`) | - -`gc_should_collect` (`gc_free_threading.c:2117`) is: - -```c -if (count <= threshold || threshold == 0 || !gc_enabled) return false; // :2123 -if (gcstate->old[0].threshold == 0) return true; // :2126 -if (count < gcstate->long_lived_total / 4) return false; // :2131 -return gc_should_collect_mem_usage(gcstate); // :2136 +| schedules on | `generations[0].count > threshold` only (`gc.c:1866`) | `gc_should_collect` (`:2153`) | +| re-checked when the checkpoint runs it | `gc_select_generation`, gen 0 on count alone (`:1258`) | `gc_should_collect` **again** (`:2328`) | +| process-memory gate | none | `gc_should_collect_mem_usage` (`:2080`) | +| `long_lived_total / 4` | oldest generation only (`:1300`) | every gen-0 decision (`:2131`) | + +`gc_should_collect_mem_usage` returns true if `deferred > threshold * 40` (`:2089`) or +`(footprint − last_mem) > max(last_mem/10, 128)` (`:2100`); otherwise it zeroes `young.count` into +`deferred_count` (`:2109-2113`) and returns false. `footprint` is `get_process_mem_usage()`, on +macOS `task_info(TASK_VM_INFO).phys_footprint` (`:2010`): the **whole process**, i.e. the JVM. +`last_mem` is written only after a collection (`:2301`). Twenty thousand small Python lists cannot +move a JVM's footprint by `last_mem/10`, and the escape (`deferred_count > 40 × threshold0` = 80,000 +at default) is not reached by ~20,000 allocations. **This is also the latch:** early in a JVM's life +the footprint is climbing and the gate opens; once a collection records the settled footprint, the +gate is shut one-way. Scope shape is irrelevant, consistent with §8b. + +Experiments (all in `FreeThreadedGCGateTest`; residue of 20,000 and collections run): + +| experiment | free-threaded | GIL | +|---|---|---| +| `gc.get_count()[0]` shape across workload | sawtooth `[1728, 1488, 976, 464, 2464, 1488, 976, 464, 2464, 1488]`, 0 collections, residue 20,000 (memory gate declines) | — | +| threshold `(2000,10,10)` | 20,000 — 0 collections | 1,968 — 9 collections | +| `(2000,0,0)` (`old[0].threshold == 0` short-circuits `:2126`) | **1,318 — 8 collections** | 1,966 — 5 collections | +| `(2000,10,10)` again (A/B/A, one JVM) | 20,000 — 0 | 1,970 — 9 | +| `threshold0` 2000 / 500 / 100 (bar 80,000 / 20,000 / 4,000) | 20,000 / 20,000 / **1,466 (3 collections)** | scales smoothly: 9 / 36 / 172 collections | +| baseline | 20,000 — 0 | 1,972 — 9 | +| 400 MB ballast faulted in midway | **9,754 — 1 collection** | 1,988 — 9 | +| ballast dropped | 20,000 — 0 | 1,968 — 9 | + +The ballast result reproduced identically in three consecutive JVMs: JVM memory growth, and nothing +else, collected Python cycles. Probe pitfall recorded: sampling `gc.get_count()` every 500 +allocations flushes the per-thread buffer (`gc_get_count_impl`, `Modules/gcmodule.c:215`; +`LOCAL_ALLOC_COUNT_THRESHOLD` = 512), so the collector is never asked and the trace looks linear; +sample every 2,000. + +Not confirmed: the scaling prediction (10,000 / 30,000 / 60,000 cycles at default thresholds gave +1 / 2 / 2 collections; the 10,000 arm should have given 0 but ran first in its JVM while the +footprint was still climbing). Untested, not confirmed. + +What changed in code/tests: `FreeThreadedGCGateTest` holds free-threading to the same "reclaims +more than half" bound as the GIL build once the gate is open, and asserts the matched no-op on the +GIL build so a CPython change to either gate fails it. `measureCyclicGarbageWithAutoDrain` asserts +no bound free-threaded in the default configuration (outcome depends on JVM footprint history). + +Advice: an embedder can open the gate (`gc.set_threshold(t0, 0, 0)`; small `threshold0` lowers the +`40 × threshold0` bar), but both trade throughput and tune CPython internals the Stable ABI does not +promise. The supported way is `PyGC_Collect()` (§4). + +## How to reproduce + +```bash +# GIL build +./gradlew :python-multiplatform:desktopTest --tests 'python.multiplatform.ref.GCSchedulingMeasurementTest' \ + --tests 'python.multiplatform.ref.CycleCollectionTest' --console=plain > .tmp/gc-gil.log 2>&1; echo "EXIT=$?" +# free-threaded build (3.14t, desktop only) +./gradlew :python-multiplatform:desktopTest -PpythonFreeThreaded=true \ + --tests 'python.multiplatform.ref.FreeThreadedGCGateTest' --tests 'python.multiplatform.ref.EvalCheckpointTest' \ + --console=plain > .tmp/gc-ft.log 2>&1; echo "EXIT=$?" ``` -and `gc_should_collect_mem_usage` (`:2080`) is, in substance: - -```c -if (deferred > threshold * 40) return true; // :2089 -if ((footprint - last_mem) > Py_MAX(last_mem / 10, 128)) return true; // :2100 -young.count = 0; deferred_count += young.count; return false; // :2109-2113 -``` +(Package names assumed from the directory layout; adjust to the test files' `package` line.) Run on +a quiet machine, per-module and unfiltered when quoting numbers; the footprint-gated results depend +on JVM history, so run the whole suite for a cut. -`footprint` is `get_process_mem_usage()`, which on macOS is -`task_info(TASK_VM_INFO).phys_footprint` (`:2010`) — **the whole process**, i.e. the JVM. -`last_mem` is written in exactly one place, after a collection (`:2301`). - -So, in a JVM-hosted embedder: 20,000 small Python lists cannot move a JVM's physical footprint by -the required `last_mem / 10`, the gate returns false, and on the way out it **zeroes `young.count` -into `deferred_count`** — which means the count has to climb from zero to 2,000 again before the -question is even asked again. The only escape is `deferred_count > threshold * 40` = 80,000 -container allocations at the default threshold, and this workload makes ~20,000. Checkpoints are -irrelevant to all of it; the bit gets read and `gc_collect_main` declines at `:2328`. - -**This is also the latch.** `last_mem` only moves when a collection runs, and it moves to the -whole process's current footprint. Early in a JVM's life the footprint is still climbing steeply, -so the gate opens and collections happen; once a collection records the JVM's settled footprint, -the bar becomes a tenth of a large number that Python-side allocation cannot move, and it is shut -one-way. Nothing about the scope shape is involved, which is consistent with §8b rejecting it. - -#### Confirmed by experiment, not by reading - -The repo has been wrong before about causes derived from source reading, so each claim above was -turned into a prediction that could fail. All of it is in -`python-multiplatform/src/desktopTest/.../ref/FreeThreadedGCGateTest.kt`. - -**Which gate declines** — `gc_should_collect_mem_usage`'s failing branch zeroes `young.count` -(`:2109`); the `long_lived_total/4` branch (`:2131`) does not. So the shape of `gc.get_count()[0]` -across the workload tells them apart. Free-threaded it is a **sawtooth** -(`[1728, 1488, 976, 464, 2464, 1488, 976, 464, 2464, 1488]`) with 0 collections and residue 20,000: -the memory gate, repeatedly. - -> This probe took two attempts, and the first one was wrong in the direction that would have -> produced a confident false answer. `gc_get_count_impl` (`Modules/gcmodule.c:215`) flushes the -> per-thread allocation buffer and sets `gc->alloc_count = 0`, and `record_allocation` only -> consults `gc_should_collect` when that buffer reaches `LOCAL_ALLOC_COUNT_THRESHOLD` = 512 -> (`:69`, `:2147`). Sampling every 500 allocations therefore stopped the collector from ever being -> asked, and produced a perfectly linear trace (`226, 726, 1226, … 19726`) that reads exactly like -> proof of the *other* hypothesis. Sampling every 2,000 allocations instead gives the sawtooth. - -**The one-line falsifier** — `:2126` says `old[0].threshold == 0` short-circuits the whole chain. -Nothing in `gc.c` reads that field while scheduling. So `gc.set_threshold(2000, 0, 0)` — a setting -about *generation 1* — must switch *generation 0* collection on free-threaded and do nothing at all -with the global lock. Run A/B/A in one JVM so the latch cannot explain the difference; residue out -of 20,000, and the number of collections that actually ran: - -| arm | free-threaded | global lock | -|---|---|---| -| `(2000, 10, 10)` | 20,000 — 0 collections | 1,968 — 9 collections | -| `(2000, 0, 0)` | **1,318 — 8 collections** | 1,966 — 5 collections | -| `(2000, 10, 10)` again | 20,000 — 0 collections | 1,970 — 9 collections | - -**The `deferred_count` arithmetic** — the escape bar is `40 × threshold0`, and `deferred_count` -grows *only* in the memory gate's failing branch. Against a ~20,000-allocation workload that -predicts no collection at threshold0 = 2000 (bar 80,000) or 500 (bar 20,000), and collection at -100 (bar 4,000). Free-threaded: 20,000 / 20,000 / **1,466 with 3 collections**. The same sweep with -the global lock just scales smoothly with the threshold (9 / 36 / 172 collections), because there -is no such bar. This also proves the `long_lived_total/4` gate at `:2131` is being *passed* — -nothing else feeds `deferred_count`. - -**The latch, driven directly** — if the gate is whole-process footprint, then faulting in 400 MB of -ballast partway through the workload must open it, while changing no threshold, allocating no extra -Python container and touching nothing the collector tracks. Dropping the ballast must shut it -again. Residue out of 20,000: - -| arm | free-threaded | global lock | -|---|---|---| -| baseline | 20,000 — 0 collections | 1,972 — 9 collections | -| 400 MB faulted in midway | **9,754 — 1 collection** | 1,988 — 9 collections | -| ballast dropped | 20,000 — 0 collections | 1,968 — 9 collections | - -Reproduced identically to the object in three consecutive JVMs. **JVM memory growth, and nothing -else, collected Python cycles.** That is the finding: on a free-threaded build the Python cyclic -collector's schedule is a function of the host process's memory, which for an embedder is not -Python's memory at all. - -One experiment did **not** confirm what it was built to test and is recorded as such: scaling the -workload (10,000 / 30,000 / 60,000 cycles) at default thresholds to cross the 80,000 bar gave -1 / 2 / 2 collections, but the 10,000-cycle arm should have given 0. It ran first in its JVM, while -the footprint was still climbing, so the memory gate — the very thing being controlled for — -supplied the collections. The scaling prediction is untested, not confirmed. - -#### What this changes - -- The free-threaded case now has a **bound**, where §8 could only say the residue was bimodal and - refuse to assert one. `FreeThreadedGCGateTest` holds free-threading to the same - "reclaims more than half" bound as the GIL build *once the gate the source names is open*, and - asserts the matched no-op on the GIL build so that a CPython change to either gate fails the test - and says which one. -- `GCSchedulingMeasurementTest.measureCyclicGarbageWithAutoDrain` still asserts no bound - free-threaded in the default configuration, and that is now a documented consequence rather than - an unknown: at default thresholds the outcome depends on the JVM's footprint history, so it is - genuinely not boundable, and the two arms that reclaimed in §8b's single-scope row were the JVM - still warming up. -- §8b's rejection stands and is now explained: scope shape cannot matter because the gate never - looks at thread state. - -#### The advice does not change - -An embedder can open the gate deliberately — `gc.set_threshold(t0, 0, 0)` restores count-driven -collection free-threaded, and a small `threshold0` lowers the `40 × threshold0` bar — but both -trade throughput for it, and both are tuning of CPython internals that the Stable ABI does not -promise. The supported way to collect cycles in an embedder that runs no Python bytecode is still -the one that works identically on both builds, documented in §4: `PyGC_Collect()`, at a cost -proportional to the heap. +The full original, with the stepwise corrections, is +[`docs/archive/investigations/gc-scheduling-investigation.md`](../archive/investigations/gc-scheduling-investigation.md). diff --git a/docs/investigations/gil-parking-investigation.md b/docs/investigations/gil-parking-investigation.md index d91b5e623..20f30ee1b 100644 --- a/docs/investigations/gil-parking-investigation.md +++ b/docs/investigations/gil-parking-investigation.md @@ -1,367 +1,82 @@ -# Why enabling `PyEval_SaveThread()` in `Python3.initialize()` crashes desktopTest - -Research only. No source in this repo was changed to produce this document; all CPython claims -below are cited from `python/cpython` at tag `v3.14.0` on GitHub, fetched read-only. - -## 0. Summary and confidence - -**Ruled out, with source evidence:** the failure is not "`PyGILState_Ensure` creates a second, -competing `PyThreadState`" on the initialising thread. It reuses the *same* `PyThreadState*` that -`Py_Initialize()` created. This is CPython's documented, intended mechanism for exactly this -pattern (park, then re-enter from a C thread later) — see §2 and §3. - -**Established, with source evidence, but not sufficient by itself:** `PyEval_SaveThread()` leaves -the calling OS thread's slot in the GILState TSS (`autoTSSkey`) pointing at the parked -`PyThreadState`. CPython's own comment in `pystate.c` says so explicitly: "We do not unbind the -gilstate tstate here. It will still be used in `PyGILState_Ensure()`." Reusing the same, -still-registered thread state is a supported idiom (it's what `Py_BEGIN_ALLOW_THREADS` / -`Py_END_ALLOW_THREADS` do on every existing embedding), so this fact **describes the mechanism -correctly but does not, by itself, explain a segfault.** Something else that the reuse *doesn't* -restore is the more likely proximate cause, and I could not pin that down from source reading -alone. See §5 for the two candidates and their weaknesses. - -**Confidence: medium on the mechanism (thread-state identity across the park/re-enter boundary), -low on the exact reason it becomes memory-unsafe.** The minimal experiment in §6 is designed to -close that gap cheaply, without running the full suite. - -## 1. Same thread, established rather than assumed - -`PythonTestFixture.available` is a Kotlin `by lazy` property; `Python3.initialize()` runs on -whichever thread first touches it. `desktopTest`'s Gradle configuration -(`python-multiplatform/build.gradle.kts:724`) sets no `maxParallelForks`, no -`junit.jupiter.execution.parallel.*` property exists anywhere in the repo, and no -`junit-platform.properties` exists. Gradle's `Test` task therefore runs the whole suite -sequentially inside one worker JVM, on the JVM's single test-execution thread — which is named -`"Test worker"` by Gradle, matching the thread name in today's crash report verbatim. So -`Python3.initialize()` and `EmbedApiLowLevelTest.importReturnsANewReferenceAndMissingModulesReportFailure` -run on the **same OS thread**, in the same process, with nothing else attaching in between. - -This directly determines which branch of `PyGILState_Ensure()` fires later (§3): the -"already-registered" branch, not the "first time on this OS thread" branch. - -## 2. What `PyEval_SaveThread()` does to the calling thread's state - -Source (`Python/ceval_gil.c`, v3.14.0): - -```c -PyThreadState * -PyEval_SaveThread(void) -{ - PyThreadState *tstate = _PyThreadState_GET(); - _PyThreadState_Detach(tstate); - return tstate; -} -``` - -`_PyThreadState_Detach` (`Python/pystate.c`) calls `detach_thread(tstate, _Py_THREAD_DETACHED)`: - -```c -static void -detach_thread(PyThreadState *tstate, int detached_state) -{ - assert(_Py_atomic_load_int_relaxed(&tstate->state) == _Py_THREAD_ATTACHED); - assert(tstate == current_fast_get()); - if (tstate->critical_section != 0) { - _PyCriticalSection_SuspendAll(tstate); - } - ... - tstate_deactivate(tstate); - tstate_set_detached(tstate, detached_state); - current_fast_clear(&_PyRuntime); - _PyEval_ReleaseLock(tstate->interp, tstate, 0); -} -``` - -and `tstate_deactivate`: - -```c -static inline void -tstate_deactivate(PyThreadState *tstate) -{ - ... - tstate->_status.active = 0; - - // We do not unbind the gilstate tstate here. - // It will still be used in PyGILState_Ensure(). -} -``` - -So: the `PyThreadState` object itself is **not destroyed**. It is marked inactive/detached, the -GIL is released, and the per-OS-thread "current thread state" pointer -(`current_fast_get`/`current_fast_clear`, a `thread_local` variable) is cleared to `NULL`. What is -**not** touched anywhere in this path: `tstate->gilstate_counter`, and the GILState TSS slot -(`gilstate_tss` / historically `autoTSSkey`) that `PyGILState_Ensure` consults. That slot is a -*separate* piece of per-OS-thread storage from the "current thread state" pointer, and CPython's -own comment above states outright that detaching deliberately leaves it alone. - -By the end of `Py_Initialize()` (before the parking call), the calling thread's GILState TSS slot -is already bound to the main thread's `PyThreadState`, with `gilstate_counter == 1` — confirmed by -`_PyGILState_SetTstate()`'s debug assertions (`Python/pystate.c`): - -```c -assert(runtime->gilstate.autoInterpreterState == tstate->interp); -assert(gilstate_tss_get(runtime) == tstate); -assert(tstate->gilstate_counter == 1); -``` - -(`bind_gilstate_tstate` is what performs that binding, called from `tstate_activate` — i.e. from -inside `_PyThreadState_Attach`/`PyEval_RestoreThread` — the first time the main thread is -attached during startup.) - -## 3. What `PyGILState_Ensure()` then does on that same thread - -Source (`Python/pystate.c`, v3.14.0): - -```c -PyGILState_STATE -PyGILState_Ensure(void) -{ - ... - PyThreadState *tcur = gilstate_tss_get(runtime); - int has_gil; - if (tcur == NULL) { - tcur = new_threadstate(runtime->gilstate.autoInterpreterState, - _PyThreadState_WHENCE_GILSTATE); - ... - bind_tstate(tcur); - bind_gilstate_tstate(tcur); - tcur->gilstate_counter = 0; - has_gil = 0; - } - else { - has_gil = holds_gil(tcur); - } - if (!has_gil) { - PyEval_RestoreThread(tcur); - } - ++tcur->gilstate_counter; - return has_gil ? PyGILState_LOCKED : PyGILState_UNLOCKED; -} +# Enabling `PyEval_SaveThread()` in `Python3.initialize()` + +Question: why did parking the initialising thread's state (so other threads can attach) appear to +crash `desktopTest`? + +## Conclusion + +**The investigation's own diagnosis was incomplete, and parking is on.** What this document +established from CPython 3.14.0 source (read-only, no code run) holds, but it did not find the +cause, and the cause turned out not to be in the runtime: + +- `PyGILState_Ensure` on the initialising thread after `PyEval_SaveThread()` does **not** create a + second `PyThreadState`. `PyEval_SaveThread` detaches the state and releases the GIL but leaves the + GILState TSS slot bound (CPython's own comment in `pystate.c`: "We do not unbind the gilstate + tstate here. It will still be used in `PyGILState_Ensure()`"). `Ensure` finds the same state, + `has_gil` is false, so it calls `PyEval_RestoreThread` on it; `gilstate_counter` goes 1 → 2 and + `Release` brings it back and re-parks. Park/re-enter on one thread is the supported idiom + (`Py_BEGIN/END_ALLOW_THREADS` does it). +- `PyGILState_Release` is never handed an invalid state in this flow. +- The thread is the same: Gradle runs the whole suite on one `"Test worker"` thread, which is also + where `Python3.initialize()` runs. + +**The actual cause (ROADMAP §1, "Closed"):** several tests wrapped a pointer they had only been +*lent* with `borrowed = false`, so two wrappers owned one pointer and both decremented it. That +double-free corrupted CPython's free lists and surfaced later in an unrelated test. With parking +off, the cleaner thread blocked forever on the GIL, so the extra decrements queued and never ran, +which is why enabling parking appeared to *cause* the crash. The `SIGSEGV` in +`_TAIL_CALL_DICT_MERGE` under `PyImport_ImportModule("sys")` was a victim of the corruption, not +the fault site. Two earlier attempts read the evidence as "parking is broken" and reverted. + +The two candidates this document left open (stale per-thread C-stack recursion limits calibrated +once at first attach; Python-level import bookkeeping on a cache hit) were never confirmed and, +given the cause above, are not needed to explain anything. The stack-limit staleness is a real +property of reusing one `PyThreadState` (`_Py_InitializeRecursionLimits` runs only while +`c_stack_hard_limit == 0`) but no failure has been attributed to it. + +**Status: acted on, parking enabled unconditionally.** +`python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/Python3.kt`: +`initialize()` calls `PyEval_SaveThread()` after `Py_Initialize()` (guarded only by +`mainThreadState == null`); `finalize()` reclaims it with `PyEval_RestoreThread`. Verified by +reading the file at `5db4168c`. + +Discrepancy to resolve elsewhere: `docs/SPEC.md` C-4 says "Releasing the GIL after initialisation +is not enabled." That contradicts the code above and ROADMAP §1. This document does not edit SPEC. + +Related rule that survives: every C API call must be inside `withGIL{}`/`withPython{}`. Free-threaded +builds do not remove it (a thread must still be attached before touching any object, `Py_IncRef` +included). `Python3.kt` documents this at the `PyEval_SaveThread()` call site. + +## Numbers + +None. This was a source-reading investigation. The only quantitative record is the ROADMAP §1 +test counts when parking was enabled: desktop 164, iOS 157, zero failures (a historical count; the +suites have grown since). + +## How to reproduce / guard + +The bisection experiment proposed here (park, re-enter on the same thread with bare re-entry → pure +C call → `PyImport_ImportModule("sys")` → `PySys_GetObject` + `PyDict_GetItemString`) exists as +`python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/GilParkingTest.kt` +(`parkThenReenterSameThreadIsolatesTheFailure`). It asserts only that nothing crashes (SPEC C-4). +Run alone on a quiet tree: + +```bash +./gradlew :python-multiplatform:desktopTest --tests 'python.multiplatform.ffi.GilParkingTest' \ + --console=plain > .tmp/gilparking.log 2>&1; echo "EXIT=$?" ``` -Because the GILState TSS slot on this OS thread is still bound (§2), `tcur` is **not** `NULL`, so -the "new thread" branch (which would create a genuinely second `PyThreadState`) is **not** taken. -`tcur` is the exact same `PyThreadState*` `Py_Initialize()` created. `holds_gil(tcur)` compares -`tcur` against `current_fast_get()`, which is `NULL` after parking, so `has_gil = 0`, and -`PyEval_RestoreThread(tcur)` re-attaches that same object (`current_fast_set`, GIL re-acquired, -`tstate_activate` — which is a no-op on the already-bound gilstate slot). `gilstate_counter` goes -1 → 2. `PyGILState_Release()` mirrors this exactly: decrements 2 → 1, and because the saved -`oldstate` was `PyGILState_UNLOCKED`, it re-parks with another internal `PyEval_SaveThread()` call. -This is self-consistent and, cycled repeatedly, returns to the same `gilstate_counter == 1`, -detached baseline every time. **Nothing in this path frees, reallocates, or otherwise clears the -frame stack, exception state, or recursion bookkeeping that the object already carried** — it is -literally the same object, reused. - -This rules out "two thread states corrupting each other's interpreter frame stack" as the -mechanism, at least on the initialising thread — there is only ever one `PyThreadState` in play -here. It does not, by itself, identify what *does* go wrong. - -## 4. Whether `PyGILState_Release` is handed an invalid state - -No. Traced above: the `PyGILState_STATE` value threaded through `withGIL`'s `tState.state` / -`PyGILState_Release` call is always the value `PyGILState_Ensure` just returned for the current, -still-registered `tcur`. There is no thread-identity mismatch and no stale/deleted state reaching -`Release` in this flow. (Contrast with the *different*, already-fixed bug documented in -`GCLeakTest.desktop.kt:6-11`: an earlier version called `PyEval_SaveThread()` from `forceGC()` on -a thread that was not attached at all. `PyEval_SaveThread`'s body dereferences -`_PyThreadState_GET()` unconditionally — on a PGO/LTO release build, `assert()`s such as -`detach_thread`'s `assert(tstate == current_fast_get())` compile out, so a `NULL` tstate reaches -`detach_thread` and is dereferenced (`tstate->critical_section`) with no check. That is a plain -null-pointer crash from calling `PyEval_SaveThread` while unattached, and it is a different bug -shape from today's: today's parking call is made while the thread *is* attached, immediately -after `Py_Initialize()`, so this specific failure mode does not apply.) - -## 5. What CPython 3.14 changed here, and the two remaining candidates - -The whole attach/detach vocabulary read above (`_PyThreadState_Attach`/`Detach`, -`tstate_activate`/`deactivate`, the `_status.active` / `_status.bound_gilstate` bitfield, -`current_fast_get`/`set`/`clear`) is the free-threading-era (PEP 703) rewrite of thread-state -lifecycle management, unified so the same source serves both GIL and `Py_GIL_DISABLED` builds. I -did not verify the exact CPython version this landed in (older CPython used a simpler -`PyEval_AcquireThread`/`ReleaseThread` pair over a single `interp->tstate_current` swap), but it is -recent relative to most public documentation and tutorials on `PyGILState_*`, which is worth -knowing before trusting older blog posts or Stack Overflow answers about this API. - -Two candidates for what actually breaks, neither confirmed: - -**Candidate A — stale per-thread C-stack recursion limits.** Each `_PyThreadStateImpl` carries -`c_stack_top` / `c_stack_soft_limit` / `c_stack_hard_limit`, computed **once** by -`_Py_InitializeRecursionLimits()` from the current machine stack pointer, and only if -`c_stack_hard_limit == 0`: - -```c -void -_PyThreadState_Attach(PyThreadState *tstate) -{ - ... - if (_tstate->c_stack_hard_limit == 0) { - _Py_InitializeRecursionLimits(tstate); - } - ... -} -``` - -Because our thread reuses the same `PyThreadState` object across every park/re-enter cycle, this -calibration happens exactly once — at whatever C call depth `Py_Initialize()`'s *first* internal -attach happened at — and is **never** recomputed on later re-entries from a structurally different -call depth (the JVM/Panama boundary is a different C stack shape than CPython's own -`Py_InitializeFromConfig` call chain). `_Py_CheckRecursiveCall` compares the live stack pointer -against this stale `c_stack_hard_limit` on every recursive C call in the eval loop -(`if (here_addr < c_stack_hard_limit) Py_FatalError(...)`). This is a real, citable staleness bug -in the reuse path — but by itself it explains a *miscalibrated guard*, and the failure it produces -when it fires is a `RecursionError` or a `Py_FatalError` with the text "Unrecoverable stack -overflow", neither of which matches today's raw `SIGSEGV` (no such message was reported), and a -single-frame `PyImport_ImportModule("sys")` call is not deep enough to plausibly exhaust several -hundred KB to MB of C stack on its own. **I could not make this candidate fit the observed crash -without more evidence, and flag it mainly because it's a real, separate latent hazard in the reuse -pattern that the fix should be aware of regardless of whether it explains today's crash.** - -**Candidate B — something the eval loop touches on the initial (never-parked) startup path that -`import sys`'s cache-hit-with-Python-level-bookkeeping path exercises differently.** -`PyImport_ImportModule("sys")` for an already-loaded module does not simply `PyDict_GetItemString` -and return — `import_ensure_initialized()` (in `Modules/import.c`, not fetched in this -investigation) can invoke Python-level machinery (attribute lookups on `__spec__`, method calls), -which is consistent with the reported crash frames (`PyObject_CallMethodObjArgs` → -`_PyFunction_Vectorcall` → `DICT_MERGE`) being genuine bytecode execution rather than a pure C -lookup. I could not determine, from source reading alone, why *this* bytecode execution — on a -thread whose `PyThreadState` is otherwise identical to the one `Py_Initialize()` used successfully -— would be unsafe. This is the open question the experiment in §6 is aimed at. - -I was not able to reduce this to one explanation. Ranked: reuse of the same `PyThreadState` across -the park/re-enter boundary is established fact and is the right place to keep looking; which piece -of state that reuse fails to restore is unresolved, with Candidate A eliminated by the mismatch in -failure shape and Candidate B unconfirmed. - -## 6. Minimal experiment (does not require the full suite) - -Add a single desktop test — not to run today, but to hand to the next attempt — that escalates in -four steps on **one thread**, printing/asserting between each so a failure localizes to a specific -step instead of reporting "0 tests run": - -```kotlin -@Test -fun parkThenReenterSameThreadIsolatesTheFailure() { - Python3.initialize(silent = true) // must NOT be parked yet at this point - val savedBeforeGuess = PyGILState_GetThisThreadState() // record identity #1, still attached - println("before park: $savedBeforeGuess") - - val parked = PyEval_SaveThread() // manual parking, mirroring the - // commented-out call in initialize() - println("PyEval_SaveThread returned: $parked") - - // Step 1: bare re-entry, no Python C API call at all inside the guard except one that - // reads state without touching bytecode. - withGIL { - val after = PyGILState_GetThisThreadState() - println("after Ensure: $after") - // EXPECT: after == savedBeforeGuess (same PyThreadState*, confirms/refutes §3 empirically - // rather than from source reading alone). A mismatch would mean the "new thread" branch - // fired -- i.e. the TSS slot was NOT still bound, contradicting the CPython source read - // here, and pointing at something specific to this embedding (e.g. the JVM's native - // thread setup interacting with TSS) rather than to CPython's generic behaviour. - } - // Step 1 crashing here would mean the attach/detach cycle itself is broken -- contradicts - // this document's source reading, and the desktop Panama bindings for PyEval_SaveThread / - // PyGILState_Ensure/Release (bindings.kt:41-50) would be the next thing to audit, since this - // project has a documented history of invoke/invokeExact and ADDRESS/JAVA_LONG mismatches. - - // Step 2: a pure-C call that cannot execute Python bytecode. - withGIL { - Py_IsInitialized() - } - // Step 2 crashing but not Step 1 would point at something broken in ordinary C API use after - // reattachment, unrelated to bytecode execution -- undermining Candidate B. - - // Step 3: the exact call that crashed today. - withGIL { - PyImport_ImportModule("sys") - } - // Step 3 crashing but not Steps 1-2 confirms it's specifically bytecode execution (not mere - // attachment, not mere C API use) that's unsafe after reattachment -- supports Candidate B - // and argues against Candidate A (see the manual-lookup step below). - - // Step 4 (only if Step 3 crashes): bypass import_ensure_initialized's Python-level path - // entirely, staying in pure C, to see whether a plain cache lookup alone is safe: - withGIL { - val modules = PySys_GetObject("modules") - PyDict_GetItemString(modules!!, "sys") - } - // If Step 4 is safe where Step 3 is not, that isolates the crash to whatever Python-level - // code import_ensure_initialized runs on a cache hit -- worth reading Modules/import.c's - // import_ensure_initialized next, specifically for anything that inspects C-stack depth, - // per-thread caches, or recursion state. -} -``` - -This is deliberately not `@Test`-annotated as ready to run blind — the next attempt should run it -alone (`--tests` filter), on a quiet tree, and stop at the first step that crashes rather than -proceeding to the next. That directly gives a call-depth-bounded bisection of the failure, which -the "audit every call site" and "run the whole suite" attempts so far have not produced. - -`PyGILState_GetThisThreadState` and `PySys_GetObject`/`PyDict_GetItemString` are already declared -in `EmbedAPI.kt` (`commonMain`); their desktop `actual`s were not verified as part of this -investigation (see §7). - -## 7. What the fix would look like, under each candidate - -- **If Candidate A (stale C-stack calibration) turns out to matter**, no amount of call-site - guarding fixes it — the fix has to either recalibrate `c_stack_hard_limit` on every reattachment - (not something this project's API surface exposes; would need a non-limited-API call or a - CPython-side change) or ensure the *first* attach happens at a call depth structurally - representative of where Python will actually run later. Concretely: call `Py_Initialize()` - itself from inside the same trampoline depth used by ordinary `withGIL` calls, rather than - directly from `Python3.initialize()`'s own stack frame, so the one-time calibration isn't - systematically shallower or deeper than steady-state use. This is a real design change, not a - one-line fix. - -- **If Candidate B (something Python-level import bookkeeping depends on) turns out to matter**, - the fix likely isn't about *this* call site specifically — `PyImport_ImportModule("sys")` is - representative of "any first bytecode execution after reattachment", not a special case — so - narrowing further requires the bisection in §6 before a fix can be scoped at all. - -- **If the experiment shows Step 1 or Step 2 already crashes** (i.e. even bare reattachment or a - pure-C call is unsafe), that overturns the "reuse is fine, something else breaks" framing of this - whole document, and the next place to look is the desktop Panama bindings for exactly these four - functions (`bindings.kt:41-50`) — this project has a documented, repeated history of exactly this - class of bug (`invoke` vs `invokeExact`, `ADDRESS` vs `JAVA_LONG`), and while a quick read during - this investigation found `PyEval_SaveThread`/`RestoreThread`/`PyGILState_Ensure`/`Release` all - using `invokeExact` and the project's `P = JAVA_LONG` pointer convention correctly, that read was - not exhaustive (the `find()` symbol-lookup helper and the underlying `MethodHandle` descriptors - it builds were not traced end to end). +Tests nearest the real cause (double release / ownership; named, not re-run here): `commonTest/.../ref/DoubleReleaseTest.kt`, +`OwnershipLeakTest.kt`, `GCLeakTest.kt`. -- **A structurally different fix, independent of which candidate is right:** if reusing the - initialising thread's `PyThreadState` across a park/re-enter boundary turns out to be - fundamentally what's unsafe in this embedding (as opposed to something narrower), the design - question becomes whether the initialising thread should ever call `PyGILState_Ensure` again at - all. An alternative shape: run `Py_Initialize()` on a dedicated thread that parks once and is - never reused for Python work afterward — every subsequent call, including from what is currently - "the same thread" in tests, would then go through the "first time on this OS thread" branch of - `PyGILState_Ensure`, which creates a genuinely fresh `PyThreadState` rather than resurrecting the - init-time one. That is a bigger change than uncommenting the parking line (it changes where - `Py_Initialize()`/`Py_Finalize()` are allowed to be called from, and `Python3.finalize()`'s - `PyEval_RestoreThread(mainThreadState)` call would need to run on that same dedicated thread - too), and it should not be attempted until the experiment in §6 has actually localized the fault - — building it on a guess would repeat the pattern this document was asked to stop. +## Open -## 8. What could not be determined +- Which CPython version introduced the attach/detach vocabulary and the C-stack-address recursion + guard was not verified (it reads as the PEP 703 rewrite). +- iOS: `GCLeakTest`'s earlier simulator failure was not GIL-related (cleaners ran and reached + `Py_DecRef`: 101001 of 101002). ROADMAP §1 reports the iOS suite green (157) once parking was + enabled; this document does not explain the earlier iOS symptom. -- **The exact opcode-level or memory-level reason `DICT_MERGE` faults.** Both candidates in §5 are - plausible mechanisms consistent with *some* of the evidence, but neither was confirmed against - the specific crash. This needs the experiment in §6, or a debugger session (`lldb` on the core, - or a build with `Py_DEBUG` to turn the compiled-out `assert()`s back on) — deliberately not - attempted here per the "prefer reading over running" instruction for this task. -- **Whether `import_ensure_initialized`'s Python-level path (Candidate B) is real** — I did not - fetch `Modules/import.c`, so the claim that a `sys.modules` cache hit still runs Python-level - code is inferred from general CPython import-machinery knowledge, not confirmed against 3.14 - source in this session. -- **Which CPython version introduced the current attach/detach vocabulary and the C-stack-address - recursion guard.** Both read as free-threading-era (PEP 703) work; I did not verify version - numbers, so treat "this is new" as directionally right but not dated. -- **Whether the desktop Panama bindings for the four functions in play are fully correct end to - end.** `bindings.kt:41-50` looked right on inspection (consistent `invokeExact`, consistent - `JAVA_LONG` pointer convention), but the `find()` helper and `MethodHandle` descriptor - construction underneath were not traced, and this project has a specific documented history of - bugs in exactly this area. -- **iOS.** This document is about the desktop crash only. ROADMAP §1 already separately notes that - the iOS simulator's `GCLeakTest` failure is not GIL-related and is a distinct, unexamined defect; - nothing here bears on it. +The full original, with the CPython source excerpts, the same-thread argument and the experiment +sketch, is +[`docs/archive/investigations/gil-parking-investigation.md`](../archive/investigations/gil-parking-investigation.md). diff --git a/docs/investigations/jni-call-convention-audit.md b/docs/investigations/jni-call-convention-audit.md index 84a5692dd..2c2e6bde6 100644 --- a/docs/investigations/jni-call-convention-audit.md +++ b/docs/investigations/jni-call-convention-audit.md @@ -1,218 +1,70 @@ -# JNI Calling Convention Audit +# JNI calling convention audit + +Android binds CPython through `RegisterNatives`. `@CriticalNative` and `@FastNative` both stop the +ART collector for the duration of the call, and `@CriticalNative` receives no `JNIEnv`. So neither +may be used for a function that can re-enter the runtime, which means any function that can run +Python (`__del__`, `__getattr__`, a module's top-level code) — with upcalls live, that Python can +be Kotlin. + +## Conclusion + +1. **No re-entrant function is live under a GC-blocking convention.** The audit's headline + ("six re-entrant functions registered `@CriticalNative`, demote them") was a false alarm: it + read a block-commented region of dead pre-migration code in `EmbedAPI.android.kt` as live, and + it read the registration table as the configuration. `jni_onload.def` registers all three + conventions per function (bare name = `@CriticalNative`, `F` = `@FastNative`, `N` = ordinary); + the convention that runs is chosen at the call site, and the six already call the `N` variant. +2. **The proposed promotion to `@CriticalNative` is rejected.** It is a loss on API 34+ (table + below), is not a relabel (needs a new raw-symbol table entry, declaration and call-site + branch), and three of its eight targets are not leaves. +3. **One change was made instead:** `PyList_GetItemRaw` got a `@FastNative` twin + (`PyList_GetItemRawF`) and a `preferFastNative` branch; it had been the only declaration that + ignored the device axis, on the per-element path of bulk list iteration. +4. **The classification is now derived, not written.** The hand-written table in the original + audit (71 rows) went stale on the next commit and its "Unresolved entries: none" claim was + wrong (header prototypes can condemn a function but never clear one: `void Py_Initialize(void)` + and `int PyRun_SimpleString(const char*)` mention no `PyObject *` and both run arbitrary + Python). A test re-derives it on every desktop build. + +**Status: acted on.** +- Classification and its guards: `python-multiplatform/src/desktopTest/.../native/ffi/JniCallConventionClassificationTest.kt` + (`noLiveCallSiteReachesAGcBlockingBindingOutsideThePromotedSet`, + `everyGcBlockingDeclarationIsEitherPromotedOrQuarantined`, `quarantinedDeclarationsHaveNoCallSiteAnywhere`, + `criticalNativeRegistrationsTakeNoJniEnvAndTheOtherTwoDo`, `theSignatureRuleDoesNotClaimToProveLeafness`). +- Decision procedure for a new call site: `androidMain/README.md`, "Choosing a convention for a new + call site" (five steps, enforced by the test above). +- `PyList_GetItemRawF`: `androidMain/.../bindings.kt`; branch in `EmbedAPI.android.kt` + (`preferFastNative`); device check `androidInstrumentedTest/.../JniWiringTest.kt` + `bothListGetItemConventionsAgree`. SPEC C-3 is the contract. +- The `PyList_GetItemRawF` change was compile-verified only when written; whether + `bothListGetItemConventionsAgree` has been run on a device since is not recorded here. -Android binds CPython through `RegisterNatives`. Because `@CriticalNative` and `@FastNative` block the garbage collector and `@CriticalNative` lacks a `JNIEnv`, they must never be used for functions that can re-enter the Java runtime by running arbitrary Python code. - -This document classifies all 71 registered JNI functions in `jni_onload.def`. - -> ## Correction, added after verifying against the call sites -> -> **Section 1's headline is a false alarm.** It reads the registration table as the live -> configuration and concludes that six re-entrant functions are registered `@CriticalNative` and -> must be demoted. They are not live. -> -> `jni_onload.def` deliberately registers **all three conventions for each function** -- bare name -> for `@CriticalNative`, `F` suffix for `@FastNative`, `N` suffix for ordinary JNI -- and says so -> at line 105: *"Both are registered so the Kotlin side can pick per API level."* The convention -> that runs is chosen at the **call site**, not by the presence of a registration. -> -> Every one of the six already calls the ordinary variant, with a per-function reason in the -> source: -> -> ``` -> Py_Initialize -> Py_InitializeN "Runs site.py ... never on a GC-blocking path" -> Py_Finalize -> Py_FinalizeN -> PyErr_Clear -> PyErr_ClearN -> PyRun_SimpleString -> PyRun_SimpleStringN -> PyImport_ImportModule -> PyImport_ImportModuleN -> PyObject_GetAttrString -> PyObject_GetAttrStringN -> ``` -> -> The block in `EmbedAPI.android.kt` that appears to call the bare `@CriticalNative` names is -> **inside a `/** */` comment** -- dead code kept from before the migration. Reading it as live is -> what produced the finding. -> -> **Section 2 is a different matter and may be real.** Its promotion targets (`Py_IncRefN`, -> `PyEval_SaveThreadN`, and the rest) are registered *only* under `N`; there is no -> `@CriticalNative` registration to switch to. Acting on it means adding registrations as well as -> moving call sites, which is genuine work rather than a relabel. -> -> Two things to check before acting on it. The cost claim -- "~45ns on API 26-31, only ~2-4ns on -> API 34+" -- points the opposite way from what this repo measured: `@CriticalNative`'s net cost -> is near zero up to API 33 and rises sharply after (24.35ns on 34, 44.05ns on 36 hardware). If -> those measurements hold, promotion helps old devices and *hurts* new ones, which inverts the -> recommendation. And "Unresolved entries: none" overstates the confidence available from reading -> names and documentation alone; the classification below is a starting point for a human, not a -> settled answer. -> -> **What survives is the classification itself**, which is the useful part: it is the rule for -> choosing a convention when a *new* call site is written, and that is where the crash ROADMAP §3 -> warns about would actually be introduced. -> -> Section 2 was checked against those measurements and **rejected**; see *Resolution* at the end of -> this document for the judgement, the three classifications in it that are wrong, and the one -> change that was made instead. - - -## 1. Mismatches - -These functions are currently registered as `@CriticalNative` (no suffix) or `@FastNative` (F suffix), but they can execute arbitrary Python code. These are latent crashes and must be demoted to ordinary JNI. - -- **`Py_Initialize`** / **`Py_InitializeF`** - - **Re-entry path**: Imports the `site` module during startup by default, which executes module-level Python code. -- **`Py_Finalize`** / **`Py_FinalizeF`** - - **Re-entry path**: Invokes `atexit` callbacks and executes `__del__` methods of surviving objects during teardown. -- **`PyErr_Clear`** / **`PyErr_ClearF`** - - **Re-entry path**: Decrements the refcount of the current exception object, type, and traceback. If the refcount drops to zero, `__del__` is called. -- **`PyRun_SimpleString`** / **`PyRun_SimpleStringF`** - - **Re-entry path**: Parses, compiles, and executes the provided string in the `__main__` module. -- **`PyImport_ImportModule`** / **`PyImport_ImportModuleF`** - - **Re-entry path**: Executes the top-level Python code of the module being imported. -- **`PyObject_GetAttrString`** / **`PyObject_GetAttrStringF`** - - **Re-entry path**: Attribute lookup dispatches to `__getattribute__`, `__getattr__`, or descriptor `__get__` methods written in user code. - -## 2. Safe Promotions - -These functions are registered as ordinary JNI (with the `N` suffix or implicitly ordinary) but are provably leaves or non-blocking, and therefore safe to promote. Given that `@CriticalNative` is practically free on API 26-33 but costs ~24ns net on API 34+ (while `@FastNative` is ~2ns on 34+), the value of promoting these depends on the target device. - -- **`Py_IncRefN`, `Py_NewRefN`, `Py_XNewRefN`** - - **Argument for promotion**: These simply increment the `ob_refcnt` struct field. They do not trigger deallocations, do not allocate memory (no GC), and do not call Python code. Safe for `@CriticalNative`. -- **`PyGILState_ReleaseN`, `PyEval_SaveThreadN`** - - **Argument for promotion**: Unlocking the GIL mutex does not block and does not invoke Python code. Safe for `@CriticalNative`. -- **`PyGILState_GetThisThreadStateN`, `PyThreadState_GetDictN`** - - **Argument for promotion**: These only read from Thread Local Storage or read a pointer from a struct. No Python execution. Safe for `@CriticalNative`. -- **`PyEval_InitThreadsN`** - - **Argument for promotion**: Initializes locking (or is a no-op). Does not run Python code. Safe for `@CriticalNative`. -- **`asmListToArray`** - - **Argument for promotion**: This composed function only calls `PyList_Size` and `PyList_GetItem`, both of which are O(1) leaf array reads that do not mutate refcounts. Because it accesses a Java `jlongArray`, it still requires `JNIEnv`, so it cannot be `@CriticalNative`. It is safe to promote to `@FastNative`. - -## 3. Full Classification Table - -| Function Name | Current Convention | Should Be | Can Reach Python? (Mechanism) | -|---|---|---|---| -| `Py_Initialize` | Critical | Ordinary | Yes. Imports `site` module during startup. | -| `Py_IsInitialized` | Critical | Critical | No. Reads a global boolean. | -| `Py_Finalize` | Critical | Ordinary | Yes. Runs `atexit` callbacks and `__del__` on remaining objects. | -| `PyErr_Clear` | Critical | Ordinary | Yes. Decrements exception refcount; dropping to 0 runs `__del__`. | -| `PyLong_FromLongLong` | Critical | Critical | No. Primitive `malloc`, does not trigger cyclic GC. | -| `PyList_Size` | Critical | Critical | No. Reads `ob_size` struct field. | -| `PyRun_SimpleString` | Critical | Ordinary | Yes. Executes arbitrary Python string. | -| `Py_GetVersion` | Critical | Critical | No. Returns static string pointer. | -| `PyImport_ImportModule` | Critical | Ordinary | Yes. Executes top-level module code. | -| `PyObject_GetAttrString` | Critical | Ordinary | Yes. Invokes `__getattr__` or `__get__`. | -| `PyErr_Occurred` | Critical | Critical | No. Reads thread state. | -| `echo0` | Critical | Critical | No. Probe, returns argument. | -| `echo1` | Critical | Critical | No. Probe, returns argument. | -| `echo2` | Critical | Critical | No. Probe, returns argument. | -| `echoNormal` | Ordinary | Ordinary | No. Probe, requires JNIEnv. | -| `echoFast` | FastNative | FastNative | No. Probe, requires JNIEnv. | -| `PyList_SizeFast` | FastNative | FastNative | No. Leaf read. | -| `PyList_SizeNormal` | Ordinary | Ordinary | No. Leaf read. | -| `Py_InitializeF` | FastNative | Ordinary | Yes. Imports `site` module during startup. | -| `Py_IsInitializedF` | FastNative | FastNative | No. Reads a global boolean. | -| `Py_FinalizeF` | FastNative | Ordinary | Yes. Runs `atexit` callbacks and `__del__`. | -| `PyErr_ClearF` | FastNative | Ordinary | Yes. Decrements exception refcount; dropping to 0 runs `__del__`. | -| `PyLong_FromLongLongF` | FastNative | FastNative | No. Primitive `malloc`, does not trigger cyclic GC. | -| `PyList_SizeF` | FastNative | FastNative | No. Reads `ob_size` struct field. | -| `PyRun_SimpleStringF` | FastNative | Ordinary | Yes. Executes arbitrary Python string. | -| `Py_GetVersionF` | FastNative | FastNative | No. Returns static string pointer. | -| `PyImport_ImportModuleF` | FastNative | Ordinary | Yes. Executes top-level module code. | -| `PyObject_GetAttrStringF` | FastNative | Ordinary | Yes. Invokes `__getattr__` or `__get__`. | -| `PyErr_OccurredF` | FastNative | FastNative | No. Reads thread state. | -| `PyList_GetItemRaw` | Critical | Critical | No. Borrows reference from array index (leaf). | -| `ffiDirectBufferAddress` | Ordinary | Ordinary | No. Uses JNIEnv. | -| `asmExec` | Ordinary | Ordinary | Yes. Composes `PyRun_String`. | -| `asmGetAttr` | Ordinary | Ordinary | Yes. Composes `PyObject_GetAttrString` and `PyErr_Clear`. | -| `asmListToArray` | Ordinary | FastNative | No. Only calls leaf functions (`PyList_Size`, `PyList_GetItem`). | -| `Py_InitializeN` | Ordinary | Ordinary | Yes. Imports `site` module during startup. | -| `Py_FinalizeN` | Ordinary | Ordinary | Yes. Runs `atexit` callbacks and `__del__`. | -| `PyErr_ClearN` | Ordinary | Ordinary | Yes. Decrements exception refcount; dropping to 0 runs `__del__`. | -| `PyRun_SimpleStringN` | Ordinary | Ordinary | Yes. Executes arbitrary Python string. | -| `PyImport_ImportModuleN` | Ordinary | Ordinary | Yes. Executes top-level module code. | -| `PyObject_GetAttrStringN` | Ordinary | Ordinary | Yes. Invokes `__getattr__` or `__get__`. | -| `Py_DecRefN` | Ordinary | Ordinary | Yes. Reaching refcount 0 deallocates and runs `__del__`. | -| `Py_IncRefN` | Ordinary | Critical | No. Only increments struct field `ob_refcnt`. | -| `Py_NewRefN` | Ordinary | Critical | No. Only increments struct field `ob_refcnt`. | -| `Py_XNewRefN` | Ordinary | Critical | No. Only increments struct field `ob_refcnt`. | -| `PyGILState_EnsureN` | Ordinary | Ordinary | No, but blocks. Blocking while suspending JVM GC risks deadlock. | -| `PyGILState_ReleaseN` | Ordinary | Critical | No. Unlocks mutex (non-blocking). | -| `PyGILState_GetThisThreadStateN` | Ordinary | Critical | No. Reads TLS. | -| `PyEval_SaveThreadN` | Ordinary | Critical | No. Unlocks GIL mutex (non-blocking). | -| `PyEval_RestoreThreadN` | Ordinary | Ordinary | No, but blocks waiting for GIL. Risks deadlock if GC is suspended. | -| `PyEval_InitThreadsN` | Ordinary | Critical | No. Initializes locking. | -| `PyThreadState_GetDictN` | Ordinary | Critical | No. Reads struct pointer. | -| `PyTuple_NewN` | Ordinary | Ordinary | Yes (inferred). GC tracked container allocation can trigger cyclic GC, which can run `__del__`. | -| `PyTuple_SetItemN` | Ordinary | Ordinary | Yes. Overwriting an existing item drops its refcount, potentially running `__del__`. | -| `PyDict_NewN` | Ordinary | Ordinary | Yes (inferred). GC tracked container allocation can trigger cyclic GC. | -| `PyDict_SetItemStringN` | Ordinary | Ordinary | Yes. Key hashing can invoke `__hash__` / `__eq__`, replacing values decrefs. | -| `PyDict_SetItemN` | Ordinary | Ordinary | Yes. Key hashing can invoke `__hash__` / `__eq__`, replacing values decrefs. | -| `PyDict_GetItemStringN` | Ordinary | Ordinary | Yes. Dictionary lookup can invoke `__eq__` on keys. | -| `PyUnicode_AsUTF8` | Critical | Critical | No. Primitive string allocation/cache. | -| `PyUnicode_AsUTF8F` | FastNative | FastNative | No. Primitive string allocation/cache. | -| `PyUnicode_FromString` | Critical | Critical | No. Primitive string allocation/cache. | -| `PyUnicode_FromStringF` | FastNative | FastNative | No. Primitive string allocation/cache. | -| `PyImport_AddModuleRefN` | Ordinary | Ordinary | Yes. Dict lookup/insertion can invoke `__eq__` or GC. | -| `PyRun_StringN` | Ordinary | Ordinary | Yes. Executes arbitrary Python code. | -| `PyObject_StrN` | Ordinary | Ordinary | Yes. Invokes `__str__`. | -| `PyObject_IsTrueN` | Ordinary | Ordinary | Yes. Invokes `__bool__` or `__len__`. | -| `PyObject_CallNoArgsN` | Ordinary | Ordinary | Yes. Invokes `__call__`. | -| `PyObject_CallObjectN` | Ordinary | Ordinary | Yes. Invokes `__call__`. | -| `testUpcallPrimitive` | Ordinary | Ordinary | Yes (JNI upcall). Explicitly calls into Java runtime. | -| `testUpcallString` | Ordinary | Ordinary | Yes (JNI upcall). Explicitly calls into Java runtime. | -| `testUpcallUnattached` | Ordinary | Ordinary | Yes (JNI upcall in new thread). | -| `testThreadCreateFloor` | Ordinary | Ordinary | No (thread floor probe), but uses ordinary. | -| `testAttachWithoutDetach` | Ordinary | Ordinary | Yes (attaches a pthread and calls back into Java). | - -## 4. Rules of Thumb - -Based on CPython's documented semantics, the following rules apply when classifying new functions: - -1. **Any decref is re-entrant**: If a function can drop a reference count (e.g. `Py_DecRef`, `PyTuple_SetItem` dropping the overwritten item, `PyDict_SetItem`, `PyErr_Clear`), that reference can hit zero, deallocate the object, and execute arbitrary Python `__del__` methods. -2. **GC allocations are re-entrant**: Allocating GC-tracked containers (tuples, lists, dicts, instances) using functions like `PyTuple_New` or `PyDict_New` can cross the allocation threshold and trigger the cyclic garbage collector. The cyclic GC reclaims unreachable cycles, executing `__del__` on them. -3. **Blocking requires ordinary JNI**: Functions that acquire the GIL (`PyGILState_Ensure`, `PyEval_RestoreThread`) do not run Python, but they *must* stay on ordinary JNI. A thread waiting for a lock while marked `@CriticalNative` prevents the JVM from reaching a safepoint, which deadlocks the entire process if the thread holding the GIL triggers a JVM garbage collection. -4. **Dictionary lookups are re-entrant**: Even reading from a dictionary (`PyDict_GetItemString`) can invoke user code if custom keys exist that define `__eq__` or `__hash__`. -5. **Non-GC allocations are leaves**: Allocating primitives (e.g. `PyLong_FromLongLong`, `PyUnicode_FromString`) uses standard memory allocation (`malloc`) and does not participate in cyclic GC, making them pure leaves. -6. **Increments are leaves**: Functions that strictly increase reference counts (`Py_IncRef`, `Py_NewRef`) do not deallocate and thus do not trigger callbacks. - -## Report Summary - -- **Total Functions Classified**: 71 registrations. -- **Mismatches Found**: 12 (6 unique C functions exposed as both Critical and Fast). These are the dangerous ones: `Py_Initialize`, `Py_Finalize`, `PyErr_Clear`, `PyRun_SimpleString`, `PyImport_ImportModule`, and `PyObject_GetAttrString`. They can run arbitrary Python and must be moved to ordinary JNI. -- **Safe Promotions**: 8 functions (`Py_IncRefN`, `Py_NewRefN`, `Py_XNewRefN`, `PyGILState_ReleaseN`, `PyEval_SaveThreadN`, `PyGILState_GetThisThreadStateN`, `PyThreadState_GetDictN`, `PyEval_InitThreadsN`) can safely be moved from Ordinary to Critical, and `asmListToArray` to FastNative. -- **Cost of Being Wrong**: - - Promoting a re-entrant function to `@CriticalNative` is a fatal crash when upcalls exist. - - Leaving a leaf as Ordinary wastes ~45ns per call on API 26-31, but only ~2-4ns on modern API 34+. Safe promotions are therefore an optimization mostly relevant for older devices or high-volume loops, and should never override correctness. -- **Unresolved Entries**: None. All 71 registrations were resolved confidently based on documented CPython semantics (e.g. GC thresholds on container allocation). - - **This line is wrong, and it is the reason the audit is no longer the source of truth.** Header - prototypes can condemn a function and never clear one: `void Py_Initialize(void)` and - `int PyRun_SimpleString(const char*)` name no `PyObject *` and both run arbitrary Python. A - re-derivation over the table as it stands leaves **65 undecided**, and the confidence recorded - here is what put three re-entrant functions on the safe-to-promote list above. - `JniCallConventionClassificationTest` now derives the classification on every desktop build and - asserts that bucket stays non-empty, with those two functions named in it. - ---- - -# Resolution +## The eight live GC-blocking call sites -Written after checking section 2 against this repository's own measurements and against the -registration table as it stands at 187 entries (the audit read 71). +All eight are leaves on the success path; all can allocate a GC-tracked exception on the failure +path. That second column is conditional and second-order (the exception instance is GC-tracked, so +it can cross the collection threshold, so the collector can run a `__del__`); it is not a reason +to demote them, they are the measured hot path. It is why "provably a leaf" is not the promotion +test — the honest test is *leaf on the success path, and measured worth it*. -**The table holds 366 today**, so this resolution is itself a snapshot — roughly 180 registrations -have never been classified by any hand-written pass. That is why the classification moved into -`JniCallConventionClassificationTest`, which re-derives it from `bindings.kt`, `jni_onload.def` and -the bundled CPython headers on every desktop build. Its re-derivation agrees with the judgement -recorded here — the eight GC-blocking call sites reproduce exactly, no function was wrongly -promoted, and all 366 prologues match their convention — so what was missing was never correctness, -only a way for it to stay correct. +| call site | success path | failure path | +|---|---|---| +| `Py_IsInitialized` | reads a global | — | +| `Py_GetVersion` | static `const char*` | — | +| `PyErr_Occurred` | reads thread state, borrowed | — | +| `PyLong_FromLongLong` | non-GC allocation | preallocated `MemoryError` | +| `PyUnicode_FromString` | non-GC allocation | `UnicodeDecodeError` | +| `PyUnicode_AsUTF8` | caches UTF-8 form | `UnicodeEncodeError` on lone surrogates | +| `PyList_Size` | reads `ob_size` | `PyErr_BadInternalCall` | +| `PyList_GetItem` | indexes item array | `IndexError` | -## The promotion to `@CriticalNative` is rejected +## Numbers -Section 2 argues its eight promotions are worth taking because `@CriticalNative` costs "~45ns on -API 26-31, only ~2-4ns on API 34+". That is backwards. Measured here, net of the Kotlin floor -(`docs/design/downcall-design.md`, `androidMain/README.md`), the *change* a promotion buys is: +Change in per-call cost, net of the Kotlin floor, from moving a function off ordinary JNI +(source: `docs/design/downcall-design.md`, `androidMain/README.md`; ART emulators / device per those +documents — conditions are in them, not re-measured here): -| API | ordinary → `@FastNative` | ordinary → `@CriticalNative` | +| API | ordinary → `@FastNative` (ns) | ordinary → `@CriticalNative` (ns) | |---|---|---| | 26 | −6.64 | −43.47 | | 30 | −41.33 | −73.33 | @@ -221,118 +73,63 @@ API 26-31, only ~2-4ns on API 34+". That is backwards. Measured here, net of the | 34 | −5.84 | **+16.54** | | 36 | −14.77 | **+25.73** | -`@CriticalNative` is a **loss** on the two newest levels measured. Taking section 2 as written -would slow every one of those eight functions down on any device from Android 14 onward. - -It could be had behind the existing `preferFastNative` branch — critical below 34, ordinary above. -That is not being done, for three reasons. +`@CriticalNative` is a loss on API 34 and 36. `PyList_GetItemRaw` motivation: ROADMAP §5 measured +`list → LongArray`, 1000 elements, at 50065.89 ns on API 36 hardware (50 ns/element) against a +`@CriticalNative` net cost of 44.05 ns there (an upper bound on the convention's share, since the +50 ns also covers the loop and pointer conversion). Predicted after the twin: 44.05 → 3.55 ns per +element on API 36 — a prediction, not a recorded measurement. -**It is not a relabel.** `@CriticalNative` receives no `JNIEnv` or `jclass`, so it cannot reuse the -`f_*N` wrappers, which all have them. Each promotion needs a new table entry pointing at the raw -CPython symbol, a new `@CriticalNative` declaration, and a branch at the call site. Compare with a -`@FastNative` promotion, which is one annotation: the C signature is already right and the -registration name does not change. +## The three rejected promotions that were not leaves -**Three of the eight cannot be promoted at all.** They are not leaves: - -| | audit says | actually | +| | audit said | actually | |---|---|---| -| `PyGILState_Release` | "unlocks mutex (non-blocking)" | the release that drops the counter to zero deletes the thread state; clearing a thread state decrefs its dict and exception state, so `__del__` runs | -| `PyThreadState_GetDict` | "reads struct pointer" | allocates the thread dict with `PyDict_New` on first call per thread — a GC-tracked allocation, so rule 2 applies | -| `PyEval_InitThreads` | "initializes locking" | a no-op retained for the stable ABI (removed from the C API in 3.13; still exported by the bundled `libpython`, confirmed with `nm -D`). Promoting a no-op buys the convention delta and nothing else | - -That leaves `Py_IncRef`, `Py_NewRef`, `Py_XNewRef`, `PyGILState_GetThisThreadState` and -`PyEval_SaveThread` as genuine leaves — and of those, only the three refcount operations are on a -path called often enough for tens of nanoseconds to accumulate. - -**The audit's own confidence claim does not hold.** "Unresolved entries: none" is not available -from reading names and documentation, which is exactly how the three above were misread. - -## What was changed instead: `PyList_GetItem` - -The static sweep found one place where the current configuration contradicts the measurements, and -it is a demotion rather than a promotion. - -`PyList_GetItemRaw` was registered `@CriticalNative` with **no `@FastNative` twin and no -`preferFastNative` branch** — the only declaration in `bindings.kt` that ignores the device axis. -It is also the per-element call of bulk list iteration, so the penalty multiplies by N. ROADMAP §5 -measured `list → LongArray`, 1000 elements, at 50065.89 ns on API 36 hardware: 50 ns per element, -against a `@CriticalNative` net cost of 44.05 ns on that device. That 50 ns also covers the loop and -the `toNativePointer` conversion, so 44.05 is an upper bound on the convention's share rather than -an attribution — but it is an upper bound of 88%, which is enough to act on. - -So a `@FastNative` twin was added (`f_PyList_GetItemRaw` in `jni_onload.def`, -`PyList_GetItemRawF` in `bindings.kt`) and the call site now branches on `preferFastNative`, the -same shape as `PyList_Size` directly above it. Predicted effect on API 36: 44.05 → 3.55 ns per -element. Nothing else was promoted or demoted. - -This is compile-verified only. `JniWiringTest.bothListGetItemConventionsAgree` asserts the two -registrations return identical pointers at every index of `sys.path`, which catches both a missing -table entry and a wrapper written without the leading `JNIEnv*, jclass`; it has not been run, -because the device was in use. - -It also narrows §5. If a large share of the 11x bulk-iteration win came from the per-element call -running under the slowest available convention on that device, then part of the remaining case for -composition was paying for a configuration mistake rather than for crossings. - -## The registrations added since the audit: 116, all ordinary, no defects - -The audit read 71 entries. The table has 187 — 74 added at `9abc8edd`, 42 at `aea09585`, none -removed. - -**Every one of the 116 is registered `N`-suffixed and declared plain `@JvmStatic external fun`.** -Not one has a `@CriticalNative` or `@FastNative` twin registered, so no call site can be reaching a -GC-blocking variant of them: there is none to reach. The failure mode ROADMAP §3 warns about cannot -be present in the new surface. Verified by extracting the table names from all three revisions and -parsing the annotations in `bindings.kt`. - -Classifying the 42 anyway, since the point of the exercise is the rule for the *next* one. All 42 -are string-carrying; 31 are re-entrant and correctly ordinary: - -| mechanism | functions | -|---|---| -| executes module or arbitrary code | `PyImport_ExecCodeModuleN`, `PyImport_ExecCodeModuleExN`, `PyImport_ExecCodeModuleWithPathnamesN`, `PyImport_ImportModuleLevelN`, `PyImport_ImportModuleNoBlockN` †, `PyImport_ImportFrozenModuleN`, `Py_CompileStringN` | -| runs a Python-level hook | `PyObject_HasAttrStringN`, `PyObject_HasAttrStringWithErrorN` (`__getattr__`), `PyMapping_GetItemStringN`, `PyMapping_SetItemStringN`, `PyMapping_HasKeyStringN`, `PyMapping_HasKeyStringWithErrorN` (`__getitem__`/`__contains__`), `PySequence_FastN` (`__iter__`), `PyUnicode_TranslateN` (`__getitem__` on the table), `PySys_AuditTupleN` (audit hooks), `PyErr_WarnExplicitN` (the `warnings` machinery) | -| Python codec lookup / error handler | `PyUnicode_AsEncodedStringN`, `PyUnicode_FromEncodedObjectN`, `PyUnicode_DecodeLocaleN`, `PyUnicode_EncodeLocaleN`, `PyUnicode_DecodeFSDefaultN` | -| drops a reference, or allocates GC-tracked | `PyDict_DelItemStringN`, `PySys_SetObjectN`, `PyErr_NewExceptionN`, `PyErr_NewExceptionWithDocN`, `PyErr_SetFromErrnoWithFilenameN`, `PyErr_SyntaxLocationN`, `PyErr_SyntaxLocationExN`, `PyUnicodeTranslateError_SetReasonN` | -| never returns | `Py_FatalErrorN` | - -† `PyImport_ImportModuleNoBlockN` has since been unregistered: CPython 3.15 removed -`PyImport_ImportModuleNoBlock`, which had been an exact alias of `PyImport_ImportModule` since 3.3 -(ROADMAP §9). The counts above are left as they were read, since this section records what the -table held at `9abc8edd`/`aea09585` rather than what it holds now. The same migration also -unregistered `PySys_ResetWarnOptionsN` and renamed `PyWeakref_GetObjectN` to `PyWeakref_GetRefN`; -none of the three changed anyone's convention, and the replacement is registered ordinary like the -function it replaces. - -The remaining 11 are leaves on the success path: `PyBytes_FromStringN`, `PyBytes_AsStringN`, -`PyByteArray_AsStringN`, `PyUnicode_EqualToUTF8N`, `PyUnicode_CompareWithASCIIStringN`, -`PyUnicode_InternFromStringN`, `PySys_GetObjectN`, `PyImport_GetMagicTagN`, `PyEval_GetFuncNameN`, -`PyEval_GetFuncDescN`, `Py_EnterRecursiveCallN`. None of them is promoted, and none should be: -every one is cold error-reporting or introspection surface, where the whole available win is under -40 ns on a call that happens once. Step 4 of the decision procedure in `androidMain/README.md` -covers exactly this case. - -## The eight live GC-blocking call sites - -Outside the `/** */` block of dead pre-migration code, `EmbedAPI.android.kt` reaches a -`@CriticalNative` or `@FastNative` binding in eight places. All eight are leaves on the success -path and all eight can allocate a GC-tracked exception on the failure path: - -| call site | success path | failure path | -|---|---|---| -| `Py_IsInitialized` | reads a global | — | -| `Py_GetVersion` | returns a static `const char*` | — | -| `PyErr_Occurred` | reads the thread state, borrowed | — | -| `PyLong_FromLongLong` | non-GC allocation | preallocated `MemoryError` | -| `PyUnicode_FromString` | non-GC allocation | `UnicodeDecodeError` on malformed UTF-8 | -| `PyUnicode_AsUTF8` | caches the UTF-8 form | `UnicodeEncodeError` on lone surrogates | -| `PyList_Size` | reads `ob_size` | `PyErr_BadInternalCall` on a non-list | -| `PyList_GetItem` | indexes the item array | `IndexError` out of range | - -That second column is second-order and conditional — the exception instance is GC-tracked, so it -can cross the collection threshold, so the cyclic collector can run a `__del__`, which with upcalls -live can be Kotlin. It is not a live defect and it is not a reason to demote these eight, which are -the measured hot path. It is the reason step 4 of the decision procedure does not pretend "provably -a leaf" is reachable: the honest promotion test is *leaf on the success path, and measured worth -it*, not *cannot run Python*. +| `PyGILState_Release` | "unlocks mutex (non-blocking)" | the release that drops the counter to zero deletes the thread state; clearing it decrefs its dict and exception state, so `__del__` runs | +| `PyThreadState_GetDict` | "reads struct pointer" | allocates the thread dict with `PyDict_New` on first call per thread — a GC-tracked allocation | +| `PyEval_InitThreads` | "initializes locking" | a no-op kept for the stable ABI; promoting it buys only the convention delta | + +The genuine leaves among the eight are `Py_IncRef`, `Py_NewRef`, `Py_XNewRef`, +`PyGILState_GetThisThreadState`, `PyEval_SaveThread`; only the three refcount operations are hot +enough for tens of nanoseconds to accumulate. + +## Rules of thumb for a new function + +1. Any decref is re-entrant (the count can hit zero → `__del__`). +2. Allocating a GC-tracked container (`PyTuple_New`, `PyDict_New`, instances) can cross the + collection threshold and run `__del__` on unreachable cycles. +3. Anything that blocks (`PyGILState_Ensure`, `PyEval_RestoreThread`) stays ordinary: a thread + waiting for a lock while marked `@CriticalNative` keeps the JVM from a safepoint, and deadlocks + if the GIL holder triggers a JVM GC. +4. Dict lookups are re-entrant (`__eq__`/`__hash__` on keys). +5. Non-GC allocation (`PyLong_FromLongLong`, `PyUnicode_FromString`) is a leaf on success. +6. Pure increments (`Py_IncRef`, `Py_NewRef`) are leaves. + +## Registrations added after the audit + +The audit read 71 registrations; the table later held 187 (116 added in `9abc8edd` and +`aea09585`) and 366 when the resolution was written. All 116 are registered `N`-suffixed with a +plain `@JvmStatic external fun` and no `@CriticalNative`/`@FastNative` twin, so no call site can +reach a GC-blocking variant of them. Of the 42 string-carrying ones, 31 are re-entrant (run module +or arbitrary code, a Python-level hook, a codec lookup, drop a reference, or never return) and 11 +are leaves on success (`PyBytes_FromStringN`, `PyBytes_AsStringN`, `PyByteArray_AsStringN`, +`PyUnicode_EqualToUTF8N`, `PyUnicode_CompareWithASCIIStringN`, `PyUnicode_InternFromStringN`, +`PySys_GetObjectN`, `PyImport_GetMagicTagN`, `PyEval_GetFuncNameN`, `PyEval_GetFuncDescN`, +`Py_EnterRecursiveCallN`); none should be promoted (cold error/introspection surface, whole win +under 40 ns on a call that happens once). Later unregistrations (CPython 3.15 removals, per ROADMAP +§9): `PyImport_ImportModuleNoBlockN`, `PySys_ResetWarnOptionsN`; `PyWeakref_GetObjectN` became +`PyWeakref_GetRefN`. + +## How to reproduce + +Classification (desktop, source-level, no device): + +```bash +./gradlew :python-multiplatform:desktopTest --tests 'python.native.ffi.JniCallConventionClassificationTest' \ + --console=plain > .tmp/jni-classification.log 2>&1; echo "EXIT=$?" +``` + +Per-convention costs: `androidInstrumentedTest/.../native/ffi/JniOverheadBenchmark.kt` on +`pmp_api26` and `pmp_api36` (harness: `./benchmarks/cost-table.sh`, see +[`cost-table.md`](cost-table.md)). + +The full original, with the 71-row table, the false-alarm reasoning and the 42-function breakdown, +is [`docs/archive/investigations/jni-call-convention-audit.md`](../archive/investigations/jni-call-convention-audit.md). diff --git a/docs/locale/README_ko.md b/docs/locale/README_ko.md index 978f25259..74f270bc9 100644 --- a/docs/locale/README_ko.md +++ b/docs/locale/README_ko.md @@ -184,7 +184,7 @@ Python → Kotlin 바인딩 코드가 없다. 자세한 내용: [설계 문서]( ## 📊 상태 -정직한 요약이다 — 전체 기록은 [로드맵](../roadmap/ROADMAP.md) 에 있다. +정직한 요약이다 — 남은 작업은 [로드맵](../roadmap/ROADMAP.md) 에 있다. | 영역 | 상태 | |---|---| diff --git a/docs/platforms/graal-native-image-verification.md b/docs/platforms/graal-native-image-verification.md index c0f5fc587..12ccbb28c 100644 --- a/docs/platforms/graal-native-image-verification.md +++ b/docs/platforms/graal-native-image-verification.md @@ -6,6 +6,12 @@ under GraalVM's closed-world assumption, and that choice is only confirmed by bu This file is both the **procedure** (how to reproduce) and the **record** of the last run. +Status: a manual verification (SPEC U-6). The image itself is not run by any automated test; what +guards it automatically is `ReachabilityMetadataTest.kt` (SPEC C-2), which checks that every +`FunctionDescriptor` the desktop backend links is declared in the generated reachability metadata. +The record below is dated 2026-08-17 and has not been re-run since this document was last reviewed +(the "Worktree" path in it predates the rule that worktrees live under `.worktrees/`). + --- ## Procedure diff --git a/docs/platforms/python-version-acquisition.md b/docs/platforms/python-version-acquisition.md index cbfe5a79d..ddb77bfbb 100644 --- a/docs/platforms/python-version-acquisition.md +++ b/docs/platforms/python-version-acquisition.md @@ -25,12 +25,20 @@ The project currently uses `python-build-standalone` (maintained by Astral). For * **Prebuilt-cpython Status:** The `python/prebuilt-cpython` repository is currently a planning repository containing no artifacts, so it is not a source. * **Conclusion:** We acquire Android artifacts directly from python.org, and iOS artifacts from BeeWare's Python-Apple-support, eliminating the need for kivy toolchain dependency. -## 4. Stable ABI Implications (PEP 803) - -* **The Constraint:** This project relies entirely on the CPython Stable ABI for its ~330 bindings. -* **Python 3.13 & 3.14:** The traditional Stable ABI (`abi3`) **does not** support free-threaded (GIL-disabled) builds. If we attempt to use a free-threaded 3.14 build, our bindings relying on `abi3` will fail or crash due to opaque structural changes in `PyObject`. -* **Python 3.15 (PEP 803):** Python 3.15 introduces `abi3t`, a new variant of the Stable ABI specifically for free-threaded builds. -* **Conclusion (superseded 2026-08-12):** this was stale. `Py_LIMITED_API` is not defined here, so `abi3t` is not a blocker; 3.14t works on desktop with `-PpythonFreeThreaded=true` (236 tests, 0 failures). See `docs/roadmap/ROADMAP.md` §9. +## 4. Stable ABI and free-threaded builds + +* **The constraint:** the ~330 bindings target the CPython Stable ABI, and CPython's Limited API / + `abi3` is not offered by free-threaded builds in 3.13 and 3.14 (3.15 adds `abi3t`, PEP 803). +* **Why it does not block free-threading here:** the bindings are symbols resolved dynamically at run + time and `Py_LIMITED_API` is not defined, so no compile-time ABI contract applies. 3.14t works on + desktop with `-PpythonFreeThreaded=true` (the whole desktop suite: 236 tests, 0 failures, at the time + of ROADMAP §9; SPEC T-2). An earlier version of this section concluded the opposite ("fails or crashes + due to opaque structural changes", "wait for 3.15t") and was wrong; see + [`../design/threading-and-abi.md`](../design/threading-and-abi.md) and + [`../archive/threading-and-abi-3.15t-gate.md`](../archive/threading-and-abi-3.15t-gate.md). +* **Scope:** free-threaded prebuilts exist only for the desktop targets (section 2); Android (python.org) + and iOS (Python-Apple-support) publish none, so the flag is desktop-only and defaults to `false` + (`gradle.properties`). The interpreter version is `pythonVersion` in `gradle.properties`. ## 5. Security and Integrity Verification @@ -42,3 +50,12 @@ The project currently uses `python-build-standalone` (maintained by Astral). For * **iOS (BeeWare, ≤ 3.14) — nothing exists to verify against.** The `Python-Apple-support` releases publish five `tar.gz` assets and nothing else: no checksums, no signatures, and no GitHub attestations. The lockfile pin is the only honest instrument available here, and no amount of build wiring changes that. * **The lockfile is not superseded by any of this.** The two gates prove different things — the lockfile says "these are the exact bytes this repository reviewed and pinned", Sigstore says "these are the bytes the release manager actually signed". The lockfile is also the only check that works offline and the only one covering every source, so it stays unconditional and Sigstore is layered on top of it. * **The verification is known to fail when it should**, which is the only thing that makes it worth having. Two negative controls were run against the real 3.14.7 Android bundle: flipping one base64 character in `messageSignature.signature` gave `KeylessVerificationException: Artifact signature was not valid`, and pointing the identity map at the 3.13 release manager gave `No provided certificate identities matched values in certificate`. Both failed the build with a non-zero exit. + +## 6. WebAssembly + +No distributor ships a `python.wasm` that exports `wasmExports` and `wasmMemory` to JavaScript, which the +`wasmJs` target needs. CPython is therefore **built**, not downloaded: `tools/wasm/build-cpython.sh` builds +CPython 3.14.2 for `wasm32-emscripten` (Emscripten 5.0.3, matched to the `pyemscripten_2026_0` platform of +PEP 783) under the git-ignored `.caches/`, and `python-multiplatform-wasm-runtime` is the published zip of +the result. This is deliberately a different patch release from the native `pythonVersion`; see +[`wasm-design.md`](wasm-design.md). diff --git a/docs/platforms/wasm-design.md b/docs/platforms/wasm-design.md index 7359c602a..414eb1b67 100644 --- a/docs/platforms/wasm-design.md +++ b/docs/platforms/wasm-design.md @@ -1,1375 +1,132 @@ -# WebAssembly (WASM) Feasibility and Design +# WebAssembly (`wasmJs`) design -This document details the feasibility and architectural design for supporting WebAssembly (WASM) in this Kotlin Multiplatform library, specifically addressing the challenges of embedding CPython 3.13 in a Kotlin/Wasm environment. +How CPython runs under Kotlin/Wasm in this library, why it has this shape, and what was measured to +get there. The rules a contributor must follow are in +[`python-multiplatform/src/wasmJsMain/README.md`](../../python-multiplatform/src/wasmJsMain/README.md); +the behavioural contract is `docs/SPEC.md` (§0, C-6, M-3, U-5). This document is the design record. +The chronological experiment log it was distilled from — including the stretch where wasm was +"parked" — is [`../archive/wasm-design-experiment-log.md`](../archive/wasm-design-experiment-log.md). -## 1. Kotlin/Wasm Target Reality +**Status: integrated and experimental.** `wasmJs` is a Kotlin target (`python-multiplatform/build.gradle.kts`), +`src/wasmJsMain` implements the `EmbedAPI` `actual`s, and `src/wasmJsTest` runs the suite on Node +(`wasmJsNodeTest`, the whole suite) and on a browser (`wasmJsBrowserTest`, a filtered subset). `:sample` +has a browser wasm target. `wasmWasi` is not a target (when last assessed it was blocked by Kotlin/Wasm tooling, see the table below). -Based on the project's Kotlin version **2.0.20**: -- **Supported Targets:** Kotlin provides both `wasmJs` (for the browser and Node.js with JS host) and `wasmWasi` (for standalone Wasm environments). Both are classified as Alpha in 2.0.20 (though `wasmJs` matures in later 2.x releases). -- **JS Interop:** Kotlin/Wasm uses the WebAssembly Garbage Collection (WasmGC) proposal. It interoperates with JavaScript via `external` declarations and the `JsAny` type hierarchy. - - **Strings:** Translated seamlessly between Kotlin `String` and JavaScript `string`. They are not passed as `const char*` raw pointers. - - **Pointers & Opaque Handles:** WebAssembly linear memory pointers (like `PyObject*`) are simply 32-bit integers (`Int` or JS `Number`). They cannot be directly mapped to Kotlin's WasmGC objects. - - **Callbacks:** Kotlin lambdas or `@JsExport` functions are exposed as JavaScript function objects, not raw C function pointers. +## Decisions -## 2. The Two Candidate Approaches for `wasmJs` - -Since Kotlin/Wasm and a WASM CPython build are separate modules that do not share linear memory, they must communicate through the JS host environment. - -### (a) Pyodide's High-Level JS API -This involves using `pyodide.runPython`, `pyodide.globals`, and `PyProxy` objects. -- **API Coverage:** Almost **none** of the ~330 `expect` C-API functions can be implemented this way. Pyodide abstracts away the C API. -- **Object Model:** This approach completely destroys the `PyObject`/`PyType` model the library is built on. WASM would have to use `JsAny` wrappers of `PyProxy` instead of `NativePointer`. -- **Verdict:** **Impractical.** It would result in a fundamentally different architecture and semantics for WASM compared to Android, Desktop, and Native. - -### (b) Raw CPython C Symbols Exported from a WASM Build -This involves reaching Emscripten's `EXPORTED_FUNCTIONS` (e.g., `Module._PyList_Size`), passing memory addresses as 32-bit integers. -- **API Coverage:** Can implement the entire `expect` surface, mapping 1-to-1 with the Stable ABI. -- **Object Model:** Preserves the exact same `NativePointer` and `PyObject` semantics as other platforms. -- **Per-Call Cost:** High. Every call must cross from Kotlin WasmGC -> JS -> Emscripten Linear Memory. However, since the signature census shows only 14 simple ABI shapes, this crossing can be tightly optimized. - -## 3. Does Pyodide Export What We Need? - -**No, a stock Pyodide build does not export the full Stable ABI.** -Pyodide relies on Emscripten's dead-code elimination. Symbols not used by Pyodide's internal bindings or explicitly listed in `EXPORTED_FUNCTIONS` are stripped. Furthermore: -- **CPython 3.13 Status:** The `wasm32-emscripten` build was dropped from official CPython support in 3.13. (Pyodide maintains it out-of-tree). `wasm32-wasi` was promoted to Tier 2 support. -- **Custom Build Required:** To use the raw C API approach, we would need to generate a custom Emscripten build (or a custom Pyodide distribution) that explicitly adds our ~330 required C API symbols to `EXPORTED_FUNCTIONS` (or use `EMSCRIPTEN_KEEPALIVE`). - -## 4. Pointer Width and `NativePointer` - -WASM32 uses **32-bit pointers**. -- The current abstraction `data class NativePointer(val address: Any)` uses `Any`. Passing a 32-bit integer pointer into `Any` will box it (as a `Double` on JS or a boxed primitive in WasmGC), adding allocation overhead. -- The `toRawValue(): Long` signature assumes 64-bit addresses. Since a 32-bit unsigned integer fits perfectly into a 64-bit `Long`, the abstraction mathematically holds. However, Kotlin/Wasm code implementing `toRawValue()` will need to cast the JS/Wasm 32-bit `Int` to a Kotlin `Long` carefully to avoid sign-extension bugs. - -## 5. Upcalls on WASM (Python calling Kotlin) - -Python expects a raw C function pointer (an integer address pointing to executable memory). We can preserve the "routing by data" design from other platforms, but the plumbing is different: -1. We write our ~4 static trampoline shapes in Kotlin and export them to JS using `@JsExport`. -2. In JS glue code, we use Emscripten's `addFunction(jsFunction, signature)` to inject the JS function into Emscripten's WebAssembly function table. -3. `addFunction` returns an integer (the table index). This integer *is* the C function pointer. -4. We pass this integer to CPython (e.g., in `tp_new` or `PyCFunction_NewEx`). When CPython calls it, Emscripten routes the call out to JS, which calls our Kotlin WasmGC function. The context pointer (`self` or `closure`) is passed as an integer, allowing us to route to the correct Kotlin object. - -## 6. `wasmWasi` - -**Separate implementations are required.** -Pyodide and Emscripten rely entirely on JavaScript glue code. If the project targets non-browser WASM via `wasmWasi` (pairing with CPython's Tier 2 `wasm32-wasi`), there is no JS host. -Currently, Kotlin/Wasm cannot dynamically link directly to an arbitrary C WebAssembly module without a JS host bridge, as the Wasm Component Model and native static linking are not yet mature enough in the Kotlin toolchain. Thus, `wasmWasi` is genuinely a second, entirely different port that is currently **blocked** by Kotlin/Wasm tooling limitations. - -## 7. Recommendation and Staged Plan - -**Recommendation:** Pursue **Approach (b) — Raw C Symbols via JS Bridge** for the `wasmJs` target. It is the only path that preserves the multiplatform object model. Ignore `wasmWasi` until Wasm Component Model linking matures in Kotlin. - -**Caveat:** WASM will have permanently higher per-call overhead than Native/JVM due to the WasmGC ↔ JS ↔ Emscripten boundary crossings. - -### Staged Plan -1. **Toolchain Customization:** Create a custom Pyodide/Emscripten build script that passes all ~330 `EmbedAPI` functions to Emscripten's `EXPORTED_FUNCTIONS`. -2. **Downcalls (Kotlin to Python):** Write a Kotlin `external` module that binds to `Module._Py*` functions on the Emscripten JS object. Implement `EmbedAPI.wasmJs.kt` to call these. -3. **Upcalls (Python to Kotlin):** Use Emscripten's `addFunction` via JS interop to register 4 static Kotlin `@JsExport` trampolines, returning function pointers to hand to Python. -4. **Pointer Optimization:** If boxing in `NativePointer(val address: Any)` proves too slow in WasmGC, consider refactoring the `expect` class to use a concrete `Long` across all platforms (which JVM/Native already use natively) to avoid allocations. - ---- - -## Addendum: correcting two framing errors - -### Kotlin/Native once had a wasm target, and it was removed - -The analysis above assumed Kotlin/Wasm is the only route. That is true today, but not for a -fundamental reason. - -Kotlin/Native used to ship a `wasm32` target built through LLVM, producing linear-memory -WebAssembly with a C ABI — the same memory model an Emscripten build of CPython uses. That target -could in principle have used `cinterop`, exactly as the iOS target does today. JetBrains -**deprecated it in Kotlin 1.8.20 and removed it in 1.9.20**, in favour of the Kotlin/Wasm -toolchain, which skips LLVM and is built on WasmGC. - -The consequence is that the JS bridge is forced by *tooling*, not by WebAssembly itself: - -- Kotlin/Wasm is WasmGC-based, so its memory model differs from C's linear memory. -- Kotlin/Wasm has no `cinterop` at all. `wasmJs` offers JS interop; `wasmWasi` offers WASI syscalls. - -This project targets Kotlin 2.0.20, so the `wasm32` target is long gone and pinning to 1.9.10 is -not viable — the rest of the toolchain requires 2.0.x. - -### The dependency is CPython's own WASM support, not Pyodide - -Pyodide is a downstream distribution carrying its own patches. A library that intends to be a -standard multiplatform binding should build CPython from source for a WASM target rather than -depend on Pyodide. - -Where CPython's own support actually stands: - -| Target | CPython 3.13 | CPython 3.14 | -|---|---|---| -| `wasm32-wasi` | Tier 2 (officially supported) | Tier 2 | -| `wasm32-emscripten` | Not a PEP 11 platform | Tier 3, via PEP 776 | - -[PEP 776](https://peps.python.org/pep-0776/) is Active, targets 3.14, and formalises Emscripten at -Tier 3. Three of its details bear directly on this design: - -1. **Only static linking is supported.** The PEP states it is only supported to statically link the - interpreter; dynamic linking is unsupported because `EM_JS` functions behave differently in - dynamic builds. CPython therefore cannot be dynamically loaded into another WebAssembly module. -2. **The build produces an `.mjs` plus `.wasm` pair.** A JavaScript runtime layer is intrinsic to - the Emscripten build; it is not something an integration can route around. -3. **No official python.org distribution is mandated** — binaries continue to come from downstream, - i.e. Pyodide. Avoiding Pyodide as a *dependency* therefore means building CPython from source - ourselves, not that an upstream binary exists to use instead. - -Because Kotlin/Wasm cannot statically link C code, and CPython cannot be dynamically linked into -another module, the browser architecture is necessarily three parts: the CPython Emscripten module, -the Kotlin/Wasm module, and JavaScript coordinating them. That holds regardless of which approach -is taken. - -`wasm32-wasi` is the better-supported target on the CPython side, but there is no JavaScript host to -act as glue, and Kotlin/Wasm's `wasmWasi` cannot link a C module. Bridging the two awaits the WASM -Component Model. - -### Practical consequence - -Official Emscripten support arrives in **CPython 3.14**. This project is pinned to 3.13, where -Emscripten has no PEP 11 status at all. WASM support therefore implies moving the embedded -interpreter to 3.14 or later. - ---- - -# Why WASM is parked, and what it would look like - -Recorded so the reasoning survives. Nothing here is implemented. - -## The blocker is Kotlin's toolchain, not WebAssembly - -Every other platform calls CPython directly — cinterop on iOS and androidNative, Panama on -desktop, JNI on Android. Kotlin/Wasm cannot: `wasmJs` offers only JS interop and `wasmWasi` -only WASI syscalls, and neither can link a C library. - -Kotlin/Native once had a `wasm32` target that went through LLVM and emitted linear-memory -wasm — the same memory model Emscripten's CPython uses, which is exactly what would have made -cinterop work. It was deprecated in 1.8.20 and removed in 1.9.20. So the JS bridge is a -consequence of a toolchain decision, not of WebAssembly. - -**The deeper barrier is memory, not calls.** Two wasm modules can call each other directly if -the host wires module B's exports to module A's imports at instantiation — JS is needed once, -not per call. But Kotlin/Wasm is WasmGC: its objects live on a managed heap, and it has no way -to read another module's linear memory. CPython's entire API is pointers into exactly that -memory. So JS is not a convenience in the call path — it is where the marshalling has to -happen. - -JetBrains has a **Multiple Memory** proposal under consideration that would address this, and a -C-interop example PR against `kotlin-wasm-examples` was opened in March 2024. Neither has a -committed date, and neither appears on the published roadmap, so neither can be planned around. - -## emscripten, not wasi - -| | `wasm32-emscripten` | `wasm32-wasi` | +| Decision | Choice | Why | |---|---|---| -| CPython support | Tier 3 from 3.14 (PEP 776) | **Tier 2** | -| bridge between the two modules | **JS exists** | none — needs Component Model, which Kotlin does not emit | -| C extension modules | **dlopen works** | no dlopen in preview1; extensions must be linked in | -| matches this project | **browser; the sample already targets `wasmJs`** | server/edge | - -The decisive row is the third. This project exists to share libraries, and on WASI a user could -not add a compiled package after the fact — the interpreter would have to be rebuilt. That -guts the goal on that platform, which outweighs WASI's better support tier. - -The second row is nearly as decisive: WASI has no JS host, so there is no mechanism at all to -connect a Kotlin/Wasm module to a CPython module there. - -## The shape it would take - -Put our C code inside CPython's module, so it can handle pointers, and keep the boundary to -integers. - - +- Emscripten module (one linear memory) -----+ - | CPython + our C shim | pointers handled here - +---------------------+-----------------------+ - | i32 only - +-----+-----+ - | JS glue | string copying only - +-----+-----+ - | - +--------+---------+ - | Kotlin/Wasm | - +------------------+ - -Pointers are 32-bit on wasm32, so a `PyObject*` fits in a Kotlin `Int`. The shim exposes -composed operations, the same ones measured on Android: - - int pmp_getattr(int obj, int namePtr); // lookup + error handling, one crossing - int pmp_exec(int codePtr); // __main__ + globals + run - int pmp_list_items(int list, int outPtr, int cap); // bulk: fills a caller buffer - int pmp_str_utf8(int obj, int outLenPtr); - void pmp_decref(int obj); - int pmp_scratch(void); // reusable buffer address, once - -**Interning applies here and matters more.** A Kotlin string lives on the WasmGC heap and a C -string in linear memory, so copying costs a JS hop — but attribute and module names are -repeated literals, so caching the linear-memory address makes every call after the first cost -nothing. This is the same fix measured on Android at 2238 ns → 148 ns, against a hop that is -more expensive here than JNI's. - -**Bulk is where composition earns its keep on this platform.** It was rejected on desktop and -superseded by interning on Android; here it is the only mitigation left, because -`Module.HEAP32.subarray(p, p+n)` fetches N items in one hop where per-element access would cost -N. - -Nothing above changes the layering: `wasmJsMain` supplies `actual`s for the same EmbedAPI -`expect`s, and the object model does not move. - -## What survives if the toolchain improves - -If Multiple Memory lands, the C shim stays — composition is wanted for its own sake — and only -the JS glue is replaced by direct memory access. The `actual` signatures do not change. That -bounds the throwaway work to a few dozen lines of glue, which is the argument for this shape -over waiting indefinitely. - -## Unresolved - -- `NativePointer` holds `Any`, so a wasm32 `Int` boxes on every conversion, and - `toRawValue(): Long` needs sign-extension care. -- CPython must be built with our shim and an explicit `EXPORTED_FUNCTIONS` list; a stock - Pyodide build strips the C API by dead-code elimination. That is a second build pipeline. -- Upcalls: `addFunction(jsFn, sig)` gives a callable function pointer, but re-entering - Kotlin/Wasm from it goes through JS. Unmeasured, and likely worse than every other platform. -- Lifetime: whether Kotlin/Wasm offers a GC hook equivalent to `Cleaner`/`createCleaner` has not - been checked. Without one, only explicit `close()` works. - -## Start condition - -Not now. `wasmJsMain`'s `actual`s cannot be written until it is settled what they implement — -§1/§4 (GIL and automatic release) and §7 (upcall shape) are both in flight, and building against -them now means building twice. - -## Packaging: a side module, against our own CPython - -Two independent choices, easily conflated: - -1. **How our C code is packaged** — statically linked into CPython, or a side module loaded by - `dlopen` at import time. -2. **Where CPython comes from** — built by us with Emscripten, or Pyodide's distribution. - -### Side module, on both counts of its own - -| | statically linked | **side module** | -|---|---|---| -| changing one binding | rebuild all of CPython | rebuild our module | -| release cadence | tied to CPython's | independent | -| shape | a CPython fork, effectively | an ordinary extension module (`PyInit_*`) | -| runtime | no indirection | dynamic-link indirection (small) | - -Having to rebuild the interpreter to change a binding is the same trap this project already hit -elsewhere — a `.def` edit needing a forced link and staging pass before it reached a device. An -extension module is what CPython already has a mechanism for; we are not doing anything unusual. - -### Our own CPython, and no attempt at Pyodide compatibility - -The earlier draft argued for keeping Pyodide compatibility in reach, on the grounds that Pyodide -*is* the package ecosystem on the web. That argument does not survive contact with how Pyodide -wheels are tagged: `cp312-cp312-pyodide_2024_0_wasm32` pins the Python version, the Emscripten -version **and** Pyodide's own ABI together. What you get is not "the Python ecosystem" but -"packages someone built for exactly this Pyodide release". - -So the choice is not ecosystem-versus-no-ecosystem. It is whose tag to be locked to, and -targeting Pyodide would mean the web platform tracking their release cadence while every other -platform is pinned to 3.14 from source. That is a real cost for a conditional benefit. - -The loss from building our own is narrower than it first appears: - -| | on our own build | -|---|---| -| pure-Python wheels (`py3-none-any`) — no ABI tag | work unchanged | -| C extensions (numpy, pandas, lxml, cryptography) | must be built for our ABI | - -And the second row is true of Pyodide too, for anything outside the list they happen to have -built. Pyodide's advantage is a prebuilt catalogue, not a different rule. - -**Decision: our own Emscripten build of CPython 3.14, packaged against by a side module.** -Compiled third-party extensions are a known limitation on this platform, recorded rather than -solved. - -### Correction: PEP 783 unbinds the tag from Pyodide - -The section above is right about the decision and wrong about its cost. It was written from the -`pyodide_2024_0_wasm32` tag and concluded that any wheel compatibility means being locked to a -Pyodide release. That stopped being true. - -[PEP 783 — Emscripten Packaging](https://peps.python.org/pep-0783/) is **Accepted**. It replaces -the Pyodide-owned tag with a standard platform tag series that PyPI accepts: - -``` -pyodide_2024_0_wasm32 Pyodide's tag, Pyodide's platform -pyemscripten_2026_0_wasm32 standard platform tag (PEP 783) -``` - -Three things follow, and they change the trade-off rather than the conclusion. - -**The platform is versioned per Python feature release, not per distribution release.** The -current series is `pyemscripten_2024_0` (3.12), `pyemscripten_2025_0` (3.13), `pyemscripten_2026_0` -(3.14). The earlier worry — "the web platform tracks Pyodide's cadence while every other platform -is pinned to 3.14" — does not apply: the platform version *is* the Python version. Pinning to 3.14 -from source and claiming `pyemscripten_2026_0` are the same act. The patch component exists as -an escape hatch the PEP hopes never to use. - -**Compatibility is a build recipe, not a distribution.** The PEP states that the tags "can be used -by Python interpreters compiled and linked with the specified version of Emscripten and with the -specified ABI-sensitive flags". So our own build can claim the platform. The ABI-sensitive surface -the PEP enumerates: - -| | | +| How Kotlin reaches CPython | Raw Stable ABI symbols as `@WasmImport`s against the Emscripten module's own wasm exports — **no JS in the call path** | Preserves the exact object model (`NativePointer`, `PyObject`) of every other target; a high-level Pyodide-style JS API (`pyodide.runPython`, `PyProxy`) would replace the whole object model with `JsAny` wrappers and covers almost none of the ~330 C functions | +| Memory | **One shared linear memory.** Kotlin imports Emscripten's `Module.wasmMemory` as its `intrinsics.memory` | Needs Kotlin ≥ 2.4.20-Beta2, whose `wasmJs` modules **import** `intrinsics.memory` (`min=0, max=none`). Releases up to 2.4.10 exported an unbounded memory, which Emscripten cannot import — the earlier plan to patch the `.wasm` memory section (and a YouTrack issue) is obsolete ([`../archive/wasm-youtrack-issue.md`](../archive/wasm-youtrack-issue.md)) | +| Where CPython comes from | **Our own Emscripten build**, `tools/wasm/build-cpython.sh`: CPython **3.14.2**, Emscripten **5.0.3**, matched to the `pyemscripten_2026_0` platform of PEP 783 | No distributor ships a `python.wasm` exporting `wasmExports,wasmMemory`. The version is deliberately not the 3.14.x pinned for native targets (`pythonVersion` in `gradle.properties`); see `docs/design/ecosystem.md` | +| C shim / composition | **None** | The hops that composition amortised are gone; measured gain ≤ 2.9 ns per operation (below). It also removes a second build pipeline | +| `EXPORTED_FUNCTIONS` list | **Not needed** | `-sMAIN_MODULE` is `LINKABLE`, so the whole Stable ABI is exported (all 310 C symbols `EmbedAPI.kt` declared at the time were found in the build's 8287 exports). The build adds `wasmExports,wasmMemory` to `-sEXPORTED_RUNTIME_METHODS`, which is not ABI-sensitive | +| Upcalls (Python → Kotlin) | `@WasmExport` trampolines placed in CPython's `__indirect_function_table` with `WebAssembly.Table.set`; the table index is the C function pointer. Routing is by data (a handle in `self`/`closure`), through the generated table, as on every platform | A funcref is a funcref: `call_indirect` reaches the Kotlin export directly. Measured 3.1 ns vs 10.9 ns for the `addFunction` + JS-closure route | +| Reclamation | `registerCleaner` over a JS `FinalizationRegistry` + `WeakRef`, reached through `toJsReference()` | The Kotlin/Wasm stdlib has no finalisation hook, but the host does; `toJsReference()` does not pin the object (200 objects, `alive 0 / 200`) | +| `Py_ssize_t` | 32-bit; handled at the boundary | The only ABI divergence of wasm32 (SPEC C-6, `WasmPySsizeTBoundaryTest`) | +| `wasmWasi` | Not a target | Kotlin/Wasm cannot link a C module there and WASI has no JS host to glue them; CPython-on-WASI has no `dlopen`, so users could not add compiled packages, which defeats the purpose | + +### Why the architecture is what it is + +- Kotlin/Wasm is **WasmGC**: a Kotlin object cannot live in linear memory, so CPython can never hold + a Kotlin object pointer. The answer is the same handle table as everywhere + ([`object-lifetime.md`](../design/object-lifetime.md)). Kotlin reads CPython's memory with + `Pointer(addr)` (plain `i32.load`/`i32.store`). +- Kotlin never uses its own linear memory (0 pages across strings, collections, exceptions, a 10 MiB + `ByteArray`, a 100k-object graph, the `ArrayBuffer` bridge and coroutines), so handing the memory to + Emscripten costs nothing. The one exception is `withScopedMemoryAllocator`, which allocates from + address 0 on top of Emscripten's static data: **it must never be called** (CPython allocates, + Kotlin dereferences). +- A wasm memory *import* survives CPython growing the memory underneath Kotlin; a JS `TypedArray` + view would detach. +- Kotlin/Wasm has no `call_indirect`, so *calling* a C function pointer (`tp_traverse`'s `visitproc`, + `tp_free`) goes through two small functions in `cpython.mjs` (`pmpCallVisit`, `pmpCallFree`). Both are + cold paths. +- The generated entry module is patched at build time (`patchKotlinWasmOutputForCPython`): one + substitution for the memory, one for the upcall handoff (`pmpSetKotlinExports(exports)`, before + `exports._start()`). An application must also declare the three `@WasmExport` trampolines itself — + `@WasmExport` is honoured only in the compilation that produces the `.wasm` (`ProxyTypeExportNames`; + generating that file from the Gradle plugin is not done). +- **JSPI must not reach the embedding.** `cpython.mjs` deletes `WebAssembly.promising` / + `WebAssembly.Suspending`; with JSPI live the first blocking syscall (`select.poll().poll(0)`, which + `selectors.py` does at import) fails with `SuspendError` and surfaces as `RuntimeError: unreachable`. + See the wasmJsMain README for the cost of this global mutation. + +## Measurements + +All from the experiment and the tests that replaced it (Node 24 / Emscripten 5.0.3 unless stated; +current figures: run `WasmCrossingOverheadTest`, `WasmUpcallRouteOverheadTest`, +`WasmMarshallingOverheadTest`; `docs/investigations/cost-table.md` has the cross-platform table). + +| Question | Result | |---|---| -| Emscripten compiler version | pinned per platform version | -| statically linked libraries | fixed set | -| stack unwinding ABI | fixed selection | -| dependency lookup handling | specified | -| `-pthread` | prohibited | -| `-sWASM_BIGINT` | required | - -`-sWASM_BIGINT` is worth noting for us specifically: it is what makes an i64 cross the JS boundary -as a BigInt rather than being split into two i32s, which is the same `NativePointer` width question -§4 above raises. The platform requires the setting we would have wanted anyway. - -**The full specification currently lives in Pyodide's documentation**, which the PEP references -normatively while noting the wording was chosen to be "more forwards compatible to a future where -the definition of the platform moves upstream". So Pyodide still *authors* the spec even though it -no longer *owns* the tag. That is a real residual coupling, but it binds a flag list, not a runtime. - -#### What this changes - -| | earlier draft | with PEP 783 | +| Direct `@WasmImport` vs JS trampoline, 10 M calls of a trivial C function | **5.0 ns** vs 13.6–16.9 ns | +| One real crossing on CPython (`PyErr_Occurred`) | 2.9 ns | +| Bulk read, 1000 `i32`: shared memory `Pointer.loadInt()` vs `Module.HEAP32` | 0.7 vs 7.1 ns per element | +| Composition against the interpreter, reading one global (200 000×): naive / + interned strings / + hoisted module+dict | 259.5 / 185.4 / **65.2** ns; a composed call would save 2.9 ns of that | +| Upcall, 10 M indirect calls issued from C: `table.set` vs `addFunction` + JS closure vs C→C control | **3.1** vs 10.9 vs 0.7 ns | +| Kotlin `METH_O` function vs C builtin `abs` through `PyMethodDef` | 87 vs 73 ns (14 ns is the JS frame CPython's own trampoline imposes — see below) | +| String argument, `"version"` (7 chars) / 54 chars: scratch vs interned | 30.6 / 157.0 ns vs 22.2 / 26.4 ns (`WasmMarshallingOverheadTest`, 200 000 iterations) | +| `char*` → `String`, 27 / 4000 bytes: ASCII `CharArray`+`concatToString()` vs `ByteArray`+`decodeToString()` | 103 / 5 875 ns vs 405 / **58 271** ns — never `decodeToString()` a large buffer | + +Corrections that came out of measuring (they changed the design): + +- Shared memory does **not** make string marshalling "disappear". Kotlin/Wasm strings already are JS + strings, and what shared memory removes is the intermediate WasmGC `ByteArray` allocation (82.7 ns + → 30.6 ns per short argument), not a copy. +- Interning is worth much more for long names than short ones (a cache hit is flat in length, an encode + is linear). The marshalling rules for the other platforms are in + [`marshalling-design.md`](../design/marshalling-design.md); wasm uses the same two primitives + (`Wasm.internedUtf8`, `Wasm.scratchUtf8`, four scratch slots). +- CPython's own wasm trampoline for `PyMethodDef` / getset calls never installs on this toolchain pair + (its `EM_JS` initialiser needs `wasmTable`/`wasmMemory` before they exist and the `LinkError` is + swallowed by a bare `catch`). Every C extension pays the same JS frame, so the cost is symmetric. + `WasmCallTrampolineTest` pins this and goes red the day upstream fixes it. Prefer type slots installed + through `PyType_FromSpec`, which CPython invokes by plain `call_indirect`. + +## The interpreter build and the platform tag + +`tools/wasm/build-cpython.sh` builds CPython 3.14.2 with CPython's own `Tools/wasm/emscripten` driver +plus the changes that make it claim `pyemscripten_2026_0` (PEP 783): + +| | stock Tier 3 build | this build | |---|---|---| -| what the tag pins | Pyodide release | Python version + a published flag set | -| our own build + C-extension wheels | mutually exclusive | **compatible, if we match the flags** | -| Pyodide as a dependency | the price of wheels | not required | -| pure-Python wheels | work unchanged | work unchanged | - -The decision stands — our own Emscripten CPython 3.14 — but the recorded limitation was -overstated. Compiled extensions are not lost by building our own; they are lost only if we build -with flags that diverge from `pyemscripten_2026_0`. **Matching that flag set should be a build -constraint from the start**, because retrofitting it means rebuilding the interpreter. - -Evidence that the path is real rather than paper: [pypa/packaging #804](https://github.com/pypa/packaging/pull/804) -implements the tag handling, [pypi/warehouse #19804](https://github.com/pypi/warehouse/pull/19804) -adds PyPI support, [maturin #3163](https://github.com/pyo3/maturin/pull/3163) adds it to a build -backend, and [pydantic publishes Emscripten wheels to PyPI under it](https://pydantic.dev/articles/emscripten-wheels-pydantic). - -#### Resolved: the flag list, read off the source rather than the rendered page - -`pyodide.org/en/stable/development/abi.html` still returns 403 to automated fetching. The same -documents are in the repository and `raw.githubusercontent.com` serves them, which is how the -following was read. Sources are `github.com/pyodide/pyodide` at tag **`314.0.4`** — -`docs/development/abi/314.md`, `docs/development/abi/flags.md`, `Makefile.envs` — and -`github.com/pyodide/pyodide-build` `pyodide_build/config.py`. - -`Makefile.envs` at that tag pins the whole platform in four lines: - -``` -PYODIDE_VERSION ?= 314.0.4 -PYODIDE_ABI_VERSION ?= 2026_0 -PYVERSION ?= 3.14.2 -PYODIDE_EMSCRIPTEN_VERSION ?= 5.0.3 -``` - -| | `pyemscripten_2026_0` (3.14) | for comparison, `2025_0` (3.13) | -|---|---|---| -| Emscripten | **5.0.3** | 4.0.9 | -| unwinding | `-fwasm-exceptions -sSUPPORT_LONGJMP=wasm`, at **compile and link** | same | -| `-sWASM_BIGINT` | required, but **default since Emscripten 4.0.0** — not typed explicitly | same | -| `-pthread` | prohibited; "if `-pthread` is used, the resulting libraries will not load" | same | -| dependency lookup | full `RPATH` support, and `RPATH` is the *only* mechanism | same | -| Rust | stable `1.93.0`; no nightly, no custom sysroot | nightly + custom sysroot | -| new static libs | `sqlite3` 3.39.0, `lzma`/xz 5.2.2 | — | - -Statically linked into the main binary: `zlib`, `bzip2`, `sqlite3`, `lzma`, `zstd`, `libffi`, -`libhiwire`, and the Emscripten JS libraries (`libGL`, `libegl.js`, `libwebgl.js`, `libhtml5*.js`, -`libsdl.js`, `libwebsocket.js`, `libeventloop.js`, `liblz4`, the filesystem backends). **OpenSSL is -not** — Pyodide builds it as a shared side module (`libssl.so`/`libcrypto.so`, OpenSSL 1.1.1w) -found through `RPATH`. - -Extension modules are side modules. Pyodide documents `-sSIDE_MODULE=2` plus an explicit -`-sEXPORTED_FUNCTIONS=["_PyInit_…"]` as the recommendation, though its own C default is -`-sSIDE_MODULE=1`. That is exactly the shape the packaging decision above already chose for our -shim. - -**PEP 783 itself does not enumerate any of this.** It lists the ABI-sensitive *surface* and then -defers normatively to Pyodide's documentation for the values. It does specify how the tag is -derived, and that is the part that bites: `packaging` reads -`sysconfig.get_config_var("PYEMSCRIPTEN_PLATFORM_VERSION")`. **That variable does not exist -anywhere in CPython 3.14.2** — `grep -r PYEMSCRIPTEN` over the whole checkout returns nothing. So a -stock CPython Emscripten build cannot advertise the platform even if it matches the flags; the -variable is something Pyodide's build injects. - ---- - -## Correction: the JS layer is not mandatory - -Everything above assumes Kotlin/Wasm can only reach CPython through JavaScript — that crossings -cost a JS hop and that string data must be copied by JS between two isolated memories. Reading -the Kotlin compiler shows both halves of that are wrong on the `wasmJs` target. This section -records what was checked and where it stops. - -### 1. Kotlin can call another wasm module's exports directly - -`@WasmImport(module, name)` and `@WasmExport(name)` exist in the stdlib -(`libraries/stdlib/wasm/src/kotlin/wasm/Annotations.kt`), gated behind -`@ExperimentalWasmInterop` since 2.2. The annotation doc says the function is imported "without -type adapters". The compiler's own codegen test `callingWasmDirectly.kt` states the intent -outright: - -> Here we pass export of another Wasm module to our import directly without JS layer. -> This enables strict type check without JS conversions. - -So `pmp_getattr(i32, i32) -> i32` can be a direct wasm-to-wasm call. JS is still involved in -*instantiating* the modules and wiring the import, but not in the call. This is exactly the -narrow integer interface the shape above already proposes — it just costs less than assumed. - -The constraint is the one that makes it work: no type adapters means wasm primitive types only. -No strings, no objects. Which is what the composed shim signature already looks like. - -### 2. On `wasmJs`, Kotlin *imports* its linear memory - -`WasmCompiledModuleFragment.createAndExportMemory` branches on a flag that -`wasmCompiler.kt` sets as `importWasmMemoryInsteadOfExport = isWasmJsTarget`: - -| target | what the module does with its memory | -|---|---| -| `wasmWasi` | **defines and exports** it — `WasmExport.Memory("memory", …)`, and the source comments that the name is a WASI ABI convention | -| `wasmJs` | **imports** it, as `WasmImportDescriptor("intrinsics", "memory")`, declared with 0 initial pages | - -That inverts the premise. A `wasmJs` module does not own a memory it defines; the host hands one -in. And `kotlin.wasm.unsafe.Pointer` is an `i32` into precisely that memory. - -So the question "can Kotlin read CPython's `char*`" reduces to "can the host supply Emscripten's -memory as that import" — at the wasm level, both are just a `WebAssembly.Memory` in an import -object, and Emscripten exposes its own as `Module.wasmMemory`. **If that works, string -marshalling does not merely get cheaper, it disappears** — which matters more here than anywhere -else, because every measurement on Android and desktop found marshalling, not crossings, to be -the dominant cost. - -### 3. The blocker is the allocator, not the memory - -`kotlin/wasm/unsafe/MemoryAllocation.kt` creates the top-level allocator as -`ScopedMemoryAllocator(0, parent = null)` and extends it with `wasm_memory_grow`. **It starts at -address 0 and assumes it owns everything.** In Kotlin's own memory that is correct. In a memory -shared with Emscripten, address 0 is the null page and the low region holds Emscripten's static -data, so `withScopedMemoryAllocator` would hand out addresses CPython already owns and corrupt -the heap. - -This does not sink the idea, but it bounds it: - -- **Reading and writing CPython-owned buffers** — addresses that CPython allocated and handed - back as `i32` — is safe. -- **Allocating from Kotlin inside the shared memory** is not, unless the allocator can be given - a base address it does not currently accept. - -Whether a `Pointer` can be constructed from an arbitrary `Int` returned by the shim, rather than -only from `allocate()`, has not been checked and decides whether the safe half is usable at all. - -### 4. The impossible direction stays impossible, and is already handled - -None of this changes that a WasmGC reference cannot be stored in linear memory — that is a -spec-level guarantee, not a maturity gap, so CPython can never hold a Kotlin object pointer in a -`PyObject` field. The answer there remains an index into a Kotlin-side table, which is what -`docs/design/object-lifetime.md` already specifies for Python→Kotlin. Kotlin/Wasm even blesses the -mechanism: `JsReference` passes a Kotlin object as an opaque reference. - -### What this changes - -| | as recorded above | after reading the compiler | -|---|---|---| -| Kotlin → CPython call | JS hop | **direct wasm call** (`@WasmImport`) | -| string marshalling | JS copies between two memories | **possibly none** — if the memory is shared | -| Kotlin allocating in that memory | assumed fine | **unsafe** — allocator starts at 0 | -| CPython holding a Kotlin object | impossible | impossible (unchanged) | - -The composed-shim design does not change shape. What changes is its cost, and the reason to build -it: composition was proposed here to amortise JS hops, and if the hops are gone it has to justify -itself on crossing count alone — the same argument that closed desktop composition in ROADMAP §6. - -### Unverified, in the order that decides the design - -1. Can the host supply Emscripten's `wasmMemory` as Kotlin's `("intrinsics", "memory")` import? - The generated JS glue builds that import object; whether it can be overridden is unknown. -2. Can `Pointer` be constructed from an arbitrary `Int`? If not, the read-only half is unusable - and only `@WasmImport` survives. -3. Is `("intrinsics", "memory")` stable? It is an internal name, not public API. -4. Does `@WasmImport` work when the exporting module is an Emscripten `MAIN_MODULE`, whose - exports are ordinary wasm functions but whose instantiation Emscripten's own glue controls? - -None of these needs code in this repo to answer; all four are answerable with a small standalone -experiment, and together they decide whether WASM looks like iOS or like the JS-bridge design. - ---- - -## Measured: what the released toolchain actually does - -The section above was written from the Kotlin compiler at master and got one thing wrong in a way -that matters. `wasm-experiment/` builds a trivial `wasmJs` module on Kotlin 2.4.10, runs it, and -parses the emitted binary. Findings, in the order they change the design. - -### The memory import is master-only; the released direction is the opposite - -`importWasmMemoryInsteadOfExport = isWasmJsTarget` is in the compiler source but **not in any -released version**. Parsing the 2.4.10 output: the only length-prefixed `intrinsics` import is -`intrinsics.tag` (the exception tag), there is no memory import, and the export section contains -`memory`. Same on 2.2.20. - -So Kotlin does not import a memory today. It exports one. - -**That inverts the plan, and the inverted plan is better** — because it needs nothing unreleased -on either side. Emscripten has `-sIMPORTED_MEMORY`, which makes the CPython module import -`env.memory` rather than define it. Instantiate Kotlin first, take `exports.memory`, grow it to -Emscripten's initial size, hand it over as `env.memory`. Both halves are shipped features. - -### Kotlin's linear memory is empty, and stays empty - -This is what makes handing it over safe, and it is measured rather than assumed: - -``` -pages at startup 0 -pages after string interop 0 -pages after collections 0 -pages after exception 0 -pages after explicit allocate 2 (allocated at 0x0) -``` - -The module declares `min_pages = 0`, and all 40 of its data segments are **passive** — they -initialise WasmGC arrays through `array.new_data` and are never placed in linear memory. String -interop was the obvious suspect for a hidden consumer and it is not one; Kotlin strings are -WasmGC arrays and cross to JS as `externref`. - -So a `wasmJs` module's linear memory is unused space that happens to be attached to it. Giving it -to Emscripten costs Kotlin nothing. - -### The allocator hazard is real but avoidable - -The last probe line confirms the source reading: `withScopedMemoryAllocator` grew the memory and -allocated at **address 0**, exactly on top of where Emscripten keeps its static data. In a shared -memory this corrupts the heap on first use. - -The mitigation is not a workaround, it is the natural design: **CPython allocates, Kotlin only -reads.** `Pointer(addr)` has a public constructor and its `loadInt`/`storeByte` members compile to -plain `i32.load`/`i32.store` with no allocator involvement — confirmed by compiling and running -it. Every address we would ever dereference comes back from a C call. - -`withScopedMemoryAllocator` must therefore never appear in `wasmJsMain`. That belongs in the -source-set README as a hard rule, in the same category as desktop's "always `invokeExact`". - -### What the picture looks like now - -| | earlier draft | measured | -|---|---|---| -| Kotlin → CPython call | JS hop | direct wasm call (`@WasmImport`) | -| who owns the shared memory | CPython, imported by Kotlin (master-only) | **Kotlin exports it, Emscripten imports it** | -| Kotlin reading `char*` | impossible | `Pointer(addr).loadByte()`, no JS, no allocator | -| string marshalling | dominant cost | **none, if the memory is shared** | -| CPython holding a Kotlin object | impossible | impossible (unchanged — handle index) | - -If this holds end to end, WASM stops being the outlier and lands close to the iOS shape: no -boundary for data, a direct call for control, and a handle table for the one direction WasmGC -forbids. The composition argument weakens accordingly — it was proposed here to amortise JS hops, -and without them it must justify itself on crossing count alone, which is what closed desktop -composition in ROADMAP §6. - -### Still unverified - -1. **Does `@WasmImport` bind to an Emscripten module's export?** The mechanism is proven by the - compiler's own `callingWasmDirectly.kt` codegen test, but against a hand-built module, not one - whose instantiation Emscripten's glue controls. Needs `emsdk`, which is not installed here. -2. **Does Emscripten accept a memory that starts at 0 pages and is grown by JS before - instantiation?** `-sIMPORTED_MEMORY` normally pairs with a memory JS created at the right size. -3. **Does anything in a larger Kotlin program touch linear memory?** The probe covers strings, - collections and exceptions on a trivial module. Coroutines, `ByteArray` interop and the JS - `ArrayBuffer` bridges are untested. - ---- - -## Proven end to end: Kotlin and CPython-side C can share one memory - -The section above listed three unverified items and said the first needed `emsdk`. It is -installed, both experiments are built and run, and both pass. `wasm-experiment/` reproduces them. - -### Test A — `@WasmImport` binds to an Emscripten export - -Kotlin called `add_two` compiled by `emcc` and got 42. The compiler emits a real wasm import, -`(import "./probeA-wrapper.mjs" "add_two" (func …))`, and wires the ES module into the import -object. The value supplied prints as `function 2() { [native code] }` — a raw wasm export, not a -JS wrapper — so the call is wasm-to-wasm with no JS frame, which is what the compiler's own -`callingWasmDirectly.kt` test intends. - -### Test B — Emscripten imports the memory Kotlin exports - -``` -C: get_static_message() -> 0x400 -Kotlin: readCStringAt(same address) -> "hello-from-the-cpython-side" PASS -C: alloc_message() -> 0x10628 -Kotlin: writeCStringAt(addr, "written-by-kotlin") -C: str_len(addr) -> 17, UTF8ToString(addr) -> "written-by-kotlin" PASS -``` - -Both directions, static data and `malloc`'d heap. **No copying and no JS in the data path.** -Kotlin dereferences a C address with `Pointer(addr)` and gets the bytes C put there. - -### The single blocker, and it is one value - -Emscripten refused Kotlin's memory at first: - -``` -LinkError: memory import has no maximum limit, expected at most 4294967295 -``` - -Kotlin emits `WasmLimits(0, null)` — no maximum — and wasm requires a supplied memory's limits to -sit inside the importer's, so an unbounded memory can never satisfy any Emscripten import. -Patching section 5 to `{min: 0, max: 32768}` made everything link and both directions work. -Nothing else about the pairing needed changing. - -**That is a compiler-chosen constant, not a design problem.** It is worth a YouTrack issue; a -`-Xwasm-memory-maximum`-style knob, or simply emitting the wasm32 ceiling, would remove the need -to post-process the binary. Until then a build step can patch it, which is ugly but bounded — the -memory section is six bytes and nothing in the format holds an absolute offset. - -### The constraint that shapes the design - -A and B cannot be combined in one instantiation graph while Kotlin owns the memory: - -``` -Kotlin @WasmImport needs Emscripten's exports at instantiation time -Emscripten -sIMPORTED_MEMORY needs Kotlin's memory before that -``` - -Wasm imports are supplied up front, so that is a cycle. Three ways out: - -1. **Kotlin imports the memory instead of exporting it.** No cycle — Emscripten instantiates - first, then Kotlin receives both the memory and the functions. This is - `importWasmMemoryInsteadOfExport = isWasmJsTarget`, present at master and in no release. It is - the configuration this design wants, and the second reason to talk to JetBrains. -2. **JS trampolines for calls, shared memory for data.** Available today. Costs a JS hop per - call and keeps the data path free — and the data path is the half that dominated every - measurement on Android and desktop, so this captures most of the win. -3. Copy through JS, as originally designed. - -### What §10 looks like now - -| | original design | proven | -|---|---|---| -| control path | JS hop per call | direct wasm call, **or** JS trampoline if sharing memory | -| data path | JS copies between two memories | **none — one memory** | -| string marshalling | dominant cost, needs interning | **absent** | -| composition | needed to amortise JS hops | **weakly motivated** — same argument that closed §6 | -| CPython holding a Kotlin object | impossible | impossible — handle index, per `object-lifetime.md` | - -WASM stops being the outlier. With option 2 it is roughly Android's shape with a cheaper data -path; with option 1 it is close to iOS. - -### Rules this produces for `wasmJsMain` - -- **`withScopedMemoryAllocator` must never be called.** It allocates at address 0, on top of - Emscripten's static data. Measured: the probe grew the memory to 2 pages and handed back `0x0`. - CPython allocates; Kotlin only dereferences addresses it was given. -- `Pointer(addr)` is the only way in, and its public constructor plus raw `i32.load`/`i32.store` - members make that free. - -### Resolved - -- **Linear memory is strictly unused.** The probe was expanded to include large `ByteArray` allocations (10MB), large object graphs (100,000 items), `Uint8Array` JS ArrayBuffer bridges, and coroutines (`kotlinx.coroutines.GlobalScope.launch`). **None of them touched linear memory.** The page count stayed exactly at 0 until `withScopedMemoryAllocator` was explicitly called. This confirms that a shared linear memory design is perfectly safe and won't be corrupted by normal Kotlin runtime behavior. -- **JS trampoline cost is ~2.7x.** A microbenchmark running 10,000,000 iterations found a direct `@WasmImport` call took ~48.7 ms, while routing the call through a JS trampoline took ~132.9 ms (2.73x slower). This shows the JS boundary imposes measurable overhead, but since we eliminate copying overhead for strings and data, this cost is a worthwhile tradeoff to achieve the shared memory model. -- **Unbounded Memory bug.** Kotlin's compiler emits `WasmLimits(0, null)`, causing Emscripten to reject the imported memory. This is confirmed to be the only blocker. A draft YouTrack issue has been written to `docs/platforms/wasm-youtrack-issue.md` requesting a compiler flag (or a default Wasm32 ceiling) to fix this. - -### Still open - -- Upcalls. `addFunction` still routes through JS to re-enter WasmGC, and nothing here changes - that; §7 still needs measuring before `wasmJsMain` is written. - -### Correction: the coroutine reading was contaminated - -The result above was nearly reported wrong. `runPromise` resolves after `main()` returns, so the -coroutine's page count was taken *after* the `withScopedMemoryAllocator` demo had already grown -the memory to 2 pages. The probe printed `pages inside coroutine 2` and that number said nothing -about coroutines — it was the allocator's growth, observed late. - -Gating the allocator demo off and re-running gives the reading that means something: - -``` -pages at startup 0 -pages after string interop 0 -pages after collections 0 -pages after exception 0 -pages after large ByteArray 0 (10 MiB) -pages after large obj graph 0 (100k strings) -pages after ArrayBuffer bridge 0 -pages observed inside coroutine 0 <- uncontaminated -``` - -So the conclusion holds, and now for coroutines too: **nothing in ordinary Kotlin/Wasm touches -linear memory.** `MEASURE_ALLOCATOR` in the probe is off by default for this reason, and the -coroutine line must be read as meaningless in any run where it is on. - ---- - -## Resolved upstream: 2.4.20-Beta2 imports the memory - -The YouTrack issue drafted in `wasm-youtrack-issue.md` should not be filed. The thing it asks for -has already shipped, and in a better form than the request. - -`importWasmMemoryInsteadOfExport = isWasmJsTarget` was read on the compiler's `master` branch and -recorded here as unreleased, on the evidence that 2.2.20 and 2.4.10 both export a memory. That was -right about those two versions and wrong about the conclusion: **2.4.20-Beta2 imports it.** The -experiment builds against it and the binary says so plainly: - -``` -imports intrinsics.memory (kind 0x02, memory) - intrinsics.memory (kind 0x03, global) - intrinsics.tag -memory definitions 0 <- defines none of its own -non-function exports none <- no longer exports one -``` - -And the generated glue hands it in as an ordinary JavaScript value: - -```js -intrinsics: { - memory: new WebAssembly.Memory({ initial: 0 }), - tag: wasmTag -}, -``` - -Replacing that one expression with Emscripten's `Module.wasmMemory` is the whole integration. No -binary patching, no compiler flag, no `max` limit to negotiate — `patch-memory-max.py` becomes -history rather than a build step. - -### It also removes the constraint that shaped the design - -The instantiation cycle is gone. While Kotlin exported the memory, Emscripten had to import it -while Kotlin simultaneously needed Emscripten's exports through `@WasmImport`, and wasm supplies -imports up front, so the two could not be satisfied together. The recorded workaround was JS -trampolines for the calls, measured at 2.73x a direct call. - -With Kotlin importing, the order resolves: instantiate Emscripten first, then hand Kotlin both the -memory and the functions. **Direct wasm-to-wasm calls and a shared linear memory at the same -time** — which is the configuration this design wanted and had written off as unavailable. - -### What that leaves - -The last structural obstacle to §10 is gone. Remaining work is ordinary: pin the Kotlin version at -2.4.20-Beta2 or later (the library targets 2.0.20 today, so this is a real upgrade with its own -cost), build CPython for `pyemscripten_2026_0`, write the composed shim, and supply -`Module.wasmMemory` in place of the placeholder above. - -Two facts from earlier still hold and still constrain the shape. Kotlin never touches its linear -memory on its own — measured across strings, collections, exceptions, a 10 MiB `ByteArray`, a -100k-object graph, the `ArrayBuffer` bridge and coroutines — so handing it to Emscripten costs -nothing. And `withScopedMemoryAllocator` must never appear in `wasmJsMain`, because it allocates -from address 0 upward, on top of Emscripten's static data. - ---- - -## Test C: run, and the cycle is gone - -The section above predicted this from reading the binary. `wasm-experiment/` now builds and runs -it. `native/run.sh` reproduces; Test C is `native/combined-test.mjs`. - -### The binary confirms the shape - -Parsing `wasm-experiment.wasm` built on 2.4.20-Beta2: - -``` -import './probeA-wrapper.mjs' . 'add_two' kind=0 func <- direct wasm call -import 'intrinsics' . 'tag' kind=4 tag -import 'intrinsics' . 'memory' kind=3 global externref -import 'intrinsics' . 'memory' kind=2 memory min=0 max=none -memory definitions 0 -non-function exports none -``` - -`max=none` on the *import* is what makes this direction work. A wasm import must be satisfied by -a memory whose limits sit inside the declared ones, so an unbounded import accepts anything — -including Emscripten's bounded memory. The old direction was unsatisfiable for exactly the -mirror-image reason, and `patch-memory-max.py` existed to force it. It is no longer used. - -### The integration is one substitution - -`patch-import-object.py` rewrites one expression in the generated glue: - -```js -intrinsics: { memory: new WebAssembly.Memory({ initial: 0 }), tag: wasmTag } - -> memory: .wasmMemory -``` - -Ordering comes free from ES modules. `wasm-experiment.import-object.mjs` imports the wrapper, the -wrapper has a top-level `await factory()`, so Emscripten is fully instantiated before Kotlin's -import object is built. Emscripten is compiled exactly as it would be alone — no -`-sIMPORTED_MEMORY`, no binary patching. Kotlin is the side that adapts. - -### Both halves hold at once - -``` -Emscripten's memory object Memory, 258 pages -Kotlin's intrinsics.memory Memory, 258 pages PASS same buffer -Kotlin called cAddTwo(40, 2) -> 42 PASS @WasmImport, memory shared -C: get_static_message() -> 0x400 -Kotlin: readCStringAt(that address) -> "hello-from-the-cpython-side" PASS -Kotlin: writeCStringAt(malloc'd addr, "written-by-kotlin") -C: str_len -> 17, UTF8ToString -> "written-by-kotlin" PASS -``` - -The value handed to the import still prints as `function 2() { [native code] }` — a raw wasm -export, no JS wrapper — so the control path is wasm-to-wasm with the memory shared. - -### And it survives the memory growing - -Not previously checked, and it is not optional: CPython links with `-sALLOW_MEMORY_GROWTH --sINITIAL_MEMORY=20971520`, so the memory *will* grow underneath Kotlin in production. A JS -TypedArray view detaches when that happens. Kotlin holds the memory as a wasm import rather than -a view, and does not: - -``` -C: malloc(64 MiB) -> 0x10638, 258 pages -> 1026 pages - PASS -- the shared memory actually grew - PASS -- Kotlin still reads the pre-growth address correctly - PASS -- Kotlin writes into memory that did not exist when it was instantiated -``` - -### Measured: the control path - -10,000,000 calls of the same trivial C function, one process, Node 24, Emscripten 5.0.3. - -| | ns/call | -|---|---| -| direct `@WasmImport` | **5.0** (stable across runs) | -| JS trampoline | 13.6 – 16.9 (2.7x – 3.4x) | - -So removing the cycle is worth about 9–12 ns per crossing. That is the whole benefit of option 1 -over option 2, and it is now available. - -### Measured: the data path — and the earlier conclusion was wrong - -Everything above said string marshalling "does not merely get cheaper, it disappears". **It does -not.** Both loops below run in Kotlin and end with a Kotlin `String` built from the same C -address; only the route differs. - -| ns per read | 27 bytes | 4000 bytes | -|---|---|---| -| shared: scan to NUL, build nothing | 193 | 1 649 | -| shared: `ByteArray` copy, no decode | 399 | 4 875 | -| shared: `ByteArray` + `decodeToString()` | 405 | **58 271** | -| shared: `CharArray` + `concatToString()` (ASCII) | **103** | 5 875 | -| shared: `StringBuilder.append(Char)` per byte | 311 | 51 348 | -| copied: `UTF8ToString` through JS | 125 | **3 827** | - -Two things account for it, and neither is the boundary: - -1. **Kotlin/Wasm strings already *are* JS strings.** The generated glue compiles with - `builtins: ['js-string']`. `UTF8ToString` produces a JS string, and handing that to Kotlin is - not a conversion. The "copy through JS" path was assumed to pay for a copy that no longer - exists. -2. **`ByteArray.decodeToString()` is pathological on long input** — 53 µs of the 58 µs figure at - 4000 bytes, about 13 ns per byte, against 0.2 ns per byte at 27 bytes. `concatToString()` on a - `CharArray` costs a tenth of it. Worth a separate report upstream. - -The honest rule that comes out of this is not "marshalling disappears" but: - -- **`char*` → `String`: use the ASCII `CharArray` + `concatToString()` route, and fall back to - `UTF8ToString` for anything long or non-ASCII.** Never `decodeToString()` on a large buffer. -- **`char*` → `ByteArray` (the `PyBytes` case): shared memory, unambiguously.** No decode, and no - JS-side equivalent that avoids a copy. -- Shared memory is still what makes writing *into* C buffers free, which has no JS equivalent at - all short of `Module.HEAPU8.set`. - -### Measured: bulk, which is what the composed shim was for - -1000 `i32` out of a C-owned array, 20,000 repetitions: - -| | ns per element | -|---|---| -| shared memory, `Pointer.loadInt()` | **0.7** | -| through JS, `Module.HEAP32[a >> 2]` | 7.1 | - -So the composition arithmetic is now fully determined. Walking a list without a shim means N -direct `PyList_GetItem` calls at **5.0 ns** of crossing each. With a shim it is one crossing plus -N shared reads at **0.7 ns**. **Composition buys ~4.3 ns per element and nothing else** — the -CPython-side work is identical either way. - -ROADMAP §6 closed desktop composition at a 10% saving. Test D below measures the same question -against the real interpreter and answers it more sharply. - ---- - -## Test D: it works against real CPython, and it closes the composition question - -Tests A and C run against a 4 KB toy compiled by `emcc`. Test D runs against **CPython 3.14.2 -built for `wasm32-emscripten`** — 10 MB of wasm, 8191 exports, `-sMAIN_MODULE`, a growable memory, -instantiated by Emscripten's own glue. The build is `Tools/wasm/emscripten` (PEP 776, Tier 3 from -3.14) under Emscripten 5.0.3; see the build notes below. - -``` -python.wasm exports 8191 -CPython's memory 320 pages -Kotlin's intrinsics.memory 320 pages PASS same buffer - -Kotlin: pyExec("answer = 6 * 7; greeting = ...") -> 0 PASS -Kotlin: pyGlobalInt("answer") -> 42 PASS -Kotlin: pyGlobalString("greeting") -> "hello-from-cpython-42" PASS -Kotlin: pyExec("bytearray(48 MiB)") -> 0, 320 -> 934 pages PASS - Kotlin still reads CPython correctly after the growth PASS - a pre-growth string still reads correctly PASS -``` - -Kotlin wrote the Python source **directly into CPython's heap** with `Pointer.storeByte` on an -address `malloc` returned, called `PyRun_SimpleString` as a direct wasm import, then dereferenced -`PyUnicode_AsUTF8`'s `char*` to get the string back. No copying, no JS in either direction, and it -survives the interpreter growing its own memory by 600 pages underneath. - -### The one build change CPython needed, and it is not ABI-sensitive - -The Stable ABI symbols are all there — `-sMAIN_MODULE` is `LINKABLE`, so Emscripten exports -everything and emits `EXPORTED_FUNCTIONS is not valid with LINKABLE set` for CPython's own list. -**This retires the recorded worry that "CPython must be built with an explicit -`EXPORTED_FUNCTIONS` list; a stock Pyodide build strips the C API by dead-code elimination."** The -supported CPython build already exports the whole surface. - -What it does *not* do is let JavaScript reach them. `Module` carries 36 keys and none of the -interesting ones, so there is nothing to hand to `@WasmImport`. One flag fixes it: - -``` --sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY,wasmExports,wasmMemory - ^^^^^^^^^^^^^^^^^^^^^^ -``` - -After a relink, `Module.wasmExports.PyRun_SimpleString` prints as `function 4426() { [native -code] }` — a raw wasm export — and `Module.wasmMemory` is the `WebAssembly.Memory` Kotlin's -`intrinsics.memory` needs. Both are JS-glue settings and neither appears in PEP 783's -ABI-sensitive list, so this costs nothing against the platform tag. - -### Composition, measured against the interpreter - -Reading one global out of `__main__`, 200,000 times, on the same shared-memory direct-call setup. -Each row adds a pure-Kotlin optimisation: - -| | ns | | -|---|---|---| -| naive | 259.5 | 2 `malloc` + 2 `free` + 2 string writes + 4 API calls | -| + interned C strings | 185.4 | names allocated once — the Android fix, for free here | -| + module/dict hoisted | **65.2** | 2 API calls — the floor | -| one crossing (`PyErr_Occurred`) | **2.9** | direct `@WasmImport`, real interpreter | - -**75% of the naive cost is removable without shipping a single line of C.** What is left is 65 ns -of two CPython calls, of which the crossings are 5.8 ns — 9% of the floor, 2% of the naive figure. -A composed `pmp_getattr` could merge those two calls into one and save **2.9 ns**. - -The bulk case ends the same way. Composition there buys the difference between a crossing and a -shared-memory read, 2.9 ns against 0.7 ns per element, and CPython does identical work either way. -The constraint that kept the case alive — `PySequence_Fast_ITEMS` is a macro over -`PyListObject->ob_item` and `abi3t` makes `PyObject` incomplete, so shared memory does *not* let -Kotlin walk a list's storage — turns out not to matter, because N calls at 2.9 ns is cheap. - -**Verdict: no composed shim on WASM.** Same answer as ROADMAP §6 gave desktop, reached the same -way, and with a wider margin: desktop's composition was worth 10%, this is worth 1–2% after the -free wins are taken. The signatures sketched earlier — `pmp_getattr`, `pmp_exec`, `pmp_list_items`, -`pmp_str_utf8`, `pmp_decref`, `pmp_scratch` — were motivated by amortising JS hops, and the hops -are gone. What replaces them is ordinary `wasmJsMain` Kotlin: `@WasmImport` per Stable ABI -function, an interning cache for repeated names, and `Pointer` reads for everything else. - -That also removes the second build pipeline. No extension module to build, version and ship -against a specific interpreter — which was the largest recurring cost in the plan. - -### What is still not answered - -Upcalls. `addFunction` re-entering WasmGC still goes through JS and is still unmeasured; §7's -shape has to settle before `wasmJsMain` is written. Nothing in Tests C or D touches it. - ---- - -## Building CPython 3.14.2 for Emscripten: what worked - -`/Volumes/macMini/wasm-build/build-cpython-emscripten.sh` reproduces it. The driver is CPython's -own `Tools/wasm/emscripten` (`build` runs configure-build-python → make-build-python → libffi → -mpdecimal → configure-host → make-host). It succeeded end to end on the first attempt against -Emscripten 5.0.3, and `python.sh --version` prints `Python 3.14.2`. - -``` -sysconfig.get_platform() emscripten-5.0.3-wasm32 -sysconfig.get_config_var('PYEMSCRIPTEN_PLATFORM_VERSION') None -Checked 114 modules (85 built-in, 6 shared, 17 n/a, 1 disabled, 5 missing, 0 failed on import) -python.wasm 10 MB python.mjs 570 KB -``` - -### What the stock build passes, and how far it is from `pyemscripten_2026_0` - -Read off the actual command lines, not the documentation: - -| | stock CPython 3.14.2 | `pyemscripten_2026_0` | -|---|---|---| -| Emscripten | 5.0.3 (we pinned it) | **5.0.3** — matches | -| `-sWASM_BIGINT` | passed explicitly in `LDFLAGS_NODIST` | required; default since Emscripten 4.0 — matches | -| `-pthread` | absent; `configure` even reports `emcc accepts -pthread... no` | prohibited — matches | -| `-fPIC` | yes (required by `MAIN_MODULE`) | yes — matches | -| dynamic linking | `--enable-wasm-dynamic-linking` → `-sMAIN_MODULE`; extensions built `-shared -sSIDE_MODULE=1` | `MAIN_MODULE=1` / side modules — matches | -| **unwinding** | **nothing** — no `-fwasm-exceptions`, no `-sSUPPORT_LONGJMP=wasm` | **required at compile and link** — **DIVERGES** | -| **static libs** | zlib, bzip2, sqlite3, libffi, mpdecimal; **no lzma, no zstd** | + lzma 5.2.2, zstd — **DIVERGES** | -| **OpenSSL** | **absent** — "Could not build the ssl module" | shared side module, OpenSSL 1.1.1w — **DIVERGES** | -| platform variable | `PYEMSCRIPTEN_PLATFORM_VERSION` is **not defined anywhere in CPython 3.14.2** | `packaging` reads it to emit the tag — **DIVERGES** | - -So the stock Tier 3 build is close but cannot claim the tag, and the gap is real work rather than a -flag flip: the unwinding ABI has to be added at both compile and link, `lzma`/`zstd`/OpenSSL have to -be built as ports or side modules, and something has to define `PYEMSCRIPTEN_PLATFORM_VERSION`. - -A practical trap for whoever does it: the driver hardcodes `CFLAGS=-DPY_CALL_TRAMPOLINE --sUSE_BZIP2` in its `configure` argv and appends user arguments *after*, and autoconf takes the -last assignment — so passing `CFLAGS=…` to add `-fwasm-exceptions` silently drops -`-DPY_CALL_TRAMPOLINE`. The full string has to be repeated. - -**Not verified:** that the ABI-matching build succeeds, or that a PyPI `pyemscripten_2026_0` wheel -loads into it. Only the stock build was run. - -### Environment note - -Installing Emscripten 5.0.3 through `emsdk install` **replaces `~/emsdk/upstream` in place** — the -previously active 6.0.6 now reports as not installed and would have to be re-downloaded. The active -version is a global, shared setting, not per-project. - ---- - -## Proven: the tag claim is real — a PyPI `pyemscripten_2026_0` wheel loads - -The section above listed four divergences between the stock Tier 3 build and -`pyemscripten_2026_0` and marked the whole thing "not verified". It has now been built and -tested against an actual wheel. `/Volumes/macMini/wasm-build/build-cpython-abi.sh` reproduces -it, into `cpython314-abi/` so the stock build survives for comparison. - -**Result: `pydantic_core-2.48.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl`, downloaded from -PyPI, imports and runs.** - -``` -python 3.14.2 emscripten-5.0.3-wasm32 -IMPORTED pydantic_core 2.48.0 - .../pydantic_core/_pydantic_core.cpython-314-wasm32-emscripten.so -validate_python(42) -> 42 -raised ValidationError -``` - -The `.so` is 4.4 MB of Rust/PyO3. The `ValidationError` matters as much as the `42`: it is an -exception propagating out of the side module, so the unwinding ABI is exercised and not merely -linked. - -### Only one of the four divergences actually gated it - -| | matched? | did it gate the wheel? | -|---|---|---| -| unwinding ABI | **yes, this build** | **yes — this was the whole gate** | -| `PYEMSCRIPTEN_PLATFORM_VERSION` | **yes, via a one-line CPython patch** | no — gates *claiming* the tag, not loading | -| lzma, zstd | still missing | no | -| OpenSSL | still missing | no | - -The negative control is exact. The same wheel on the stock build: - -``` -ImportError: could not load dynamic lib: .../_pydantic_core...so -LinkError: WebAssembly.Instance(): Import #204 "env" "__cpp_exception": - tag import requires a WebAssembly.Tag -``` - -Reading the binaries says why before running anything. The wheel's `.so` imports a **wasm tag**: - -``` -tag env.__cpp_exception <- the side module needs it -memory env.memory -table env.__indirect_function_table -``` - -and the two interpreters differ in exactly that: - -``` -cpython314-abi/python.wasm section id 13 (tag), exports __cpp_exception, __c_longjmp -cpython314/python.wasm no tag section, exports neither -``` - -So `-fwasm-exceptions -sSUPPORT_LONGJMP=wasm` is not a nice-to-have that affects code -generation quality — it is a link-level contract, and it is checked. `lzma`/`zstd`/OpenSSL are -ABI-sensitive in the Pyodide flag list but only bind wheels that link *them*; they cost the -stdlib modules `_lzma`, `_zstd`, `_hashlib`, `_ssl` and nothing else here. - -The build-driver trap recorded earlier is real and was avoided by repeating the whole string. -The result is visible in the generated Makefile: - -``` -CONFIGURE_CFLAGS= -DPY_CALL_TRAMPOLINE -sUSE_BZIP2 -fwasm-exceptions -sSUPPORT_LONGJMP=wasm -CONFIGURE_LDFLAGS= -fwasm-exceptions -sSUPPORT_LONGJMP=wasm -``` - -### `PYEMSCRIPTEN_PLATFORM_VERSION`, and a trap that silently produces a wrong tag - -`packaging` 26.3 implements PEP 783 as literally as the PEP describes: - -```python -def _emscripten_platforms(): - v = sysconfig.get_config_var("PYEMSCRIPTEN_PLATFORM_VERSION") - if v: - yield f"pyemscripten_{v}_wasm32" -``` - -Before defining it, the interpreter loads the wheel but cannot claim the tag — its platform tag -is `emscripten_5_0_3_wasm32` and no `pyemscripten_*` tag is produced at all. - -The obvious way to define it is a Makefile variable, because that is what `sysconfig ---generate-posix-vars` reads. **That silently produces the wrong tag.** `sysconfig` coerces any -value that parses as an integer, and Python accepts underscores inside integer literals: - -```python -int('2026_0') == 20260 int('2025_0') == 20250 int('2024_0') == 20240 -``` - -so the config var comes out as the integer `20260` and `packaging` emits -`pyemscripten_20260_wasm32` — a plausible-looking tag that matches nothing. Every version in -the PEP's series has this shape, so it is not specific to 2026. - -`sysconfig` already has the opt-out — `_ALWAYS_STR`, which holds the two Apple deployment -targets for the same reason. Adding one name to it is the whole fix: - -```python -_ALWAYS_STR = { - 'IPHONEOS_DEPLOYMENT_TARGET', - 'MACOSX_DEPLOYMENT_TARGET', - 'PYEMSCRIPTEN_PLATFORM_VERSION', # int('2026_0') == 20260 -} -``` - -After which the interpreter's first tag is exactly the wheel's: - -``` -config var = '2026_0' -first tag = cp314-cp314-pyemscripten_2026_0_wasm32 -wheel tag = cp314-cp314-pyemscripten_2026_0_wasm32 MATCH -``` - -This is worth reporting upstream: CPython defines no way to set the variable, and the one -mechanism a packager would naturally reach for corrupts it without any error. - -### What this changes for the plan - -The earlier decision — our own Emscripten CPython 3.14, no Pyodide dependency — stands, and the -recorded limitation shrinks again. "Compiled third-party extensions are a known limitation on -this platform" is now false for anything published under the PEP 783 tag. The remaining -limitation is narrower and ordinary: packages that need `lzma`, `zstd` or OpenSSL need those -built, and packages not published for the tag still have to be built. - ---- - -## Upcalls: measured, and better than every prediction in this document - -§5 and §7 step 3 specify one mechanism — `@JsExport` the trampoline, wrap it in a JS closure, -hand it to `addFunction` — and "Unresolved" writes it off as "goes through JS, unmeasured, and -likely worse than every other platform". Tests E and F measure it, and measure an alternative -this document does not consider. - -### There is no JS in the call path, because a funcref is a funcref - -`@WasmExport` puts a Kotlin function in the **wasm** export section with no type adapters, so -what JS receives is an exported wasm function, not a closure: - -``` -String(raw.kotlin_upcall_add) -> function 5097() { [native code] } -``` - -`WebAssembly.Table.prototype.set` accepts any exported wasm function whatever instance produced -it, so it can go straight into Emscripten's `__indirect_function_table`, and Emscripten's -`call_indirect` reaches it directly. JS appears **once, at registration** — to put a funcref in -a table — and never in a call. - -``` -table.grow(1) -> 2; table.set(2, kotlin_upcall_add) accepted -C: call_fp(2, 40, 2) -> 42 -``` - -Test E, 10,000,000 indirect calls issued from inside C so no JS is in the timed loop: - -| | ns/call | crossing | -|---|---|---| -| C → C (control, same module) | 0.7 | — | -| **C → Kotlin, `table.set`** | **3.1** | **2.4 ns** | -| C → Kotlin, `addFunction` + JS closure (the design's mechanism) | 10.9 | 10.2 ns | - -The design's mechanism is **3.5x** the direct one. And the direct upcall at ~3 ns is the mirror -of the direct downcall at ~5 ns — the two directions cost the same order, which is the thing -this document assumed could not be true. - -`addFunction` handed the wasm export directly works too, and deduplicates onto the entry -`table.set` already installed — so it is the same path, not a third one. - -### Re-entrancy holds: the WasmGC heap is live inside a frame CPython pushed - -The routing-by-data design needs the upcall to find a Kotlin object from a context integer and -write into linear memory. Both work from inside a C-pushed frame: - -``` -Kotlin registry[7] = "routed-through-a-cpython-frame" -C: call_fp(route, ctx=7, out=0x10628) -> 30 -C: UTF8ToString(out) -> "routed-through-a-cpython-frame" -``` - -An unrouted context returns cleanly rather than trapping, and allocating on the WasmGC heap -inside the upcall is fine. - -**Signatures must match exactly.** `call_indirect` is statically typed and does not coerce: -calling a `(i32)->i32` export through a `(i32,i32)->i32` slot raises `RuntimeError: null -function or function signature mismatch`. That is a runtime trap, not a compile error, so the -Kotlin trampoline's arity is a correctness requirement rather than a style question. - -### Test F: real CPython calls Kotlin, through a real `PyMethodDef` - -A toy's `call_indirect` is not what CPython does. CPython is built `-DPY_CALL_TRAMPOLINE`, and -`pycore_emscripten_trampoline.h` routes every `PyCFunction`, `getset` getter and setter through -`_PyEM_TrampolineCall`, which dispatches on arity. So the mechanism has to survive that. - -It does. A Kotlin `@WasmExport` of shape `(PyObject*, PyObject*) -> PyObject*`, put in CPython's -own table and registered in a `PyMethodDef` that Kotlin builds in linear memory: - -``` -CPython's __indirect_function_table: Table, length 13518 -table.grow(1) -> 13518; table.set(13518, kotlin_pycfunction) accepted -Kotlin: installPyCallback("kotlin_cb", fp=13518) -> 0 - -Kotlin: pyExec("cb_result = kotlin_cb(41)") -> 0 -Kotlin-side upcall counter: 0 -> 1 -Kotlin: pyGlobalInt("cb_result") -> 42 -type(kotlin_cb).__name__ = "builtin_function_or_method" -``` - -Python sees an ordinary builtin. Variadic arguments arrive intact, 1000 calls in a Python loop -all return correctly, and a bad argument raises rather than corrupting anything. The return -value is built by Kotlin calling `PyLong_FromLong` — **a downcall issued from inside an -upcall**, so both directions are live on one stack. - -### But CPython's own wasm trampoline never installs, and that is an upstream defect - -`Python/emscripten_trampoline.c` tries to replace its JS trampoline with a wasm one that -dispatches using `__builtin_wasm_test_function_pointer_signature` (a wasm-gc `ref.test`). A -four-argument Kotlin export matches none of the four shapes it tests, so it discriminates: - -* wasm trampoline → all four `ref.test`s fail → `SystemError: Handler takes too many arguments` -* JS fallback → `wasmTable.get(func)(arg1, arg2, arg3)` → called anyway, fourth argument zero - -``` -kotlin_cb_4(1) -> "returned 9906512" -``` - -It was called. **The JS fallback is live**, and instrumenting the swallowed `catch` says why: - -``` -[instr] fell back: LinkError: WebAssembly.Instance(): Import #0 "env" "memory": - memory import must be a WebAssembly.Memory object -[instr] wasmTable is undefined | wasmMemory is undefined -``` - -The `EM_JS` initialiser runs while `python.mjs` is still evaluating, and needs `wasmTable` and -`wasmMemory` — which in this configuration are **exports** of `python.wasm`, assigned only after -instantiation. The `LinkError` is swallowed by a bare `catch (e) {}` and the JS fallback is kept -silently. - -This is an instantiation cycle, not an ordering slip — the same shape as the one Kotlin's memory -import dissolved, pointing the other way. The trampoline module imports `env.memory` and -`env.__indirect_function_table` from the very module whose import object needs the trampoline. -`-sIMPORTED_MEMORY` was tried: it fixes the memory half and the failure moves to the table, - -``` -[instr] fell back: LinkError: Import #1 "env" "__indirect_function_table": - table import requires a WebAssembly.Table -[instr] wasmTable undefined wasmMemory object -``` - -and Emscripten has no imported-table setting to finish the job. So on this toolchain pair the -wasm trampoline cannot install at all. - -**The consequence is symmetric, which is why it does not change the design.** Every -`PyCFunction` call in the build crosses a JS frame — a C extension's exactly as much as -Kotlin's. Measured like for like, `abs()` (a C builtin declared `METH_O`) against a Kotlin -`METH_O` function with the identical wasm signature: - -| | ns/call | -|---|---| -| pure Python `def py_cb(x): return x+1` | 59 | -| C builtin `abs(x)` — `METH_O` | 73 | -| **Kotlin `kotlin_cb_o(x)` — `METH_O`** | **87** | -| | **14 ns attributable to the crossing** | -| Kotlin, `METH_VARARGS` | 161 | -| Kotlin, `METH_VARARGS` + tuple read | 183 | - -The `METH_VARARGS` rows carry CPython's argument-tuple construction; that cost is identical for -a C extension and is not the crossing. 14 ns against `abs` matches Test E's `addFunction` -figure, as it should — both are the JS-frame path. - -### What this means for §7 - -| platform | how Python re-enters the host | -|---|---| -| iOS | a plain function call | -| Android | a JNI callback | -| desktop | a Panama upcall stub | -| **WASM, slot reached by `call_indirect`** | **a plain `call_indirect` — ~3 ns** | -| **WASM, `PyMethodDef` today** | the same JS trampoline every C extension pays — ~14 ns over C | - -**§7's design holds on WASM, and the caveat it carries is wrong.** "WASM will have permanently -higher per-call overhead than Native/JVM due to the WasmGC ↔ JS ↔ Emscripten boundary" describes -a boundary that is not in the path. Routing by data through a small set of static trampoline -shapes works unchanged; the context integer, the WasmGC lookup and the linear-memory write all -survive re-entry. - -Two things change in step 3 of the staged plan: - -* **`@WasmExport` and `WebAssembly.Table.set`, not `@JsExport` and `addFunction`.** Same number - of moving parts, 3.5x cheaper, and JS touched once per trampoline rather than once per call. -* **Prefer slots CPython calls directly.** Type slots installed through `PyType_FromSpec` are - invoked as ordinary `call_indirect`; `PyMethodDef` entries and getset descriptors go through - `_PyEM_TrampolineCall`. Only the first gets the ~3 ns figure while the upstream defect stands. - (The slot/trampoline split is read from `pycore_emscripten_trampoline.h`, which wraps exactly - `_PyCFunction_TrampolineCall`, `descr_get_trampoline_call` and `descr_set_trampoline_call`; - the ~3 ns figure for the direct path is Test E's measurement, not a measurement of a type slot.) - -### Still open - -Nothing structural. `wasmJsMain` can be written against §7 as amended. What is *not* answered: - -- Whether a Kotlin exception thrown inside an upcall unwinds safely through CPython's frames. - Every trampoline here returns normally or returns `NULL`. -- Reference-count ownership across the boundary, which is a contract question rather than a - toolchain one. -- ~~Whether Kotlin/Wasm offers a GC hook equivalent to `Cleaner`/`createCleaner`.~~ **Answered.** - The stdlib does not, but the *host* does, and `toJsReference()` reaches it without pinning the - object it hands over — measured at `alive 0 / 200` with JS holding only a `WeakRef` and a - `FinalizationRegistry` entry. `registerCleaner` is built on that. The second finding cost more: - a collection is observable only across a **macrotask** boundary, because a `WeakRef` outlives the - job that created it and the registry callback is delivered as a task, and - `FinalizationRegistry.prototype.cleanupSome()` — which would have made it synchronous — has been - removed from V8. See `src/wasmJsMain/README.md` and ROADMAP §10 for the numbers. - - -### Verification status of the wheel claim - -**Demonstrated, reproducibly, as Test G in `native/run.sh`.** The wheel is the real PyPI file -- -`pydantic_core-2.48.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl`, 1,403,190 bytes, sha256 -`4fc45a49334c54541cbc97bf416d9300b4b1d3b2840dfb079b1123dc3f9ef5a6`, matching -`https://pypi.org/pypi/pydantic-core/json` exactly. It contains a 4,408,639-byte -`_pydantic_core.cpython-314-wasm32-emscripten.so` (Rust/PyO3, the compiled extension). - -Against the ABI build's own `python.sh` (no Kotlin involved -- this is a CPython Emscripten -loader question, not a Kotlin one): - -``` -pydantic_core imported OK, version: 2.48.0 -validate_python("42") -> 42 -raised: ValidationError -``` - -Against the stock build, same wheel, same `python.sh` harness, as the negative control: - -``` -ImportError: could not load dynamic lib: .../_pydantic_core.cpython-314-wasm32-emscripten.so -LinkError: WebAssembly.Instance(): Import #204 "env" "__cpp_exception": tag import requires a WebAssembly.Tag -``` - -identical to what the original report quoted. Comparing the two `python.wasm` binaries directly: -the stock build has no tag section and no `__cpp_exception` symbol, the ABI build has two tags and -the symbol -- and that difference is exactly what separates the pass above from the failure. The -unwinding ABI (`-fwasm-exceptions -sSUPPORT_LONGJMP=wasm`) is both present only in the ABI build -and the thing that gates loading a compiled `pyemscripten_2026_0` wheel; PEP 783's premise that -matching the ABI unlocks compiled extensions holds up on this tree, for this wheel. - -Scope of what this settles: one wheel (pydantic-core), one path (import + a validator call + -exception raise). It does not check every C-API surface a compiled extension might touch, nor -whether other wheels on this tag rely on something the ABI build's flag set doesn't cover. +| Unwinding ABI `-fwasm-exceptions -sSUPPORT_LONGJMP=wasm` at compile **and** link | absent | **present — this is the one that gates loading a compiled wheel** (the side module imports the `__cpp_exception` tag) | +| `PYEMSCRIPTEN_PLATFORM_VERSION` | not defined anywhere in CPython 3.14.2 | defined via a patch to `sysconfig` (`_ALWAYS_STR`; otherwise `int('2026_0') == 20260` silently yields the tag `pyemscripten_20260_wasm32`) | +| `wasmExports,wasmMemory,wasmTable` in `-sEXPORTED_RUNTIME_METHODS` | no | yes (not ABI-sensitive) | +| `lzma`, `zstd`, OpenSSL | missing | **still missing** — ABI-sensitive per Pyodide's flag list but they did not gate the wheel tested; costs `_lzma`, `_zstd`, `_hashlib`, `_ssl` | + +Evidence: a real PyPI wheel, `pydantic_core-2.48.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl`, imports +and runs on this build (`validate_python(42) -> 42`, a `ValidationError` raised and caught — the +exception exercises the unwinding ABI), and fails with `tag import requires a WebAssembly.Tag` on the +stock build. This is one wheel and one code path; it does not show that every compiled extension +loads. `WasmCompiledWheelTest` runs it (`tools/wasm/build-cpython.sh wheels` fetches the pinned wheels); +`WasmInterpreterAbiTest` checks the claims, with `tools/wasm/build-cpython.sh stock` as the negative +control. Emscripten is pinned to 5.0.3 because that is what the platform specifies, and installing it +replaces `~/emsdk/upstream` in place (a global setting). + +Platform notes that still hold: PEP 776 makes `wasm32-emscripten` Tier 3 from CPython 3.14 and supports +only static linking of the interpreter (CPython cannot be dynamically loaded into another module); the +build yields a `.mjs` + `.wasm` pair, so a JS layer exists for instantiation but not for calls. Pure-Python +wheels work unchanged; compiled third-party extensions load if published for the PEP 783 tag. + +## Browser + +Node and a browser differ in the filesystem and three build steps, all supplied by the library +(`stageWasmBrowserRuntime`): `cpython.mjs` in the consumer's webpack context, `python.mjs`/`python.wasm` +beside the page, the standard library as `python3..zip` installed into MEMFS (no NODEFS in a +browser; no `thisProgram`, so `sys.prefix` stays `/`), and the two substitutions on the compile-sync output. +Details and the postcondition that guards the entry-module ordering are in the wasmJsMain README. + +## Not verified / open + +- Whether a Kotlin exception thrown inside an upcall unwinds safely through CPython's frames: every + trampoline in the experiment returned normally or returned `NULL`. +- Whether compiled wheels other than pydantic-core load. +- Compose (`@Composable` from Python) on wasm is `planned` (SPEC B-6, N-2); `suspend` upcalls are + `planned` on wasm because there are no threads (SPEC U-5). +- Absolute wasmJs nanoseconds depend on suite scope and host lifetime (a filtered run is 55–74% + slower in pure-Python rows); quote ratios, or full-suite figures labelled as such + ([`downcall-design.md`](../design/downcall-design.md), "Warmup"). +- Test count: the wasmJsMain README records 403 tests, 0 failures, 0 skipped on `wasmJsNodeTest` and says + to re-run rather than trust the number; I did not re-run it. diff --git a/docs/roadmap/ROADMAP.md b/docs/roadmap/ROADMAP.md index c343e1724..136925281 100644 --- a/docs/roadmap/ROADMAP.md +++ b/docs/roadmap/ROADMAP.md @@ -1,4179 +1,389 @@ # ROADMAP -What is not done yet, why it matters, and what is known about it. Ordered by what blocks what, -not by size. - -Items are written so the reason survives without the conversation that produced it. Where a -claim came from a measurement, the number is here; where it is a judgement, it says so. - -**Current test state — see §14a for the six run paths, how each is counted, and the numbers as -last verified (2026-08-13).** The table that used to be here (`desktopTest` 107/106, -`iosSimulatorArm64Test` 104/103, `GCLeakTest` "red on purpose") was stale on every figure: §1 is -closed and `GCLeakTest` passes now, and re-running the two JVM/native suites today gives -`desktopTest` 327/0 (1 skipped) and `iosSimulatorArm64Test` 289/0. A frozen number rots the moment -someone adds a test, which is exactly what happened here — §14a gives the counting recipe instead -of a number to carry forward uncorrected. +**What is left to do in this repository, and nothing else.** + +- **Behaviour status** — what is implemented, partial or planned, with the tests that assert it — + is in [`../SPEC.md`](../SPEC.md). This file does not repeat it; it lists the work that remains and + points at the test or document that proves the part already done. +- **History** — how each item was found, measured and closed — stays in git. The last full version + of the old progress log is `4ed98143:docs/roadmap/ROADMAP.md`; the whole trail is + `git log -p -- ROADMAP.md docs/roadmap/ROADMAP.md`. +- **Section numbers 1–16 are the old ones**, kept because other files cite them ("ROADMAP §9", + "§15d", …). A closed section keeps its heading and one line saying where its evidence is. The old + sub-sections (§2a…§16f) are mapped at the head of each section. +- Each remaining item says: what is left, its status (**open** — not started or not working; + **partial** — part done, the rest named; **unverified** — could not be decided from code and tests), + and the evidence path. Paths are relative to the repository root; `PM` = `python-multiplatform/src`, + `GP` = `python-multiplatform-gradle-plugin/src`. +- Established against code and tests on 2026-10-03 (`develop` @ `5db4168c`) **without running + Gradle**. A cited test is one that exists and asserts the thing; whether it passes in this revision + was not observed here. --- ## 1. Release the GIL after initialisation -**Closed.** `Python3.initialize()` parks its thread state with `PyEval_SaveThread()`, and both -suites are green with it enabled — desktop 164, iOS 157, zero failures. `GCLeakTest` passes for the -first time. - -The cause was never the runtime. Several tests wrapped a pointer they had only been lent with -`borrowed = false`, so two wrappers owned one pointer and both decremented it. That double-free -corrupted CPython's free lists and surfaced later in an unrelated test. With parking off the -cleaner thread blocked forever, so the extra decrements queued and never ran — which is exactly -why turning parking on appeared to *cause* the crash. Two earlier attempts read it that way and -reverted. - -**It unblocked §4, multithreaded use, and made three latent bugs visible** (a GIL-less -`PyType_FromSpec` call among them). Expect more of that shape: anything that only worked because -the main thread never let go. - -**State, checked against the code rather than carried forward: enabled, unconditionally.** -`Python3.kt`'s `initialize()` calls `PyEval_SaveThread()` on every call with no flag guarding it, -and the comment at that call site points here for the history. An earlier draft of this section -said "disabled ... with the reason in a comment there" directly under the "Closed" declaration -above — that line described an intermediate state from partway through the debugging history -below and was never updated when the section was actually closed. Anyone reading only the opening -paragraph and that one line together would have concluded the opposite of what the code does. - -`Py_Initialize()` leaves its calling thread holding the GIL. Until that thread parks its state -with `PyEval_SaveThread()`, no other thread can attach, and any thread that tries blocks -forever in `PyGILState_Ensure`. - -That is not theoretical. On desktop `GCLeakTest` reports `cleanup actions started: 1, reached -Py_DecRef: 0` — the cleaner thread starts one release, blocks, and never processes another. - -**Correction: "on any platform" was wrong.** Running the same test on the iOS simulator gives -`started: 101002, reached Py_DecRef: 101001`. The cleaners run there and they do reach the -decref; the target's count still does not move (202 before, 202 after). So iOS is not blocked on -the GIL, and fixing §1 will not turn it green. One test name is covering two unrelated defects, -and either the iOS decrefs land on objects other than the one being measured or the measurement -itself is wrong. Nobody has looked yet. - -**The recorded diagnosis is wrong, or at least incomplete.** Enabling the parking call was tried -again today and the result rules out the explanation this section gives. - -``` -desktopTest with PyEval_SaveThread() enabled - 0 tests run, JVM aborts (exit 134) - SIGSEGV in libpython3.14.dylib _TAIL_CALL_DICT_MERGE - <- _PyFunction_Vectorcall - <- PyObject_CallMethodObjArgs - Java frame: EmbedAPI_desktopKt.PyImport_ImportModule - <- EmbedApiLowLevelTest.importReturnsANewReferenceAndMissingModulesReportFailure -``` - -That call site is this: - -```kotlin -val sys = Python3.withPython { PyImport_ImportModule("sys") } -``` - -**It is already inside the guard.** So "find the C API call that is outside `withGIL`" is not a -sufficient theory, and an audit of call sites will not find the bug. `withGIL` itself reads -correctly — depth-counted `PyGILState_Ensure`/`PyGILState_Release`, outermost acquires — so the -next attempt has to look at the *interaction*: what `PyGILState_Ensure` does on a thread whose -state was parked with `PyEval_SaveThread`, and whether the initialising thread and the calling -thread are the same one. Note this repo has already been bitten once by mixing the raw -thread-state scheme with the refcounted one. - -The crash frame is inside CPython's import machinery, so the GIL is acquired far enough to start -executing Python before something goes wrong. That is a different shape from "no GIL held". - -The older explanation, kept because it is still part of the picture: every C API call outside -`withGIL` becomes a -segfault far from its cause: closing the gap in `BenchmarkTest.testAttributeAccess` -(`PyImport_Import`) moved the crash from `PyImport_Import` to `_PyObject_Malloc`, with no Java -frame naming the new site. - -Doing this properly needs a pass over every call site rather than a one-line change. Two things -make that cheaper than the last attempt: `ReleaseCounter` distinguishes "cleaner never ran" from -"cleaner ran and could not release", and the suites are large enough now to localise a -regression. +**Closed.** `Python3.initialize()` parks its thread state with `PyEval_SaveThread()` unconditionally +(`PM/commonMain/kotlin/python/multiplatform/ffi/Python3.kt`, `initialize`); background cleaners +release through `withGIL` (`PM/commonTest/.../ref/GCLeakTest.kt`). -**Second attempt, and what it bought.** An audit of `commonMain` concluded the sweep was -complete and enabled the line. It was not: `DesktopOverheadBenchmark` in `desktopTest` called -`PyImport_ImportModule`, `PyObject_GetAttrString` and `PyList_Size` bare, which was harmless -only while `initialize()` kept the GIL to itself. Those are now guarded — a real gap closed and -kept. - -With them fixed the crash moved to `_TAIL_CALL_DICT_MERGE`, which is CPython's `DICT_MERGE` -opcode: **Python bytecode executing without the GIL**, i.e. something that runs Python code -rather than merely touching an object. Parked again at that point, with the suite back to -118/117. - -One caveat on that last frame: two other agents were editing the tree concurrently, one of them -in `ffi/conversion/`. An unguarded call only crashes once the GIL is actually released, so a -gap introduced by in-flight work would look exactly like a gap that was always there. The next -attempt should start from a quiet tree. - -Pattern worth stating plainly, because it has now repeated three times: each fix moves the -crash somewhere further from its cause. Grep for the EmbedAPI function names across every source -set — `desktopTest` and `commonTest` included, which is where both misses have been so far — -rather than auditing by reading. - -Note that free-threaded builds do **not** remove this requirement. Dropping the global lock -removes contention, not the rule that a thread must be attached before touching any object, -`Py_IncRef` included. +- **1.1 A test that fails when a C API call runs without the GIL — partial.** Every C API call is + supposed to hold the GIL (AGENTS.md §16), and three past regressions were exactly an unguarded call + that only crashed once the GIL was really released. The only direct test, + `PM/commonTest/.../ffi/GilParkingTest.kt`, asserts "does not crash". What is left is a guard that + names the unguarded call (SPEC C-4). Note: SPEC C-4's sentence "Releasing the GIL after + initialisation is not enabled" contradicts the code above. ## 2. Finish the Android JNI surface -**Closed, on the measure the section itself set.** Both emulators run 213 tests with zero -failures, and the registration surface is complete: 363 of 367 `external fun` are bound through -`RegisterNatives`, with the four exceptions deliberate — `ffiAllocUtf8`/`ffiFreeUtf8`/`ffiReadUtf8` -are hand-written with the correct `JNIEnv*`/`jclass` prologue and cannot move into C because the -buffers belong to Kotlin/Native, and `echoCriticalNamed` exists in order to be name-linked, so -registering it would delete the measurement. - -Descriptor consistency is checked against the compiled bytecode rather than the Kotlin source — -`javap` on `bindings.class` joined to the table by name, zero mismatches — and there are no -duplicate wrappers or table names, which is a failure that broke cinterop once during this work. - -**The history below is why this took three passes**, and it is worth keeping. The section was -closed once on `AssembledApiTest` passing, which measured its own scope: those 19 tests only -exercised the 71 functions that had been registered. Wiring `commonTest` into Android (§11b) took -discovery from 19 tests to 168 and the suite died on its 2nd, then its 12th. Two defects behind -that had been invisible because nothing on Android had ever run those tests — an unpackaged stdlib -and a `RegisterNatives` count hardcoded to 138 against a table of 145. - -**Reopened once. Marking it closed the first time was wrong.** - -It was closed on the strength of `AssembledApiTest` passing — 19 tests on `pmp_api26` and -`pmp_api36`, 0 failed. That was true and it did not mean what it was taken to mean. Those 19 tests -only ever exercised the 71 functions that had been registered. **The passing acceptance test -measured its own scope.** - -Connecting `commonTest` to Android (§11b) raised discovery from 19 tests to 168 per device and the -suite died on the **2nd** one, with a hard SIGSEGV on both API levels. Then, after that fix, on the -12th: - -``` -PyImport_AddModule fault addr 0xc24110e8 EmbedAPI_androidKt -> PythonTestFixture.mainGlobals -PyErr_GetRaisedException fault addr 0xc2411160 PyException.fromCurrentError -> PyObject.getAttr -``` - -Both truncated pointers, which is the signature of the name-linked `@CName` fallback described -below. An unregistered call does not fail cleanly; it corrupts a pointer and kills the process. - -**The real numbers**, recounted from the two files rather than carried forward (parse -`external fun` out of `androidMain/.../bindings.kt`, parse the `JNINativeMethod` table out of -`artMain/cinterop/jni_onload.def`, join on the method name and compare JNI signatures): - -``` - was now -bindings.kt external fun 369 365 -registered 145 187 -unregistered 224 178 -signature mismatches 0 0 (all 187 entries agree with their Kotlin declaration) -``` - -**The string half of it is finished.** Declarations that still carry a `jstring` across the -boundary went from **52 to 6**, and of those six none is a defect: - -- `asmExec`, `asmGetAttr`, `testUpcallString` are registered *with* a jstring signature and their - C wrappers use `GetStringUTFChars` — that is the composition-probe design, not the broken path -- `ffiAllocUtf8`, `ffiReadUtf8` are name-linked but hand-written in `artMain/JNIOnLoadExporter.kt` - with the correct `JNIEnv*`/`jclass` prologue, so the convention matches -- `ffiSymbolRaw` is the one genuine leftover on the name-linked `@CName` path. It has no caller - anywhere in `src/` — `jvmMain`'s `ffiSymbol` that wraps it is itself unreferenced — so it is a - landmine rather than a live crash. Fixing it means a `dlsym` wrapper taking a jlong, and it - belongs with whatever revives the shape vocabulary on Android. - *(Done — see below. It is `ffiSymbolRawN(name: Long)` now, and the shape vocabulary went with it.)* - -The 42 functions migrated in this pass are listed with their per-argument intern/scratch -judgement in `docs/design/marshalling-design.md`. Every symbol they bind was checked to exist in the -shipped `libpython3.14.so` before registering (`llvm-nm -D --defined-only`), and the linked -`libmultiplatform_python3.14.so` has no unresolved `Py*` symbol. - -### The unregistered surface is now empty - -The 178 that were left are registered. Recounted the same way, against the compiled class this -time rather than against the Kotlin source — `javap -p -s` on `build/tmp/kotlin-classes/debug/ -python/native/ffi/bindings.class`, joined to the `JNINativeMethod` table on the method name: - -``` - before after -bindings.kt external fun 369 367 (two dead declarations removed) -registered 191 363 -unregistered 178 4 -descriptor mismatches 0 0 (all 363 agree with the compiled bytecode) -non-static registrations 0 0 -``` - -The four that remain are deliberate, and none of them is on the broken path: - -- `ffiAllocUtf8`, `ffiFreeUtf8`, `ffiReadUtf8` — name-linked, but hand-written in - `artMain/.../JNIOnLoadExporter.kt` with the correct `JNIEnv*`/`jclass` prologue and a real - `jstring`. They cannot move into `jni_onload.def`: the buffers they allocate and read belong to - Kotlin/Native. -- `echoCriticalNamed` — the probe that exists *to* be name-linked, so that a benchmark can compare - name-based linking against `RegisterNatives` at the same calling convention. Registering it would - delete the measurement. - -Two declarations were removed rather than registered, both provably uncalled and both already -having a registered equivalent: `PyObject_Call` (superseded by `PyObject_CallN`) and -`PyList_GetItem` (superseded by the `PyList_GetItemRaw` / `PyList_GetItemRawF` pair). - -**All 157 CPython functions in this pass were unreachable from `commonMain`/`commonTest`.** That -is not a reason it was safe to leave them — it is the finding that the earlier passes had already -covered the whole reachable surface, so what remained was landmines only. The check: take the 312 -`expect fun` in `EmbedAPI.kt`, keep the 109 named anywhere in `commonMain`, `commonTest`, -`androidInstrumentedTest` or `jvmMain`, map each through its `EmbedAPI.android.kt` actual to the -`bindings.*` it calls, and intersect. The intersection with this pass is empty; the same procedure -returns `true` for `PyDict_Clear`, `PyObject_Type` and `PyList_Append`, which were registered in -the previous pass, so the procedure does find reachable functions when they exist. - -`docs/investigations/android-unregistered-surface.md` is kept for the call-graph traces, but its counts and its -"71 reachable" framing are historical. - -The shape vocabulary went with it. The 14 `downcall_*` trampolines and `ffiSymbolRaw` used to reach -`nativeMain`'s `@CName` exports by name, which is the same broken path and worse — a shape -function's first argument is the *target function pointer*, so a shifted argument register is a -wild call rather than a bad `PyObject*`. The indirect call is done in `jni_onload.def` now, which -removes the fallback and the hop through Kotlin/Native at once. `ffiSymbolRaw` became -`ffiSymbolRawN(name: Long)` over a `dlopen(NULL, RTLD_NOW)` handle, which is the jlong-taking -`dlsym` wrapper this section asked for above. - -Verification actually run: `compileKotlinAndroidNativeArm64`, `compileDebugKotlinAndroid` and -`linkMultiplatform_python3.14DebugSharedAndroidNativeArm64` all clean; the linked -`libmultiplatform_python3.14.so` has 317 undefined `Py*` symbols and every one of them is defined -in the shipped `libpython3.13.so` (`objdump -T`); desktop 171 tests, 0 failed. - -~~**Not yet run: the device suite.** §2's own measure is how far the 168 `commonTest` cases get on -`pmp_api26`/`pmp_api36`, and that has not been re-run since this change — another agent held the -emulators. The registration surface is complete and internally consistent; whether the suite -advances past the 12th case is unmeasured. **Do not close this item on the strength of the -consistency check alone** — that is the same mistake as closing it on `AssembledApiTest`.~~ -**Since run, repeatedly, and past far more than the 12th case.** The "Android had no usable -temporary directory" entry below records `connectedDebugAndroidTest` at 492/3 then 493/0 on -`pmp_api26`; §14b item 7 records `AndroidCleanerPathTest` on both `pmp_api26` and `pmp_api36`; §7's -"Android landed next" records `UpcallEntryTest` at 251/0 on both API levels. The device suite is -not merely unblocked — it has run clean, several times, on both emulators. - -Earlier in this section: four were migrated first (`PyImport_AddModule`, `PyErr_SetString`, -`PyObject_SetAttrString`, `PyObject_DelAttrString`), which moved the suite from 10 tests to 12. -**Progress on this item is measured by how far the 168 get, not by whether a chosen test passes** — -that is the mistake that closed it prematurely. - -The history below is kept because the three failure modes it records are exactly what the two -crashes above are, and because §3 still has to classify whatever §2 binds. - -**Was:** 39 of 380 `external fun` declarations bound through `RegisterNatives`. The rest kept the -original wiring, which does not work. - -The original wiring is broken three ways, all measured or read off the code: - -- arguments arrive shifted, because the `@CName` exports declare no `JNIEnv*`/`jobject` while - ART's ordinary convention passes them (`JniWiringTest` proved this on device: sent - `0x5A5A12345678`, received a pointer) -- 78 exports take or return a Kotlin/Native `String`, which is not a JVM `jstring` -- 182 declarations use `JNIPointer?`, a boxed `java.lang.Long` on the JVM, against a raw `jlong` - -The 39 that work do so through `JNI_OnLoad` → `RegisterNatives`, binding CPython's own C -functions directly with no trampoline. Migrating the rest is mechanical **except** for one -judgement per function — see §3. - -**This is not a latent problem — the production API is unusable on Android today.** Calling -`Python3.exec` crashes the process on its very first call, on both API 26 and API 36, because -the functions it reaches (`PyImport_AddModuleRef`, `PyObject_GetAttrString`, `PyRun_String`) -still take a Kotlin `String` straight across JNI. Every Android test that passes does so by -going through `bindings` directly and reconstructing the path by hand; none of them had ever -called the real object model, so nothing caught it. A benchmark added against `Python3.exec` -found it immediately. - -The string-marshalling work (interning) converted three functions. The rest of the several -hundred still carry the original wiring. +**Closed.** 363 of 367 `external fun` are registered through `RegisterNatives`; the four exceptions +are deliberate (`ffiAllocUtf8`/`ffiFreeUtf8`/`ffiReadUtf8`, `echoCriticalNamed`). Evidence: +`PM/androidInstrumentedTest/kotlin/python/native/ffi/JniWiringTest.kt`, +`PM/desktopTest/kotlin/python/native/ffi/JniCallConventionClassificationTest.kt`, and the +`commonTest` suite running on device (`connectedDebugAndroidTest`, see SPEC §0). ## 3. Classify the remaining functions leaf vs re-entrant -**Closed.** The classification is in `docs/investigations/jni-call-convention-audit.md`; the decision procedure a -new call site has to pass is in `androidMain/README.md`. - -`@CriticalNative` and `@FastNative` both stop the collector for the call, and `@CriticalNative` -has no `JNIEnv`, so nothing under it can re-enter the runtime. A function that can execute -arbitrary Python — a module's top-level code, a `__getattr__`, a `__del__` reached by dropping -the last reference — must therefore stay on ordinary JNI. - -**No function was promoted, and that is the finding.** The audit proposed eight promotions to -`@CriticalNative` on the argument that it is expensive below API 34 and cheap above. Measured here -it is the reverse: promoting costs **+16.54 ns on API 34 and +25.73 ns on 36**, against savings of -43-73 ns on 26-31. Three of the eight are not leaves either — `PyGILState_Release` deletes the -thread state on the last release and runs `__del__` through it, `PyThreadState_GetDict` allocates -the thread dict lazily, and `PyEval_InitThreads` is a no-op kept for the stable ABI. - -The 116 registrations added since the audit (71 → 187) are **all ordinary, with no `@CriticalNative` -or `@FastNative` twin registered for any of them**, so the crash this section warns about cannot be -reached through the new surface. - -**The table is at 366 now, and that claim was re-derived rather than carried forward** — it still -holds, and so does every other substantive claim in this section. What did not hold is the numbers: -the audit classified 71 by hand, this section quoted 187, and roughly 180 registrations had never -been classified by anything. `JniCallConventionClassificationTest` derives the classification on -every desktop build, so the count stops being something a document has to remember. It also records -what the audit could not: **65 registrations are undecided**, because a header prototype can condemn -a function and never clear one. - -One change came out of it, in the opposite direction to the audit's: `PyList_GetItem` was pinned to -`@CriticalNative` with no `preferFastNative` branch — the only binding ignoring the device axis, and -the per-element call of bulk iteration, where §5 measured 50 ns per element on API 36 against a -`@CriticalNative` net cost of 44.05 ns there. It now has a `@FastNative` twin and branches like -every other pair. Compile-verified; the device assertion -(`JniWiringTest.bothListGetItemConventionsAgree`) is written but has not been run. +**Closed.** The classification is derived on every desktop build by +`JniCallConventionClassificationTest`; the procedure is in `PM/androidMain/README.md` and +`docs/investigations/jni-call-convention-audit.md`. Measured reason no function was promoted to +`@CriticalNative`: promoting costs +16.54 ns on API 34 and +25.73 ns on API 36. -The default is ordinary JNI, and the promoted set stays enumerated rather than derived: every C API -function can fail, and CPython reports failure by allocating a GC-tracked exception, so "provably -cannot run Python" is not a property any of them has. Guessing wrong toward ordinary costs 6-41 ns; -guessing wrong the other way is a crash once upcalls exist. - -**Was:** depends on §2. Of the 11 migrated at that point, 5 were leaves and 6 re-entrant; -`PyErr_Clear` and `PyObject_GetAttrString` both look like leaves and are not. +- **3.1 `JniWiringTest.bothListGetItemConventionsAgree` on a device — unverified.** The + `PyList_GetItem` `@FastNative` twin was compile-verified when added; whether this assertion has + passed on `pmp_api26` and `pmp_api36` is not recorded per test. Undecided registrations (65 at the + last count) stay on ordinary JNI by default, which is the safe direction; no action is needed for + them. ## 4. Automatic reference release -**Closed.** A thousand Python lists holding a target object are dropped with no explicit -`close()`, Kotlin's collector takes the wrappers, and the target's count falls as the lists are -destroyed. Verified on both collectors — desktop through `java.lang.ref.Cleaner`, iOS through -Kotlin/Native's `createCleaner`. Android below API 33 uses the `PhantomReference` path, which is -now covered on every desktop build — the loop moved to `jvmMain` because nothing in it is -Android-specific and `desktopTest` can reach it there. It could not be tested where it lived: both -emulators are API 36, so they take the `Cleaner` branch and no test in the repository reached the -other one. It held a defect the whole time, and the shape of that defect is the argument for the -move — the drain loop caught `InterruptedException` alone, so one throwing release killed the -thread and every later release leaked in silence. - -**One thing still cannot be freed**: wrappers outliving `Py_Finalize()` are skipped -deliberately, leaving stale pointers if the interpreter is restarted. - -~~And cross-boundary cycles through a *real* application proxy still leak — §7's `tp_traverse` -slot wiring exists and is tested on every target (`ProxyTypeFactory`), but nothing yet makes a -production-generated proxy class an instance of that type, so the mechanism and the object model -it needs to protect are not yet connected. See §7's "What is not done".~~ **Closed by -`work/handleslot` (`99acd830`, 2026-08-18, merged `a1fb825c`)** — `PythonProxySource.renderClass` -now bases a generated class on `_PmGcObject`, which subclasses `_pm_proxy_base` -(`ProxyTypeFactory.createProxyType()`'s published type), whenever `ReflectedClass.hasTraverse` is -set. Production proxies are instances of the traversable type; see §7's account (the paragraph -after `CycleCollectionTest.aPythonSubclassOfTheProxyTypeSurvivesBeingUntrackedTwice`) and §14b -item 5. Cycles that close entirely on the Kotlin side (no Python-side edge for `tp_traverse` to -report) are the part that remains open — `docs/design/object-lifetime.md` has it. - -**The third — the exception window — was not structural, and is closed.** "A raw pointer leaks -if an exception lands between the C call returning it and the wrapper taking ownership" -described real bugs, not a property of the architecture. `OwnershipLeakTest` forces two of them -and measures the count; both were red before the fix by exactly the number of attempts — **+50 -references over 50 attempts in each case**, one lost per attempt, and no collector can reach -them because no Kotlin object ever owned them. - -- `PyDict.fromMap` allocated the dict and populated it afterwards. One unhashable key makes - `PyDict_SetItem` fail partway, and the half-built dict — by then holding a reference to every - key and value already stored — was still a bare pointer. Measured on both the stored value and - the already-stored key: +50 each. -- `PyType.getInstance` is handed a new reference on every call and, on a **cache hit**, dropped - it. The comment argued this was harmless because types are immortal — true of `int` and - `list`, false of every user-defined class, which leaked one reference per `PyObject.Type` - read. Measured against a class defined in Python: +50 over 50 reads. - -The fix is `adoptingNewReference` in `PyObject.kt` — release the reference if, and only if, the -block that was going to adopt it throws — plus `try/finally` at the sites that hold a new -reference across a second C call: `repr`, `toString`, `richCompare`, `PyType.name`, -`snapshotElements`, `PyDict.snapshotEntries`, the four `fromList`/`fromSet` scratch tuples, -`deriveTypeAndRelease` (which abandoned its scratch instance outright when `PyObject_Type` -failed), `PyException.messageOf` and `isNoneObject`. `PyException.fromExceptionInstance` now -wraps the exception instance **first**, so a failure while building the message or walking -`__context__`/`__cause__` can no longer strand the exception itself — in the one path that -exists to report failures. - -One rule came out of it, and it is written on the helper: **it must never wrap a constructor -that can throw.** `PyAutoCloseable` registers the cleaner in its own constructor, before any -subclass initialiser runs, so a wrapper that throws from `init` has *already* queued a release; -releasing again would be the double free of §1. `PyType.getInstance` therefore validates before -it constructs rather than from `init`. - -Only the two above are proven by a red-then-green test; the rest are windows that no current -call site can force (they need `PyUnicode_AsUTF8` or `PyTuple_GetItem` to fail on a live -object). They are fixed as structure, not as measured leaks, and are marked as such here. - -**Was:** depends on §1. - -**Closed.** Both the explicit and automatic (GC-driven) release paths are fully functional on JVM and Kotlin/Native. The block recorded in §1 (GIL locking on parked thread) was resolved, unblocking the background cleaners. - -Measurements prove that GC-driven release actually drops CPython reference counts: -- `GCLeakTest.testReferenceCountDecreasesOnGC` verifies a simple single-object wrapper lifecycle. -- `GCLeakTest.testCascadingReleaseOnGC` verifies complex nested structures: appending a target object to 1000 Python lists, then dropping the Kotlin wrappers for those outer lists. The Kotlin GC correctly collects the list wrappers, their cleaners call `Py_DecRef` on the lists, which cascades into CPython freeing the lists and decrementing the target object's reference count. - -**Platform Differences:** -- **JVM (`desktopMain`)**: Uses `java.lang.ref.Cleaner`, driven by a dedicated background thread. -- **Kotlin/Native (`iosMain`, etc)**: Uses `kotlin.native.ref.createCleaner`, which runs on a dedicated worker thread, allowing asynchronous non-blocking cleanup. -- **Android (`androidMain`)**: Uses `Cleaner` where available (API 33+), with a fallback to `PhantomReference` requiring background polling on older devices. (Testing on device requires `androidInstrumentedTest` setup — see §11b). - -**What fundamentally cannot be released in this architecture:** -1. ~~**Uncaught exceptions during FFI allocation**: If Kotlin code calls a C-API function that returns a new reference (e.g., `PyObject_GetAttrString`), and a Kotlin exception disrupts the control flow *before* that raw pointer is wrapped in `PyObject(..., borrowed = false)` or explicitly `Py_DecRef`'d via a `finally` block, the CPython reference leaks permanently.~~ **Closed — see above.** This was a list of missing `finally` blocks, not a limit of the architecture. -2. **Post-Finalize GC**: When `Py_Finalize()` executes, it frees CPython's heap. If Kotlin wrappers are GC'd *after* this, their cleaners see `!Python3.isInitialized` and exit early to avoid segfaults. While safe for shutdown, if the interpreter is later re-initialized via `Py_Initialize()`, those dangling wrappers will retain pointers that either point to unmapped memory or alias newly allocated CPython objects. -3. **Cross-boundary cycles**: A Python object holding an upcall proxy to a Kotlin object, which in turn holds a `PyObject` pointing back to the Python object. Neither language's GC can trace through the other, leading to a permanent leak unless broken manually or addressed via `tp_traverse` (see §7). +Closed for the ordinary case: wrappers release on GC on every target +(`PM/commonTest/.../ref/GCLeakTest.kt`, `OwnershipLeakTest.kt`, `PM/jvmTest/.../ref/PhantomCleanerRegistryTest.kt`, +`PM/androidInstrumentedTest/.../ref/AndroidCleanerPathTest.kt`), and cross-boundary cycles with a +Python-side edge are collected (§7, SPEC M-3). + +- **4.1 Cycles that close entirely on the Kotlin side — open.** `HandleTable` holds every entry + strongly by construction (`PM/commonMain/kotlin/python/multiplatform/reflection/HandleTable.kt`, + KDoc "This table leaks by construction"), so a cycle with no Python-side edge for `tp_traverse` to + report never collects. The mechanism (weak table entries, the proxy supplying the strength) is + named but not worked through in `docs/design/object-lifetime.md`. +- **4.2 Wrappers that outlive `Py_Finalize()` — open (known limitation).** Their cleaners see + `!Python3.isInitialized` and skip the release (`Python3.finalize`), which is safe at shutdown but + leaves stale pointers if the interpreter is ever initialised again. No re-initialisation test + exists; `finalize` is `internal` and documented as "will not be able to re-initialize". ## 5. Composition (조립) -**Measured, on device:** - -| | per-call | composed | | -|---|---|---|---| -| `getAttr`, API 36 hardware | 3929.84 ns | 713.34 ns | 5.5x | -| `list → LongArray` 1000 elems, API 36 hardware | 50065.89 ns | 4532.55 ns | 11.0x | -| `getAttr`, API 26 emulator | 2431.34 ns | 244.70 ns | 9.9x | -| `list → LongArray`, API 26 emulator | 4890.83 ns | 4113.13 ns | 1.2x | - -Note what this does and does not say. Four crossings at ~7ns is 28ns, which does not explain -3929 against 713 — most of the single-call win is the `ffiAllocUtf8` malloc/copy/free round trip -disappearing, not the crossing count. The crossing count genuinely dominates only in the bulk -case, where it scales with N. - -API 26 barely benefits in the bulk case because its per-element crossings run under -`@CriticalNative` and are already free there. Composition pays most where per-crossing cost is -highest, which is modern Android. - -**State:** `artMain/JniExport.kt` still exports composed `Python3` functions -(`initialize`, `finalize`, `internalIsInitialized`, …) and **nothing on the JVM side calls -them** — the `external fun` declarations that used to live in `PyObject.android.kt` were -removed by `0fae961a`. Probes exist (`asmGetAttr`, `asmListToArray`, `asmExec`) and are -benchmarked, but the production path does not use composition anywhere. - -Composed operations belong in the FFI layer as `expect`/`actual`, not in the object model — -`PyObject` must not reference `bindings` (see `docs/design/architecture.md`). Restoring composition -therefore does **not** require reopening `PyObject` as `expect`/`actual`. - -`Python3.exec` was measured too, and it contradicts the reasoning above: - -| | per-call | composed | | -|---|---|---|---| -| `exec("x = 1 + 1")`, API 36 emulator | 17799.46 ns | 10123.94 ns | **1.76x** | -| `exec("x = 1 + 1")`, API 26 emulator | 17798.43 ns | 9570.26 ns | **1.86x** | - -Composing a realistic binder call — one that parses, compiles and executes Python — still -nearly halves it. The prediction beforehand was "a few percent", on the argument that Python's -own work would dominate. It does not: about 7.7 µs of the 17.8 µs is boundary cost. - -The mechanism is visible in the numbers. 7.7 µs over ~11 crossings would be 700 ns each, two -orders of magnitude above the 7 ns a crossing actually costs. And the per-call figure is -identical on API 26 and API 36 (17798 vs 17799) despite those devices differing by 20x in -per-crossing cost. So this is not crossings — it is the three `ffiAllocUtf8` round trips, each -a JNI call plus `GetStringUTFChars` plus malloc plus copy plus free. - -Which means composition is worth extending past the object model, and that **string-carrying -operations are the ones to compose first**, regardless of how much real work they do. - -### Most of it can be had without composing at all +String-carrying operations: **closed** — interning beats composing on every API level (`getAttr` +160 ns on API 36, 167 ns on API 26 interned; `docs/design/marshalling-design.md`). -`getAttr`, measured three ways on the same device: - -| | API 36 | API 26 | -|---|---|---| -| per-call (`ffiAllocUtf8`) | 2238.25 ns | 2488.62 ns | -| composed (`asmGetAttr`) | 463.60 ns — 4.83x | 243.08 ns — 10.24x | -| **direct buffer, no composition** | **425.90 ns — 5.26x** | 656.81 ns — 3.79x | - -On modern Android the marshalling fix alone **beats** composition, and it needs none of the -machinery: no `artMain` export, no per-operation composed function, no JVM-side wiring. One -`DirectByteBuffer` per thread, its address taken once, and every call afterwards is encode-and- -copy inside the JVM followed by passing a `long`. It applies to every string-carrying operation -automatically rather than one at a time. - -This is the same thing Panama does on desktop, which is why desktop's marshalling is ~200 ns -per string against Android's ~2500 ns. PanamaPort would have brought it too — its -`allocateFrom(String)` runs through `AndroidUnsafe.allocateMemory` → `sun.misc.Unsafe`, a JVM -intrinsic with no crossing. The earlier decision to skip PanamaPort was argued on downcall stubs -alone and missed that its memory model was the part that mattered here. - -API 26 still favours composing, by roughly 2.7x. But `@CriticalNative` is already free at that -level, so old devices were never the ones paying for crossings, and they are a shrinking share. - -**So §5 shrinks.** Composition stays justified only where crossing count genuinely dominates — -bulk iteration, which scales with N (11x at 1000 elements). For single string-carrying calls, -fix the marshalling instead. - -### Interning settles it, on every API level - -The direct-buffer path lost to composition on API 26, and the reason was not crossings — both -make exactly one ordinary-JNI call there. It was where the string gets encoded: composition -hands the jstring to `GetStringUTFChars` and encodes natively, while the direct-buffer path -encodes in Java, and API 26's ART is much worse at that. - -Attacking the encode rather than the crossing: - -| `getAttr`, ns | API 36 | API 26 | -|---|---|---| -| composed | 528.85 | 255.19 | -| direct buffer + `toByteArray` | 694.03 | 800.96 | -| direct buffer + ASCII fast path | 373.18 | 248.94 | -| **interned C string** | **160.25** | **167.33** | - -**Interning wins on both, and wins uniformly** — 160 against 167 ns, where composition differs -by 2x between the same two devices. It is 3.3x better than composing on API 36 and 1.5x better -on API 26, so the API-26-versus-modern split disappears entirely. - -That is not surprising in hindsight: attribute and module names are repeated literals, so the -encode is pure waste after the first time. CPython interns its own strings for the same reason. - -The ASCII fast path alone already matches or beats composition on both levels (373 vs 528, -249 vs 255), which makes it the right fallback for strings that are not worth caching. - -**Design consequence.** The `expect`/`actual` memory abstraction should be built around -`String → cached C string address`, not around a general `Arena`. A bound on the cache is -required — callers can pass arbitrary strings — with the ASCII path handling everything that -misses. `Arena`-style allocation is still needed for non-string buffers, but it is not the -centre of the API. - -**This closes composition for string-carrying operations at every API level.** What remains of -§5 is bulk iteration only, where crossing count genuinely scales with N. +- **5.1 Bulk iteration through a composed call — open (optional optimisation).** Composition pays + only where crossings scale with N: `list → LongArray` of 1000 elements measured 11.0x faster + composed on API 36 hardware (50 065.89 → 4 532.55 ns, + `PM/androidInstrumentedTest/kotlin/python/native/ffi/CompositionBenchmark.kt`). The probe + (`asmListToArray`, `PM/androidMain/.../bindings.kt`) exists; no production path uses it. Also left: + `PM/artMain/kotlin/python/multiplatform/ffi/JniExport.kt` still exports composed `Python3` + functions that nothing on the JVM side calls — decide whether to wire or delete them. ## 6. Composition on desktop — measured, and the answer is no -**Measured.** The same question Android answered at 1.8x, asked on desktop without building a -composed path: run `exec`'s three C calls with the C strings allocated per iteration, then with -them allocated once. The gap is what composing could remove. - -| | ns | -|---|---| -| `exec("x = 1 + 1")`, strings allocated per call | 5768.04 | -| same, strings hoisted out of the loop | 5161.29 | -| **marshalling share** | **606.74 (10.5% of the call, 1.12x)** | - -Against Android's 43% and 1.8x. The difference is what the two platforms do to produce a C -string: Android's `ffiAllocUtf8` is a JNI round trip — a crossing, then `GetStringUTFChars`, -then malloc, copy, free — at roughly 2.5 µs per string. Desktop's `withUtf8` is a Panama -`allocateFrom` with no crossing at all, about 200 ns per string. +**Closed** (composing would buy ~10 % of an `exec`; not worth new native targets). -So composing on desktop would buy ~10%, and cost new Kotlin/Native targets for macOS and Linux, -a packaging path for the resulting library, and a permanent asymmetry where Windows cannot -participate. **Not worth it.** This item is closed unless something changes the premise. - -That cheaper win has since been taken: all 45 string-carrying wrappers moved to `invokeExact`, -so `bindings.kt` now has 310 exact calls and zero inexact ones. Marshalling went from -606.74 ns to **440.03 ns** (10.5% → 8.1% of the call). Kotlin passes the `Long` out of -`withUtf8`'s lambda as a primitive `long`, so no descriptor changes were needed. - -The benchmark that produced those numbers is now `@Ignore`d. Running `exec` 200,000 times left -the JVM aborting inside `PyDict_New` and took the suite from 108 tests down to 2. Clearing the -error indicator on its failure paths recovered most of that — a pending exception was part of -it — but an abort still followed the run, and the cause is not pinned. Re-enable only after -that is understood. - -Also worth recording for whoever revisits this: the Stable ABI closes the obvious bulk shortcut -regardless. `PySequence_Fast_ITEMS` is a macro reading `PyListObject->ob_item`, and `abi3t` -(3.15 free-threaded) makes `PyObject` an incomplete type, so there is no supported way to -extract list items in one call without shipping our own native code. +- **6.1 `DesktopOverheadBenchmark.stringMarshallingShareOfARealisticCall` is `@Ignore`d — open.** + Running `exec` 200 000 times aborts the JVM inside `PyDict_New`; clearing the error indicator on + failure paths recovered most of it but the abort's cause is not pinned + (`PM/desktopTest/kotlin/python/native/ffi/DesktopOverheadBenchmark.kt`). Re-enable only once + understood (SPEC X-1). ## 7. Python → Kotlin binder (upcalls) -**Both halves now exist, and the design premise is proven rather than argued.** - -The runtime is in `reflection/` — `HandleTable` (slot plus generation, so a released handle -cannot alias onto whatever takes its slot), `UpcallTable`, `ExposedCallable`, `ObjectReference`. -The generator is `python-multiplatform-ksp/`, wired into a user module by -`python-multiplatform-gradle-plugin/` (one `id(...)`, no per-target `add("ksp", ...)`). -Fixture modules under `ksp-fixtures/` run 35 tests against a table KSP actually generated, not a -hand-written one — 29 on desktop, and 6 more from `ksp-fixtures/android`, the fixture that carries -an Android plugin (§13). - -**It survives a GraalVM native image**, which is the condition the whole design was chosen for: - -``` -KOTLIN: table = 19 entries, 3 classes, from io_github_thisisthepy_sample -PYTHON: resolved handle = 4294967314 -PYTHON: invoke result = 7 -PYTHON: @PythonInternal entry resolves to -1 -PYTHON: UPCALL_OK -``` - -**Re-verified after the upcall runtime, the KSP table and the artefact walker landed** — the claim -above was made before all three, so it was worth re-running rather than carrying forward: - -``` -KOTLIN: table = 41 entries, 4 classes, from io_github_thisisthepy_sample -PYTHON: invoke result = 7 -PYTHON: invoke_args result = 'presses x3 = 21' -PYTHON: @PythonInternal entry resolves to -1 -PYTHON: UPCALL_OK / PROXY_OK -``` - -19 entries became 41 because `ExposedToPython.kt` grew; the exposure policy still refuses -`@PythonInternal` with `-1` under the closed world, which is the part that had to hold. The -procedure, including the toolchain path and why `JAVA_HOME` must stay on JDK 21 while native-image -runs on 25, is in `docs/platforms/graal-native-image-verification.md`. - -Python builds a function pointer with `ctypes`, resolves a Kotlin declaration by name through -`HandleTable`, and calls back into Kotlin — inside a closed world where runtime reflection is -forbidden. Every wall hit getting there was metadata or wiring; the lookup and invoke path itself -needed no reflection registration, because it uses none. `sample` carries the build path -(`nativeCompile`, `runNativeUpcallDemo`, Liberica NIK). - -**The table it resolves against is generated now, not hand-written.** `NativeImageMain.kt` used to -carry its own `FunctionTableFragment` returning a constant `42`, with a comment saying the -generator did not exist yet — and a hand-written fragment passes whether or not KSP ever runs, so -the check measured less than it appeared to. It installs `python.multiplatform.generated.FunctionTable` -— generated from `:sample`'s own sources — instead, and the number Python reads back (`7`) is one the process -produced by calling the Kotlin object seven times, so a stub returning a constant cannot satisfy -it. The `-1` line is the same check applied to `@PythonInternal`: an opted-out declaration must -not resolve. - -**The metadata is generated, not maintained.** `reachability-metadata.json` was a checked-in file, -and a checked-in file rots without saying so: a native image links a `FunctionDescriptor` the -metadata does not declare, builds clean, and dies on the first call with -`MissingForeignRegistrationError`. It had already rotted — it declared one upcall descriptor while -`Panama` builds three, so `ProxyTypeFactory` (the `tp_traverse` wiring below) would have failed in -any image that reached it. `generateDesktopReachabilityMetadata` now derives the `foreign` section -from `bindings.kt`, `ShapeDowncalls.desktop.kt` and `Panama.kt`, and fails the build on a -declaration form it cannot read rather than dropping the descriptor. `ReachabilityMetadataTest` -re-checks the result against the `MethodHandle.type()`s the loaded classes actually produce, and -`runNativeUpcallDemo` now asserts on the `UPCALL_OK` marker instead of on the exit status. The -failure mode was confirmed by deliberately mis-declaring one descriptor and watching the binary -die at runtime, not argued from the documentation. - -**The binary is 16.7 MB, down from 35.4 MB.** The largest single item in the image was CPython -itself: a registered resource is baked into the image heap as a `byte[]`, so the binary carried -19.4 MB of `libpython` only so that `manager.kt` could write it back out to a temporary file at -startup. It has to be on disk anyway — `PYTHONHOME` must point at a prefix with a matching stdlib -for `Py_Initialize` to get past `encodings` — so `manager.kt` loads it from that prefix when the -classpath copy is absent (`PYTHON_MULTIPLATFORM_LIBPYTHON` overrides), and the resource -registration is gone. - -**What is not done**, and should not be read as done: - -- ~~`tp_traverse` functions are generated and tested, but nothing wires them into CPython's actual - `tp_traverse` slot, and `tp_clear` and Kotlin-side cycle closing are untouched.~~ **The slot - wiring is closed** — `work/cycles` (`83233767`, merged `aa5436d1`, 2026-08-14) built - `ProxyTypeFactory.createProxyType()` on every target: a real `PyType_FromSpec` heap type with - `Py_TPFLAGS_HAVE_GC`, `tp_traverse`/`tp_clear`/`tp_dealloc` slots that call the KSP-generated - traverse function through `ClassLookup`, and `tp_dealloc` releasing the `HandleTable` entry. - `CycleCollectionTest` (desktop, Kotlin/Native, ART) and `WasmCycleCollectionTest` build a real - `P -> proxy -> handle -> Kotlin -> PyObject -> P` cycle, drop it, call `gc.collect()`, and check - the handle is gone — re-verified on this tree 2026-08-18 (both tests pass; see this section's - audit entry below). ~~The production proxy classes `PythonProxySource` renders never construct an - instance of `ProxyTypeFactory`'s type.~~ **Closed on desktop first** (`work/handleslot`, - 2026-08-18), **and by the end of the same day on every other target too — see the "Both are - closed too" and "Closed on iOS and androidNative too" entries below; this heading undersold the - scope even as it was being written.** `dbf54e21` (same day, `work/untrack`) had already answered the risk this item used - to end on — a Python subclass of the heap type survives being `PyObject_GC_UnTrack`ed twice, so - `subtype_dealloc`'s own untrack ahead of `ProxyType.tp_dealloc`'s is safe — and had also found - that inheritance alone buys nothing: the handle has to reach the type's own storage before a - `tp_traverse` inherited through a Python subclass has anything to read, and nothing wrote it - there. That is what closes here, on desktop only: - - - `ProxyTypeFactory` (desktop) grew a `Py_tp_members` slot: one `PyMemberDef` named `_pm_handle`, - `Py_T_LONG`, carrying `Py_RELATIVE_OFFSET` (CPython 3.12+, exactly the flag documented for a - type extending storage under multiple/deep inheritance — precisely this case). Offset 0 is the - same base `PyObject_GetTypeData` already computes for `tp_traverse`/`takeHandle`, so - `self._pm_handle = ...` on an instance of this type — including a further Python-level - subclass — writes the C slot directly, with **zero change** to the constructor line - `PythonProxySource` already renders. - - `ProxyTypeFactory.installGcBase()` (new `expect`/`actual`, `false` on every target but desktop) - publishes the type into `__main__` as `_pm_proxy_base` via `PyObject_SetAttrString` — the type's - own address is a valid `PyObject*`, so this needs no `ctypes` round trip (unlike the - proof-of-concept in `dbf54e21`'s test) and nothing new for wasm's missing `ctypes` to block. - - `PythonProxySource` gained `_PmGcObject`: `_pm_proxy_base` subclassed if the bootstrap set it, - else `_PmObject` (today's behaviour, unchanged, on every non-desktop target). `renderClass` uses - it instead of `_PmObject` exactly when `cls.hasTraverse` is true **and** the class has no - metaclass — `_pm_owned_new` only ever adds `_PmObject` to a metaclassed class's bases, and the - two owners cannot be combined (`class _PmGcObject(_pm_gc_base, _PmObject):` is a real - `TypeError: multiple bases have instance lay-out conflict`, both being solid bases). A class - with no traversable `PyObject` field, or one with a metaclass, renders exactly as it did before - — `RefHolderCycleCollectionTest.anOrdinaryClassWithNoPyObjectFieldsKeepsRenderingOnTheOldOwner` - pins `class Counter(_PmObject):` unchanged. - - Proved against a *real* KSP-exposed class, not the hand-built proxy the existing cycle tests - construct through direct C-API calls: `ksp-fixtures:app`'s new `RefHolderCycleCollectionTest` - installs the generated table, `exec`s the real generated `fixture.library.RefHolder` proxy, and - builds `_rh -> (opaque handle) -> Kotlin field -> [a plain list] -> _rh` — a cycle whose only - edge back to `_rh` lives on the Kotlin heap, unlike `ProxyHandleLifetimeTest`'s existing - `c.box -> box -> c`, which closes entirely through ordinary Python attributes `subtype_traverse` - already walks and so does not exercise this gap at all. Red before the fix - (`HandleTable.liveCount` stayed at `baseline + 1` after `gc.collect()` — the leak, observed, not - inferred), green after (three tests, 0 failed). - - ~~A real, pre-existing, and unrelated bug surfaced while building that test and is **not fixed - here**: `_pm_unwrap` unwraps *any* `_PmObject`-derived argument to its raw `HandleTable` handle - before a call, which is correct for an owned-result argument but wrong for a `PyObject`-typed - Kotlin parameter — `_rh.primary = _rh` (as opposed to the list-wrapped version the test uses) - resolves to the underlying Kotlin object on the way in and fails the property's own cast - (`ClassCastException`, observed). Filed here rather than fixed because it is orthogonal to the - handle slot and was not this task's spec.~~ **Fixed** (`886a8e8f`): `PythonProxySource.argValues` - now unwraps a `TypeTag.OBJECT` argument only when the parameter's *declared* type is not - `python.multiplatform.ffi.PyObject`, read off `ExposedCallable.paramTypeNames[i]` — the same - field the return direction already used for this ambiguity — instead of testing the value's - shape. `ProxyObjectArgumentTest` (`ksp-fixtures:app`) pins both `_rh.primary = _rh` and the - mirror-image case a base-class test had been hiding by accident (a traversable class's proxy is - never an `_PmObject`, so testing only that base meant every proxy of a class with a `PyObject` - field skipped unwrapping regardless of what its parameter declared). - One instance was left latent and has since been fixed too, in this branch: - `PythonCallables.Fragment`'s `pythonx.runtime.newFunction` declared its `body` slot - `paramTypeNames[0] = "kotlin.Any"` while its callable does `args[0] as PyObject` — the same - defect from the other type name, reachable only through a caller that does not exist in this - repository (`PythonxAdapter._make_function` calls `newFunction` by resolved handle directly, - never through a `PythonProxySource`-rendered wrapper). Reproduced deliberately by building that - caller (`PythonCallablesProxyArgumentTest`, `desktopTest`: registers `PythonCallables.Fragment` - into the same `UpcallTable` as a real proxy class and calls the rendered - `pythonx.runtime.newFunction` with a proxy in `body`) — before the fix it raised - `RuntimeError: class ...ProxyCounter cannot be cast to class ...PyObject`, observed, not - inferred. Fixed by declaring `body` as `python.multiplatform.ffi.PyObject`; - `PythonxTableSourceTest` pins the corrected row. Checked before committing to the fix: nothing - dispatches this entry through `PythonxAdapter`'s own `_coerce`/`_TABLE` allowlist (it is always - invoked by literal resolved handle), so the changed declared type does not move any Python-side - behaviour that exists today — it only fixes the row a future `_coerce`-routed caller would see. - - **Left exactly where it was** on Android, iOS, androidNative and wasm: `installGcBase()` is a - literal `false`, `PythonProxySource` still falls back to `_PmObject` there, and nothing about - those four targets' `ProxyTypeFactory` changed. What each one still needs, per `ProxyTypeFactory`'s - now-updated class doc: Android already has `ProxyTypeFactory.setHandle` (a native function a - generated `__init__` could call — option (B)) but no Python-visible base to subclass and no - `__init__` rewrite to call it; iOS/androidNative's shared `nativeMain` factory has neither a - `Py_tp_members` slot nor a setter, and would need `PyMemberDef`/`PyType_Slot` filled in by - cinterop-generated structs rather than `sun.misc.Unsafe`, which is most of why desktop went - first; wasm has the `PyObject_SetAttrString` publication route available (no `ctypes` needed, - same as desktop) but is missing the `Py_tp_members` slot itself, and its `PyMemberDef`/ - `PyType_Slot` are 20/8-byte 32-bit structs, not the 64-bit layout desktop's `Unsafe` code - assumes — the pointer-width trap `dbf54e21` already paid for once. - - Verified: `desktop 485/0`, `wasm 440/0` (browser 10, node 430), `ksp-fixtures:app:desktopTest` - 64→67/0 (three new tests), `compileKotlinAndroidNativeArm64` and `compileDebugKotlinAndroid` - both green — `git diff --stat` touches only the five `ProxyTypeFactory.kt` files, - `PythonProxySource.kt`, and the one new test file. - - **Closed on iOS and androidNative too** (`work/hs-native`, 2026-08-18), confirming the desktop - doc's own prediction that the two cinterop targets would be *easier* than desktop rather than - harder: `PyType_Spec`/`PyType_Slot`/`PyMemberDef` come out of cinterop as real structs with - named fields, so `nativeMain`'s `ProxyTypeFactory.createProxyType()` fills in a `Py_tp_members` - slot — `_pm_handle`, `Py_T_LONG`, `Py_RELATIVE_OFFSET` — by field assignment instead of - `sun.misc.Unsafe` offset arithmetic, and `installGcBase()` publishes the type into `__main__` - exactly as desktop's does (`PyObject_SetAttrString` on the type's own address, no `ctypes`). - Because `nativeMain` is shared, this is one implementation for both targets rather than two. - - Proved with a native test rather than a KSP fixture — `ksp-fixtures/app` has no iOS leaf, so - the RefHolder-through-KSP route the desktop proof used does not reach this platform. Instead - `nativeTest/.../CycleCollectionTest.kt` gained - `aHandleWrittenFromPythonClosesACycleThroughAKotlinField`: it publishes `_pm_proxy_base`, - defines a Python subclass (`__slots__ = ()`, so no `__dict__` to fall back to), and writes - `self._pm_handle = h` — the exact line a generated `__init__` emits — then reads the C slot - directly (`PyObject_GetTypeData`) to confirm the write landed there rather than merely - succeeding as an ordinary attribute, before closing a cycle through a Kotlin field and - collecting it. Red before the fix (`installGcBase()` returning `false` fails the first - assertion), green after — `iosSimulatorArm64Test` 429/0 (verified as the same run, not - inferred: the new test's own `` entry carries no failure or error node), `desktop - 500/0` unchanged, `compileKotlinAndroidNativeArm64` and `compileTestKotlinAndroidNativeArm64` - both green (androidNative has no test device attached in this environment, so only compiled, - per this file's own standing note on §11.1's device constraint). `git diff --stat` touches only - `ProxyTypeFactory.kt` (nativeMain) and the one test file. - - ~~**Still open**: Android and wasm, per the per-target notes above — unchanged by this round.~~ - **Both are closed too** (`a5387511` for wasm, `1544f095` for Android — both landed before the - iOS/androidNative round above, and this section was never updated to carry them forward; caught - while auditing this file against the code rather than the commit log). Verified by reading the - current source, not by re-running the commits' own numbers: `wasmJsMain/ProxyTypeFactory.kt`'s - `createProxyType()` fills the `Py_tp_members` slot with `PY_T_LONGLONG` for `_pm_handle` (eight - bytes on wasm32, where a plain `long` is only four — the corruption `a5387511` traces losing the - top half of the handle on assignment), and its `installGcBase()` publishes `_pm_proxy_base` via - `PyObject_SetAttrString` exactly as desktop's does, no longer a literal `false`. - `artMain/cinterop/jni_onload.def` declares the same member (`"_pm_handle", Py_T_LONG, - Py_RELATIVE_OFFSET`, wired at `Py_tp_members` slot 3) as a real C struct compiled against the - JNI headers, and `androidMain/ProxyTypeFactory.kt`'s `installGcBase()` is a real - `createProxyType()` + `PyObject_SetAttrStringN` call, not a stub. `Py_T_LONG` rather than - `Py_T_LONGLONG` is correct here and not the wasm bug repeating: Android is a 64-bit-only target, - where C `long` is already eight bytes. - **So the handle slot is now real on all five targets** (desktop, wasm, Android, iOS, - androidNative). The one gap left is the one `581ec772` already named: - androidNative compiles (`compileKotlinAndroidNativeArm64`, - `compileTestKotlinAndroidNativeArm64`) but has never run its tests on a device or emulator in - this environment — nothing attached, same standing constraint as §11.1. Desktop, wasm, Android - (emulator) and iOS simulator all have a green test run cited in this section; androidNative does - not. -- ~~Companion-object members, interfaces, enums and annotation classes are not exposed.~~ - **Closed**, except annotation classes, which are now deliberately excluded — see below. -- ~~The aggregator uses `Dependencies.ALL_FILES`, correct but reprocessed every build.~~ - **Measured, and the aggregator turned out not to be the cause** — see below. -- ~~No convenience Gradle plugin; user modules wire KSP per target by hand.~~ - **Closed:** `python-multiplatform-gradle-plugin/`, applied by id. ~~**Except on Android**, where - applying it is a configuration-time crash until AGP moves to 8.10.~~ **Fixed too, same day** — - `gradle/libs.versions.toml` pins AGP `8.10.1` (KSP 2.3.11's declared minimum) and Gradle `8.11.1`; - see "The convenience plugin could not be applied to an Android module — closed" in §13. - -**Was:** entirely unimplemented — `ClassLookup.kt`, `ObjectReference.kt` and `ReflectedClass.kt` -held 1–3 lines each, and this was README's only unchecked box. - -Design is settled in `docs/design/upcall-design.md`: build-time generated function table (runtime -reflection is impossible on Kotlin/Native and under GraalVM's closed world), blacklist exposure -(all `public`, minus an opt-out annotation), name resolved once with the handle cached in the -Python proxy's instance data, KSP running in user modules too. - -**Module fragment collection is settled** and demonstrated, not inferred. Library modules -generate fragments into a well-known package; the app module's KSP finds them with -`getDeclarationsFromPackage` and emits an aggregator holding explicit references. No -`ServiceLoader`, no reflection, no `@EagerInitialization`. A three-module experiment under -`ksp-fixtures/` compiles and runs, discovering fragments across a module boundary. Two caveats -carried over: `.klib` discovery on a Native target is inferred rather than tested, and -`getDeclarationsFromPackage` is `@KspExperimental`, so keep that step swappable. See -`docs/design/upcall-table-design.md`. - -**The generator exists now.** `python-multiplatform-ksp/` is the shipped KSP processor (library -role emits a fragment, app role emits a fragment for itself then aggregates every `Fragment_*` -it can see). `ksp-fixtures/{library,app}` is the TDD harness: `GeneratedTableTest` (11 tests, JVM) -runs `UpcallTableTest`'s exact scenarios — constructor/method/getter/setter through -`HandleTable`, `@PythonInternal` exclusion, narrow `Int`/`Float` boundary widening, `tp_traverse` -field detection — against a table KSP generated rather than a hand-written fragment. `.klib` -discovery is no longer inferred: the same pair on `androidNativeArm64` found the library's -fragment from its compiled `.klib` and linked a real test binary (compiled and linked only, not -run — no device in this workspace). See `docs/design/upcall-table-design.md` for the full account, -including the one place the doc's own sketch was wrong (fragments must be `public`, not -`internal` — Kotlin enforces `internal` per module, and the app module compiling generated code -that references a library's fragment is a different module even inside one Gradle build). - -**Tree shaking is not solved, but it is now measured.** A table referencing every `public` -declaration does defeat dead-code elimination, and that is inherent to blacklist exposure plus -static linking. 200 synthetic exposed functions cost ≈1.84 KB/entry in a stripped -`androidNativeArm64` binary (368,640 bytes total) — a floor, since the synthetic functions were -trivial one-liners and a real function's body adds its own size on top. Whether that floor plus a -real library's bodies is acceptable is a product judgement the measurement informs but does not -settle. See `docs/design/upcall-table-design.md` §4. - -**The declaration surface is now the whole one, and what is left out is left out on purpose.** -Companion members and `object` members become receiver-less entries under the *owner's* name -(`Owner.member`, two new `CallableKind`s: `STATIC_GETTER`/`STATIC_SETTER`); interfaces get entries -and no constructor, so a Kotlin object reaching Python through a handle is callable through the -interface even when its concrete class does not redeclare the member; an `enum class` gets one -`STATIC_GETTER` per entry plus `name`, `ordinal` and `valueOf`, and its `ReflectedClass` carries -`enumEntryNames` so the Python side can build an `enum.Enum` mirror without the boundary having to -marshal a collection. Nested declarations and top-level properties came along with it. -**Annotation classes are exposed nowhere**: applying one is a compile-time act and reading one -back needs runtime reflection, which is the single thing this design cannot have, so an instance -Python could construct would have nothing to attach to. `values()`/`entries` are out for the same -kind of reason — they return collections the boundary cannot carry. - -Three latent generator bugs surfaced while widening the surface, each of which produced a -*generated file that did not compile* rather than anything the processor could detect: a cast to a -generic type without its arguments (`args[0] as kotlin.collections.List`), a constructor entry for -an abstract class, and a declaration over a type parameter (`args[0] as T`). Parameter types now -render their arguments; abstract, sealed and `inner` classes get no constructor entry; generic -declarations are not exposed at all. `data class` `copy`/`componentN` are dropped too — KSP does -report them (it does not report `equals`/`hashCode`/`toString`), and `binding-policy.md` already -said compiler-generated members stay out. - -**Incremental aggregation: measured, and the aggregator was never the problem.** KSP's own dirty -set (`ksp.incremental.log=true`, `build/kspCaches/.../kspDirtySet.log`) reports **100% dirty on a -one-file change in a two-file module both before and after** narrowing the aggregator's -`Dependencies` from `ALL_FILES` to the fragment files it actually reads. The cause is one level -down: a module's own `Fragment_` is legitimately an aggregating output over *every* source -file — blacklist exposure means any file can add an entry — so any change regenerates it, and KSP -then marks every file that maps to it dirty. `ALL_FILES` on the aggregator stays, because the -narrower form buys nothing measurable and its safety under a classpath-only change was never -established. The only route to real incrementality is per-file fragments (one isolating output per -source file), which changes fragment naming, `UpcallTable`'s per-module idempotency and duplicate -detection — a design change, not a tweak. Recorded in `docs/design/upcall-table-design.md` §11.5. - -**Cycle collection: the generator's half is done, and so — separately — is the slot-wiring half.** -`tp_traverse` on the Python proxy must reach through the handle into the Kotlin object's -`PyObject`-typed fields, which means the generator emits a traverse function per exposed class -alongside the call entries. That part is implemented and tested (`FragmentScanner` detects -`PyObject`-typed — including subclass-typed — fields via `isPyObjectType`, emits -`traverse = { obj, visit -> ... }`, and `ReflectedClass.hasTraverse` / `.traverse(...)` are -exercised end to end in `ksp-fixtures`). This section used to say the *rest* — the proxy type's -`tp_traverse` slot wiring, `tp_clear` mutating Kotlin state — was untouched. It was written before -`work/cycles` (merged `aa5436d1`, 2026-08-14): `ProxyTypeFactory.createProxyType()` now builds a -real `Py_TPFLAGS_HAVE_GC` heap type on every target with those two slots (plus `tp_dealloc`) wired -to exactly this generated function, and `CycleCollectionTest`/`WasmCycleCollectionTest` prove a -real cycle collects (re-run on this tree 2026-08-18: both pass). ~~Re-verifying that claim for this -task surfaced the part that is genuinely still open and is *not* what this section used to -describe: **nothing routes a production proxy through that type.** `PythonProxySource` — the -renderer `install()` actually `exec`s — builds every class on a plain Python `_PmObject` base, and -`ProxyTypeFactory.createProxyType()` is called from no non-test code on any target.~~ **Closed by -`work/handleslot` (`99acd830`, 2026-08-18, merged `a1fb825c`), the same day this was written.** -`PythonProxySource.renderClass` now emits `class $className(_PmGcObject):` instead of `_PmObject` -whenever `ReflectedClass.hasTraverse` is set and there is no metaclass -(`PythonProxySource.kt:1168`, `val gcEligible = metaclassName == null && cls.hasTraverse`), and -`_PmGcObject` subclasses `_pm_proxy_base`, which every platform's `ProxyTypeFactory.createProxyType()` -publishes into `__main__` (`ProxyTypeFactory.kt:23`, and the per-platform `actual`s at -`desktopMain/.../ProxyTypeFactory.kt:253`, `androidMain/.../ProxyTypeFactory.kt:174`, -`nativeMain/.../ProxyTypeFactory.kt:345`, `wasmJsMain/.../ProxyTypeFactory.kt:311`) — see §14b -item 5 for the full account, including the `subtype_dealloc` double-untrack test. Cycles that close -on the Kotlin side (the handle table holding weakly) are separately still untouched, as -`docs/design/object-lifetime.md` already recorded. - -Cost is not yet measured. The table lookup is an array index and is not the expense; the -boundary is. Note that iOS and androidNative have no boundary here at all — Python and Kotlin -share one binary — so this is a JVM-only cost. +Closed in its core: build-time table, KSP generator, one entry point per platform, generated proxies, +`tp_traverse` handle slot on all five targets, GraalVM native image (SPEC U-1…U-6, M-3, M-4; +`ksp-fixtures/app/src/desktopTest/`, `PM/commonTest/.../reflection/UpcallTableTest.kt`, +`PM/*Test/.../ref/CycleCollectionTest.kt`). Upcall cost is measured +(`PM/commonTest/.../reflection/UpcallOverheadTest.kt`, `PM/commonTest/.../ffi/upcall/GeneratedProxyCostTest.kt`). + +- **7.1 Generated proxies exercised outside desktop — partial.** SPEC U-3: the full proxy behaviour + (construct, methods, properties, companions) is asserted on desktop only; outside it there are + `ksp-fixtures/app/src/androidNativeArm64Test/.../NativeSmokeTest.kt` and + `ksp-fixtures/android/.../GeneratedAndroidTableTest.kt`. The KSP-generated-proxy cycle proof + (`ksp-fixtures/app/src/desktopTest/.../RefHolderCycleCollectionTest.kt`) likewise exists on desktop + only, because `ksp-fixtures/app` has no iOS or device leaf; other targets prove the slot with a + hand-built subclass (`PM/nativeTest/.../CycleCollectionTest.kt`). +- **7.2 Extension functions as methods on KSP-generated proxies — partial.** SPEC U-7: asserted for a + hand-written table (`PM/commonTest/.../pythonx/PythonxAdapterTest.kt`) and for walked jars + (`ksp-fixtures/artifact/.../WalkedArtifactComposeModifierTest.kt`); not for a table KSP generated. +- **7.3 Incremental KSP — open (deferred design change).** A one-file change regenerates the + module's whole `Fragment_` (100 % dirty, measured with `ksp.incremental.log`). The only + route is per-file fragments, which changes fragment naming and `UpcallTable`'s duplicate detection; + `docs/design/upcall.md` §2.7. +- **7.4 Binary-size cost of blacklist exposure — open (product decision).** A table referencing every + `public` declaration defeats dead-code elimination: ≈1.84 KB per entry in a stripped + `androidNativeArm64` binary (200 synthetic entries, 368 640 bytes; `docs/design/upcall.md` §2.6). + The measurement informs, but does not settle, whether that is acceptable. +- **7.5 Silent exclusions without a KSP warning — open.** A `suspend` function *type* in a signature + now warns (`FragmentScanner.warnUnexposableType`); a declaration dropped by + `BindingPolicy.isExposedFunctionShape` (an extension receiver, type parameters) is still dropped with + no warning. `docs/design/upcall.md` §8 item 4. +- **7.6 Kotlin calling a Python override — open.** Python can subclass a proxy, but a Kotlin caller of a + virtual method never reaches the Python override; nothing implements or asserts it + (`docs/design/upcall.md` §4.4). +- **7.7 `BYTES` marshalling is per item — open.** `PyBytes_AsStringAndSize` is not declared in + `EmbedAPI.kt`, so a `ByteArray` crosses one byte at a time; correct (NUL bytes survive, SPEC U-2) but + slow (`docs/design/upcall.md` §3.3). +- **7.8 Fragment discovery from transitive and published artefacts — unverified.** Discovery is proven + for project modules and a `.klib` (`NativeSmokeTest`), not for a fragment arriving through a + transitive Maven dependency (`docs/design/upcall.md` §2.9). +- **7.9 Proxy-cost tables predate the 100 000-iteration warmup — open (re-measure).** + `docs/design/upcall.md` §7.5/§7.6 absolutes were taken before the warmup fix; re-cut with + `GeneratedProxyCostTest` through `./benchmarks/cost-table.sh`. Android/androidNative proxy-cost rows + are missing. ## 7b. Finish `PyValue` -**Closed for the eager path.** `PyContext` was already complete — `withContext` restores the -strategy even on throw, and all five `ConversionStrategy` variants dispatch. `PyValue`/`PyProxy` -now is too: `toKotlin`, `toKotlinOrNull` and `toPython` are implemented, no TODO stub remains in -`PyProxy.kt`, and `ConversionTest` passes on desktop (in the 118-test run) and on the iOS -simulator (111 tests). - -**Closed for the lazy path too.** The per-type lifetime rule this section named as the last -missing piece is written — `docs/design/object-lifetime.md`, "Conversion caching, and where it stops", -one row per source type, derived from what each conversion actually returns rather than from what -the type looks like — and the cache is implemented against it: - -- scalars and the decoded `String` copy themselves out of CPython and are cached; a converted - container is cached as a **snapshot**, independent of Python memory precisely because it is a - copy, with `invalidateNativeCache()` as the stated way to re-read a container that has since - been mutated; -- `memoryview` is **refused**, because a copy still would not carry its format, shape and strides. - `bytes` and `bytearray` were refused alongside it on the ground that their native form is a - pointer into the object's own buffer, until a `ByteArray` copy showed that ground does not reach - them — both convert now, `bytearray` as a snapshot; -- a bare `NativePointer` — what `ConversionStrategy.RAW` returns — is rejected by `PyValue`'s - constructor: it is an address, not a reference, and storing it would fail only once the address - had been reused; -- `isIndependentOfPythonMemory` is the rule as code, and every store to `cachedNativeValue` on - the conversion path goes through it. - -`PyValueLazyConversionTest` (13 tests, commonTest) is the coverage `ConversionTest` never had: -lazy conversion of an *untyped* source for all nine builtins the walk understands, a cached value -outliving the release of its source with the heap churned underneath it, the snapshot going stale -and then being invalidated, and the refusals — buffers, user-defined classes, builtin subclasses, -`complex`, `None` — each asserted to leave the cache empty rather than throw an NPE. - -**The lazy path also caught a borrowed reference.** `PyContext.proxyConvert` handed the `PyValue` -the *caller's* wrapper whenever `typedWrap` had no dedicated wrapper to build, so the `PyValue` -owned nothing of its own and dangled as soon as the caller closed that wrapper. Measured as a -missing `+1` on `sys.getrefcount` (3 where 4 was required) and fixed by taking an independent -reference on that path. - -**Was:** `PyValue`/`PyProxy` incomplete — `toKotlin()` and `toPython()` were TODO stubs ending in -`cachedNativeValue!!`, so a cache miss was an NPE. It was said to work only because the basic -types bypassed it — `PyInt`, `PyFloat` and the rest convert inside their own `cachedNativeValue` -accessors and never reach the stub, so `TYPED` conversion of a builtin succeeded while anything -without a dedicated wrapper would fail. Both halves are gone: no `!!` on that field remains in the -library sources, and an untyped source now converts through the generic walk (observed, per -builtin). - -**What is still not converted**, refused rather than guessed at: `memoryview`, subclasses of -builtins (the dispatch is by exact type, mirroring `PyLong_Check` rather than `isinstance`), -`complex`, and any user-defined class — `TYPED` stops at the `PyObject` for those by design. +**Closed** for the eager and lazy paths (`PM/commonTest/.../conversion/ConversionTest.kt`, +`PyValueLazyConversionTest.kt`). `complex`, builtin subclasses, `memoryview` and user-defined classes +are refused by design (SPEC O-7). -`bytes` was on that list with the note that a `ByteArray` copy would also be correct and was not -implemented. It is implemented now, and `bytearray` with it: once the conversion is a copy, the -buffer-pointer objection stops applying to either, so refusing one and not the other would have -been arbitrary. - -`complex` was on that list under the subclasses-of-builtins reason, and that reason was wrong for -it. `complex` **is** an exact type and `PyComplex` exists; it is refused only because neither -`typedWrap` nor `pyObjectToNative` carries an entry for it. It stays refused — there is no Kotlin -counterpart type to convert into — but not for the reason given here. +- **7b.1 NATIVE conversion of a user-defined object — open (design question).** `pyObjectToNative`'s + fallback returns `str(obj)`, which is lossy and one-way. Three candidates — `__dict__` projection, + an upcall-table handle, or documenting NATIVE as builtins-only — are written at the branch + (`PM/commonMain/kotlin/python/multiplatform/ffi/types/collections/CollectionSupport.kt`, + `TODO(open design question)`), with what to measure first. ## 8. `jvmMain` unification -**No longer a performance item.** `invokeExact` was reached on desktop without it (1015.95 ns → -2.65 ns), so what remains is maintenance debt: 380 Android + 315 desktop hand-written -declarations. - -The duplication has caused real bugs — desktop's `find()` carried the symbol name separately -from the `actual`, which is how `Py_RunMain` came to be bound to `Py_FinalizeEx`. - -**Attempted at scale (2026-08-18, `work/jvmmain`) and the "~330 → 14 shapes" estimate above does -not hold.** `EmbedAPI.android.kt`/`EmbedAPI.desktop.kt` were parsed (not grepped — a hand-rolled -parser that walks brace/paren balance and `if`/`else` continuations, checked against the 314 -`commonMain` `expect` declarations) and every `actual` body compared by exact text after -normalizing whitespace. Of 314 common declarations, **3 have literally identical bodies**: -`toAddressValue`, `toRawValue`, `AddressValue.toNativePointer`. The other 311 differ for a -structural reason, not a cosmetic one — Android's JNI dispatch needs a symbol name per calling -convention (`PyErr_ClearN`/`PyErr_ClearF` vs desktop's unsuffixed Panama handle `PyErr_Clear`) and -often a `preferFastNative` branch besides, so almost every wrapper's body names a different callee -on each platform. That is "different wiring" in exactly the sense this document already warns -about — not a case the recipe was wrong to rule out, but evidence the duplication is smaller than -it looks from the outside. - -Of the 3 identical wrappers, only **1 was actually movable**: `toRawValue`, moved to -`jvmMain/kotlin/python/native/ffi/EmbedAPI.jvm.kt` along with the `toPlatformPointer()` helper it -calls (itself hoisted, since `jvmMain` code cannot reference a symbol that only exists in a leaf -source set — `androidMain`'s `toPlatformPointer(): JNIPointer` and desktop's `toPlatformPointer(): -Long` are the same behavior under the `JNIPointer = Long` typealias, so hoisting changed nothing -observable). The other two, despite identical wrapper text, both construct or unwrap -`NativeAddressValue` — a class whose `toString()` genuinely differs per platform -(`"JNIPointer(raw=0x…)"` on Android vs `"NativeAddress(0x…)"` on desktop, and Android's uses signed -`ptr.toString(16)` where desktop's uses `ptr.toULong().toString(16)`). Hoisting the wrapper would -require also hoisting that class, which would force one platform's `toString()` to change — -disallowed by "동작이 바뀌면 안 된다." They stay duplicated, correctly. - -**The recipe (remove `inline` from the `commonMain` `expect`, delete the two platform `actual`s, -add one in `jvmMain`) is confirmed safe, not just asserted.** It was applied to `toRawValue` and -compiled clean on both `compileKotlinDesktop` and `compileDebugKotlinAndroid`. The Kotlin 2.0.20 -`Internal error in file lowering` crash this section used to warn about **did not reproduce even -with `inline` deliberately left in place** on this function (tested and reverted) — for this -specific case at least, the risk turned out to be smaller than documented, though the sample size -is one trivial no-argument function and the crash may still be real for a shape not tried here. -Test counts unchanged (`desktopTest` 485/0/1, `ksp-fixtures:app:desktopTest` 64/0 — note these are -higher than a `484/0/1` figure recorded elsewhere in this document, which was stale before this -task started, not a regression this task caused; verified by re-running the suite with the change -stashed out). `compileKotlinAndroidNativeArm64` unaffected (this item is JVM-only). +**Closed, nothing further recommended.** Of 314 common declarations only 3 have identical +Android/desktop bodies and only `toRawValue` was movable (now in +`PM/jvmMain/kotlin/python/native/ffi/EmbedAPI.jvm.kt`); the rest differ in real wiring (per-convention +JNI callee names). Re-opening it means redesigning Android's convention selection, not moving code. ## 9. Free-threading -**Nothing is being waited on. It builds, it runs, and what it breaks is now known.** -`./gradlew :python-multiplatform:desktopTest -PpythonFreeThreaded=true` runs the whole desktop -suite against `cpython-3.14.7+20260807--freethreaded-install_only`: - -| build | tests | failing | skipped | -|---|---|---|---| -| default (GIL) 3.14.7 | 236 | 0 | 1 | -| free-threaded 3.14.7 | 236 | 0 | 1 | - -This section used to say "3.15t" and "waiting on upstream"; neither was true. 3.14 free-threaded -is what was measured here, and those artefacts have been on the python-build-standalone release -all along. - -**It is now green on both.** It used to report two failures. Deferred deallocation was the first -and is fixed by the checkpoint below. `CycleCollectionTest` was the second, and it was written off -twice — first as a premise the runtime does not honour, then as a queued-decrement problem a -checkpoint would flush. Both were wrong, and it is fixed; see below and -`docs/investigations/gc-scheduling-investigation.md`. - -### The flag did nothing at all until three silent defects were fixed - -None of these produced an error message; all three made the build quietly do something other -than what was asked. - -- **Extraction was not keyed by what it extracted.** Every unpack is guarded by "is the - destination directory empty?", and the destination was a flat `extracted/`. Passing - `-PpythonFreeThreaded=true` therefore downloaded the free-threaded archive, verified its - checksum, and then *discarded* it, because the GIL build had already filled the directory. The - build then ran the GIL interpreter while every log line named the free-threaded tarball. The - identical hazard applied to `-PpythonVersion`. Extraction is now - `extracted//[-freethreaded]` (`extractedDir`, `desktopFlavourSuffix`). -- **A free-threaded install renames everything looked up by name.** It ships - `libpython3.14t.dylib`, `lib/python3.14t/` and `bin/python3.14t`, and does *not* ship the - un-suffixed names. `manager.loadLibPython` asked for `libpython3.14.dylib`, which such an - install does not contain. `Versions.abiFlags` / `taggedVersionString` now carry the `t`, fed by - a new `BuildConfig.pythonFreeThreaded`. -- **`updatePythonChecksums` would have destroyed the lockfile.** It wrote `"\\n"` — a literal - backslash and an `n`, not a newline — so the whole file came out as one physical line beginning - with `#`, which `Properties` reads as a single comment. One run would have deleted every - checksum and left the next build failing "Missing checksum" for all of them. The committed file - was intact only because nobody had run the task since it was written. - -### Free-threading defers deallocation to the owning thread, and a pure embedder never gets there - -`GCLeakTest.testCascadingReleaseOnGC` used to fail. This is the important finding in this section, -and it is the free-threading counterpart of §1: turning the feature on exposed an assumption, not a -typo. **It now passes, unmodified**, because the library reaches the checkpoint the runtime is -waiting for. The diagnosis below stands; the last subsection records the fix. - -The test builds 1000 Python lists each holding one target object, drops the Kotlin wrappers, and -waits for the cleaner to release them. Measured on the free-threaded build (the cleaner is -confirmed to have run — `ReleaseCounter.released` rose by 2003): - -| after | `sys.getrefcount(target)` | -|---|---| -| 1000 lists built, each holding `target` | 1002 | -| 50 forced JVM GCs; cleaner called `Py_DecRef` 2003 times | 1002 | -| 500 further C API round trips on the owning thread | 1002 | -| 2 s of wall clock | 1002 | -| `Python3.exec("pass")` — one trivial bytecode frame | **2** | - -So the decrements happened, and the objects were not freed. Not time, and not C API traffic: -entering the eval loop *once*, for a body that does nothing, released all thousand at once. That -is the signature of the free-threaded build's biased reference counting — a decref from a thread -that does not own the object cannot run `tp_dealloc` there, so the object is queued to its owner, -and the owner drains that queue at an eval-loop checkpoint. - -Under the GIL there is no such queue and the cleaner's decref frees immediately. The consequence -for this library is specific and unpleasant: **an embedder that drives CPython entirely through -the C API never reaches a checkpoint, so memory released by the cleaner is never actually -reclaimed.** It is not a leak in the sense of a lost pointer — the refcount is correct and one -line of Python flushes it — but a Kotlin process that only ever calls `PyObject_Call` will grow -without bound. - -### The checkpoint is the fix, and the Stable ABI does not offer a way to reach one without bytecode - -The open question above was *what to call to force a drain, and from where*. Both halves have -answers now, and the first one is a "no" that had to be established before the second made sense. - -**CPython exposes no non-bytecode way to merge the queue on the current thread.** The merge lives -behind `_PY_EVAL_EXPLICIT_MERGE_BIT`, and `Python/ceval_gil.c` clears that bit in exactly one -function: - -```c -int _Py_HandlePending(PyThreadState *tstate) { - ... -#ifdef Py_GIL_DISABLED - if ((breaker & _PY_EVAL_EXPLICIT_MERGE_BIT) != 0) { - _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_EXPLICIT_MERGE_BIT); - _Py_brc_merge_refcounts(tstate); - } - if (_Py_qsbr_should_process(((_PyThreadStateImpl *)tstate)->qsbr)) { - _PyMem_ProcessDelayed(tstate); - } -#endif - if ((breaker & _PY_GC_SCHEDULED_BIT) != 0) { ... _Py_RunGC(tstate); } - ... -} -``` - -`_Py_HandlePending` has one caller family: the `_CHECK_PERIODIC` / `_CHECK_PERIODIC_IF_NOT_YIELD_FROM` -uops in `Python/bytecodes.c`, which open every Python-level frame (`RESUME`) and close every call -instruction. There is no public entry point. - -`Py_MakePendingCalls` is the obvious Stable ABI candidate and it **does not work**. Read the -source and it cannot: it forwards to `_PyEval_MakePendingCalls`, which handles `handle_signals` -and `make_pending_calls` and returns — the merge bit is not among them. Measured, on the -free-threaded build with 1000 releases queued, it returns 0 and leaves `sys.getrefcount` at -1002. `EvalCheckpointTest` asserts exactly that, so the day CPython changes its mind, the test -says so. - -`PyGC_Collect` **does** work, and is the only Stable ABI function that does, because -`gc_collect_internal` stops the world and walks every thread state: - -```c -_Py_FOR_EACH_TSTATE_BEGIN(interp, p) { - _PyObject_MergePerThreadRefcounts((_PyThreadStateImpl *)p); - merge_queued_objects((_PyThreadStateImpl *)p, state); // <-- the drain -} -``` - -That is the one option that reclaims on behalf of a thread *other* than the caller. It is also -the most expensive thing here by three orders of magnitude, because it costs a heap walk rather -than a queue pop. Both functions are now bound (`Py_MakePendingCalls`, `PyGC_Collect`); both are -in `Misc/stable_abi.toml`. - -So the drain has to be an eval-loop entry, and the only question left is how cheap one can be -made. `exec("pass")` recompiles a module every time. A *cached* zero-argument Python function -whose body is `pass`, called through `PyObject_CallNoArgs`, does not — and it still reaches the -checkpoint after specialisation, because `RESUME_CHECK` deopts back to `RESUME` whenever -`eval_breaker != version`, and the merge bit makes them differ. Measured on the free-threaded -build (`EvalCheckpointTest.testCheckpointCostAgainstTheAlternatives`, macOS arm64): - -| | ns/op | | -|---|---:|---| -| `withGIL { }` — attach and detach, nothing else | 154.6 | the floor | -| `Python3.drainPendingReleases()` | 283.0 | **the checkpoint, ~128 ns over the floor** | -| `withGIL { Py_MakePendingCalls() }` | 183.1 | cheap, and does not merge | -| `Python3.exec("pass")` | 6 458.9 | 23× — it recompiles | -| `withGIL { PyGC_Collect() }` | 256 447.1 | 906× — heap walk, but drains every thread | - -**Where it is called from.** Not the cleaner — that was ruled out before the mechanism was known -and the measurement confirms the instinct was right for a second reason: the queue belongs to the -thread that *owns* the object, and the cleaner owns nothing, so a checkpoint taken there would run -Python on a cleaner thread (§1's deadlock) and drain an empty queue for it. -`EvalCheckpointTest.testCleanerActivityAloneTakesNoCheckpoint` pins that suppression. - -It rides on the next ordinary call instead. `withGIL` takes a checkpoint at its outermost entry, -behind two gates: at most one per `Python3.autoDrainInterval` outermost scopes (32), and skipped -entirely unless `ReleaseCounter.released` has moved since the last one, so a workload that drops -no wrappers pays a field compare. Amortised that is ~4 ns on a 155 ns scope. `drainPendingReleases()` -is public for anyone who wants to force one, and `autoDrainInterval = 0` turns the automatic path -off. - -The automatic path defaults **on for free-threaded builds and off otherwise**, which is why the -default suite's numbers are unchanged. One guard is worth naming: the checkpoint declines to run -when `PyErr_Occurred()` is non-null. Several call sites read the error indicator *after* their -`withPython { }` scope closes — `PyObject.getAttr` is one — and entering the eval loop can replace -it, since a pending signal handler or a finaliser raising is enough. - -**This is not only a free-threading problem.** `_PY_GC_SCHEDULED_BIT` is cleared in the same -function, and `_Py_ScheduleGC` is how allocation triggers a collection on *both* builds since 3.12. -An embedder that never reaches a checkpoint therefore never runs the cyclic collector either, on -either build. That is not what §9 is about and nothing here depends on it, but it is the same root -cause, and `drainPendingReleases()` covers it for anyone who turns the automatic path on. - -### Heap types are deferred-reference-counted, and a collection is what makes the count readable - -This was the last free-threaded failure, and it is fixed. -`CycleCollectionTest.testHandleReleasedWhenProxyDiesWithoutCycle` had two independent defects, and -its own guard assertion caught both — the one documented as "not decoration: it is what proves the -probe reads a real refcount". It refused to proceed twice, exactly as designed. - -- **The probe read eight bytes at offset 0 of `PyObject`.** Measured on 3.14.7: on the default - build that word is `ob_refcnt` and steps with the count; free-threaded it is `ob_tid` and never - moves, while the count is `ob_ref_local` (a `uint32` at +12) plus `ob_ref_shared >> 2` (at +16). - The failure reported `6171668704` for both "before" and "alive" — a thread id, read as a - refcount. The probe now branches on `BuildConfig.pythonFreeThreaded`. -- **Fixing the offset was necessary and not sufficient.** With the right fields, "before" and - "alive" still read identically — `1152921504606846978` — because a free-threaded heap type - carries `_PyGC_BITS_DEFERRED` and `PyType_GenericAlloc`'s `_Py_INCREF_TYPE` is a no-op for a - deferred type on its owning thread. Measured on the proxy type: `ob_gc_bits == 0x41` - (`TRACKED | DEFERRED`) and `ob_ref_shared == 0x3ffffffffffffffd`, a shared count of - `PY_SSIZE_T_MAX / 8` — the deferred sentinel. Creating 100 instances left all six header words - bit-identical. - -**The conclusion this section used to draw from that is what was wrong.** It said the invariant -"each live instance of a heap type holds one reference to that type" *does not exist* on a -free-threaded build, and that no change to this library could repair the test. It does exist. It is -simply not **observable** until a collection materialises the deferred references — measured, after -one `PyGC_Collect()` the same 100 instances show up as exactly `100 << 2` on `ob_ref_shared`, and -`tp_dealloc` then gives back exactly 100, returning the total to its pre-instantiation value to the -unit. The test now takes a collection on either side of its instantiation loop, free-threaded only. -**The assertions are unchanged** — the same "rises by exactly 100" and "returns to exactly where it -started" hold on both builds. - -**A checkpoint is not the fix here, and this is worth keeping straight.** The obvious guess, given -the rest of this section, is that the missing decrements were queued by biased reference counting -and that `drainPendingReleases()` would flush them. Measured: it does not. The header is -bit-identical before and after a checkpoint, because there is nothing queued to merge — the -increments were never made. Deferred reference counting and BRC queueing are different mechanisms -with different remedies, and only the second is what §9's checkpoint is for. - -This also revisits what this section previously claimed about `abi3t`. The library proper is clean — -`ProxyTypeFactory` writes only into memory it obtained from `PyObject_GetTypeData` — but that test -does read inside a `PyObject`, and it is the one place that would have to change. That is a -property of using a direct probe, not of the invariant; counting through `sys.getrefcount` would -also work, at the price of the argument's own temporary reference. - -The full trace, with the header dumps, is in `docs/investigations/gc-scheduling-investigation.md`. That document -also records the half of this that is **not** free-threading-specific: `_Py_ScheduleGC` only sets -`_PY_GC_SCHEDULED_BIT`, so a pure C API embedder never runs the cyclic collector on *either* build. - -### What each platform can actually get - -Checked against the live release listings, 2026-08-12: - -| target | free-threaded prebuilt | source | -|---|---|---| -| desktop (macOS/Linux/Windows) | **yes** | `python-build-standalone` `20260807`, `…-freethreaded-install_only`, for 3.14.7 and 3.15.0rc1 alike | -| Android | no | python.org's `python---linux-android.tar.gz` contains `libpython3.14.so` / `libpython3.15.so` only — checked in both 3.14.7 and 3.15.0rc1 | -| iOS | no | neither BeeWare's Python-Apple-support nor python.org's XCframework defines `Py_GIL_DISABLED` | - -So free-threading is a desktop-only capability for as long as that holds, and the flag should stay -off by default. - -### `abi3t` is not a blocker, because nothing here asks for the Limited API - -`abi3t` is Final for 3.15 (PEP 803), and 3.14 free-threaded has no Limited API at all. That was -recorded as the reason to wait. It is not one: `Py_LIMITED_API` is never defined anywhere in this -build — desktop binds symbols by name at runtime through Panama, and the native targets cinterop -against the full headers. "abi3" in this codebase means a self-imposed rule about *which* -functions to call, not a compilation mode. The rule is what keeps one binding working across -versions, and it is unaffected. - -### 3.15.0rc1: the three functions are migrated, on every target - -`-PpythonVersion=3.15.0rc1` gives **208 tests, 0 failures, 1 skipped** on desktop — the same as -the default. The first attempt failed 4, all of them assertions hardcoding `"3.14"` against the -reported interpreter version (`EmbedApiLowLevelTest`, `InterpreterAvailabilityTest`, -`Python3Test`, `DesktopPythonTest`). They now compare against -`Versions.currentVersion.compactVersionString`, which is strictly stronger: it checks that the -interpreter loaded is the one the build configured, instead of pinning a release line that has to -be hand-edited on every bump. (Two other `"3.14"` literals nearby are `math.pi`, left alone.) - -`compileKotlinAndroidNativeArm64` and `compileKotlinIosSimulatorArm64` both fail on 3.15, with the -*same* four errors, from three functions 3.15 removed: - -| removed in 3.15 | replacement | -|---|---| -| `PySys_ResetWarnOptions` | none — the `PyConfig` API covers it | -| `PyImport_ImportModuleNoBlock` | `PyImport_ImportModule` (an alias since 3.3) | -| `PyWeakref_GetObject` | `PyWeakref_GetRef` — **present in 3.14 too**, so the migration can be made without dropping 3.14 | - -Why desktop does not notice: the symbols are **still exported from the shared library** — checked -with `nm` on `libpython3.15.dylib`, all three are there — and were removed only from the headers. -Panama resolves by symbol name at run time, so nothing breaks; cinterop and the hand-written C in -`jni_onload.def` compile against headers, so they do. - -That made the porting cost concrete rather than open-ended: three functions, each appearing in -the `commonMain` `expect`, four platform `actual`s, `jni_onload.def` (declaration, thunk and table -entry) and the wasmJs `@WasmImport` block. Roughly 20 sites, no behavioural change on 3.14. - -**Done.** All three were still present when this was picked up, and all three are now migrated: - -| was | is now | -|---|---| -| `PySys_ResetWarnOptions` | removed outright — there is no C replacement, and the documented migration (configure `sys.warnoptions` through `PyConfig`, or clear it from Python) is not a binding | -| `PyImport_ImportModuleNoBlock` | kept as a `@Deprecated` **plain `commonMain` function** forwarding to `PyImport_ImportModule`, with no C symbol behind it. It had been an exact alias since 3.3, so forwarding is behaviour-preserving rather than an approximation, and existing source still compiles through the bump | -| `PyWeakref_GetObject` | replaced by `PyWeakref_GetRef`, which 3.13 and 3.14 also export, so nothing was given up to gain it | - -**The out-parameter is the part that was not free.** `PyWeakref_GetRef` is -`int f(PyObject *ref, PyObject **pobj)` — a status *and* a write-through slot, where the removed -function was a single return. Three of the four targets already had somewhere to put that slot and -needed no new machinery: `jni_onload.def`'s thunk uses a C local (so the JNI boundary still carries -only primitives, per `androidMain/README.md`), cinterop uses `memScoped`, and wasmJs takes four -bytes from CPython's own heap through the `malloc`/`free` it already imports. Desktop had nothing — -Panama here reaches native memory only through `allocateUtf8String`/`readUtf8String`, neither of -which can carry a pointer, since a pointer is not NUL-terminated. So `Panama` gained -`allocatePointerSlot`/`readPointerSlot`/`freePointerSlot`, built entirely from handles the modern -backend already had (`malloc` plus the same `MemorySegment.copy` that backs `allocateUtf8Freeable`, -run in the other direction). **No new reflective lookups**, so no new JDK-version surface. This will -be reused: CPython is converting more borrowed-reference getters to the same shape -(`PyDict_GetItemRef`, `PyObject_GetOptionalAttr`). - -The JDK 16-18 incubator backend throws instead. Allocating there is easy; *reading* is not, and the -class that could (`MemoryAccess`) changed shape across 16, 17 and 18. A guessed read yields a wrong -`PyObject *`, i.e. a use-after-free landing somewhere unrelated — so it refuses at the allocation -step, before anything has been allocated. That branch is already documented as an unverified -best-effort mirror, and JDK 19+ resolves the modern one. - -Signature at the Kotlin level is `PyWeakref_GetRef(ref): NativePointer?` — a new strong reference, -or null. C's `0` (dead) and `-1` (error) both arrive as null because an out-parameter does not -survive the JNI boundary, but **no information is lost**: the error case sets the error indicator -and the dead case does not, so `PyErr_Occurred` separates them. - -Three tests in `EmbedApiLowLevelTest` (`commonTest`, so they run on desktop, iOS and Android): -`PyWeakref_GetRef` on a live referent returns the referent *and* raises its reference count by one, -which is what distinguishes the new function from the borrowed-reference one it replaces — an -implementation still wired to `PyWeakref_GetObject` would pass an identity check and then corrupt -the heap on the caller's matching `Py_DecRef`; a dead referent returns null with a *clear* error -indicator; and the `ImportModuleNoBlock` shim resolves to the same module object as the function it -aliases. - -Writing them turned up a trap worth recording: the first version built its referent from -`types.SimpleNamespace`, which is **not weakly referenceable** (nor are `int`, `str`, `tuple`, -`list` or `dict`; `set` and `frozenset` are). `PyWeakref_NewRef` answered by returning null and -raising `TypeError` — and because every class in the binary shares one interpreter, the unhandled -indicator then took down two *unrelated* tests, which is what surfaced first. The assertions now -clear the indicator before failing, so a failure in this file can no longer be mistaken for a -failure elsewhere. - -Verified: `compileKotlinAndroidNativeArm64`, `compileKotlinIosSimulatorArm64`, -`compileDebugKotlinAndroid` and `compileKotlinWasmJs` all compile; `desktopTest` **236 tests, -0 failures, 1 skipped** and `iosSimulatorArm64Test` **222 tests, 0 failures** (both counted from a -cleaned `build/test-results/`). The `jni_onload.def` invariants hold — no duplicate thunk or table -name, registration count still taken with `sizeof`, and every table entry still has a matching -`external fun` and vice versa. - -**The default stays 3.14.7.** A release candidate is not a default, and this migration is -version-neutral: `PyWeakref_GetRef` exists on 3.14, so none of it waits for the bump. - -### iOS: the source has to change with the version, and 3.15 is the switchover - -python.org began publishing an official iOS `Python.xcframework` with 3.15 — -`ftp/python/3.15.0/python-3.15.0rc1-iOS-XCframework.tar.gz`, first appearing at 3.15.0b1. -BeeWare's Python-Apple-support, the only previous source, stops at `3.14-b10` and has no 3.15 -release. The two do not overlap, so this is a hard switch on the version rather than a -preference: **3.14 and earlier can only come from BeeWare, 3.15 and later only from python.org.** -`iosFromPythonOrg` in the build now picks between them at ≥ 3.15, and both archives are pinned in -`python-checksums.properties`. - -Swapping is otherwise free, because the trees are layout-compatible everywhere this build reaches -into them — `Python.xcframework//Python.framework/Headers`, `-F …/`, -`Python.xcframework/lib/pythonX.Y` and `…//lib-arm64/pythonX.Y` all exist in both. The -differences are in parts nothing reads: BeeWare adds `platform-config/` (cross-compilation -sysconfig data for building wheels) and a `VERSIONS` file. The header-set differences -(`module.modulemap`, `lock.h`, `monitoring.h`, `typeslots.h` on one side; `pyabi.h`, `slots.h`, -`slots_generated.h` on the other) are 3.14-vs-3.15, not packaging. - -Neither source publishes checksums this build can use — BeeWare publishes none at all, python.org -publishes sigstore material that is not reasonable to verify in Gradle — so both stay pinned by -the local lockfile. - -The swap was verified as far as it could be while 3.15 is not the default: -`cinteropPythonIosSimulatorArm64` **succeeds** against the python.org framework, so its headers and -`Python.framework` are consumed exactly like BeeWare's. The build then failed in -`compileKotlinIosSimulatorArm64` — on the three removed functions above, with byte-identical errors -to `androidNativeArm64`, i.e. in shared `nativeMain` source and not in anything iOS-specific. -`compileKotlinIosSimulatorArm64` on the default 3.14 (BeeWare) still passes, so no regression was -introduced for the version actually in use. **That compile blocker is now gone** (see above), which -leaves the acquisition path as the thing to be sure of. - -#### The acquisition path is ready; four smaller things are not - -Re-read against the build on 2026-08-12. The switchover itself is wired and pinned: - -* the predicate is `python-multiplatform/build.gradle.kts`, `iosFromPythonOrg` — it splits the - configured version and compares major/minor only, so the `rc1` suffix on `3.15.0rc1` does not - reach `toInt()` and the gate answers correctly; -* `pythonOrgReleaseDir` strips the pre-release suffix separately, because python.org publishes - `3.15.0rc1` **under `ftp/python/3.15.0/`** while the archive keeps the full name; -* the extraction directory is keyed by version, so 3.14 and 3.15 trees cannot collide; -* `python-checksums.properties` already pins `ios-3.15.0rc1-pythonorg`, both Android ABIs, and all - four desktop platforms at 3.15.0rc1. A plain `-PpythonVersion=3.15.0rc1` downloads nothing - unpinned. - -What a bump would still hit, none of it iOS-specific and none of it large: - -1. **Free-threading has no 3.15 checksums.** There are `-freethreaded` entries for 3.14.7 only, so - `-PpythonVersion=3.15.0rc1 -PpythonFreeThreaded=true` fails on a missing checksum. Consistent - with the table above — 3.15 free-threading is desktop-only anyway — but it is a lockfile gap, - not a deliberate refusal. -2. ~~**Three Android *instrumented* tests still hardcode `"3.14"`**, plus - `PythonOnDevice.PYTHON_DIR`~~ — **fixed.** The earlier pass that moved four assertions onto - `Versions.currentVersion.compactVersionString` covered the desktop and common tests and missed - these, because nothing compiles them without an SDK configured. The three assertions now derive - from `compactVersionString` and `PYTHON_DIR` from `taggedVersionString`, matching what - `MainActivity` already did — and `taggedVersionString` additionally carries the `t` suffix for a - free-threaded build, which the literal could not express. - **Compile-verified only** (`compileDebugAndroidTestKotlinAndroid`); not run, because the - emulators were reserved for other work. -3. **The two python.org paths disagree about pre-release layout.** iOS strips the suffix to build - the ftp directory; the Android download uses the configured version verbatim - (`ftp/python/3.15.0rc1/…`). A checksum is pinned for the Android archive, so that URL evidently - resolves today — but only one of the two can be right in general. -4. ~~**The iOS *app* packaging path has no producer.**~~ — **half of this was wrong, and the other - half is worse than stated.** The *framework* half has a producer and did when this was written: - `sample/build.gradle.kts`'s `prepareIosFrameworks` is a `Sync` from - `:python-multiplatform:downloadPython_ios`'s extraction into `sample/build/xcode-frameworks`, and - every `link*Ios*` task depends on it. Running it populates `Python.xcframework` exactly as both - the Gradle `linkerOpts` and the Xcode project expect. - - What has no producer is the **standard library** half, and that is what actually stops the app — - see §13's "The sample runs on iOS" below. The Xcode project's "Install Target Specific Python - Standard Library" phase rsyncs from - `Python.xcframework/ios-arm64_x86_64-simulator/lib/`, which in this distribution holds - `libpython3.14.dylib` **and nothing else**; the stdlib lives in `Python.xcframework/lib/python3.14` - plus `ios-arm64_x86_64-simulator/lib-arm64/python3.14`, which is precisely the union - `extractIosSimulatorStdlib` builds for the *test* path. So the app bundle has never carried a - stdlib, and nothing in the app path sets `PYTHONHOME` either. "iOS works" still should not be read - as covering the app — but the missing piece is the stdlib, not the framework. - -Also stale, and cheap to correct when touched: `iosMain/README.md` still says the stdlib comes from -BeeWare unqualified, and `extractIosSimulatorStdlib`'s KDoc still says `lib/python3.13`. Neither -affects behaviour — the code is `$libVersion`-derived — and `extractIosSimulatorStdlib` does hardcode -the simulator slice name and `lib-arm64`, which is correct for the arm64 host but would need -`lib-x86_64` for an x86_64 simulator. +3.14t runs the whole desktop suite with `-PpythonFreeThreaded=true` (opt-in; `gradle.properties` +default `pythonFreeThreaded=false`): 236 tests, 0 failures, 1 skipped when last recorded, the same +count as the GIL build (table at `4ed98143:docs/roadmap/ROADMAP.md`:1013; SPEC T-2). Deferred deallocation is +drained by the eval-loop checkpoint (`PM/commonTest/.../ref/EvalCheckpointTest.kt`, +`docs/investigations/gc-scheduling-investigation.md`); `FreeThreadedGCGateTest` guards the GC probe. +The 3.15 migration (`PyWeakref_GetRef`, the two removed functions) is done +(`PM/commonTest/kotlin/python/native/ffi/EmbedApiLowLevelTest.kt`), and 3.15.0rc1 checksums, including +free-threaded ones, are pinned in `python-checksums.properties`. + +- **9.1 `autoDrainInterval`'s default on the GIL build — open (decision).** Today + `if (BuildConfig.pythonFreeThreaded) 32 else 0` (`Python3.kt`). Off means a pure C-API embedder never + runs the cyclic collector: 20 000+ cyclic objects left with the automatic path off against ~1 970 + with it on (`docs/investigations/gc-scheduling-investigation.md` §6–§7). On means a `__del__` or + weakref callback can run at a `withGIL` exit the caller did not ask to yield from. The checkpoint + costs ~128 ns over a 154.6 ns `withGIL` floor, ~4 ns amortised + (`EvalCheckpointTest.testCheckpointCostAgainstTheAlternatives`). A one-line change once decided. +- **9.2 Free-threading beyond desktop — open (blocked upstream).** No free-threaded CPython prebuilt + exists for Android (python.org ships `libpython3.14.so`/`3.15.so` only) or iOS (neither BeeWare + nor python.org defines `Py_GIL_DISABLED`), checked 2026-08-12. SPEC N-1. ## 10. WASM -**There is no JS bridge, and the interpreter is running.** Every framing this section previously -carried — that Kotlin/Wasm cannot reach C, that the data path must be copied through JS, that -direct calls and shared memory cannot be had together — is disproved by running code. -`wasm-experiment/` reproduces all of it; `docs/platforms/wasm-design.md` has the detail. - -**Test D — Kotlin/Wasm against CPython 3.14.2, built here for `wasm32-emscripten`:** - -``` -python.wasm exports 8191 Kotlin's intrinsics.memory == CPython's memory -Kotlin: pyExec("answer = 6 * 7; greeting = ...") -> 0 PASS -Kotlin: pyGlobalInt("answer") -> 42 PASS -Kotlin: pyGlobalString("greeting") -> "hello-from-cpython-42" PASS -Kotlin: pyExec("bytearray(48 MiB)") 320 -> 934 pages, reads still correct PASS -``` - -Kotlin wrote the Python source straight into CPython's heap and dereferenced -`PyUnicode_AsUTF8`'s `char*` out of it. Direct wasm-to-wasm calls, one linear memory, no copying -and no JS in either direction — and it survives the interpreter growing its memory by 600 pages. - -What made it possible: Kotlin **2.4.20-Beta2 imports** its linear memory instead of exporting it. -That reverses the ownership behind the instantiation cycle, so Emscripten instantiates first and -Kotlin receives both the memory and the exports. The integration is one substitution in the -generated glue — `intrinsics.memory` from a placeholder `new WebAssembly.Memory({initial: 0})` to -`Module.wasmMemory`. `patch-memory-max.py` and the YouTrack issue it was going to justify are both -obsolete: the memory *import* declares no maximum, so any memory satisfies it. - -**Composition is closed, the same way §6 closed it on desktop.** Reading a global from `__main__`, -measured against the real interpreter: - -| | ns | -|---|---| -| naive | 259.5 | -| + interned C strings | 185.4 | -| + module/dict hoisted | **65.2** | -| one crossing (`PyErr_Occurred`, direct) | **2.9** | - -75% of the naive cost comes off with pure Kotlin. What is left is CPython's own work; a composed -`pmp_getattr` could merge two calls into one and save 2.9 ns. Bulk ends the same way — 2.9 ns per -crossing against 0.7 ns per shared-memory read. **No C shim, and therefore no second build -pipeline.** - -**Rules for `wasmJsMain`**, both measured: - -- **`withScopedMemoryAllocator` must never be called.** It allocates from address `0x0`, on top of - Emscripten's static data. CPython allocates; Kotlin only dereferences addresses it was handed. -- **`char*` → `String`: `CharArray` + `concatToString()` for ASCII, Emscripten's `UTF8ToString` - otherwise. Never `ByteArray.decodeToString()` on a large buffer** — 13 ns/byte at 4 KB, ten times - `concatToString`. The earlier claim that "string marshalling disappears" was wrong: Kotlin/Wasm - strings *are* JS strings under the `js-string` builtins, so the JS route was never paying for the - copy the design assumed it paid for. - -**The CPython build works and does not yet claim the platform tag.** PEP 783's -`pyemscripten_2026_0` pins Emscripten **5.0.3** (Pyodide `Makefile.envs` at tag `314.0.4`), which -is what was used. The stock Tier 3 build (`Tools/wasm/emscripten`, PEP 776) already matches on -`-sWASM_BIGINT`, no `-pthread`, `-fPIC` and `MAIN_MODULE`/`SIDE_MODULE`. It diverges on the -unwinding ABI (`-fwasm-exceptions -sSUPPORT_LONGJMP=wasm` absent), on static libs (no lzma, zstd or -OpenSSL), and on `PYEMSCRIPTEN_PLATFORM_VERSION`, which does not exist anywhere in CPython 3.14.2 — -so a stock build cannot advertise the tag even where the flags line up. Also settled: -`-sMAIN_MODULE` is `LINKABLE` and exports all 8191 symbols, so **no custom `EXPORTED_FUNCTIONS` -list is needed** for the Stable ABI. Only `wasmExports` and `wasmMemory` have to be added to -`-sEXPORTED_RUNTIME_METHODS`, and neither is ABI-sensitive. - -### The target is on, and `commonTest` runs on it - -`wasmJs` is a real target now — a leaf directly under `commonMain`, beside `jvmMain` and -`nativeMain`. It is not a partial bring-up: **all 310 C symbols have `actual`s**, and the whole -shared object-model suite runs against a live interpreter. - -``` -:python-multiplatform:wasmJsNodeTest 238 tests, 0 failed, 0 skipped - desktopTest, unchanged 233 tests, 0 failed, 1 skipped -Embedded CPython version: 3.14.2 ... [Clang 23.0.0git] on wasm32-emscripten -``` - -(190 when the target first came up, then 214 with 3 failing, then 234 with the same 3. The three -were always `GCLeakTest`'s; they pass now — see "Lifetimes" below — and the four added with them are -`WasmFinalizationTest`'s.) - -Nothing skipped: `PythonTestFixture.available` was true, so `Python3.initialize()` brought CPython -up through an ordinary `@WasmImport` call to `Py_Initialize` — the library's own bring-up path, -rather than a JS-side `Py_InitializeEx` as in `wasm-experiment/`. - -Three questions the experiment did not have to answer, and how they came out: - -* **`Py_ssize_t` is 32-bit on wasm32**, and `EmbedAPI.kt` types it as `Long`. Fourteen functions are - affected (`PyList_*`, `PyTuple_*`, `PyDict_Size`, `PySet_Size`, `PyObject_Size/Length`) and - `EmbedAPI.wasmJs.kt` converts at the boundary. This is the **only** ABI divergence across the - whole surface — and it is not a compile error, it is a `LinkError` at instantiation, so - `bindings.kt`'s types are read off `python.wasm`'s own type section rather than transliterated - from the `expect`s. -* **All 310 symbols really are exported**, checked against the ABI build's 8287 — which confirms the - `-sMAIN_MODULE`/`LINKABLE` reading above on the surface this library actually needs. -* **The Gradle integration is the one substitution** the design predicted. The compiler emits - `intrinsics: { memory: new WebAssembly.Memory({ initial: 0 }), … }` into `*.import-object.mjs`; - a `doFirst` on the test task rewrites that to Emscripten's `wasmMemory` and stages - `python.mjs`/`python.wasm` beside the bundle. No binary patching, no `-sIMPORTED_MEMORY`. - *(There are two now. Upcalls need a second one, on the generated entry module — see below.)* - -### Lifetimes: closed. The candidate route was measured, and it holds - -The stdlib has no finalisation hook — that part was right, and re-checking -`kotlin-stdlib-wasm-js-2.4.20-Beta2.klib` still turns up no `FinalizationRegistry`, no `WeakRef` and -no `Cleaner`. **The conclusion drawn from it was wrong.** The *host* has all three, and a -`JsReference` reaches them. `registerCleaner` is built on one now, and `GCLeakTest`'s three cases -pass: - -``` -:python-multiplatform:wasmJsNodeTest 238 tests, 0 failed, 0 skipped - desktopTest 233 tests, 0 failed, 1 skipped -``` - -(was 234 / 3 failed — the three were `GCLeakTest`'s, and the four added are `WasmFinalizationTest`.) - -**The question that decided it: is `JsReference` a strong reference?** If handing a Kotlin object to -JS pinned it, registering it with a `FinalizationRegistry` would keep it alive forever and there -would be no route at all. Measured, not reasoned: - -| | measured | -|---|---| -| 200 Kotlin objects handed over with `toJsReference()`, held from JS only by a `WeakRef` and a registry entry | **`alive 0 / 200`** — not strong | -| `FinalizationRegistry` callbacks for those WasmGC objects | **200 / 200** | -| 200 real `PyObject` wrappers dropped without `close()`, judged by CPython's own refcount | **`2 → 202 → 2`** | - -**The second finding cost more than the first: nothing on this platform can observe a collection -without yielding to the host.** Node 26 / V8 14.6, same object, `gc()` at each step: +The target runs the shared suite under Node and a browser subset under Chromium; lifetimes, upcalls, +cycles, interning and `Py_ssize_t` are closed (`PM/wasmJsTest/`: `WasmFinalizationTest`, +`WasmCycleCollectionTest`, `WasmPySsizeTBoundaryTest`, `WasmBrowserRuntimeTest`, +`WasmSelectorsImportTest`); the plugin stages the runtime for consumers +(`GP/main/kotlin/python/multiplatform/gradle/WasmBrowserRuntimeStaging.kt`, +`GenerateWasmProxyExportsTask.kt`). The interpreter is built by `tools/wasm/build-cpython.sh` +(`docs/platforms/wasm-design.md`). Old §10c/§10d (browser test, CI split) are closed; their open +remainder is 10.1. + +- **10.1 The wasm test suite does not run in CI — open.** `.github/workflows/wasm.yml` runs the + `test` job only when the repository variable `WASM_RUNTIME_URL` is set. It is not set (observed + 2026-10-03: `gh variable list` empty; the latest `develop` run shows "WasmJS Node + browser tests: + skipped", compile job green). What closes it is uploading the + `python-multiplatform-wasm-runtime` zip (7.0 MB) somewhere a runner can fetch and setting the + variable; see also 15.1. +- **10.2 `suspend` upcalls and `asyncio` on wasm — open.** SPEC U-5 is `planned` on wasm (no threads), + and `import asyncio` traps the instance (`asyncio.run` wants `socketpair`); the sample skips + section 7 there (`sample/src/wasmJsMain/.../bindings/ProxyDemo.wasmJs.kt`). +- **10.3 A wasm consumer must hand-write the `pmp_invoke` export — open.** `@WasmExport` is honoured + only in the compilation that produces the final `.wasm`, so the library cannot export the general + upcall dispatcher for a consumer; `GenerateWasmProxyExportsTask` generates only the three + proxy-type slots. Without the three-line export (`PM/wasmJsTest/.../UpcallExports.kt` shows it), + `UpcallBootstrap.publishToGlobals()` throws. Generating it from the plugin, as was done for the slots, + is the obvious step. +- **10.4 A consumer's bundle in a real browser / production bundler — unverified.** The only browser + execution evidence is the in-repo `:sample` and `wasmJsBrowserTest`. An external consumer + (`consumer-plugin-android`) was verified to *build* `wasmJsBrowserDistribution` with the runtime + staged by the plugin; it was never run in a browser, and no production bundler other than the + Kotlin Gradle webpack setup was tried (also `docs/design/ecosystem.md` §5 "Unverified"). +- **10.5 The wasm CPython build lacks lzma, zstd and OpenSSL — open.** Listed as "NOT DONE" in + `tools/wasm/build-cpython.sh`'s header; ABI-sensitive per the Pyodide flag list, though they did not + gate the one compiled wheel tested (`WasmCompiledWheelTest`). +- **10.6 The sample's wasm upcall line (`presses x3 = 0`) was never cross-checked against desktop — + unverified, minor.** -``` -same job, after gc() WeakRef ALIVE registry callbacks 0 -one microtask WeakRef ALIVE registry callbacks 0 -two microtasks WeakRef ALIVE registry callbacks 0 -one macrotask WeakRef CLEARED registry callbacks 200 -``` - -A `WeakRef` keeps its target alive for the job that created it, and the registry callback is -delivered as a task. `FinalizationRegistry.prototype.cleanupSome()` would have made it synchronous -and **has been removed from V8** — `--harmony-weak-refs-with-cleanup-some` is rejected as an -unrecognised flag, from the command line and from `v8.setFlagsFromString` alike. So the three -`GCLeakTest` cases could not have passed no matter what the mechanism was: a synchronous test body -is one job, and a job that has not ended sees nothing. That is why they read as "wasm has no -finalisation hook" — the hook was the first problem, and the harness was the second. - -`commonTest` therefore drives them through `collectorTest`, which is a blocking loop on every target -whose finalisation runs on a thread and a chain of host turns on `wasmJs`. **No assertion changed.** -`kotlin-test`'s wasm adapter awaits a returned `Promise` (`TeamcityAdapterWithPromiseSupport`), -confirmed by a control that returned `Promise.reject` and duly failed; `WasmFinalizationTest` keeps a -standing `@AfterTest` control so a regression cannot silently turn those assertions into ones nobody -runs. `forceGC()` is `globalThis.gc()`, with `--expose-gc` added to the test task's `nodeJsArgs`. - -Two consequences worth carrying forward: - -- **`PyAutoCloseable.wasmJs.kt` is the only `js(…)` in `wasmJsMain`, and it has to be.** Every other - declaration this target makes is a `@WasmImport`, which carries primitives only — and what must - cross here is a *reference* to the object whose reachability is the question. The cost is confined - to construction: `registerCleaner` + `close()` is **88.6 ns**, of which the two `toJsReference()` - crossings are **44.1 ns** each; nothing on the C API call path touches it. -- **The callback cannot arrive inside a Python call.** JS tasks run only after the stack unwinds and - every call into CPython here is synchronous, so the decref never re-enters the interpreter from - within another call. That is §1's hazard, and this target does not have it. - -`WasmCleanerStats` (`registered`, `released`, `finalized`, `outstanding`) is the observability half, -and stays useful on a host with no `FinalizationRegistry`, where `outstanding` is a leak count. - -Kotlin/Native once had a `wasm32` target that could have shared CPython's linear memory; it was -deprecated in 1.8.20 and removed in 1.9.20. That history no longer costs anything — `@WasmImport` -plus an imported memory reaches the same place. - -### Upcalls: closed. CPython calls Kotlin, and cycles are collected - -`ProxyTypeFactory` builds a real `PyType_FromSpec` heap type with `tp_traverse`, `tp_clear` and -`tp_dealloc` filled by Kotlin `@WasmExport`s installed into CPython's own -`__indirect_function_table`. `wasmJsTest/.../WasmCycleCollectionTest` runs the same two cases -`desktopTest/CycleCollectionTest` does, and both pass: a Kotlin↔Python cycle is broken by -`gc.collect()`, and 100 proxies that die *without* a cycle release their `HandleTable` entries and -leave the heap type's own refcount exactly where it started. - -The measurement this section already had — 3.1 ns through `call_indirect`, against 10.9 ns for the -`addFunction`-and-JS-closure route §5 specified — was the mechanism. What this pass added was the -wiring, and it turned up three constraints worth carrying forward: - -- **`@WasmExport` is honoured only in the compilation that produces the `.wasm`.** Measured, not - inferred: the identical annotation on the identical function exports from `wasmJsTest` and does - not from `wasmJsMain`, whose klib is linked in — the test binary's export section came out with - `startUnitTests` in it and nothing else. **A library cannot export its own trampolines.** The - executable module must declare three delegating lines; `wasmJsTest/.../ProxyTypeExports.kt` was - that file, kept in the suite precisely so the tests exercise the path an application takes. - ~~Generating it from `python-multiplatform-gradle-plugin` is the obvious next step and is not - done.~~ **Closed 2026-08-18 (`112b54d0`).** `GenerateWasmProxyExportsTask` - (`python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/GenerateWasmProxyExportsTask.kt`) - generates the three delegating lines from a slot list; `PythonBindingsPlugin.kt:710` registers it - and wires its output into `wasmJsMain` automatically. `ProxyTypeExports.kt` was deleted in the - same commit — see §14b item 8. -- **Registration needs the Kotlin instance's raw exports, and only the generated entry module has - them.** `cpython.mjs` cannot fetch them: it is imported *by* Kotlin's import object, so importing - the entry module back would be an ES cycle across a top-level await. So `build.gradle.kts` appends - `pmpSetKotlinExports(exports)` to the entry module. This target now has **two** generated-file - substitutions, not the one this section used to advertise. -- **Kotlin/Wasm has no `call_indirect`,** so calling a C function *pointer* — `visitproc`, `tp_free` - — goes through `WebAssembly.Table.get` in JS. Both are cold (a cyclic collection, a deallocation); - neither the downcall path nor CPython's path *into* Kotlin touches JavaScript. - -Also recorded because it cost time: `Module.UTF8ToString` is not available. This build exports -`wasmExports` and `wasmMemory` and nothing else from the Emscripten runtime, deliberately, so any -other helper fails as `undefined is not a function` inside a wasm import and arrives in Kotlin as an -empty `JsException`. - -### Interning: taken, and the answer here is not Android's - -All 59 string-argument positions across 49 `actual`s go through `Wasm.internedUtf8` or -`Wasm.scratchUtf8` now, with the per-argument judgement identical to `desktopMain`'s and -`androidMain`'s on all 59. Measured at 200 000 iterations: - -| ns per argument | 7-char name | 54-char name | -|---|---|---| -| `malloc` + `encodeToByteArray` + copy + `free` (what it did) | **82.7** | | -| `malloc` + `free` alone | 13.0 | | -| `encodeToByteArray` alone | 59.2 | | -| `Wasm.scratchUtf8` | **30.6** | **157.0** | -| `Wasm.internedUtf8` (hit) | **22.2** | **26.4** | - -`PyObject_GetAttrString` through the `actual`, both rows back to back in one run so that only the -marshalling differs: **258.0 → 229.2 ns**. - -**The rule transferred; the reasoning did not, and neither did the size of the win.** Three -findings: - -- The expensive part was the **intermediate WasmGC `ByteArray`**, not the encode. Writing UTF-8 - straight into linear memory is what takes 82.7 ns to 30.6. Shared memory is what makes that - possible — the destination is CPython's own heap, so there is no staging buffer — which is the - *opposite* of the original design note's claim that shared memory removes a string copy. It - removes the allocation. -- **Interning's whole value is in the string length.** For a 7-character name it beats scratch by - 8 ns, nothing like Android's 15x (2238 → 148). For a 54-character one it beats it by **6x** - (157.0 → 26.4), because a cache hit is flat in the length and an encode is linear. Reporting only - the short case — which is what the first measurement here did — would have argued interning is - barely worth having. -- Negative result worth keeping: `BenchmarkTest`'s shared `PyObject_GetAttrString` row **did not - move** across this change (376.6 → 384.0 ns). That is two whole suite runs compared against a - ~29 ns per-call difference; it is not a usable A/B at this size, in either direction. The in-run - rows are the measurement. - -### `Py_ssize_t`: two guards, because the two halves fail differently - -The declaration half — `bindings.kt` against `python.wasm` — is derived now rather than remembered. -`./gradlew :python-multiplatform:verifyWasmAbiSignatures` parses the wasm binary's type, import, -function and export sections and compares all 316 `@WasmImport` declarations against it, plus the 3 -glue functions against `cpython.mjs`. It runs before `wasmJsNodeTest`. Confirmed by deliberately -mis-declaring one and watching it fail with the function named: - -``` -PyList_Size: bindings.kt declares (i32) -> (i64), python.wasm has (i32) -> (i32) -``` - -Same shape as `generateDesktopReachabilityMetadata` (§7): read the artefact that decides, every -build, never a checked-in copy of the answer. The task also asserts that every `external fun` in the -file was parsed — the first version required an explicit return type and so silently skipped the -three `Unit`-returning declarations, which is §2's "green test measuring its own scope" again. - -The conversion half links cleanly and is silently wrong, so it needed a different guard. -`EmbedAPI.wasmJs.kt` names the two directions instead of casting inline — `Int.pySsizeToLong()` -sign-extends, because -1 is the error return and the unsigned widening that is correct for pointers -here would turn it into 4294967295; `Long.toPySsize()` range-checks rather than truncating, so -`PyList_New(0x1_0000_0000)` fails instead of quietly becoming `PyList_New(0)`. -`WasmPySsizeTBoundaryTest` covers both directions and the whole `Int` range. - -### The sample runs in a real browser, and the gap that stopped it was the library's - -`c327bb54` turned on `:sample`'s wasmJs target. It compiled, and the library's own wasm suite covers -the mechanism, but **the app had never come up in a browser** — and the report that closed that -round said only that in-browser confirmation was blocked on "a Gradle staging gap that is identified -and is not a library defect". Neither half of that sentence survived being checked. The gap was -recorded nowhere in the repository, and it *is* a library defect. - -It is running now. Chromium 150 (headless, software WebGL), served over HTTP from -`sample/build/dist/wasmJs/developmentExecutable`, console verbatim: - -``` -INFO: Python initialized successfully! -runtime : 3.14.2 · sys.platform=emscripten · Web emscripten (wasm32) / Wasm Kotlin/Wasm -eval : sum(kotlin_numbers) * 2 -> int: 56 -table : 42 entries, 4 classes, from io_github_thisisthepy_sample -upcall : handle 4294967325 -> 0 · with args -> presses x3 = 0 -proxies : installed over PyCFunction (self as dispatcher): 519 lines, 1 proxy classes -Greeter('Kotlin').greet(2) -> hello Kotlin! hello Kotlin! -Greeter.PUNCTUATION -> ! -await : skipped on wasmJs -- import asyncio traps this target, see ProxyDemo.wasmJs.kt -``` - -Sections 1-6, all of them, in the browser. Section 7 stays gated, untouched: `asyncio.run` still -wants `socketpair` → `ws`. The DOM after the run holds one `` and -skiko logs `GPU stall due to ReadPixels`, so **Compose rendered as well** — CPython, Kotlin/Wasm and -skiko are three wasm modules on one page. - -**Four things were missing, and the first is what the failure looked like:** +## 11. Build wiring -``` -Module not found: Error: Can't resolve './cpython.mjs' in - '/build/wasm/packages/PythonMultiplatformMobile-sample/kotlin' -``` +**Closed.** Android staging tasks are wired by `configureEach` and `dependsOn`, no +configuration-time copies or `whenTaskAdded` remain (`python-multiplatform/build.gradle.kts`). Old +§11b (Android runs `commonTest`) is **closed**: `androidInstrumentedTest` depends on `commonTest`. -1. **`cpython.mjs` does not reach a consumer's webpack context.** It is a resource of - `python-multiplatform`'s `wasmJsMain`, and the *generated import object* of anything that links - the library carries `import * as ... from './cpython.mjs'` — because `bindings.kt` declares - `@WasmImport(MODULE, ...)` against it. Nothing propagates it. Checked rather than assumed: the - library's wasmJs klib (`build/classes/kotlin/wasmJs/main/default/resources`) carries no - `cpython.mjs`, so there is no artefact a consumer could unpack it from either. -2. **`python.mjs`/`python.wasm` are staged only next to the *library's own* test bundle**, by a - `doFirst` on `KotlinJsTest`. -3. **The stdlib had no browser route at all.** `cpython.mjs` reached it through NODEFS mounted over - the real build directory, plus `import fs from "node:fs"` — which in a browser bundle is not a - branch that is never taken but a webpack resolution failure. -4. **The `@WasmExport` handoff was inserted in the wrong place for an executable.** See below; that - one is a genuine bug, not a missing copy step. +## 12. Smaller known items -**This is the library's, on the measure that decides it: an external consumer meets all four -identically.** None is specific to the sample, and none is fixable from the consumer side without -knowing three library internals — the glue file's existence, the `intrinsics.memory` placeholder, -and the `wasmInstance.exports` handoff. The split as implemented: +Closed from the old list: Sigstore verification (`-PverifyPythonSignatures=true`), the Android +pre-release URL, `PyList.subList` (a live `PySubList`), the `TODO` triage, `runMain`/`runApp` +(`PM/commonTest/.../ffi/RunMainTest.kt`, `RunAppTest.kt`), Android `TMPDIR` +(`PythonBootstrapTest.initializeLeavesTempfileUsable`), the `EmbedAPI.kt` section order (sections +27–29 now sit in docs order), stale test headers, the sample revisit. + +- **12.1 `Python3.finalize` should call `Py_FinalizeEx` — open.** It still calls `Py_Finalize()` + (`Python3.kt`, `finalize`); `Py_FinalizeEx`'s `expect` and every `actual` already exist. Blocked + only by finalisation being untestable in a suite that shares one interpreter (its only caller is + `PM/artMain/.../ffi/JniExport.kt`). +- **12.2 `TMPDIR` for androidNative inside an app, and on an iOS device — open / unverified.** The + androidNative suite passes only because `adb shell` sets `TMPDIR=/data/local/tmp`; a binary shipped + in an app would have neither `/tmp` nor that variable. iOS passes on the simulator because `/tmp` + is the host's; a device needs `NSTemporaryDirectory()`, never run. Probe: the `scriptPath…` cases + of `RunAppTest`. +- **12.3 Linux and Windows desktop have never run the suite — open.** `.github/workflows/desktop.yml` + has `os: [macos-latest]` only, and no log anywhere shows a Linux or Windows run (SPEC P-2, N-4). Adding + the two runners to the matrix is the step. +- **12.4 CI on `develop` is not green — open (observed 2026-10-03).** All four platform workflows now + run on GitHub Actions (the old "CI has never run" is closed). The latest `develop` Desktop CI run + failed one test, `PyValueBytesConversionTest.bytesConversionCostIsPaidOncePerValueAndIsReportedAgainstStr` + (a timing assertion: 1000 cached reads must cost less than one conversion), while the same commit's + PR run passed — a timing assertion that is not stable on a shared runner. Decide whether such + assertions belong in CI. +- **12.5 BeeWare's iOS archives (≤ 3.14) carry no signature or checksum — not fixable here.** The + SHA-256 lockfile pin is the only integrity check for that source; python.org's signed XCframework + replaces it from 3.15. python-build-standalone likewise publishes no per-file signature + (`SHA256SUMS` only). `docs/platforms/python-version-acquisition.md` §5. -| | where | -|---|---| -| browser-capable `cpython.mjs` (env branch, webpackIgnore imports, stdlib zip into MEMFS) | library source | -| staging `cpython.mjs` + `python.mjs` + `python.wasm` + `python3.14.zip` + generated config | library, `stageWasmBrowserRuntime` | -| adding that directory to `wasmJsMain.resources`, patching the compile-sync output | `:sample` | +## 13. The sample, and the AGP version that shapes it -The browser stdlib route is CPython's own: `Tools/wasm/emscripten/web_example/python.worker.mjs` -fetches `python3..zip` into MEMFS at `/lib/python3.zip` under `addRunDependency`, and -sets **no `thisProgram`**, so `sys.prefix` stays `/`. `cpython.mjs` does the same, reading the -version out of the running interpreter rather than from a string that would then need keeping in -step. +The sample runs on desktop, Android (both emulators), the iOS simulator and in a browser; the bindings +plugin applies to Android modules (AGP 8.10.1); `ksp-fixtures/android` exists; the generated +`installGeneratedUpcallTable` `actual` reaches intermediate source sets; the per-platform upcall +trampolines exist. GraalVM native image upcalls are verified by hand +(`docs/platforms/graal-native-image-verification.md`, `PM/desktopTest/.../ReachabilityMetadataTest.kt`). + +- **13.1 The iOS app bundle carries no Python standard library — open.** (Was old §9 item 4 and §14b + item 11.) The Xcode "Install Target Specific Python Standard Library" phase + (`iosApp/iosApp.xcodeproj/project.pbxproj`) rsyncs `…/ios-arm64_x86_64-simulator/lib/`, which holds + only `libpython3.14.dylib`, and nothing in the app path sets `PYTHONHOME` + (`sample/src/iosMain/.../MainViewController.kt` says so). The app starts only with a manual `rsync` + plus `SIMCTL_CHILD_PYTHONHOME` (recipe in git history, old §13). To decide first: stage by Gradle or + by a corrected Xcode phase; take `PYTHONHOME` from the environment or derive it from the bundle in + `iosMain` without breaking the test task. The "Prepare Python Binary Modules" phase rewrites + `lib-dynload/*.so` into `.fwork` placeholders, so a bundle-hosted stdlib needs BeeWare's importer + or that phase disabled. A `PYTHONHOME` on an external volume hangs the sandboxed app in `open()` + rather than failing. +- **13.2 Native-image upcall verification as an automated test — open.** SPEC U-6/N-5: the procedure + is manual (`:sample:nativeCompile`, Liberica NIK); only the reachability metadata is guarded + automatically. +- **13.3 iOS sample section 7, slow path — open (sample only).** "not wired on iOS": resuming a parked + continuation needs a Kotlin/Native worker the sample does not start + (`sample/src/iosMain/.../bindings/ProxyDemo.ios.kt`). The library itself does it + (`PM/nativeTest/.../AsyncUpcallNative*Test.kt`). +- **13.4 Targets compiled but never tested — open.** `iosArm64`, `iosX64`, `androidNativeX64` (SPEC + P-1). For `iosX64`, `extractIosSimulatorStdlib` hardcodes `lib-arm64` + (`python-multiplatform/build.gradle.kts`, the `…/ios-arm64_x86_64-simulator/lib-arm64/python$libVersion` + copy); an x86_64 simulator needs `lib-x86_64`. Stale text nearby: that task's KDoc still says + `lib/python3.13`, and `PM/iosMain/README.md` says the stdlib comes from BeeWare without the + ≥ 3.15 python.org qualification. -### Appending the upcall handoff works only for a test bundle +## 14. Current state, gathered -The one real bug in that list, and the browser is what surfaced it. The entry-module patch appended - -```js -import { pmpSetKotlinExports } from './cpython.mjs'; -pmpSetKotlinExports(exports); -``` - -to the end of the generated entry module. That is correct for the library's own test bundle, whose -entry module ends at `setWasmExports(wasmExports)` — its runner calls `startUnitTests` later, from -outside. An **executable** bundle does not end there. `binaries.executable()` makes the entry module -finish with `exports._start()`, which *is* Kotlin `main()`, so the handoff ran after the whole -application had already executed: - -``` -upcall : PyException: name '_pm_resolve' is not defined -proxies : IllegalStateException: could not register wasm export 'pmp_invoke' as an upcall entry - point (pmpRegisterUpcall returned -1) -PyException: No module named 'org' -``` - -`-1` is `kotlinExports === null`. Sections 1-4 passed in that same run, which is why nothing about -the diagnosis was visible from the code: the memory substitution, the interpreter, the C API and the -generated table were all fine. The fix inserts the call *before* `exports._start()`; the `import` -stays at the top of the file, where hoisting already put it. - -Worth stating as a pattern rather than as one fix: **a test bundle and an executable bundle differ in -the generated entry module, and every wasm verification this repo had was of the first kind.** - -### The JSPI warning: the premise holds, the consequence did not fire here - -`wasmJsMain/README.md` warns that deleting `WebAssembly.promising` and `WebAssembly.Suspending` -mutates a host intrinsic globally, and that on a browser page shared with another wasm module using -JSPI this would break that module. The browser run is the first chance to check it. Measured on both -sides, same browser: - -| | `typeof WebAssembly.promising` | -|---|---| -| a page that does not load the app | `function` | -| the app's page, 20 s after load | `undefined` | - -So the delete is **not** a no-op here — Chromium 150 ships JSPI, and the property really is gone -page-wide. What did *not* happen is the breakage: Compose came up on the same page (`canvases=1`), -skiko rendered, and Kotlin/Wasm ran, because none of those three uses JSPI. The warning stands as -written — it is about a page that loads a JSPI consumer, and this page does not — but its cost is now -bounded by observation rather than by argument. - -### What is still open - -- ~~**The wiring is not in `python-multiplatform-gradle-plugin`, so an external consumer still meets - the hole.**~~ **Closed, and this bullet outlived it.** `WasmBrowserRuntimeStaging.kt` is in the - plugin and compiled, and "The plugin wiring is closed, and it needed the runtime to become a Maven - artefact first" below records the verification against an external consumer. The bullet was never - struck when that landed — the same failure this document keeps finding elsewhere, an item read as - open because nobody went back to close it. The original text follows for the history. - - `:sample` carries ~50 lines that are a copy of `patchKotlinWasmOutputForCPython`, - because one Gradle build script's functions are not visible to another project's build script. - Moving it into the plugin needs the staged runtime to be a *published artefact* first, and the - wasm CPython build is published nowhere — it is a local directory named by `-PwasmPythonDir`. That - is the same shape as §15e item 4 (`StagePythonHomeTask`), which solved the desktop version of - exactly this problem, and it is the obvious next step. -- ~~**`wasmJsBrowserTest` (karma) is not wired.**~~ **Wired, and it catches three of the four.** - See §10c below for what it does and does not guard, and for the one item that had to be closed a - different way. -- The upcall line reads `presses x3 = 0`, which is what that code prints for a freshly constructed - counter; it was not cross-checked against a desktop run in this pass. - -### 10c. `wasmJsBrowserTest`, and the one defect a browser test cannot express - -**Ten tests, 4.5 s, Chromium 150 headless, and a build failure when the browser route breaks.** -`wasmJs` declares `browser { testTask { useKarma { useChromeHeadless() } } }` alongside `nodejs()`, -and the task runs a deliberately small filter rather than the suite twice: - -| | why it is in the browser run | -|---|---| -| `python.multiplatform.browser.WasmBrowserRuntimeTest` (6) | the claims only a browser can make | -| `python.multiplatform.ffi.WasmSelectorsImportTest` (4) | JSPI deletion is a *host* decision, and its regression mode is `abort()` rather than a red test | - -The other ~344 already run under Node against the same Kotlin and the same interpreter; running them -again would buy nothing but time. What the browser adds is webpack, HTTP and no filesystem, so the -new class asserts exactly those: that the host is a browser at all (so nothing below can pass -vacuously), that `cpython.mjs` survives bundling, that `sys.prefix` is `/` and `json.__file__` is -inside `/lib/python314.zip` (the MEMFS route, which has no Node equivalent), that a `PyObject` round -trip still reads Emscripten's memory through the *bundled* import object, and that -`WebAssembly.promising` is gone while `select.poll()` still returns. - -Three pieces of plumbing were needed, and each is a browser fact: - -- **`webpackCopy`.** `cpython.mjs` loads the glue with `import(/* webpackIgnore */ './python.mjs')`, - so the *browser* resolves that specifier — relative to the bundle, which karma-webpack writes to a - temp directory with a fresh name every run. A fixed URL cannot work; the files have to be copied - next to the bundle, which is what `kotlin-web-helpers`' `webpackCopy` hook does. -- **A proxy for the stdlib zip.** `STDLIB_ZIP_URL` is document-relative because in a real - distribution the zip sits next to `index.html`. karma's document is `/context.html` at the server - root, so the proxy is what makes karma's page look like that distribution. -- **`CHROME_BIN`, resolved in Gradle from one list**, with a skip-and-say-so when nothing is found — - the same contract a missing CPython Emscripten build already has. On this machine the browser is - Naver Whale, which reports `HeadlessChrome/150.0.0.0`; that is what "Chromium 150" above was. - -**Verified by breaking it, not by watching it pass.** Three reverts, three failures: - -| reverted | how it failed | -|---|---| -| the dynamic `node:fs` import → static, as it was before §10 | `UnhandledSchemeError: Reading from "node:fs" is not handled by plugins` — webpack, before a test runs | -| the staged stdlib zip | `404: /python3.14.zip`, zero tests reported: bring-up dies inside `addRunDependency` | -| the `pmpSetKotlinExports` handoff | `IllegalStateException: ... pmpRegisterUpcall returned -1` — §10's own message | - -**And one that did not fail, which is the finding.** Reverting the entry-module *ordering* fix — the -one real bug §10 found — changed nothing: all ten tests stayed green. The reason is structural -rather than a gap in the filter. A **test** bundle's generated entry module contains no -`exports._start()` at all (`grep -c _start` → 0), so the fixed and reverted forms of -`handedOffEntryModule` produce a byte-identical file. The bug is only expressible in an -**executable** bundle, and the library builds none. - -So that item is closed by a postcondition instead: both `patchKotlinWasmOutputForCPython` and -`:sample`'s copy now refuse to write an entry module in which the handoff follows `_start()`. -Checked the same way — reverting `:sample`'s placement fails -`:sample:wasmJsBrowserDevelopmentWebpack` with the ordering message, on the real executable bundle. - -~~**Not added to `.github/workflows/wasm.yml`, and the reason applies to the job already there.**~~ -**Superseded — see §10d.** The diagnosis was right (the runner has no CPython Emscripten build, so -every wasm test task's `onlyIf` skipped and `wasmJsNodeTest` reported green having run zero tests) -but its conclusion — wait for the build to be published — turned out to be one measurement away from -unnecessary. The five staged files are enough; NODEFS was never carrying the standard library on -purpose. - -### The plugin wiring is closed, and it needed the runtime to become a Maven artefact first - -The item above used to read "the wiring is not in `python-multiplatform-gradle-plugin`, so an -external consumer still meets the hole," gated on the runtime becoming a *published artefact* — -the same shape as §15e item 4 (`StagePythonHomeTask`), which solved the desktop version of exactly -this problem. Both halves are done now. - -**(b) — is the wasm CPython build publishable?** Checked rather than assumed. `build-cpython-abi.sh` -hand-patches CPython 3.14.2's own source (`Lib/sysconfig/__init__.py`, the generated `Makefile`) to -carry `-fwasm-exceptions -sSUPPORT_LONGJMP=wasm` and a `PYEMSCRIPTEN_PLATFORM_VERSION` that does not -exist upstream, plus `wasmExports`/`wasmMemory` in `EXPORTED_RUNTIME_METHODS` — the one addition -that exists *only* so this library's `@WasmImport` declarations have something to bind to. PEP 783's -`pyemscripten_2026_0` tag is real, accepted, and PyPI-validated (confirmed via web search), and -Pyodide's own build tooling knows how to produce wheels for it — but nothing distributes a -`python.wasm` with the JS-reachable exports this library needs, and no distributor has a reason to. -So there is no URL for a `StagePythonHomeTask`-style download to point at; the desktop pattern does -not transfer as-is. - -The candidate the earlier pass named — bundle it in a library artefact — does transfer, and the -size argument favours it more cleanly than it did for desktop. Desktop's `PythonHomeStaging` chose -*not* to embed specifically because one artefact would carry all four platforms' payload for every -consumer (87.7 MB of `libpython`, of which any one consumer uses a quarter) — the defect §15d had -already found and fixed once. wasm has exactly one platform, so there is no multiplication: a -wasmJs consumer needs precisely this payload, once. Measured rather than guessed: -`stageWasmBrowserRuntime`'s output directory is 13 MB (`python.wasm` 9.2 MB, `python3.14.zip` -3.6 MB, `python.mjs` 0.56 MB, the two `cpython*.mjs` glue files); zipped, 7.0 MB — smaller than -§15d's post-fix 23 MB Android AAR. - -`python-multiplatform/build.gradle.kts` now zips that directory (`wasmBrowserRuntimeZip`) and -publishes it as its own artefact ID, `io.github.thisisthepy:python-multiplatform-wasm-runtime`, at -the *library's* version (not the plugin's — see the coordinate-derivation comment in -`generateCoordinates`, `python-multiplatform-gradle-plugin/build.gradle.kts`), so resolving the -library and resolving the runtime it needs can never drift into two hand-copied version literals. -`./gradlew :python-multiplatform:publishWasmRuntimePublicationToMavenLocal` (folded into the -existing `publishAllToMavenLocal`) lands it at -`~/.m2/repository/io/github/thisisthepy/python-multiplatform-wasm-runtime/`. - -**(a) — the plugin.** `python-multiplatform-gradle-plugin` gained -`WasmBrowserRuntimeStaging.kt`: `StageWasmBrowserRuntimeTask` resolves the runtime coordinate from -a *detached* configuration and unzips it, and `PythonBindingsPlugin` registers it unconditionally -but only resolves it — no network access, no `mavenLocal` lookup — when a `wasmJsProcessResources` -or `wasmJs*Webpack` task actually exists to depend on it (`tasks.matching`, a live view, the same -trick `isBindingKspConfiguration` already relies on). The three substitutions `:sample` used to -hand-copy — `wasmJsMain.resources.srcDir`, the `intrinsics.memory` placeholder, the upcall -handoff — are all here now, matched by task *name* rather than by Kotlin Gradle Plugin task type -(`org.gradle.language.jvm.tasks.ProcessResources` is Gradle's own class, not the KGP's, so casting -`wasmJsProcessResources` to it does not add that dependency) — consistent with why `TEST_WORD` -matches KSP configurations by name in the first place: this plugin's `kotlin-dsl` classpath does -not carry the Kotlin Multiplatform Gradle plugin at all. - -One correction made against real friction rather than assumed up front: the patch's `modulePrefix` -(`"${rootProject.name}-${project.name}"`, copied from `:sample`, which is a subproject) is wrong for -a **single-module** consumer, where the `wasmJs()` target sits on the root project itself — Kotlin -does not double the name there (`consumer-plugin-android.mjs`, not -`consumer-plugin-android-consumer-plugin-android.mjs`). First attempt failed with "No -*.import-object.mjs in .../consumer-plugin-android-consumer-plugin-android/kotlin"; fixed by -deriving the name from `project == project.rootProject` instead of assuming the subproject shape. - -A second correction, found the same way: a consumer whose reachable wasmJs code never calls into -`python-multiplatform` at all has its `@WasmImport` declarations dead-code-eliminated along with the -`./cpython.mjs` import that names them — `consumer-plugin-android`'s KSP-generated `FunctionTable` -wraps plain Kotlin functions with no path into `bindings.kt`, and hit exactly this. The prior copies -of this patch (library, `:sample`) throw when that import is missing, because both always reach the -FFI. The plugin's copy logs and returns instead — nothing to patch is a legitimate outcome for a -generic external consumer, not a broken one. - -**Verified against `/Volumes/macMini/consumer-plugin-android`, an external consumer that resolves -everything by Maven coordinates** (`io.github.thisisthepy:python-multiplatform:3.14.7-alpha01`, the -plugin `3.13.0`) **with no `-PwasmPythonDir` of its own**: - -- With no reachable Python call (the baseline fixture), `wasmJsBrowserDevelopmentWebpack` succeeds, - patch skipped, logged as such. -- With one added (`fun main() { ... Python3::initialize ... }`, forcing DCE to retain the FFI - surface), the same task's log reads: - ``` - Pointed consumer-plugin-android.import-object.mjs's intrinsics.memory at Emscripten's wasmMemory - Handed consumer-plugin-android.mjs's wasm exports to cpython.mjs for upcall registration - ``` - — both substitutions the library's own build performs on itself, now performed by the plugin on a - project three artefact resolutions away from this repository. -- `wasmJsBrowserDistribution` carries the full runtime — `python.wasm`, `python.mjs`, - `python3.14.zip`, `cpython.mjs`, `cpython-config.mjs` — next to the consumer's own bundle, staged - there by nothing the consumer wrote. - -Not verified: actually running the resulting bundle in a browser. In scope for this pass was -compilation only (desktop and wasmJs; no device, no simulator); §10's own browser run above is still -the only execution evidence this repository has. - -Repository baselines unchanged by this — desktop 360/0/1, `ksp-fixtures` 64/0, wasmJs 344/0/0, all -re-measured after the change rather than assumed carried over. - -### 10d. The wasm CI job ran zero tests and reported green, and the fix was one measurement away - -**The job did nothing, and said nothing about doing nothing.** `.github/workflows/wasm.yml` asked a -runner for `wasmJsNodeTest`; the runner has no CPython Emscripten build; the task's `onlyIf` skipped -it; the job went green. Reproduced before changing anything — -`./gradlew :python-multiplatform:wasmJsNodeTest -PwasmPythonDir=/nonexistent/cpython-emscripten` → -`BUILD SUCCESSFUL`, exit 0, `build/test-results` not even created. (On this machine the *default* -path exists, so reproducing the runner needs the property pointed somewhere absent; running "without -`-PwasmPythonDir`" reproduces the developer, not the runner.) - -**What the guard was actually reading, and whether the published zip could satisfy it.** Three -guards, three markers, all under `wasmPythonDir`: the `KotlinJsTest` tasks want `python.mjs`, -`verifyWasmAbiSignatures` and `stageWasmBrowserRuntime` want `python.wasm`. The file *list* in -`python-multiplatform-wasm-runtime` matches what the test staging copies exactly — `python.mjs`, -`python.wasm`, `python3.14.zip`, plus the two glue modules. So the zip looked sufficient. It was not, -and the reason is the finding: - - ./gradlew :python-multiplatform:wasmJsNodeTest -PwasmPythonDir= - → 344 tests, 344 failed: "CPython could not be initialized in this environment" - → node, directly: Fatal Python error: Failed to import encodings module - -**Node was never getting the standard library from anything an artefact could carry.** Printed from -a run against the real build directory: - - sys.path = ['/lib/python314.zip', - '/Volumes/macMini/wasm-build/cpython314-abi/Lib', <-- the source checkout - '.../cross-build/wasm32-emscripten/build/python/build/lib.emscripten-...'] - -Entry 0 is a file **nothing had ever created** — the browser branch writes it into MEMFS, the Node -branch did not — and the interpreter was silently falling through to entry 1, the CPython *source -checkout*, which is neither in the build directory, nor in the zip, nor anywhere a consumer could -get it. Every wasm test in this repository had been depending on it. - -So `cpython.mjs` now installs the staged zip on both hosts (`installStdlibZip`, read with `fs` under -Node and `fetch` in a browser), and sets `PYTHONHOME=/` — which both answers getpath (no more -`Could not find platform independent libraries ` when `PYTHON_DIR` is an unpacked artefact -with no `python.sh`) and stops a host shell's `PYTHONHOME`, plausibly pointing at a *desktop* CPython, -from aiming the wasm interpreter at the wrong ABI. Node and browser now resolve the stdlib -identically: `sys.prefix == '/'`, `json.__file__` inside `/lib/python314.zip`. - -**Result: the published artefact alone runs the suite.** Unpack -`python-multiplatform-wasm-runtime-3.14.7-alpha01.zip` (7.0 MB, mavenLocal) into an empty directory, -point `-PwasmPythonDir` at it, and `wasmJsNodeTest` + `wasmJsBrowserTest` are **344/0/0 and 10/0/0** — -the recorded baselines, from five files and no CPython checkout. Desktop re-measured at 360/0/1. - -**And a skip can no longer be reported as success.** `-PrequireWasmRuntime=true` (`wasmRuntimePresent` in -`python-multiplatform/build.gradle.kts`) turns every one of those `onlyIf` skips into a failure -naming the path it looked for — including `wasmJsBrowserTest`'s *other* skip, a missing -Chromium-family browser, which is the same defect by a different route. The message is logged before -it is thrown because Gradle wraps an exception raised from an `onlyIf` predicate and reports only -`Could not evaluate spec for 'Task satisfies onlyIf spec'` — checked, not assumed: the first version -of this printed exactly that and nothing else, which would have traded a green tick that says nothing -for a red one that says nothing. - -**What is still open: the runner has nowhere to download the zip from.** It has never been pushed to -a remote repository — `publishWasmRuntimePublicationToMavenLocal` is the only publish that has ever -run — and the two obvious candidates are both circular: a runner cannot `publishToMavenLocal` the -zip, nor depend on `wasmBrowserRuntimeZip`, because building it needs the very CPython Emscripten -directory the runner lacks. Nor can the runner build one: `build-cpython-abi.sh` needs emsdk 5.0.3, a -CPython 3.14.2 checkout and two hand-patches, and no upstream ships a `python.wasm` with -`wasmExports`/`wasmMemory` exported. - -The workflow is therefore split, and the split is the honest part: - -| job | runs | claims | -|---|---|---| -| `compile` | always | `compileKotlinWasmJs` + `wasmJsTestClasses`. Real coverage a runner can deliver, and it is *named* "no interpreter" | -| `test` | only when the repository variable `WASM_RUNTIME_URL` is set | the real suite, with `-PrequireWasmRuntime=true` and a floor assertion on the result XML counts (344 / 10) | - -When the variable is unset the `test` job does not run at all — grey in the checks list, which is a -different claim from green — and `compile` writes a step summary plus a `::warning` saying which -coverage is absent and what to set. **One upload of the 7.0 MB zip to a release asset closes it**; -until then nothing in this workflow reports success over an empty run. - -Not verified: the workflow on a real runner (no CI run is possible from here) and the -missing-browser branch of `-PrequireWasmRuntime=true` (this machine has Whale, and the probe list finds -it). Everything else above was reproduced locally against a pristine unpacked zip, including the -fetch-and-unpack step run verbatim out of the YAML. - -## 11. Build wiring - -**Closed.** The dependencies are declared now, and the graph was read rather than assumed: -`./gradlew :python-multiplatform:connectedDebugAndroidTest --dry-run` plans 93 tasks, and every -staging step is in it, in the right order: - -``` -16, 20 linkMultiplatform_python3.14DebugSharedAndroidNativeArm64 / X64 (before preBuild at 23) -49 copyAndroidPythonAssets (before generateDebugAndroidTestAssets 52, merge…Assets 58) -76, 81 linkAndroidNativeArm64 / X64 -82 copyAndroidPythonBinaries (before mergeDebugJniLibFolders 83, …AndroidTest… 86) -``` - -`:sample:assembleDebug --dry-run` pulls the same four in, so the app path is covered too, not -just the instrumented tests. The manual pre-step this section used to prescribe is no longer -needed. - -Three edges do the work, all of them lazy-safe: - -- `copyAndroidPythonBinaries` declares `dependsOn(linkAndroidNativeArm64, linkAndroidNativeX64, - downloadAllPythonBuilds)` -- `tasks.configureEach` attaches it to every `merge*JniLibFolders` / `merge*NativeLibs`, and - attaches `copyAndroidPythonAssets` to every task whose name ends in `Assets`. This is - `configureEach`, not the `whenTaskAdded` that used to miss tasks registered later — which is - how the stdlib went unpackaged and `Py_Initialize()` aborted. -- `preBuild.dependsOn(linkTaskProvider)` for each shared-library binary, so the `.so` is relinked - and staged into `build/android//jniLibs//` before AGP looks there. - -The configuration-time `copy {}` that staged the *previous* build's library was fixed earlier by -moving it into `doLast`; that fix is still in place and is the only `copy {}` in the script that -is not already inside a task action (checked: lines 201, 245, 281 and 475 are all `doLast`). - -**One hardcoded count of the same family was removed.** `jni_onload.def` still carried -`#define NUM_METHODS 127` after `RegisterNatives` had been switched to `sizeof(methods) / -sizeof(methods[0])`. It was dead, but it is exactly the constant that once drifted out of step -with the table and silently bound a prefix of it, so it is gone and a comment says why. - -No other drifting constant or configuration-time side effect was found in -`python-multiplatform/build.gradle.kts` or `sample/build.gradle.kts`; there are no -`whenTaskAdded` uses left in the repository. - -## 11b. Android does not run the object-model tests - -**Closed.** `androidInstrumentedTest` depends on `commonTest` and both emulators run the full -suite: 176 tests each, where discovery used to be 19. ~~Eight failures remain, tracked in §2.~~ -**Those eight are fixed too** — §2's own closing measurement, reached after this section's, has -both emulators at 213 tests with zero failures. - -Wiring it up immediately paid for itself — the suite died on its 2nd test, then its 12th, and the -two defects behind that (an unpackaged stdlib and a hardcoded `RegisterNatives` count) had been -invisible because nothing on Android had ever executed those tests. - -**Was:** the largest hole in the suite. - -`commonTest` holds 111 tests over `PyObject`, `PyDict`, `PyList`, conversion, exceptions and -refcounting. They reach desktop and iOS through the source-set graph: - - jvmTest.dependsOn(commonTest) - desktopTest.dependsOn(jvmTest) - -`androidInstrumentedTest` has no such edge, so **none of the 111 run on Android**. Its entire -coverage is 19 instrumented tests — 14 of JNI wiring and benchmarks, 5 of `AssembledApiTest`. - -That is the wrong platform to under-test. §2 was exactly a case of the object model being broken -on Android alone — the production API crashed on the first `Python3.exec` — and the only thing -that caught it was an acceptance test written by hand for the purpose. The 111 would very likely -have caught it earlier. - -They cannot be `androidUnitTest`: the native library is not loadable there, so they have to run -instrumented. Whether `kotlin.test` annotations are discovered by the AndroidJUnit4 runner, and -how the interpreter gets initialised on device, are the two things to establish. - -**Expect failures when this lands.** These tests have never run on Android; whatever they report -is the actual state of the Android object model, and that is the point of doing it. - -## 12. Smaller known items - -- ~~**Download integrity**: python.org's Sigstore bundles are not checked.~~ **Done for every - archive that publishes one.** `-PverifyPythonSignatures=true` verifies the sibling - `.sigstore` in-process with `dev.sigstore:sigstore-java`, pinning the release manager's - Fulcio SAN and OIDC issuer — both Android tarballs, and from 3.15 the iOS XCframework. The claim - this entry rested on, that Sigstore verification needs an external CLI and is therefore - unreasonable inside Gradle, was false; `docs/platforms/python-version-acquisition.md` §5 is corrected. - Three things worth keeping: - - **The identity is a version→signer map, not a constant.** 3.14/3.15 are `hugo@python.org` via - GitHub; 3.12/3.13 are `thomas@python.org` via Google. A constant would keep passing on the - series it was written for and invite loosening on any other. An unrecorded series fails hard - rather than skipping. - - **Two negative controls were run, not just the happy path.** Flipping one base64 character in - the bundle's signature gave `Artifact signature was not valid`; pointing the map at the wrong - release manager gave `No provided certificate identities matched values in certificate`. Both - failed the build with a non-zero exit. Without the identity pin the second case would pass, - which is the whole reason the pin is there. - - **It is opt-in, and the SHA-256 lockfile is untouched.** `sigstore-java` drags in - grpc-netty-shaded/protobuf/bouncycastle and needs the network for its TUF root, so a default - build resolves none of it (measured: zero mentions, exit 0). The two gates prove different - things — lockfile "the bytes we reviewed", Sigstore "the bytes the release manager signed" — - and only the lockfile works offline or covers every source. - - Still uncovered, for reasons not fixable here: **python-build-standalone** publishes no sibling - signature at all (853 assets, `SHA256SUMS` the only non-archive; provenance lives in GitHub's - attestations API, keyed by digest rather than filename and rate-limited to 60/hour unauthenticated), - and **BeeWare's Python-Apple-support** (iOS ≤ 3.14) publishes five tar.gz assets and nothing - else — no checksums, no signatures, no attestations. Desktop is still checked against the - release's own `SHA256SUMS`; for BeeWare the lockfile pin is the only instrument that exists. - The lockfile stays keyed by version *and* flavour, so a `-freethreaded` or a 3.15 archive is a - separate entry and cannot be silently accepted under an existing key. -- **Fixed in passing: the Android download URL 404'd on every pre-release.** It built - `ftp/python/$version/`, but python.org publishes `3.15.0rc1` under `3.15.0/`. It stayed - invisible because the archives were already in the download directory so the fetch was skipped; - adding the `.sigstore` fetch surfaced it on the first run. `pythonOrgReleaseDir`, which the iOS - task already used, is now declared above both tasks and used by both. -- ~~**`PyList.subList`** returns a copy, not a live view.~~ **Stale — it returns `PySubList`, - which delegates `get`/`set`/`add`/`removeAt` to the backing list, i.e. it is a live view.** - **`pyObjectToNative`**'s fallback branch is still not fully native; the open question is now - written out at the branch itself (three candidate answers, and what to measure first). -- ~~**~50 `TODO` markers** remain in `commonMain`~~ **— triaged. 18 remained, not ~50; 15 are - closed, 2 are sharpened open questions, and the work items are the four bullets below.** The - `Py_IncRef`-in-`PyObject.init` question named here is answered in place: `borrowed = true` is a - statement that the wrapper must obtain its own reference, and both directions of getting it - wrong have now been paid for (§1's double free, §4's dropped reference, and the fixture leak - below). Two bugs came out of the pass: - - **`PyType.dict` handed a `mappingproxy` to `PyDict`.** `type.__dict__` is not a `dict`, and - `PyDict_Size`/`PyDict_Items` reject a non-dict with `PyErr_BadInternalCall()` — returning - `-1`/`NULL` *and leaving the error indicator set*. Measured: `int.__dict__` reported - `size == -1`, and the next unrelated `Python3.eval` in the same suite died with - `Objects/dictobject.c:4248: bad argument to internal function`. It now copies through - `PyDict_New` + `PyDict_Update` into a real dict. `PyType_GetDict()`, which the code comment - proposed instead, is **not** an option: it is declared in `cpython/object.h`, outside the - Limited API, so it is not in the Stable ABI subset this binding restricts itself to. - - **`PythonTestFixture.mainGlobals()` leaked one reference per call**, wrapping - `PyObject_GetAttrString`'s new reference with `borrowed = true`. Measured at exactly +50 over - 50 calls — the inverse of §1's defect, in the fixture every functional test is built on. - `OwnershipLeakTest.theTestFixtureDoesNotLeakMainGlobals` is the guard. -- ~~**`Python3.runMain` is not usable as written**~~ **— done. It runs a module as `__main__` - through `runpy`, and the interpreter is alive when it returns.** - - `fun runMain(moduleName: String, args: List = emptyList()): Int`. The design question - this entry left open — what "run a module" should mean for an embedded interpreter that has to - survive the call — is answered as: *the module executes in a fresh `__main__` namespace, sees - the argv it was given, and its exit status comes back as a value.* - - **Why not the two obvious candidates.** - - `Py_RunMain()` is declared in `cpython/pylifecycle.h` — a lifecycle function, and **not part - of the Limited API** this binding otherwise targets. Its contract is to run whatever - `PyConfig.run_command`/`run_module`/`run_filename` names *and then finalize Python*; there is - no mode in which it leaves the runtime standing. Nothing an embedder calls can be built on it. - (Checked in the vendored 3.14.7 header, `include/cpython/pylifecycle.h:28`.) - - `PyImport_ImportModule(name)` fails the requirement twice: the body runs under the module's - *own* `__name__`, so the `if __name__ == "__main__":` block that is usually the entire point - never fires, and the result is cached in `sys.modules`, so a second call runs nothing at all. - `RunMainTest.runMainRunsTheModuleAgainOnASecondCall` fixes the second half of that. - - **What it does instead: `runpy.run_module(name, run_name="__main__", alter_sys=True)`** — the - standard library's own answer, and the machinery `python -m` itself goes through - (`runpy._run_module_as_main` shares `_get_module_details`/`_run_code` with `run_module`). It - touches no lifecycle function, so nothing on this path can finalize anything. Read out of the - vendored `runpy.py`, `alter_sys=True` puts the execution inside two context managers: - `_TempModule("__main__")`, which installs a **fresh** module as `sys.modules["__main__"]` for - the duration and puts the previous one back in `__exit__`; and `_ModifiedArgv0(spec.origin)`, - which points `sys.argv[0]` at the module's file and restores it. Both are `with` blocks, so the - restore happens on the raising path too. That is what makes the embedder's own `__main__` — the - one `Python3.exec` writes into — neither read nor written by the module. - - **`sys.argv` is set for the duration and put back.** It has to be set, because - `_ModifiedArgv0.__enter__` *reads* `sys.argv[0]` before writing it, and an embedded interpreter - is not guaranteed a usable one (the same soft spot the old body fell into from the other side). - It has to be restored, because it is process-global state the caller did not ask to have - rewritten — a module run is a nested activity here, not the process's reason for existing. - Guards: `RunMainTest.theModuleSeesTheArgumentsItWasGivenAndArgvIsRestoredAfterwards` and - `.argvIsRestoredEvenWhenTheModuleRaises`. - - **`sys.exit()` is a return value, not an exception.** Decided, not inherited: a module written - to be run as `__main__` uses `sys.exit(n)` to report a status, and CPython's own command line - treats it that way — `Py_RunMain` catches `SystemExit` and turns it into the process exit status - rather than printing a traceback. The half an embedder must *not* inherit is the process exit. - So `sys.exit()`/`sys.exit(None)`/falling off the end → `0`; `sys.exit(n)` for integer `n` → `n`; - `sys.exit(x)` for anything else → `x` written to `sys.stderr` and `1` returned, mirroring - `_Py_HandleSystemExit`. Everything *other* than `SystemExit` propagates as a `PyException` - carrying the module's real type, message and traceback, with the interpreter still usable. - - **Implementation note.** The Python half is a cached module-level `def` compiled once with - `Py_CompileString`/`PyEval_EvalCode` into a private globals dict — the same shape as the - eval-loop checkpoint helper in the same file, including relying on frame setup falling back to - the interpreter's builtins when the globals mapping has none. It is Python rather than C API - calls because every line of it is a `try`/`finally` or an `except`, and telling `SystemExit` - apart through the C API would need a `PyExc_SystemExit` handle this binding exposes no - accessor for. `Python3.finalize()` drops the cached pointer alongside `checkpointCallable`. - - **Tests: 12 new in `RunMainTest`**, plus the old refusal guard - `Python3Test.runMainRefusesRatherThanFinalizingTheSharedInterpreter` rewritten as - `.runMainRunsAModuleWithoutFinalizingTheSharedInterpreter` (the refusal is gone; the property - it protected is kept). Desktop 485 → 497, 0 failures, - 1 skipped; `ksp-fixtures:app` 64/0 unchanged. **Red phase observed**: all 12 failed with - `java.lang.UnsupportedOperationException: Python3.runMain is not implemented...` before the - body landed. The one red phase still not runnable is the *pre-refusal* body's — `Py_RunMain()` - with no `PyConfig.run_*` set enters the REPL on the process's stdin and finalizes the shared - interpreter, hanging the worker or crashing every class scheduled after it. - - The tests register their modules through an in-memory meta-path finder rather than files, - because `runpy` resolves via `importlib.util.find_spec` and then `spec.loader.get_code(...)` — - a `sys.modules` entry is not enough — and because an in-memory finder runs identically on iOS - and Android, where the suite has no writable path it can agree on. -- ~~**`Python3.runApp` still refuses, and the reason recorded here was the smaller half of - it.**~~ **— done. It parses a deliberately reduced subset of CPython's command line and runs - it in an interpreter that survives the call.** - - This entry used to say the blocker was that `Py_BytesMain` takes `(int argc, char **argv)` and - so needs an array-of-C-strings marshalling path that each of the four platforms does - differently. That is true and it is the cheap half. **The half that was missed: `Py_BytesMain` - finalizes the interpreter.** It is the CLI's `main`, so it reaches `Py_RunMain()` — same header, - same contract. Marshalling the argv would have bought a function with exactly the defect - `runMain` was fixed to avoid, which is why "one marshalling path away from working" was the - wrong description, and why `Py_BytesMain` is still never called anywhere in this binding. - - **The decided scope**, `fun runApp(args: List): Int`, parsing `args` (the slice - `Py_BytesMain` would see as `argv[1:]`) itself rather than reimplementing CPython's parser - against a `PyConfig` this binding does not carry: - - - `-c ` runs `cmd` in the embedder's own persistent `__main__` — the same namespace `exec` - writes into — with `sys.argv[0]` set to `"-c"`, matching CPython (verified against the system - interpreter: `python3 -c "...sys.argv..." a b` → `['-c', 'a', 'b']`). - - `-m ` is a straight delegation to `runMain`, which already *is* `python -m`'s - implementation. Nothing new was written for this branch. - - a bare script path is read via Python's own `open()` — not a new per-platform Kotlin file API, - since `nativeMain` has none of its own and CPython's already behaves identically on every - target — and run in the same persistent `__main__`. `sys.argv[0]` is the path exactly as - given, unresolved (verified: `cd /tmp && python3 argvtest.py` → `sys.argv[0] == - 'argvtest.py'`, not an absolute path). - - `--` followed by a path forces that token to be read as a script path even if it looks like an - option, mirroring CPython's own observed behaviour (`python3 -- -c args` tries to open a file - literally named `-c`, it does not run `-c` as a command). - - `-` (stdin) and anything else starting with `-` — `-E -I -S -s -B -O -OO -X -W -u -P` among - them — are refused **by name**, not silently ignored: they are `PyConfig` fields that only - mean something *before* `Py_Initialize()`, and this interpreter is already running by the time - `runApp` is called. Silently ignoring `-I` would run the caller's code under a security - posture (isolation) it did not actually get, which is worse than a loud refusal. - - empty `args` is refused the same way — CPython's own default with nothing on the command line - is also to read from stdin. - - `sys.exit(n)` is a return value rather than a process exit, `sys.argv` is set for the duration - and restored afterwards (on both the returning and the raising path), and any other exception - propagates as a `PyException` with the interpreter left usable — the identical contract - `runMain` already has, applied to the `-c` and script-path paths through a second small Python - helper (`__pmp_run_toplevel__`/`__pmp_run_file__`, compiled once and cached the same way as - `runMain`'s `__pmp_run_module__`). `-m` needs no such helper: it calls `runMain` directly. - - **Tests: 18 in `RunAppTest`**, replacing `Python3Test.runAppRefusesRatherThanSilentlyDoingNothing` - (superseded, per its own doc comment). Desktop 508 → 526, 0 failures, 1 skipped; - `compileKotlinAndroidNativeArm64` and `wasmJsTest` both clean (wasmJsNodeTest 460 → 468, 0 - failures). **Red phase observed**: 15 of 18 new tests failed against the old refusing stub with - `java.lang.UnsupportedOperationException: Python3.runApp is not implemented yet: ...`; the - remaining 3 assert only the refusal shape itself (exception type, or a message check that the - stub's own "returned silently without running ..." text happened to satisfy), so they could not - distinguish red from green and are not evidence either way — the other 15 are. - - **Three commented-out places, not two**: `EmbedAPI.native.kt` also carries a full `Py_BytesMain` - `actual` — with `memScoped`/`allocArray` marshalling already written — that *looks* live at - lines 67-74 but sits inside the nested block comment spanning lines 46-128, so it compiles to - nothing. Kotlin block comments nest, and both `EmbedAPI.kt` (lines 28-249) and - `EmbedAPI.native.kt` open one at the top that swallows an entire duplicate "Section 1"; the live - declarations are the later copies. Read either file with a nesting-aware scan before concluding - a declaration exists, or the duplicate `Py_FinalizeEx`/`Py_RunMain` pairs will mislead. -- **Android had no usable temporary directory, and `tempfile` is not optional plumbing.** Found by - running `:python-multiplatform:connectedDebugAndroidTest` on `pmp_api26` after `runApp` landed: - 492 tests, 3 failures, all `[Errno 2] No usable temporary directory found in ['/tmp', - '/var/tmp', '/usr/tmp', '/']`. `tempfile` asks `os.environ` for `TMPDIR`/`TEMP`/`TMP`, then - falls back to those hardcoded paths and finally `os.getcwd()`; none of the paths exist in an app - sandbox and an Android process's cwd is `/`. `PythonBootstrap.initialize` now sets `TMPDIR` to - `context.cacheDir` through the same `Os.setenv` `PYTHONHOME` uses — the cache directory - specifically, because that is the one Android may reclaim under storage pressure — and leaves an - embedder's own `TMPDIR` alone if one is already set. `PythonBootstrapTest - .initializeLeavesTempfileUsable` pins it; 493/0 after. - - **iOS and androidNative were checked rather than fixed, and neither needed fixing.** The three - `RunAppTest` cases that failed on Android are the ones that write and run a temporary script, so - they are the probe: `iosSimulatorArm64Test` 457/0 with `RunAppTest` 19/19 including both - `scriptPath...` cases, and `androidNativeArm64Test` 460/0 on `emulator-5554`. Both re-run on - this tree rather than taken from a report. - - Why each is unaffected, which is what stops this from being luck: - - **iOS**, on the simulator, resolves `/tmp` to the host's, so `tempfile`'s hardcoded first - candidate exists. **This says nothing about a device.** A real iOS app is sandboxed the way an - Android app is, and the answer there is `NSTemporaryDirectory()`; nothing in this repository - has run on one, so the entry below about the iOS app bundle owns that question. - - **androidNative** runs its tests through `adb shell`, which sets `TMPDIR=/data/local/tmp` in - the environment CPython inherits. That is a property of the *test harness*, not of the target: - an androidNative binary shipped inside an app would have neither `/tmp` nor that variable, and - would need the same fix `PythonBootstrap` applies. Recorded here rather than left implied, - because "the suite is green" would otherwise read as "the target is covered". - -- **`Python3.finalize` reports no error detail**, and cannot: `Py_Finalize()` returns void and - there is no interpreter left to hold an error indicator afterwards. The one improvement - available is `Py_FinalizeEx()`'s `int` (0, or -1 when flushing buffered data failed). Left - undone because finalization is untested — its only caller is `artMain/JniExport.kt`, and a test - that exercises it destroys the interpreter the rest of the suite shares. - **The FFI half of it is already done, so what remains is one line, not four platforms**: - `Py_FinalizeEx` has a live `expect` and a live `actual` on desktop, androidNative/iOS, Android - and wasmJs (`EmbedAPI.{desktop,native,android,wasmJs}.kt`), plus the desktop `MethodHandle` and - the Android `RegisterNatives` entry. Only the untestability above still blocks it, which is why - it is still open rather than done in passing. -- **`EmbedAPI.kt`'s section numbers are append order, not the C API docs' chapter order.** - Sections 1–26 follow the docs; 27 (Type Objects), 28 (Tuple Objects) and 29 (Module Objects) - were appended as needed. Documented target order: Type before Integer Objects (§16), Tuple - before List Objects (§22), Module before Iterator Objects (§25). It is a ~370-line pure-comment - move with no behavioural effect, so it should be done alone, on a quiet tree, or not at all. -- ~~**Several `commonTest` file headers still describe their subjects as `TODO` stubs "expected to - fail with `NotImplementedError`"**~~ **— done.** The ten named here (`PyObjectTest`, - `Python3Test`, `PyBasicTypesTest`, `PyModuleTest`, `PyDictTest`, `PyIteratorTest`, `PySetTest`, - `PyTupleTest`, `PyListTest`, `ConversionTest`) now say what they actually cover and that a - failure is real. A sweep of the *rest* of the test tree found five more of the same shape, which - this entry had not counted: - - `PyExceptionTest` claimed the exec/eval sites "do not yet call `PyException.fromCurrentError`" - and that its `type`-asserting cases were expected to fail. They all route through - `pyErrorOrGeneric`, and `Python3.exec` is built on `PyRun_String` specifically so the error - indicator survives to be read — the opposite of what the header said. - - `RefCountTest` said `GCLeakTest` was "blocked by the GIL still being held by the initialising - thread". §1 is closed; `initialize()` parks with `PyEval_SaveThread()` and `GCLeakTest` passes. - - `PythonTestFixture` named "CPython 3.13"; `pythonVersion` is 3.14.7 and the fixture is - version-agnostic, so it no longer names one. - - `DesktopOverheadBenchmark` (desktopTest) said desktop "currently reaches Panama through - `MethodHandle.invoke` rather than `invokeExact`". That migration is done and this benchmark is - what measured it — 1015.95 ns → 2.65 ns. - - `ConversionTest`'s inline `// Once implemented: RAW should hand back a PyObject/pointer-ish - value, NATIVE a Kotlin List` described the shipped behaviour as future work. - - Headers checked and left alone because they were already accurate: `HandleTableTest` and - `UpcallTableTest` carry red-phase *notes* that explicitly distinguish "failed to compile before - the implementation existed" from a regression, which is still the right thing to tell a reader. - Comment-only throughout: desktop stayed at 236 tests, 0 failures, 1 skipped. -- ~~**No CI.** The README badges point at a different repository.~~ **Partly stale.** Four - workflow files now exist (`.github/workflows/{desktop,ios,android-native,wasm}.yml`) and - README's badges no longer point at a different repository — it carries an explicit note instead - of a badge, with the reason (see §14b). What has not changed: `gh api - repos/thisisthepy/python-multiplatform/actions/workflows/.yml` 404s for all four, i.e. - **none of them has ever run on GitHub Actions**, because none of the commits that added or - touched them has been pushed to a branch GitHub runs workflows from. See §14b. (Re-checked - 2026-08-14: `desktop.yml` still 404s. Nothing here is fixable in a worktree — it needs a push.) -- ~~**Sample app** has not been revisited since the object model landed.~~ **Done — see §13.** - -## 13. The sample, and the AGP version that shapes it - -The sample now shows four things, each on the real API: the embedded interpreter (`Python3.version` -against `currentPlatform`), a Kotlin-built `PyList` of `PyInt` published into `__main__` and -evaluated by Python, a Python-side `ctypes` call into a Kotlin declaration resolved by name, and -the contents of the generated table read back off `UpcallTable`. Measured on `:sample:run`: - -``` -runtime : 3.14.7 · sys.platform=darwin · MacOS 26.5.1 (aarch64) / JVM 21.0.12 -eval : sum(kotlin_numbers) * 2 -> int: 56 -table : 19 entries, 3 classes, from io_github_thisisthepy_sample -upcall : Python called Kotlin through handle 4294967314 and got 0 -``` - -and, since the AGP bump below, the same four sections on Android — observed on `pmp_api36` and on -`pmp_api26`, which is `minSdk`: - -``` -1 3.14.7 · sys.platform=android · Android 16 (SDK 36, aarch64) / ART VM 0.9 -3 table hit: handle 4294967317 -> 3 (Kotlin-side call; the boundary shim is desktop-only today) -4 22 entries, 4 classes, from io_github_thisisthepy_sample - @PythonInternal held: the annotated member is absent from the table -``` - -Twenty-two rather than desktop's nineteen because the Android compilation also scans -`MainActivity`; the module name is the same one, since it is one module now. - -Section 3's parenthesis is out of date as of `docs/design/upcall-async-design.md` §13: ART does have a -boundary shim, the two C gaps in it are closed, and that section now reads -`handle 4294967327 -> 0 · with args -> presses x3 = 0` from a call **Python** makes through -`_pm_resolve`/`_pm_invoke`. Sections 5-7 run on both emulators as well. - -`:sample:run` did not work before this and it was not the sample's fault twice over: the task had -no `PYTHONHOME`, so `Py_Initialize` could not find `encodings`, and a project dependency resolves -to class directories rather than to `desktopJar`, so `manager.loadLibPython` found no bundled -`libpython` either. Both are wired in `sample/build.gradle.kts` now, the second by falling back to -`$PYTHONHOME/lib` — which is the fallback `manager.kt` already had, it just had nowhere to look. - -**What was there before:** the Compose template screen (a button, an image, the platform name), -and a `desktopMain` `main()` doing raw `PyLong_FromLongLong`/`PyRun_SimpleString` with a pointer -round-tripped through a `Double`. It compiled. Nothing in it touched the object model, and -`PyRun_SimpleString` is the call `Python3.exec` exists to avoid (it calls `PyErr_Print`, which -clears the error indicator before anything can read it). - -### The sample runs on iOS — first time, and it took two fixes that are not in the repo - -**Status: verified on the simulator (iPhone 17, iOS 26.2, `iosSimulatorArm64`), 2026-08-13.** All -seven sections produce output and the Compose UI renders. Before this the iOS sample had only ever -been *compiled*; two earlier attempts stopped before launch, and the stale -`iosApp/build/Debug-iphonesimulator/PythonDemo.app` they left behind had no executable in it. - -Neither fix is committed, because both are packaging work rather than one-line corrections, and -guessing at them is how the last two attempts produced a bundle that looked built and was not. -Reproduction, in full: - -```bash -./gradlew :sample:prepareIosFrameworks :python-multiplatform:extractIosSimulatorStdlib - -# (1) actool cannot compile the app icon -- see below -cd iosApp && xcodebuild -project iosApp.xcodeproj -target iosApp -configuration Debug \ - -sdk iphonesimulator ARCHS=arm64 SYMROOT="$PWD/build" \ - ASSETCATALOG_COMPILER_APPICON_NAME="" \ - CODE_SIGN_IDENTITY="-" CODE_SIGN_STYLE=Manual DEVELOPMENT_TEAM="" build - -# (2) the bundle carries no stdlib -- see §9 item 4 -rsync -a ../python-multiplatform/build/python-stdlib/ios-simulator/lib/python3.14 \ - build/Debug-iphonesimulator/PythonDemo.app/lib/ - -xcrun simctl install "$UDID" build/Debug-iphonesimulator/PythonDemo.app -SIMCTL_CHILD_PYTHONHOME=$(xcrun simctl get_app_container "$UDID" \ - org.thisisthepy.python.multiplatform.demo) \ - xcrun simctl launch --console-pty "$UDID" org.thisisthepy.python.multiplatform.demo -``` - -**`PYTHONHOME` on an external volume hangs the app instead of failing it.** This is the finding -worth carrying forward. `iosMain/README.md` and `extractIosSimulatorStdlib` both point -`SIMCTL_CHILD_PYTHONHOME` at `python-multiplatform/build/python-stdlib/ios-simulator`, and for the -*test* binary that works — 328 tests, 0 failures, on the same simulator, in the same session. Handed -to the *app* it does not fail, it **blocks forever**: 0.0% CPU, a blank white window, an empty -`--console-pty` capture, and no sandbox denial or Python error anywhere in `simctl spawn … log show` -(only ordinary UIKit chatter). `sample`(1) on the process — all 1554 samples on one stack: - -``` -kfun:...PythonDemo#start() PythonDemo.kt:45 -kfun:python.multiplatform.ffi.Python3#initialize(kotlin.Boolean) Python3.kt:45 -Py_InitializeEx -> Py_InitializeFromConfig -> init_interp_main - -> _PyUnicode_InitEncodings -> _PyCodec_InitRegistry - -> PyImport_ImportModule -> ... -> import_find_and_load - -> os_listdir -> __opendir2 -> open$NOCANCEL <- parked here -``` - -That is the `encodings` import, parked in `open()` on a directory under `/Volumes/`. This repo's -workspace is on an external SSD (see AGENTS.md), so the documented recipe reaches a path a -sandboxed simulator *app* cannot open, while a simctl-spawned *test* binary can. Note what the -symptom is not: the failure this section and `iosMain/README.md` both warn about is -`Fatal Python error: Failed to import encodings module`, an abort. A hang looks like "the app -launched and Compose is slow", which is why it is worth writing down. Staging the stdlib inside the -app bundle and pointing `PYTHONHOME` at the installed container clears it. - -**`actool` refuses to compile the app icon on this machine.** The installed simulator runtimes stop -at iOS 26.2 (`23C54`) and the only simulator SDK is 26.5 (`23F81a`), so: - -``` -error: No simulator runtime version from ["21A328", ..., "23C54"] available to use with - iphonesimulator SDK version 23F81a -``` - -Bisected: the trigger is `--output-partial-info-plist` *together with* `--app-icon`; either alone -compiles. `ASSETCATALOG_COMPILER_APPICON_NAME=""` skips it and costs only the springboard icon. -Environment, not this project — but it is the wall the build hits first, and -`xcodebuild -showdestinations` also intermittently reports *no* eligible simulators for the scheme -because of the same missing platform, which makes `-destination` unusable and `-target` the way in. - -**The seven sections, observed.** `MainViewController.kt` gained a `dumpDemoSections()` that prints -what each card would show: `simctl` can install and launch but cannot tap, and sections 2 and 5-7 -are behind buttons. Without it a run can only be said to have exercised the four cards that render -eagerly — which is most of how "iOS compiles" and "iOS runs" stayed indistinguishable. - -| | iOS simulator (observed) | desktop JVM, same day (`:sample:runNativeImageUpcallDemo`) | -|---|---|---| -| 1 | `3.14.6 · sys.platform=ios · iOS 26.2 (SDK 260200, arm64) / Native` | `3.14.7 · sys.platform=darwin · MacOS … / JVM 21` | -| 2 | `int: 56` | same expression, not printed by that entry point | -| 3 | `table hit: handle 4294967325 -> 1` (Kotlin-side call) | `resolved handle 4294967324`, `invoke result = 7`, `presses x3 = 21` (Python-side) | -| 4 | `42 entries, 4 classes, from io_github_thisisthepy_sample`; `@PythonInternal held` | `41 entries, 4 classes`; `@PythonInternal entry resolves to -1` | -| 5 | `installed over PyMethodDef: 519 lines, 1 proxy classes`; `Greeter('Kotlin').greet(2) -> hello Kotlin! hello Kotlin!`; `g.greetings = 99 -> AttributeError (private set held)` | `509 lines, 1 proxy classes`; same three lines | -| 6 | `Greeter.forget() -> 100, then built=0`; `Greeter('x').built -> AttributeError` | identical | -| 7 fast | `await g.greetNow(1) -> hello fast path!`, `Futures created -> 0` | identical | -| 7 slow | **`not wired on iOS`** | `hello slow path! hello slow path!`, `Futures created -> 1` | - -**Nothing diverged that was not already known to diverge.** Sections 5 and 6 are character-for-character -what desktop prints, which is the result worth having: the generated proxy module, the metaclass -companion and the private-setter refusal all behave the same over a `PyMethodDef` shim as over -`ctypes`. The only gap is section 7's slow path, and `ProxyDemo.ios.kt` already says why — resuming a -parked continuation needs a Kotlin/Native worker the sample does not start. The library's own -`PythonProxyNativeDeliveryTest` does it with a bare `pthread_create` and passes here, so this is a -sample limitation, not a platform one. - -The two count differences are both benign and neither is a defect: 42 vs 41 table entries and 519 vs -509 generated lines follow from the per-target source sets KSP scans (`iosMain` has -`MainViewController.kt`, `desktopMain` has `main.kt` and `NativeImageMain.kt`) — the *proxy class* -count, which is what the generated module is actually judged on, is 1 on both. `docs/design/upcall-async-design.md` -§13.5 records `466 lines, 2 proxy classes` for Android; that measurement is older than these two and -was not re-taken here, so the difference is not evidence of anything yet. - -**Unlike every previous run of this sample, no library defect fell out of it.** Both blockers were -in packaging and in the toolchain. That is itself the report: the iOS half of the object model, -the upcall table, the generated proxy module and the fast-path `await` all work in an app, not only -under the test runner. - -### The convenience plugin could not be applied to an Android module — closed - -This was the finding, and it was a property of the repo rather than of the sample. - -``` -java.lang.NoSuchMethodError: 'void com.android.build.api.variant - .AndroidComponentsExtension.addKspConfigurations(boolean)' - at com.google.devtools.ksp.gradle.KspConfigurations$3$1.execute(KspConfigurations.kt:114) -``` - -KSP 2.3.11 declares `MINIMUM_SUPPORTED_AGP_VERSION = 8.10.0` (read off its `agpUtils` class), and -this build pinned AGP 8.5.2, whose `AndroidComponentsExtension` has no such method (`javap` on -`gradle-api-8.5.2.jar`). So `id("io.github.thisisthepy.python.multiplatform.bindings")` — which -applies `com.google.devtools.ksp` — died at configuration time in any module carrying an Android -plugin, which is **every Android consumer of the plugin, not just this sample.** `ksp-fixtures` -never hit it because neither fixture module applied an Android plugin: the fixtures were verifying -their own shape rather than the plugin's advertised one, the same failure mode §2's -`AssembledApiTest` had. - -**Two versions moved, and nothing else had to.** - -| | was | now | why | -|---|---|---|---| -| AGP | 8.5.2 | **8.10.1** | KSP's declared minimum is 8.10.0 | -| Gradle | 8.9 | **8.11.1** | AGP 8.10's own minimum | - -Kotlin (2.4.20-Beta2), KSP (2.3.11) and the Compose Multiplatform plugin (1.6.11) are untouched — -the bump needed no chain beyond those two. `python-multiplatform`'s Android wiring is the part -that was expected to complain, since it hangs hand-written `Copy` tasks, a `tasks.configureEach` -name match on `merge*JniLibFolders`/`*NativeLibs`, and a `preBuild.dependsOn(linkTaskProvider)` -off AGP internals — none of it needed changing. `compileSdk` stays at 34. - -`:sample` applies the bindings plugin directly now and `:sample-bindings` is gone; its sources moved -back under `sample/src/*/kotlin/.../demo/bindings/`. The Android `UpcallDemo` actual, which used to -answer "unavailable on Android" to every member, is an ordinary one-line delegation like the others. - -Two things the fold-back exposed that the split had hidden: - -- **`androidMain` can name the generated table; `iosMain` cannot.** `androidMain` *is* the Android - target's source set, compiled together with each variant's KSP output, so its `actual` names - `FunctionTable` directly. `iosMain` is an intermediate source set the generating leaves depend - on, so it still needs one `actual` per leaf. Same processor, opposite answer, and the difference - is which side of the compilation the source set sits on. (Both `actual`s are generated now — - see "Two smaller things the sample found" below — but the asymmetry that made them necessary is - unchanged.) -- **Exposure is a blacklist, so applying the plugin offers the whole module to Python — including - Compose.** A `@Composable` may only be called from another composable and a generated entry is an - ordinary lambda, so a scanned `@Composable` is a compile failure of *generated* code. `:sample` - therefore sets `excludePackages` to keep `...demo.ui` out. A separate module made this invisible; - any real app has UI in the same module as its bindings. - -### `ksp-fixtures/android`, and the second defect it found immediately - -`ksp-fixtures/android` is the fixture that carries `com.android.library` — the one thing neither -existing fixture does. It applies the bindings plugin, exposes the same declaration shapes the -desktop fixture uses, and runs 6 tests against the generated `FunctionTable` as a plain JVM unit -test (`testDebugUnitTest`, no device). If the AGP/KSP pair ever drifts apart again it stops -configuring, which is the failure worth having. - -It earned itself on the first run, by failing for an unrelated reason: - -``` -6 tests completed, 5 failed -``` - -The plugin decided which KSP configurations to put the processor on with `name.endsWith("Test")`. -That holds for Kotlin target names (`kspDesktopTest`), but **AGP names its source sets with the -build type last**: - -``` -main kspAndroid kspAndroidDebug kspAndroidRelease -unit test kspAndroidTest kspAndroidTestDebug kspAndroidTestRelease -instrumented kspAndroidAndroidTest kspAndroidAndroidTestDebug kspAndroidAndroidTestRelease -testFixtures kspAndroidTestFixtures kspAndroidTestFixturesDebug kspAndroidTestFixturesRelease -``` - -so the suffix check caught three of those nine. The processor ran over the *test* sources on -`kspAndroidTestDebug` and emitted a second `Fragment_` and a second `FunctionTable` into -the test compilation, where they shadowed the real ones from `main` — a compilation's own -generated sources win over its classpath. The table the test read held the test class's own -members and nothing the module exposes. - -This is exactly the duplicate-fragment hazard the plugin's own doc comment describes, arriving -through a name shape that comment did not anticipate. `Test` is now matched as a camel-case *word* -(`(?:^|[a-z0-9])Test(?:[A-Z]|$)`), not as a suffix, so a flavour or target named `testing` still -keeps its processor. `WiringTest` pins the full observed Android configuration list. - -**Nothing without an Android plugin could have found this**, which is the point of the fixture. - -### ~~Two smaller things the sample found~~ — both closed - -Neither existing fixture had the *shape* to catch either one, which is the part worth keeping. -`ksp-fixtures` grew that shape first, and both defects reproduced there before anything was fixed. - -#### The generated table is reachable only from the source set of the target that generated it - -KSP writes `FunctionTable` into `iosSimulatorArm64Main` and its siblings, so `iosMain` — which -those leaves depend on — cannot name it, exactly as `commonMain` cannot. The rule is about -*intermediate* source sets, not about "not being `commonMain`": `androidMain` is the Android -target's own source set and names `FunctionTable` directly, which the fold-back above made visible. - -**The constraint has not moved and cannot.** A fragment scan has to see the target's own -declarations — Android's table is three entries larger than desktop's because that compilation also -scans `MainActivity` — so generating into `commonMain` through `kspCommonMainMetadata`, the usual -KMP answer, would shrink the exposed surface to whatever happens to be common. What moved is *who -writes the `actual`*: - -```kotlin -// commonMain, and nowhere else in the repository -@InstallsUpcallTable -expect fun installGeneratedUpcallTable() -``` - -`python-multiplatform-ksp` finds the annotated `expect` — a leaf compilation sees its whole source -set closure, the same fact that lets the fragment scanner pick up common declarations — and emits -one `actual` per leaf, carrying `@PythonInternal` so the generated function does not itself end up -in the table. `sample` lost its three identical `InstallTable.ios*.kt` files and gained nothing in -their place. Android's hand-written `actual` is gone too, not because Android needed one but -because a per-platform special case is precisely what shared code cannot be written against. - -`expect`/`actual` rather than a common interface plus a runtime registry because the reference -chain has to stay static: §7 keeps every fragment reachable to the Kotlin/Native linker by naming -it explicitly, with no `ServiceLoader` and no `@EagerInitialization`, and a generated object that -nothing names is dead code the linker may drop. The generated `actual` is that name. - -`ksp-fixtures/app` gained a second Native leaf (`androidNativeX64`) purely so the default hierarchy -would give it an intermediate `androidNativeMain` — `iosMain`'s shape, without Xcode — plus a -`commonMain` that installs the table. Written the obvious way first, it failed exactly as the -sample did: - -``` -e: ksp-fixtures/app/src/commonMain/kotlin/fixture/app/TableInstall.kt:4:39 - Unresolved reference 'FunctionTable'. -``` - -Neither existing fixture could have produced that: both reached the table from `desktopTest`, a -*leaf* compilation, where it resolves fine. `ksp-fixtures/android` carries the same seam, because -KSP runs once per AGP *variant* there and that is where a generated `actual` could go missing or -land twice. - -#### A `var` with a `private set` generated a fragment that did not compile - -`FragmentScanner` decided on `property.isMutable` alone and emitted a `STATIC_SETTER`/`SETTER` -assigning to it. The ROADMAP recorded this read off the scanner rather than observed; putting the -shape into `ksp-fixtures/library` produced it on the first run: - -``` -e: Fragment_..._library.kt:58 Cannot access 'topLevelPrivateSet': it is private in file. -e: Fragment_..._library.kt:442 Cannot access 'privateSet': it is private in 'RestrictedSetters'. -e: Fragment_..._library.kt:458 Cannot access 'protectedSet': it is protected in 'RestrictedSetters'. -``` - -Three errors from four properties, and **the fourth is the one that matters.** `internal set` -compiled: the fragment is generated into the same compilation as the sources it scans and -`internal` is enforced per Kotlin *module*, so the assignment is legal there — and only there. -Fixing what the build complained about would have left a table entry describing a write no -consuming module could perform. `BindingPolicy.isExposedSetter` therefore reads the *setter's* -modifiers and treats `private`, `protected` and `internal` alike. The read side is untouched: a -`var` with a restricted setter is a read-only property to Python, which is what it already is to -every Kotlin caller outside its module. - -### ~~What the sample still cannot show~~ — the trampoline exists now - -**What it was.** `UpcallStub` was desktop-only and both its stubs were `(long) -> long`, so the -entry the demo called took no arguments and returned a `Long`. The *table* had carried arity and -per-argument `TypeTag`s from the day it was written; nothing read them. Python could call Kotlin -and could not pass it anything. - -**What it is.** `python.multiplatform.ffi.upcall.UpcallTrampoline` marshals both directions and is -`commonMain` — the argument handling was never platform-specific, only the address publishing was. -Desktop reaches it through one new Panama stub. Python now calls Kotlin with real arguments of -every marshalled tag, constructs a Kotlin object and calls a method on it, and gets a Python -exception when the Kotlin side raises: `python.native.ffi.UpcallArgumentsTest`, which drives the -whole path from inside the interpreter, and `UpcallTrampolineTest` (commonTest, so it compiles for -every target) which drives the marshaller directly. Desktop: 233 tests, 0 failed. - -Three things the work settled, each recorded in `docs/design/upcall-design.md`: - -- **One shape, not a family.** Argument passing needed exactly one new C shape, - `(long, long) -> long`. Arity and types ride in the tuple and in the table entry, never in the C - signature, so the stub count does not grow with the exposed surface. It is also, exactly, - `PyCFunction` — so the generated proxy type will not need a new shape either. Every other CPython - slot shape was already in `Panama`'s vocabulary. -- **`ctypes.CFUNCTYPE` releases the GIL.** The first version segfaulted in `_PyThreadState_GET` - (`PyErr_Occurred+0x1c`) on a thread whose own `withGIL` depth counter said it held the GIL. That - counter records scopes *Kotlin* opened, and C is free to have dropped the GIL inside one. An - entry point reached from C must take its own `PyGILState_Ensure` unconditionally — which applies - to every platform's entry point, not only this one. -- **`bytes` has no fast route in this ABI subset.** `PyBytes_AsString` is bound as a - NUL-terminated UTF-8 *string* read, which destroys exactly the payloads `ByteArray` exists for, - and `PyBytes_AsStringAndSize` is in no platform's `EmbedAPI`. The trampoline goes an item at a - time — correct, and the slowest path across this boundary by a wide margin. - -**iOS and androidNative have it too.** `python.native.ffi.UpcallEntry` (`nativeMain`, so both -targets get it at once) publishes the same trampoline, and `UpcallEntryTest` (`nativeTest`) drives -it from inside the interpreter: iOS simulator 230 tests, 0 failed, 8 of them new; androidNative -compiles main and test. Desktop is unchanged at 236, 0 failed, 1 skipped. - -**androidNative now runs that suite rather than only compiling it.** `androidNativeArm64Test` -(`python-multiplatform/build.gradle.kts`) pushes the Kotlin/Native test binary and the CPython -prefix to `/data/local/tmp` and parses the runner's TeamCity output into JUnit XML, so the target is -counted like every other: **252 tests, 0 failed, on `pmp_api26` and `pmp_api36`, over five runs -each.** Nothing new broke — unlike §11b, where attaching this suite to ART for the first time killed -it at the 2nd test and again at the 12th — and the reason is that `nativeMain` was already being -exercised by the iOS simulator. What had never been executed was `artMain` and the androidNative -`cinterop` bindings, and those came up clean. The empty androidNative row in -`docs/design/upcall-design.md`'s five-platform upcall table is filled in from those runs. - -It is *not* the `@CName` + `ctypes.CDLL(None)` route this document and `docs/design/upcall-design.md` -both predicted, and the reason is two independent measurements rather than a preference: - -- `@CName` symbols are exported from the androidNative `.so` (`T` in `nm -D`) and **are not - present at all** in the iOS framework or in either target's test executable — not stripped at - link time, never emitted. -- This project's iOS `Python.framework` carries **no `_ctypes`** (no `lib-dynload`, and its - `PyInit_*` exports stop at `time`), so `import ctypes` raises there regardless. - -What it uses instead is a real `PyMethodDef` whose `ml_meth` is a `staticCFunction` and whose -`self` carries the callable handle — i.e. the `PyCFunction` slot the shape was chosen to be, and -what the generated proxy type installs anyway. The desktop `ctypes` shim was standing in for -exactly this. The `@CName` functions are kept: they are the C-host entry point, they are what -`ctypes` reaches on Android, and `UpcallEntry.invokeAddress` hands out their addresses regardless -of what the linker did with the name. - -`nativeTest` had to be wired into the source-set hierarchy for that test to exist — the directory -was there and nothing pointed at it, so its one file was dead source and had been copied -byte-for-byte into all five native target test source sets. Those copies are gone. - -**Android landed next, and not the way this section predicted.** The prediction was "a -`RegisterNatives` method behind a `PyCFunction` shim". `RegisterNatives` binds a JVM `external fun` -to a C function — the *downcall* direction — and no arrangement of it lets C call Kotlin, which is -what a `PyMethodDef` slot needs. The boundary is inverted instead, exactly as the proxy type's -`tp_traverse`/`tp_clear` already are: the entry points are C functions in -`artMain/cinterop/jni_onload.def` and *they* call `python/native/ffi/UpcallCallbacks` through -`CallStaticLongMethod`, reusing `pmp_attach` so an upcall arriving on a `threading.Thread` — a bare -pthread ART has never seen — attaches and detaches rather than silently returning `NULL`. -`RegisterNatives` survives only for the one cold `upcallPublish(long)` that installs the bootstrap. -`UpcallEntryTest` (`androidInstrumentedTest`) is green on `pmp_api26` and `pmp_api36`, 251 tests -each, 0 failed; `docs/design/upcall-design.md`'s "Android's boundary runs the other way round" has the rest. - -~~**What is still open.** wasm — a `@WasmExport` plus `Table.set` (3.1 ns, measured in §11). -Per-platform detail is in `docs/design/upcall-design.md`'s "What each platform still owes".~~ **Closed.** -wasm's general upcall entry (`UpcallEntry.kt`, `wasmJsMain/kotlin/python/native/ffi/`) publishes -five `PyCFunction` objects over the one `@WasmExport`/`Table.set` shape this note describes, and -`docs/design/upcall-design.md:207` now marks that row "already proven" rather than owed. The generated -proxy type that would let Python write `obj.method(x)` instead of going through `_pm_bind` is §7's -remaining half. - -## 14. Current state, gathered - -This document accumulates section by section, in the order work happened, so the same fact ends -up recorded in two or three places over time and a reader has to piece the current picture back -together. This section is that picture, as of a documentation audit on 2026-08-13. It adds no new -finding of its own — every number and every item below is sourced from a section above, or from -running the code — it exists so the state doesn't have to be reassembled by reading the whole file. - -### 14a. Test counts across the six run paths, and how to reproduce them - -Numbers rot the moment someone adds a test, so what matters here is the *recipe*: clean the -target's `build/test-results/` directory first (a crashed run leaves the previous run's XML behind -and a naive count reports the old, larger number as if it were current — see AGENTS.md), run the -task with output redirected to a file so the exit code is real, then sum the `tests`/`failures`/ -`skipped` attributes off every `` root under the result directory. All of the JVM and -Native targets share that shape; Android's instrumented target does not (below). - -| # | path | task | results land in | needs a device/emulator? | -|---|---|---|---|---| -| 1 | desktop | `:python-multiplatform:desktopTest` | `python-multiplatform/build/test-results/desktopTest/*.xml` | no | -| 2 | iOS simulator | `:python-multiplatform:iosSimulatorArm64Test` | `python-multiplatform/build/test-results/iosSimulatorArm64Test/*.xml` | boots the simulator (needs Xcode licence acceptance), not a physical device | -| 3 | Android (ART) | `:python-multiplatform:connectedDebugAndroidTest` | AGP's own convention, **not** `build/test-results`: `python-multiplatform/build/outputs/androidTest-results/connected/debug//TEST-*.xml`, one folder per attached device | **yes** | -| 4 | androidNative | `:python-multiplatform:androidNativeArm64Test` / `androidNativeX64Test` | `python-multiplatform/build/test-results/androidNativeTest/*.xml` — the task pushes the compiled test binary to `/data/local/tmp` and translates the Kotlin/Native runner's TeamCity output into this JUnit XML itself (`build.gradle.kts`, the `androidNative${targetSuffix}Test` task registration) | **yes** | -| 5 | wasmJs | `:python-multiplatform:wasmJsNodeTest` | `python-multiplatform/build/test-results/wasmJsNodeTest/*.xml` | no (runs under Node) | -| 6 | fixtures | `:ksp-fixtures:app:desktopTest`, `:ksp-fixtures:library:desktopTest`, `:ksp-fixtures:android:testDebugUnitTest` | `ksp-fixtures//build/test-results//*.xml` | no — `android`'s is a plain JVM unit test (`testDebugUnitTest`), not the instrumented one | - -A one-line counter, given a directory of result XML: - -```python -import glob, xml.etree.ElementTree as ET -files = glob.glob("/*.xml") -tests = sum(int(ET.parse(f).getroot().attrib.get("tests", 0)) for f in files) -failures = sum(int(ET.parse(f).getroot().attrib.get("failures", 0)) for f in files) -skipped = sum(int(ET.parse(f).getroot().attrib.get("skipped", 0)) for f in files) -``` - -### 14a-1. The state at the deadline — 2026-08-24, every path re-run from a clean results directory - -Taken in one pass on `develop` @ `115370e6`, `--rerun` throughout, each results directory deleted -before its task ran. This exists because the failure this document keeps having is not a wrong -number, it is a number nobody re-took: the entry below is from 2026-08-13 and every sweep since -has found something stale. - -| path | count | fail | note | -|---|---|---|---| -| `:python-multiplatform:desktopTest` | **529** | 0 | 1 skipped | -| `:python-multiplatform:wasmJsNodeTest` | **471** | 0 | | -| `:python-multiplatform:wasmJsBrowserTest` | **10** | 0 | | -| `:python-multiplatform:iosSimulatorArm64Test` | **457** | 0 | | -| `:python-multiplatform:connectedDebugAndroidTest` | **495** | 0 | `pmp_api26` | -| `:python-multiplatform:androidNativeArm64Test` | **460** | 0 | `pmp_api26`, on device | -| `:python-multiplatform-gradle-plugin:test` | **175** | 0 | | -| `:ksp-fixtures:app:desktopTest` | **70** | 0 | | -| `:ksp-fixtures:compose:desktopTest` | **99** | 0 | | -| `:ksp-fixtures:artifact:desktopTest` | **36** | 0 | | -| `:ksp-fixtures:android:connectedDebugAndroidTest` | **8** | 0 | first device run was 2026-08-20 | -| `compileKotlinAndroidNativeArm64` | — | — | no `e:` lines | - -**`ksp-fixtures:artifact` is in that list for a reason.** It was red on `develop` for days because -the verification set in use was plugin + app + compose, and two failures sat behind that gap: a -pinned list that `e2d75100` outdated, and a stub-shape canary that `ac8e4708` outdated. All three -fixture modules are the set; AGENTS.md now says so. - -**Sibling repositories, same pass:** `toolchain` 98 + `tcl` 9, `pypackpack` `packpack` 135 + -`cli` 26, `pythonx-compose` 71 — all 0 failures. - -**The three external consumers, which no in-repo test can stand in for**, run against a -`publishToMavenLocal` of all three producers: `consumer-plugin-test` and `consumer-plugin-android` -build clean, and `consumer-test` runs and prints `Result from Python: 56`. - -**As last verified, 2026-08-13**, immediately after merging `develop` into this branch, from a -cleaned results directory, for the five paths that need no device (§14b explains why 3 and 4 were -not run here — this audit was instructed not to use devices): - -| path | tests | failures | skipped | -|---|---|---|---| -| desktop | 327 | 0 | 1 | -| iOS simulator | 289 → **328** | 0 | 0 | -| wasmJs | 302 | 0 | 0 | -| fixtures: `ksp-fixtures:app` | 64 | 0 | 0 | -| fixtures: `ksp-fixtures:library` | 0 | 0 | 0 (no test sources — exercised through `app`, not standalone) | -| fixtures: `ksp-fixtures:android` | 8 | 0 | 0 | - -The iOS row carries two figures because it was counted twice on 2026-08-13 from two branches: 289 by -the audit above, and **328, 0 failures, 0 skipped** re-counted from a cleaned results directory on -`work/gil` after merging `develop`, alongside the iOS sample run in §13. Both are real; the second -is the one that includes `develop`'s `ProxyHandleLifetimeTest`. This is exactly the rot §14a exists -to prevent, so: count it yourself, and say which tree you counted. - -Android (ART) and androidNative were **not** re-run for this audit — both need a connected device -or emulator, which this pass was told not to use. Numbers for those two paths appear at several -points earlier in this document, taken at different times for different subsets of the suite, and -none of them should be read as "the current total": §2's own closing measurement (213 tests, 0 -failed, the measure §2 itself set), §11b's `commonTest`-on-Android wiring (176 tests per device, -8 failures tracked in §2), the upcall suite specifically (`UpcallEntryTest`, 251 tests per device, -0 failed, §13), and androidNative's equivalent (252 tests, 0 failed, over five runs, §13). These -are different checkpoints in the same section's history, not five inconsistent measurements of one -number. Whoever next has a device or emulator available should run -`connectedDebugAndroidTest` and the two `androidNative*Test` tasks and replace this paragraph with -one number each, dated. - -### 14b. Remaining open items - -Gathered from across this document rather than newly found, except where noted. Each item names -what is blocking it and what the next concrete step is. - -1. **`autoDrainInterval`'s default on the GIL build.** *(Requested explicitly for this audit.)* - Not actually blocked on anything technical — the evidence is in §9 and - `docs/investigations/gc-scheduling-investigation.md` §6-§7, and a decision has already been implemented in code - (`Python3.kt`: `if (BuildConfig.pythonFreeThreaded) 32 else 0`). What is open is whether "off" - is the right *default* to ship, not whether the mechanism works. The case for leaving it off: - turning it on lets a `__del__`/weakref callback/pending call run inside a `withGIL` scope exit - the caller never asked to yield from — a behavioural change, not a free one. The case for - turning it on: measured, a GIL-build embedder that only calls the C API directly never runs the - cyclic collector either (§9, "This is not only a free-threading problem"; the accumulation - experiment in §9 reports 20,000+ cyclic objects with the automatic path off, against ~1,970 - left over with it on), so "off by default" is not free of consequences — it is a silent leak of - a different kind, traded for not surprising the caller. - **(a) blocking it:** nothing technical; a product judgement between "surprise a caller with a - side effect" and "leak cycles silently by default" is what remains. - **(b) next step:** decide, and if the answer changes, it is a one-line change at - `Python3.kt`'s `autoDrainInterval` declaration — the reasoning to cite either way is already - written on that property's KDoc and in `docs/investigations/gc-scheduling-investigation.md` §6. - -2. **CI has never run.** *(Requested explicitly for this audit.)* Four workflow files exist - (`.github/workflows/{desktop,ios,android-native,wasm}.yml`) but `gh api - repos/thisisthepy/python-multiplatform/actions/workflows/.yml` returns 404 for all four — - confirmed during this audit. **(a) blocking it:** none of the commits that added these files has - reached a branch or pull request GitHub Actions runs against; they exist only on local branches - (this one included) so far. **(b) next step:** push and open a PR, or merge to `main`, and watch - the first run. README's badge section already anticipates this correctly (see §14c) — no README - change is needed once CI goes green, only adding the badge it already describes how to add. - -3. **The suite has never been run on Linux or Windows.** *(Requested explicitly for this audit; - newly documented — this document did not previously record it as an open item anywhere.)* The - `desktop` target is a `jvm("desktop")` target, not separate Kotlin/Native targets per OS, and - README/§9's platform table both claim macOS, Linux and Windows support through - `python-build-standalone` archives. Nothing found in this repository — no CI run, no log, no - note elsewhere in this document — shows the desktop suite having actually executed on anything - but macOS. **(a) blocking it:** access to a Linux and a Windows machine (or CI runners for - both) — this audit had neither and did not attempt it. **(b) next step:** the cheapest path is - item 2 above: if `.github/workflows/desktop.yml` already includes Linux/Windows runners (not - checked in this pass), getting CI running once resolves both items together; if it only runs - macOS runners today, it needs those OSes added before this item closes. - -4. **BeeWare's iOS (≤3.14) distribution has no signature, so Sigstore verification is impossible - for it.** *(Requested explicitly for this audit.)* Already documented in §12: BeeWare's - Python-Apple-support publishes five `.tar.gz` assets and nothing else — no checksums, no - signatures, no attestations, checked against the live release listing. **(a) blocking it:** - nothing this repository controls — the upstream project does not publish signing material of - any kind, for any version. **(b) next step:** none available today beyond what already exists, - the SHA-256 lockfile pin (`python-checksums.properties`) as the only integrity check for this - one source. This stops mattering going forward rather than getting fixed: python.org's iOS - XCframework, which *does* publish Sigstore material and *is* already wired up (§12), becomes the - only source at 3.15 and later (§9, "3.15: the source has to change with the version"), so this - item is scoped to versions ≤3.14 by construction and shrinks as the default version moves. - -5. ~~`tp_traverse` is generated but not wired into CPython's actual `tp_traverse` slot; `tp_clear` - and Kotlin-side cycle closing are untouched.~~ ~~`PythonProxySource` never constructs an instance - of that type; every rendered class was a plain Python `_PmObject` that could not carry a custom - `tp_traverse`.~~ **Both closed.** `work/cycles` (merged `aa5436d1`, 2026-08-14) wired the - generated per-class traverse function into `ProxyTypeFactory`'s `PyType_FromSpec` slot table - with `tp_clear` and `tp_dealloc` alongside it. `work/handleslot` (`99acd830`, 2026-08-18, merged - `a1fb825c`) then closed the renderer gap: `PythonProxySource.renderClass` now emits - `class $className(_PmGcObject):` instead of `_PmObject` whenever `ReflectedClass.hasTraverse` - is set and there is no metaclass (`PythonProxySource.kt:1168`, - `val gcEligible = metaclassName == null && cls.hasTraverse`), and `_PmGcObject`'s `_pm_handle` - is a member descriptor over the C type's reserved slot rather than a Python attribute - (`PythonProxySource.kt:728-758`), so `self._pm_handle = h` in the generated `__init__`/`_pm_own` - writes straight into the storage `tp_traverse`/`tp_clear` read — no call site needed to change. - Same-day follow-ups extended this from desktop-only to every target: `a5387511` (wasm, merged - `b479b407`), `1544f095` (Android, merged `0c41874d`), `581ec772` (the two cinterop/native - targets, merged `e063501f`) — all four ancestors of `develop` as of 2026-08-18. - The `subtype_dealloc`/`PyObject_GC_UnTrack` double-untrack question is answered, not assumed: - `CycleCollectionTest.aPythonSubclassOfTheProxyTypeSurvivesBeingUntrackedTwice` - (`desktopTest/.../ref/CycleCollectionTest.kt:658`) builds a Python subclass of the generated - proxy heap type, confirms `gc.is_tracked()` on the instance so the second untrack is actually - exercised, and drives it through `del` + `gc.collect()`. It passes: CPython's own vendored - header says so (`internal/pycore_gc.h`: *"See also the public PyObject_GC_UnTrack() which accept - an object which is not tracked"* — the internal variant asserts, the public one this code calls - is defined to tolerate it), and the test exercises rather than just cites that guarantee. - **What is still open:** Kotlin-side cycle closing — the handle table holds every entry with a - strong reference by construction (`HandleTable.kt:25`, "this table leaks by construction... a - strong entry here is a[nother reference in the cycle]"), so a cycle that runs entirely through - Kotlin objects (no Python-side edge for `tp_traverse` to report) is still not collectible. - `docs/design/object-lifetime.md` has the mechanism and names the hard parts. - -6. **`PyValue`'s lazy conversion path.** (§7b) **Closed.** The per-type lifetime rule this item - asked for is written (`docs/design/object-lifetime.md`, "Conversion caching, and where it stops"), the - cache is implemented against it and enforced by `isIndependentOfPythonMemory`, and - `PyValueLazyConversionTest` covers the path `ConversionTest` never touched. Refusing `bytes` - outright rather than converting it to a `ByteArray` copy is the one piece of the type table - left unimplemented, and it is recorded as such in §7b rather than as a gap here. - -7. **`ksp-fixtures/android`'s `jvmTest` 8 tests on device & API Level Path Assertions.** **Closed.** - - **API Level Path Assertions (`AndroidCleanerPathTest.kt`):** An instrumented test suite in `python-multiplatform/src/androidInstrumentedTest` reads `Build.VERSION.SDK_INT` and asserts exact path selection and runtime behavior across API levels: - - On **API 26** (`pmp_api26` - Android 8.0 < 33): `cleanable` field is `null` (reflecting on missing `java.lang.ref.Cleaner` throws `NoClassDefFoundError`), `fallback` (`PhantomCleanerRegistry.Cleanable`) is populated, `PhantomCleanerRegistry.outstanding` increments by 1, `PhantomCleanerRegistry.draining` is `true`, and `PhantomCleanerRegistry.drained` increments upon GC reclamation. This verified that the `PhantomReference` fallback path is reached and fully functional on API < 33. The premise that "needs new hardware" was incorrect: the `pmp_api26` AVD was already present and executed the fallback path. - - On **API 36** (`pmp_api36` - Android 16 >= 33): `cleanable` (`java.lang.ref.Cleaner.Cleanable`) is populated, `fallback` is `null`, and `PhantomCleanerRegistry` is completely bypassed (`drained` and `outstanding` do not increment). - - **`ksp-fixtures/android` Device Verification:** Configured `androidInstrumentedTest` source set and `testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"` in `ksp-fixtures/android/build.gradle.kts` and added `src/androidInstrumentedTest/kotlin/fixture/android/GeneratedAndroidTableTest.kt`. All 8 upcall table tests ran on device (`pmp_api26`) and passed (8/0/0). - -8. ~~wasm's `ProxyTypeExports.kt`-shaped trampoline generation is manual.~~ **Closed 2026-08-18 (`112b54d0`), plugin wiring completed.** - `GenerateWasmProxyExportsTask` (`python-multiplatform-gradle-plugin`) generates the three `@WasmExport` proxy type slot trampolines (`pmp_tp_traverse`, `pmp_tp_clear`, `pmp_tp_dealloc`) from `WasmProxyExportsRendering.kt`. The hand-written test fixture `ProxyTypeExports.kt` was deleted in `112b54d0` and replaced by the task output in `:python-multiplatform:wasmJsTest`. `PythonBindingsPlugin` now automatically registers `generateWasmProxyExports` and wires its output directory to `wasmJsMain` for consumers (including `:sample`). Note that `UpcallExports.kt` / `WasmExports.kt` remain hand-written for `pmp_invoke` (the general upcall dispatcher), which is distinct from the CPython proxy type slots. - - -9. ~~`jvmMain` unification (§8) is verified but not applied.~~ **Attempted at scale, 2026-08-18 - (`work/jvmmain`).** The premise — ~330 duplicated `actual`s collapsing into ~14 shared shape - functions — did not survive contact with an exact per-declaration comparison: of 314 common - `EmbedAPI.android.kt`/`EmbedAPI.desktop.kt` declarations, only 3 have literally identical - bodies, because Android's JNI dispatch names a different callee per calling convention - (`PyErr_ClearN`/`PyErr_ClearF` vs desktop's unsuffixed Panama handle) on almost every wrapper — - real wiring divergence, not cosmetic duplication. Of those 3, only 1 (`toRawValue`) was actually - movable without changing behavior; the other 2 depend on `NativeAddressValue`, whose `toString()` - differs by platform, so hoisting them would force a behavior change. `toRawValue` and its - `toPlatformPointer()` helper now live in `jvmMain/kotlin/python/native/ffi/EmbedAPI.jvm.kt`. The - recipe itself (remove `inline` from the `commonMain` `expect`, single `actual` in `jvmMain`) is - confirmed safe — compiles clean on both JVM leaves, and the documented Kotlin 2.0.20 - `Internal error in file lowering` crash did not reproduce even with `inline` deliberately left - in for this function, tested and reverted. **(a) blocking it:** nothing left to do — the - remaining 380+315 duplicated declarations are duplicated because they are not, in fact, the same - code. **(b) next step:** none recommended; re-opening this would mean unifying the JNI - convention-selection machinery itself (giving Android's dispatch the same unsuffixed names - desktop uses), which is a design change to `bindings.kt`'s calling-convention selection, not a - mechanical `jvmMain` move, and is not scoped here. - -10. ~~**`Python3.runMain` and `Python3.runApp` are landmines, not working functionality**~~ - **— done, both of them.** (§12) `runMain` runs a module via - `runpy.run_module(name, run_name="__main__", alter_sys=True)`, in a fresh `__main__` - namespace, with `sys.argv` set for the duration and restored, and `sys.exit(n)` reported as - its `Int` return rather than acted on. `runApp(args: List): Int` parses the reduced - CLI subset decided in §12 — `-c`, `-m` (delegates to `runMain`), a script path, `--` followed - by one, with `-` and every pre-init-only `PyConfig` option (`-E -I -S -s -B -O -OO -X -W -u - -P` among them) refused by name — and runs it with the same `sys.exit`/`sys.argv` contract. - Neither path is a lifecycle function, so the interpreter survives both. `Py_BytesMain` is - still never called: it reaches `Py_RunMain()` and finalizes, which is what made "one - marshalling path away from working" the wrong description of what `runApp` needed. §12 has - the detail. - -11. **The iOS app bundle carries no Python standard library** — restated, because the previous - wording ("the iOS *app* packaging path has no producer … no Gradle task creates it") was wrong - about which piece is missing. `prepareIosFrameworks` *does* stage - `sample/build/xcode-frameworks/Python.xcframework`, and the Xcode `link*Ios*` tasks depend on it. - What is missing is the stdlib: the Xcode "Install Target Specific Python Standard Library" phase - rsyncs from a directory that in this distribution holds only `libpython3.14.dylib`, and nothing - in the app path sets `PYTHONHOME`. The app therefore cannot start from its own bundle. (§9 item - 4, §13 "The sample runs on iOS") **(a) blocking it:** two open questions rather than missing - work — whether the stdlib should be staged by Gradle or by the existing Xcode phase with - corrected source paths, and whether `PYTHONHOME` should come from the environment (as the tests - do) or be derived on iOS from `NSBundle.mainBundle.bundlePath`, which would touch `iosMain` and - has to not break the test task that sets the variable explicitly. Note also that the - "Prepare Python Binary Modules" phase rewrites `lib-dynload/*.so` into `.fwork` placeholders, so - a bundle-hosted stdlib needs BeeWare's `.fwork` importer or that phase disabled. **(b) next - step:** decide the `PYTHONHOME` question first; the staging is mechanical once it is answered, - and §13 records the exact `rsync` + `simctl` sequence that is known to work by hand. - -### 14c. What this audit checked and could not confirm - -Left as **unconfirmed** rather than guessed: - -- Whether `.github/workflows/desktop.yml` includes Linux and Windows runners (relevant to item 3 - above) — not opened during this pass. -- The Maven Central / JitPack publication status implied by README's "Use Pre-Built Package" - section: `curl` against `repo1.maven.org/maven2/io/github/thisisthepy/` returned 404 (empty - directory listing) and JitPack's own build API reports no build record for - `com.github.thisisthepy:python-multiplatform-mobile` — both consistent with *nothing having ever - been published*, but a negative result from two APIs is not the same as a documented decision - not to publish, so README now says what was actually observed rather than asserting either way. - See §14d. -- Whether `PyList.subList`'s live-view behaviour (§12) and every other claim in §12's TODO triage - still hold — re-read for internal consistency during this pass, not re-verified against the code - a second time; nothing in that section contradicted what this audit found elsewhere. -- Whether the API-26 emulator this repo already runs against exercises the `PhantomReference` - fallback (item 7 above) or something else — not traced through `Cleaner`'s own - `Build.VERSION.SDK_INT` gate during this pass. - -### 14d. README corrections made in this pass - -Documentation only — nothing in `python-multiplatform/`, `sample/`, `ksp-fixtures/` or any test -was touched. For traceability, since some of these were load-bearing enough to be worth naming: - -- **The "Usage" example did not match this library's API at all** — `Python3Library()`, `Pointer`, - force-unwrapped `python`/`py`/`mathModule` variables declared nowhere. Replaced with the pattern - `sample/src/commonMain/.../PythonDemo.kt` actually runs: `Python3.initialize()`, `PyList.fromList` - + `PyInt.from` to publish a Kotlin list into `__main__`, `Python3.eval`/`Python3.exec` to read it - back. This was very likely the most misleading single passage in either document — a reader - trying it verbatim could not get past the first line. -- **"Build Manually" cloned the wrong repository** (`python-multiplatform-mobile`, with `@branch` - syntax that is not valid `git clone` syntax anyway) and described a `/composeApp` module this - repository does not have. Corrected to this repo's actual remote and its actual two relevant - modules (`python-multiplatform/`, `sample/`). -- **"Use Pre-Built Package" asserted a Maven Central / JitPack release that could not be confirmed - to exist** (§14c) and quoted a version number (`0.0.1`) that does not match what the build - actually produces. Reworded to state what was checked and found, rather than presenting - instructions as fact. -- **The CI badge situation, checked against README's own text: already accurate**, and did not - need correcting — README already carries a comment explaining why the badge is omitted rather - than a badge pointing somewhere wrong. This is the one place this audit found README *ahead of* - ROADMAP rather than behind it (§12's old "badges point at a different repository" line was the - stale one, fixed above). -- Left alone as already accurate: the "Supporting multiplatforms" list's entries for Android, iOS - and macOS (only Linux/Windows got the untested caveat added), and the Template ToDo list's first - three lines. +**Removed.** It was a dated audit snapshot. Test counts are re-counted per AGENTS.md §15, behaviour +status lives in SPEC.md, and its open-item list (§14b) is redistributed: item 1 → 9.1, item 2 → 12.4 +(CI now runs), item 3 → 12.3, item 4 → 12.5, item 5 → 4.1, item 11 → 13.1; items 6–10 were closed. ## 15. Can a consumer outside this repo actually use the library? -§14c/§14d established that nothing is published anywhere yet. This section is the follow-up that -was still open: given that gap, does *local* publishing even work, or would a real release hit -more breakage on top of "not published"? Tested by publishing all three components to -`mavenLocal()` and building a genuine external consumer project outside this repo -(`/Volumes/macMini/consumer-test` and `/Volumes/macMini/consumer-plugin-test`, both scratch -directories, not part of this repository) against it. - -### 15a. Publishing coverage before this pass: 1 of 3 components - -The library ships as three separately-consumed pieces — `python-multiplatform` (the library -itself), `python-multiplatform-gradle-plugin` (the convenience Gradle plugin, `id("io.github. -thisisthepy.python.multiplatform.bindings")`), `python-multiplatform-ksp` (the KSP processor the -plugin wires in by default). A consumer needs all three reachable from *some* Maven repository — -`in-repo` project substitution (`projects.pythonMultiplatformKsp`, used by `ksp-fixtures` and -`sample`) hides gaps in this because it never resolves anything by coordinates at all. - -- `python-multiplatform`: `maven-publish` applied, `./gradlew tasks --all` lists a full set of - `publishPublicationToMavenLocal` tasks. **Working**, with one target-level gap — see - §15d. -- `python-multiplatform-gradle-plugin`: `maven-publish` + `kotlin-dsl` applied — but this build is - a separate Gradle build, brought in only via `pluginManagement { includeBuild(...) }` in the - root `settings.gradle.kts`. That wires plugin-id *resolution* inside this repo; it does not add - the build's tasks to the root task graph. `./gradlew tasks --all` from the repo root lists no - `python-multiplatform-gradle-plugin:*` tasks at all — a coordinator following only the root - build's task list would conclude this component has no publishing story. It has to be published - from inside its own directory: `cd python-multiplatform-gradle-plugin && ../gradlew - publishToMavenLocal`. **Working once you know to look there** — worth a line in a future - RELEASING.md, not a code fix (the separate-build shape is deliberate, see the build file's own - comment on why the plugin needs KSP on its runtime classpath but not its compile classpath). -- `python-multiplatform-ksp`: **no publishing plugin applied at all** before this pass — plain - `kotlin("jvm")`, nothing else. `./gradlew tasks --all` confirms zero publish tasks. This is the - one true gap: `python-multiplatform-gradle-plugin`'s `generateCoordinates` task bakes - `DEFAULT_PROCESSOR_COORDINATES = "io.github.thisisthepy:python-multiplatform-ksp:"` - into the plugin — the coordinate the plugin hands KSP when a consumer doesn't override - `pythonBindings { processor.set(...) }` — and until now that coordinate named an artifact that - could never exist anywhere. **Fixed**: `python-multiplatform-ksp/build.gradle.kts` now applies - `maven-publish` with a `MavenPublication` built `from(components["java"])`; default - group/artifactId/version already matched what the generated coordinates expect, so no other - change was needed. Verified — `./gradlew :python-multiplatform-ksp:publishToMavenLocal` now - produces `~/.m2/repository/io/github/thisisthepy/python-multiplatform-ksp//`. - -### 15b. A real bug the in-repo build could never surface: `generateCoordinates` baked `"null"` as the group - -Publishing the fix from §15a exposed a second, independent bug, and it is the one worth reading -carefully — nothing in this repository's own build could ever have caught it, because nothing in -this repository resolves the processor by its published coordinates. - -`python-multiplatform-gradle-plugin/build.gradle.kts` generates `ProcessorCoordinates.kt` from: - - val generateCoordinates = tasks.register("generateCoordinates") { - val coordinates = "$group:python-multiplatform-ksp:$version" - ... - -This reads as project-scoped (the file's own doc comment says "this build's own group and -version"), but the lambda passed to `tasks.register(...)` has `Task` as its receiver, not -`Project`. `Task` has a `group: String?` property of its own — a task's category label (`"build"`, -`"verification"`, ...), always null on a task nobody assigned one to — and it shadows -`Project.group` inside that block. `Task` has no `version` property, so `$version` fell through to -`Project.version` by coincidence and looked correct. The generated constant was therefore: - - internal const val DEFAULT_PROCESSOR_COORDINATES: String = "null:python-multiplatform-ksp:3.13.0" - -Every external consumer relying on the default (i.e. not setting `pythonBindings { processor.set -(...) }`, which the plugin's own doc comment calls out as the in-repo-only override) would apply -the plugin, add no KSP dependency of their own, and get: - - Could not find null:python-multiplatform-ksp:3.13.0. - Searched in the following locations: - - file:/Users/.../.m2/repository/null/python-multiplatform-ksp/3.13.0/....pom - - https://repo.maven.apache.org/maven2/null/python-multiplatform-ksp/3.13.0/....pom - -This reproduced with a real external Gradle project (`/Volumes/macMini/consumer-plugin-test`) -applying `id("io.github.thisisthepy.python.multiplatform.bindings") version "3.13.0"` from -`mavenLocal()` — `:kspKotlin` failed with exactly that message on the first attempt. - -**Fixed**: qualified both reads as `project.group` / `project.version`. Re-published, re-ran the -same external consumer with no other change — `:kspKotlin` now runs and generates the fragment -table. Both fixes are isolated to `python-multiplatform-gradle-plugin/build.gradle.kts` and -`python-multiplatform-ksp/build.gradle.kts`; nothing in `python-multiplatform/`, `sample/`, or -`ksp-fixtures/` was touched, and `desktopTest` stayed at 352/0 (1 skipped) throughout — see §14a -for the counting recipe, this run just re-applies it. - -### 15c. The library itself, once published, works exactly as advertised for a plain JVM consumer - -With `python-multiplatform` published to `mavenLocal()`, a from-scratch external Gradle project -(`/Volumes/macMini/consumer-test`, plain `kotlin("jvm")` + `application`, *not* Kotlin -Multiplatform) depending on `io.github.thisisthepy:python-multiplatform-desktop:` and -running README's own "Usage" example verbatim: - -- **Compiled clean** against the published artifact — the object-model API (`Python3`, `PyObject`, - `PyList`, `PyInt`) resolves with no extra repositories beyond `mavenLocal()` + `mavenCentral()`. -- **Ran and produced the correct result** (`Result from Python: 56`, i.e. `sum([2,3,5,7,11]) * 2`) - once `PYTHONHOME` was pointed at a real CPython prefix. The native `libpython` shared library - itself *is* bundled in the published jar (`tasks.withType` in `python-multiplatform/ - build.gradle.kts` copies it under `lib//`) and loads correctly from the classpath with - no extra wiring — `manager.loadLibPython()` found it without complaint. Only the interpreter's - standard library (`encodings`, `lib/python3.14/`, ...) is missing; `Py_Initialize()` fails with - `Fatal Python error: Failed to import encodings module` until `PYTHONHOME` supplies it. - -So the object model, the FFI layer, the upcall marshalling, and the native-library packaging are -all consumer-ready today. **The one remaining blocker for a plain-JVM consumer is CPython -distribution**, and it is a real one: - -- `desktopMain/README.md` already documents that `PYTHONHOME` must point at a matching stdlib - prefix and that this is deliberate (the jar carries the loader library, not the interpreter - tree) — so this is a known, accepted design point, not an oversight. - But **there is no published or documented way for an external consumer to obtain that prefix.** - Every path that produces one today (`downloadAllPythonBuilds` and its per-platform - `downloadPython_*` tasks) is a task defined in `python-multiplatform/build.gradle.kts` itself, - reachable only by building this repository from source. A consumer who adds the Maven dependency - and follows README's own "Usage" section hits `Failed to import encodings module` with no next - step documented anywhere. -- This was proven the direct way, not inferred: pointing the external consumer's `PYTHONHOME` at - this repo's own `python-multiplatform/build/python-standalone/extracted/// - python` (a path that only exists because this repo's own build already ran - `downloadAllPythonBuilds`) made the same consumer project run correctly. That confirms the gap - is exactly "no distribution channel for the stdlib prefix" and nothing else — once a consumer - has *a* correct prefix from *any* source, everything downstream of it (native loading, object - model, upcalls) already works. -- **Next step, not done here** (this pass fixed what was cheaply and safely fixable; packaging and - shipping a CPython distribution is a real design decision, not a one-line fix): either (a) - publish a small companion artifact/archive per platform that a consumer's build can unpack into - a `PYTHONHOME`-shaped directory, mirroring what `downloadAllPythonBuilds` already assembles from - `python-build-standalone`, or (b) document, in README's "Use Pre-Built Package" section, the - exact python-build-standalone release URL and unpack layout a consumer needs to reproduce by - hand. (a) is better for anyone who isn't willing to hand-roll it; (b) is the cheaper stopgap. - -### 15d. `androidTarget` has no Maven publication at all - -**Closed.** `androidTarget` calls `publishLibraryVariants("release")`, -`publishAndroidReleasePublicationToMavenLocal` exists, and an Android app project outside this -repository resolves the library by coordinates, builds an APK, and **runs CPython on both -emulators** — see the end of this section for the exact line it printed. - -**Was:** `./gradlew tasks --all` listed `publishPublicationToMavenLocal` for every one of -`python-multiplatform`'s targets (`desktop`, `iosArm64`, `iosSimulatorArm64`, `iosX64`, -`androidNativeArm64`, `androidNativeX64`, `wasmJs`) **except `androidTarget`.** Kotlin Multiplatform -only auto-creates a Maven publication for an Android target when the target block calls -`publishLibraryVariants(...)`, and that block never did. Practical effect: **an Android app consumer -could not get this library from Maven at all**, published or not — orthogonal to §14c's "nothing is -published" finding, and it would have survived the fix for that one. - -It was left open because turning the line on packages a release AAR carrying per-ABI native `.so`s -and a full CPython stdlib in `assets/`, and the size of that was a design question rather than a -mechanical fix. So the first thing this pass did was stop guessing at it. - -#### The measurement, before anything was decided - -`bundleReleaseAar` with `publishLibraryVariants("release")` on and nothing else changed: -**40.10 MB on disk, 137.91 MB of entries across 5898 files.** - -| | compressed | raw | files | -|---|---|---|---| -| `assets//lib/**/test` | 7.11 MB ×2 ABIs | 32.30 MB ×2 | 1790 ×2 | -| `jni/x86_64`, `jni/arm64-v8a` | 5.31 + 5.09 MB | 14.97 + 15.29 MB | 7 + 7 | -| `assets//lib/**` (pure Python, rest) | 3.16 MB ×2 | 11.91 MB ×2 | 790 ×2 | -| `assets//lib/**/lib-dynload` | 2.46 + 2.37 MB | 7.59 + 7.30 MB | 68 + 68 | -| `assets//include` | 0.45 MB ×2 | 1.91 MB ×2 | 289 ×2 | -| **classes, manifest, R.txt** | **0.49 MB** | 0.52 MB | 5 | - -The library is 0.49 MB of a 40 MB artifact. Two more facts fell out of measuring rather than -reading, and both decided the choice below: - -- **The two ABIs' stdlibs are the same tree.** `diff -rq` over - `assets/arm64-v8a/lib/python3.14` against `assets/x86_64/lib/python3.14` reports differences in - exactly five files (`_sysconfigdata__*`, `_sysconfig_vars__*`, `build-details.json`) plus - `lib-dynload`. Everything else — 790 files, 11.91 MB — is duplicated byte for byte. -- **An App Bundle strips `lib//` per device and never strips `assets/`.** So the ABI half of - the payload is already handled by tooling the consumer has, and the duplicated half is not. - -#### The three options, and why (A) won - -- **(A) ship everything.** The consumer adds one dependency and it works. Size is the price. -- **(B) leave the stdlib out and let the consumer fetch it.** Rejected on what the code actually - is, not on taste: every path that produces a CPython prefix today - (`downloadAllPythonBuilds`, `downloadPython_android_*`) is a task in - `python-multiplatform/build.gradle.kts`, reachable only by building this repo, and - `python-multiplatform-gradle-plugin` — the one thing a consumer applies — does KSP wiring and - nothing else (`PythonBindingsPlugin.kt` is 186 lines, all of it configuration names and a - reflective `ksp { arg(...) }`). Giving it a download/verify/stage path is a feature, not a - switch. And it would not buy what it appears to: on desktop, `PYTHONHOME` can point anywhere on - the machine, which is why §15c's gap is a *distribution* gap; on Android the stdlib has to end - up **inside the APK**, so a consumer-side download stages the same ~26 MB of assets. The size - moves, it does not shrink. Worth doing for §15c item 4; it does not answer §15d. -- **(C) split per ABI.** Wrong lever, per the measurement above: `assets/` is 26.3 MB of the 37 MB - of entries and is exactly the part no ABI split touches, while `jni/` — the part a split would - address — is already stripped from the installed app by AGP. It also forces consumers into - flavour/variant matching for a library that has none. - -So (A), with the payload that is *provably* never read on a device removed. - -#### What was removed, and what deliberately was not - -`copyAndroidPythonAssets` no longer stages two things: - -- `include/python3.14` — CPython's C headers. cinterop reads them from the extraction tree - (`targetIncludePath`), never from assets; no Kotlin source in `androidMain`, `artMain`, - `androidInstrumentedTest` or `sample` opens an asset under `include/`. -- `test` — CPython's own regression suite, 32.30 MB raw per ABI and **38% of the AAR**. Nothing in - this repository imports it. - -Not removed, and the distinction matters: `lib-dynload` is the compiled extension modules and is -the only genuinely per-ABI part of the tree; `idlelib`, `ensurepip`, `tkinter` and `turtle` are -dead weight for most embedders but are ordinary stdlib a consumer may legitimately import. "CPython's -own test suite" and "C headers" are provable claims. "Nobody wants `tkinter`" is not. - -The duplicated 11.91 MB of pure Python is left duplicated too. De-duplicating it would save 3.16 MB -compressed and change the published asset layout (`/lib/python`) that `MainActivity`, -`PythonOnDevice` and now README's own Android section all depend on — a breaking change to the -consumer contract for 14% of the artifact. Recorded as a measured option, not taken. - -**Result: 40.10 MB → 23.42 MB** (22.04 MB of entries, 1740 files). A `Copy` never deletes what it -stopped copying and this directory is an AGP asset source root, so the task now clears it first — -without that, both trees would have kept shipping on every machine that had built once before. - -#### A second, larger defect the publication exposed: an 87 MB metadata jar - -Publishing to `mavenLocal` produced `python-multiplatform-3.14.7-alpha01.jar` — the `allMetadataJar`, -the artifact at the **root coordinate every Kotlin Multiplatform consumer resolves to compile -against `commonMain`** — at **87.4 MB, of which 83.0 MB was four host platforms' `libpython`** -(`lib/linux-x86_64` alone: 64.69 MB compressed, 240.23 MB raw) against 0.3 MB of actual metadata. - -The cause is a scope that reads as narrower than it is: `tasks.withType { from(extractedDir) { -... } }` sits *inside* the `jvm("desktop") { ... }` block, but `tasks` is the **project's** task -container, so it matched `allMetadataJar` too. Only the CPython libraries are now scoped to -`desktopJar`; the licence stays on every jar. `desktopJar` is unchanged at 87.7 MB, which is what -§15c verified a desktop consumer loads `libpython` out of. The root artifact is **382 KB**. - -The sources jars never picked the libraries up (`desktopSourcesJar` 218 KB, `androidReleaseSourcesJar` -212 KB, neither containing a `lib/` entry), so the narrowing changes exactly one artifact. - -#### Verified against a real external consumer - -`/Volumes/macMini/consumer-android-test` — a plain `com.android.application` + `org.jetbrains. -kotlin.android` project outside this repository, no composite include, `mavenLocal()` first — depends -on the **root** coordinate `io.github.thisisthepy:python-multiplatform:3.14.7-alpha01` and lets -Gradle module metadata redirect it: - -``` -\--- io.github.thisisthepy:python-multiplatform:3.14.7-alpha01 - \--- io.github.thisisthepy:python-multiplatform-android:3.14.7-alpha01 -``` - -Its `MainActivity` unpacks `assets//lib/python3.14` to `filesDir`, sets `PYTHONHOME`, calls -`Python3.initialize()`, hands Python a `PyList` of `PyInt` built in Kotlin, and evaluates -`sum(kotlin_numbers) * 2`. `assembleDebug` produced a 26.8 MB universal APK, and on **both** -emulators: - -``` -API 36 I PyConsumer: PYCONSUMER_OK abi=arm64-v8a version=3.14.7 platform=android result=int:56 -API 26 I PyConsumer: PYCONSUMER_OK abi=arm64-v8a version=3.14.7 platform=android result=int:56 -``` - -Compiled, assembled, installed and **ran** — the JNI surface, the object model and the stdlib -packaging all work from a published artifact, with the app doing nothing the AAR does not document. - -**Still open after this**, and not a blocker for the above: the bindings plugin is untested from -outside for an Android consumer — §15b verified it against a plain JVM project only. - -~~The Android consumer must write the asset-unpacking and `PYTHONHOME` code itself~~ — -**closed, §15f.** - -### 15f. The recipe became a helper, and the third copy of it was in this repository - -§15d left the host app writing the bootstrap by hand, and named a `PythonBootstrap` helper as the -better answer. Doing it produced the argument for it that the README paragraph could not make: -**there were three copies of that recipe, and all three were wrong about the same thing.** - -| copy | lines it owned | how it decided "already unpacked" | -|---|---|---| -| `sample`'s `MainActivity` | 55 | `target.exists() && target.list()?.isNotEmpty()` | -| `/Volumes/macMini/consumer-android-test`'s `MainActivity` | 26 | `File(target, "encodings").isDirectory` | -| `PythonOnDevice`, this repo's own instrumentation fixture | 35 | `File(target, "encodings").isDirectory` | - -Every one of them probes the *result* for a single entry. A copy interrupted partway — the app -killed during a cold start, the device out of space — satisfies all three probes with an incomplete -stdlib, and the next launch accepts it. The failure then surfaces as an `ImportError` for whichever -module sorted after the interruption, nowhere near the cause. Three authors writing from the same -README paragraph produced the same defect three times; that is the case for a helper rather than for -a better paragraph. - -`PythonBootstrap` (`androidMain/kotlin/python/multiplatform/env/PythonBootstrap.kt`) writes a stamp -— `python-multiplatform ` — **after** the last byte of -the last file, and deletes it *before* starting a rewrite. A partial tree therefore has no stamp, -and a stamp is never valid for a tree that is currently being replaced. The `lastUpdateTime` -component is what makes an APK upgrade restage: `filesDir` survives the install, so the previous -build's tree is still sitting there and its Python version may well be identical, which is exactly -the case a version-only check misses. - -It deliberately does **no** `PYTHONHOME` validation of its own. `Python3.initialize` already runs -`PythonHomeCheck` (landed in `7bf08ca6`) on every call; a second implementation of that rule is a -second thing that can drift from the layout `Py_Initialize()` actually wants. This produces the -layout, that verifies it, and `PythonBootstrapTest.stagedPrefixIsOneThatPythonHomeCheckAccepts` runs -one against the other instead of asserting each separately. A second test asserts the value the -bootstrap sets is the value the check reads — on Android those are the same environment, because -libcore's single-argument `System.getenv` delegates to `Os.getenv` rather than to the JVM's cached -no-arg map. - -#### Cost: 804 files, 20,147,280 bytes - -Measured twice over, and the two disagree in a way worth keeping. From `onCreate` of the external -consumer app — fresh install, then relaunch, which is what a host app actually pays: - -| | API 26 | API 36 | -|---|---|---| -| first launch (unpacks) | 237 ms | 480 ms | -| every later launch (stamp check) | 4 ms | 28 ms | - -From `PythonBootstrapTest`, inside an instrumentation process that is already warm and whose page -cache is already hot: - -| | API 26 | API 36 | -|---|---|---| -| first (unpacks) | 71–74 ms | 95–103 ms | -| subsequent (stamp check) | 1 ms | 1 ms | - -The in-process numbers are 2–5x optimistic, so **quote the cold-start ones**. Either way it is a -factor of 17–60, which is the entire point: the alternative to a skip that can be trusted is 20 MB -of asset decompression on every cold start. - -#### The copy loop was measured, not argued - -It departs from the hand-written one in two ways, so -`PythonBootstrapTest.optimisedCopyIsMeasuredAgainstTheHandWrittenOne` runs both, alternating, two -samples each: - -| | API 26 | API 36 | -|---|---|---| -| hand-written (`list()` per entry, `copyTo`'s 8 KB buffer) | 192 / 192 ms | 324 / 324 ms | -| `PythonBootstrap` (failed `open()` classifies, one 64 KB buffer) | 124 / 113 ms | 150 / 167 ms | - -The larger half is the classification. The obvious loop asks `assets.list(child)` of every entry to -decide whether it is a directory — a native directory lookup per entry, 800-odd of them, *on top of* -the `open()` each file needs anyway. Opening first and reading `FileNotFoundException` as "directory" -folds the question into work that had to happen regardless, so only the directories pay anything -extra. - -That behaviour is **not** in `AssetManager`'s documented contract, and AssetManager was -reimplemented wholesale in API 28, so `openBasedDirectoryDetectionAgreesWithListBased` re-derives -both classifications on device on whatever API level is running rather than trusting they agree, and -the benchmark asserts both loops wrote the same 804 files. Nothing is asserted about the *margin*: -an emulator's disk is the host's SSD behind qemu and is not a stable enough clock to fail a build -on. - -Worth recording for whoever revisits this: reading the APK as a ZIP (`ZipFile(applicationInfo -.sourceDir)`, prefix-filtered, one pass, no recursion and no per-entry classification at all) is -very likely faster still. It was not taken because it has to handle `splitSourceDirs` for App -Bundles, and 100 ms once per install did not justify that surface. - -### 15e. Summary — corrected punch list for "can a consumer use this" - -1. ~~`python-multiplatform-ksp` has no publishing plugin~~ **fixed this pass.** -2. ~~`generateCoordinates` bakes `"null"` as the processor's group~~ **fixed this pass** — was - silently broken for every external consumer of the default coordinates since the day this task - was written; the in-repo build can never exercise this path. -3. ~~`python-multiplatform-gradle-plugin` publishes fine but is invisible to `./gradlew tasks` from - the repo root because it's a separate included build — needs a documented publish step, not a - code change.~~ **fixed, §15g** — one root task (`publishAllToMavenLocal`) now reaches all - three; found while doing it that the three publish under two different versions, not one. -4. ~~No distribution channel exists for the CPython stdlib prefix a consumer's `PYTHONHOME` needs — - confirmed to be the *only* remaining blocker for a plain-JVM consumer once 1–2 are fixed. - Packaging/documenting one is real follow-up work, not done here. **Android is not affected**: - §15d ships the stdlib inside the AAR, and an external Android consumer now runs. This item is - desktop-only.~~ - — **fixed, §15h.** It was confirmed to be the *only* remaining blocker for a plain-JVM consumer - once 1–2 were fixed, and it is now a `stagePythonHome` task on the bindings plugin: §15c's own - external consumer runs, printing the same `56`, with no `PYTHONHOME` set by hand and nothing - added to `desktopJar`. **Android was never affected** — §15d ships the stdlib inside the AAR. -5. ~~`androidTarget` has no Maven publication at all~~ **fixed, §15d** — `publishLibraryVariants - ("release")`, the AAR trimmed 40.10 MB → 23.42 MB on measurement rather than taste, and verified - end to end by an external Android app that resolves by coordinates and runs CPython on both - emulators. -6. New, found while doing 5: the **root** Kotlin Multiplatform artifact was 87.4 MB because a - project-wide `tasks.withType` in the desktop target block put four host platforms' libpython - into `allMetadataJar`. **Fixed, §15d** — 382 KB now. This one hits *every* KMP consumer on every - platform, not only Android. -7. ~~The host app has to hand-write the asset unpack and `PYTHONHOME` setup~~ **fixed, §15f** — - `PythonBootstrap` in `androidMain`, one call, and the three hand-written copies (`sample`, - `PythonOnDevice`, the external consumer app) all call it now. Still open for Android - specifically: the bindings plugin has never been exercised from an external *Android* consumer — - §15b covered a plain JVM one. - -None of §1–7 were guessed at: each was reproduced against a real external Gradle project outside -this repository (`/Volumes/macMini/consumer-test`, `/Volumes/macMini/consumer-plugin-test`, -`/Volumes/macMini/consumer-android-test`) rather than inferred from reading the build scripts alone. - -### 15f. The bindings plugin, exercised from an external *Android* consumer — item 7 above, closed - -§15b verified the plugin against a plain JVM consumer only. §15d's Android consumer used the -*library* by coordinates but applied no plugin at all — it hand-wrote its own KSP wiring, so it -never exercised `id("io.github.thisisthepy.python.multiplatform.bindings")`'s Android path: the -`Test`-word configuration-name matching (this doc's own `PythonBindingsPlugin.kt` comment calls -out `kspAndroidTestDebug` as the configuration that broke before that regex existed), the -`com.android.library`/`com.android.application` role inference, and the reflective `ksp { -arg(...) }` call against whatever KSP extension shape a consumer's KSP version actually exposes. -None of that is reachable through `ksp-fixtures/android`, which is an in-repo project-substitution -consumer, not one resolving the plugin by coordinates. - -**Setup**: all three components published to `mavenLocal()` from this worktree — - -| Component | Coordinates | Published from | -|---|---|---| -| library (root, desktop, android) | `io.github.thisisthepy:python-multiplatform:3.14.7-alpha01`, `...-desktop:...`, `...-android:...` | repo root (`-PpythonVersion=3.14.7`, matching what was already extracted) | -| KSP processor | `io.github.thisisthepy:python-multiplatform-ksp:3.13.0` | repo root | -| plugin implementation | `io.github.thisisthepy:python-multiplatform-gradle-plugin:3.13.0` | `python-multiplatform-gradle-plugin/` (its own build — §15a already noted this has to be done from inside that directory) | -| plugin marker (what `id(...)` actually resolves) | `io.github.thisisthepy.python.multiplatform.bindings:io.github.thisisthepy.python.multiplatform.bindings.gradle.plugin:3.13.0` | same | - -Confirmed the marker's POM depends on exactly `io.github.thisisthepy:python-multiplatform-gradle-plugin:3.13.0`, and the KSP processor's own POM group/artifact/version match `DEFAULT_PROCESSOR_COORDINATES` verbatim — §15b's fix holds under a real Android consumer, not just the plain-JVM one it was verified against. - -**Consumer**: `/Volumes/macMini/consumer-plugin-android`, outside this repository, `pluginManagement { repositories { mavenLocal(); ... } }`, a single Kotlin Multiplatform module (`jvm("desktop")` + `androidTarget()`, `com.android.library`) applying the plugin by id and version, with `pythonBindings { role.set("app"); moduleName.set("consumer_override"); excludePackages.set(listOf("com.example.consumer.internal")) }` — deliberately exercising the override knobs item 3 of the task asked about, not just the defaults §15b already covered. - -**What ran, in the order a consumer would hit it:** - -1. `./gradlew tasks` — plugin applies and configures cleanly against `mavenLocal()`, no coordinate-resolution error at configuration time (the `dependencies.addLater` wiring is lazy, so this step doesn't yet prove the processor dependency resolves — that's step 2). -2. `./gradlew desktopTest` — `kspKotlinDesktop` ran, generated `python.multiplatform.generated.FunctionTable` and `Fragment_consumer_override.kt` into `build/generated/ksp/desktop/desktopMain/`, compiled, and a hand-written test installed the table and **invoked `add(3, 4)` through it, getting `7L` back** — the full round trip, not just generation. `@PythonInternal` on one function and `excludePackages` on a whole package both worked: neither's function resolved in the table. -3. `./gradlew assembleRelease assembleDebug` — `kspReleaseKotlinAndroid` and `kspDebugKotlinAndroid` ran (the `Test`-word regex correctly left `kspDebugUnitTestKotlinAndroid` at `NO-SOURCE`, i.e. unconfigured, exactly as the plugin's doc comment says it should), both AARs built, and the Android leaf's own `FunctionTable`/`Fragment_consumer_override.kt` generated at `build/generated/ksp/android/androidRelease/`. -4. `desktopTest`'s 3 assertions passed: module aggregation, the override name, the invocation, and the package-level exclusion. No emulator was used — compile and assemble only, per this task's constraint. - -**Nothing in this path needed a code fix.** §15a/§15b's earlier fixes (KSP publishing, the `null:` group bug) were what made step 2 possible at all; from an Android angle specifically, the plugin's `Test`-word matching and role inference — the two things §13 had already hardened against `ksp-fixtures/android` — held against a real external Android module too. - -**What *did* turn up, and is not a code bug but was undocumented until now**: the consumer's own Kotlin Gradle Plugin version has to clear two independent floors, found by deliberately trying versions other than this repo's own pinned one (`libs.versions.kotlin` = `2.4.20-Beta2`): - -- **Kotlin 2.1.0**: `kspKotlinDesktop` fails at configuration time with a bare - `'org.gradle.api.provider.Property ... .getJvmDefault()'` error — no stack trace pointing at - KSP, no mention of a version mismatch. KSP 2.3.11's Gradle plugin calls a Kotlin Gradle Plugin - API that Kotlin 2.1.0's KGP does not have. -- **Kotlin 2.2.20** (past the floor above): `kspKotlinDesktop` now succeeds, but - `compileKotlinDesktop` fails compiling the *generated* fragment — `Class - 'python.multiplatform.reflection.TypeTag' was compiled with an incompatible version of Kotlin. - The actual metadata version is 2.4.0, but the compiler version 2.2.0 can read versions up to - 2.3.0.` This is unrelated to KSP: the published library itself was compiled with this repo's - Kotlin version, and a consumer's compiler has to be new enough to *read* that metadata, - independent of whatever KSP requires. -- **Kotlin 2.4.20-Beta2** (this repo's own version): clears both, confirmed above. - -Neither failure names a required version or points at the other component — a consumer hits a -compiler internals error, not a diagnostic. **Fixed**: documented as a KDoc block on -`PythonBindingsPlugin` (`python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt`) -naming both floors and the exact errors each produces, so the next person reading the plugin's own -source finds this before hitting either error blind. A runtime version check was deliberately not -added: the two floors are two different projects' compatibility windows (KSP's plugin API surface, -and this library's own build's Kotlin version), neither owned by this plugin, and asserting a -specific range here would drift out of sync with `libs.versions.toml` the first time either -changes. No compatibility matrix beyond "match this repo's own pinned version" exists or was -produced by this pass. - -**Verified not to have regressed anything in-repo**: `:python-multiplatform:desktopTest` stayed at -360/0 (1 skipped) and `:ksp-fixtures:app:desktopTest` at 64/0 after publishing and after the KDoc -change, and `:sample:compileKotlinDesktop` still compiles — the only file this pass changed inside -the repository is the doc comment above. - -Updated item 7 of §15e's punch list: the bindings plugin **has** now been exercised from an -external Android consumer, resolving by coordinates, generating and invoking a real -`FunctionTable`, with the override knobs (`role`, `moduleName`, `excludePackages`) all confirmed -working — not just the defaults. What remains open from item 7 is unchanged: the host app must -still hand-write asset unpacking and `PYTHONHOME` setup. - -### 15g. Publishing all three components took three different commands — now it takes one - -§15a already named the shape of the problem: `python-multiplatform-gradle-plugin` is a separate -included build (`pluginManagement { includeBuild(...) }` in the root `settings.gradle.kts`), so its -tasks never join the root task graph and `./gradlew tasks --all` from the repo root shows none of -them. This section closes item 3 of §15e's punch list and records what publishing all three -actually required, before and after. - -#### What three commands were needed, before this pass - -| Component | Command | Where it has to run | -|---|---|---| -| `python-multiplatform` | `./gradlew :python-multiplatform:publishToMavenLocal` | repo root | -| `python-multiplatform-ksp` | `./gradlew :python-multiplatform-ksp:publishToMavenLocal` | repo root | -| `python-multiplatform-gradle-plugin` | `../gradlew publishToMavenLocal` | **inside** `python-multiplatform-gradle-plugin/` — it is a separate Gradle build, not a subproject; running it from the root resolves nothing | - -Confirmed directly, not inferred from the build scripts: `./gradlew :python-multiplatform:tasks ---all` and `:python-multiplatform-ksp:tasks --all` each list their own `publishToMavenLocal`, and -`cd python-multiplatform-gradle-plugin && ../gradlew tasks --all` lists a third, separate -`publishToMavenLocal` that owns two publications (`pluginMaven`, the implementation jar, and -`pythonMultiplatformBindingsPluginMarkerMaven`, the artifact `id(...)` actually resolves against). -`./gradlew tasks --all` from the repo root shows the plugin build compiling -(`:python-multiplatform-gradle-plugin:compileKotlin`, pulled in because `includeBuild` makes the -plugin's classes available to resolve the plugin id) but exactly one `publish*` line total across -the whole root output — the plugin's publishing tasks are not among it. - -#### Why the three versions aren't the two the earlier record implied - -§15f's setup table reads as if `python-multiplatform-ksp:3.13.0` and -`python-multiplatform-gradle-plugin:3.13.0` were a one-off, `-PpythonVersion=3.14.7` having been -passed for the library alone in that session. Re-run from a clean checkout with **no** version -flags at all (`git status` clean, no `-P` arguments), the same split reproduces: - -``` -python-multiplatform* 3.14.7-alpha01 (7 artifacts: root + 6 targets) -python-multiplatform-ksp 3.13.0 -python-multiplatform-gradle-plugin (+ marker) 3.13.0 -``` - -The cause is in the repo, not the invocation — `gradle.properties` commits `pythonVersion=3.14.7` -at the root: - -- `python-multiplatform/build.gradle.kts` reads it: `val configuredPythonVersion = - project.findProperty("pythonVersion")?.toString() ?: project.rootProject.version.toString()`, - then sets its own `version = "$pythonVersion-alpha01"` — i.e. `3.14.7-alpha01`. This is the only - one of the three that consults `pythonVersion` at all. -- `python-multiplatform-ksp/build.gradle.kts` sets no `version` of its own, so it falls through to - the root `build.gradle.kts`'s `allprojects { version = "3.13.0" }` — a literal, commented - "Official Python release version", that nothing keeps in step with `gradle.properties`. -- `python-multiplatform-gradle-plugin/build.gradle.kts` is a separate build that `allprojects` - never reaches. It hardcodes its own literal, `version = "3.13.0"`, next to a comment claiming - it's "kept in step with the root build's version" — true only because both were last edited by - hand at the same time. Nothing checks that they still agree. - -**Judgment: not intended, in the sense that matters.** It's defensible that -`python-multiplatform-ksp` and `python-multiplatform-gradle-plugin` — pure Kotlin/Gradle tooling, -neither one embeds CPython — shouldn't move every time `pythonVersion` changes the way the library -does. But that's not what's actually happening: they're not deliberately decoupled from -`pythonVersion`, they're just two hand-copied literals that happen to still match each other and -happen to still match `pythonVersion`'s old default (3.13.0 was this repo's original embedded -version before `gradle.properties` moved to 3.14.7). The next `pythonVersion` bump moves the -library's published version and silently leaves the other two exactly where they are, with no -comment or check pointing that out at the moment it happens. Worse, the plugin and the ksp -processor are only synchronized with *each other* by the same hand-copy discipline — §15b's fix -depends on `DEFAULT_PROCESSOR_COORDINATES` (generated from the plugin build's own `project.version`) -naming a coordinate that actually exists at `python-multiplatform-ksp`'s published version; if -either literal is edited without the other, that silently reproduces §15a's original "could never -exist anywhere" failure, and nothing at configuration time would catch it — only a consumer's -`:kspKotlin` resolution would, exactly as it did before the original fix. Not fixed here (no -version was bumped, per this task's constraint) — recorded as the sharp edge in the checklist -below. - -#### The fix: one root task, no change to what or how anything publishes - -`build.gradle.kts` (repo root) now registers: - -```kotlin -tasks.register("publishAllToMavenLocal") { - group = "publishing" - dependsOn(":python-multiplatform:publishToMavenLocal") - dependsOn(":python-multiplatform-ksp:publishToMavenLocal") - dependsOn(gradle.includedBuild("python-multiplatform-gradle-plugin").task(":publishToMavenLocal")) -} -``` - -`gradle.includedBuild(name).task(path)` is the documented way to depend on a task in an included -build from the including build; nothing about `pluginManagement { includeBuild(...) }` or any -publication's own configuration changed. Each component still owns its existing `maven-publish` -setup exactly as before. - -#### Verified: cleared `~/.m2/repository/io/github/thisisthepy` entirely, ran the one task - -`rm -rf ~/.m2/repository/io/github/thisisthepy` (this also removed the plugin marker, which lives -under the nested `io/github/thisisthepy/python/multiplatform/bindings` group, not under the -component artifact IDs), then `./gradlew publishAllToMavenLocal` from the repo root — -`BUILD SUCCESSFUL`, confirmed by exit code from a non-piped foreground run. Afterward, all of it was -back: - -- `python-multiplatform` root + 6 target artifacts (`-android`, `-androidnativearm64`, - `-androidnativex64`, `-desktop`, `-iosarm64`, `-iossimulatorarm64`, `-iosx64`, `-wasm-js`) at - `3.14.7-alpha01`. -- `python-multiplatform-ksp` at `3.13.0`. -- `python-multiplatform-gradle-plugin` (`pluginMaven`) at `3.13.0`. -- The plugin marker, `io.github.thisisthepy.python.multiplatform.bindings:io.github.thisisthepy. - python.multiplatform.bindings.gradle.plugin`, at `3.13.0` — the artifact `id(...)` actually - resolves against, and the one that's easiest to forget because it lives under a different group - path than every other artifact here. - -No regression: `build/test-results` cleared first, then `:python-multiplatform:desktopTest` + -`:ksp-fixtures:app:desktopTest` — 360/0 (1 skipped) and 64/0, unchanged from baseline. Only -`build.gradle.kts` (repo root) was touched; nothing under `python-multiplatform/`, -`python-multiplatform-ksp/`, or `python-multiplatform-gradle-plugin/` changed. - -#### Release checklist - -1. **Decide what actually moved:** - - Embedded CPython changed → bump `pythonVersion` in `gradle.properties`. This alone - re-versions `python-multiplatform`'s publications (`-alphaNN`). It does - **not** touch `python-multiplatform-ksp` or `python-multiplatform-gradle-plugin` — they have - no code path that reads `pythonVersion` at all. - - The KSP processor's own code changed → `python-multiplatform-ksp` has no `version` of its - own today; it inherits the root's `allprojects { version = "3.13.0" }` literal. Giving it an - independent version means adding a `version = ...` line to - `python-multiplatform-ksp/build.gradle.kts` — it doesn't have one to edit yet. - - The plugin's own code changed → hand-edit `version` in - `python-multiplatform-gradle-plugin/build.gradle.kts`. Nothing derives it. -2. **If the ksp processor's version moves independently of the plugin build's version**, update - both by hand and verify they still agree: `generateCoordinates` - (`python-multiplatform-gradle-plugin/build.gradle.kts`) bakes - `${project.group}:python-multiplatform-ksp:${project.version}` where `project.version` is the - *plugin build's own* version — not the ksp project's. They're equal today only because both - literals say `"3.13.0"`. Letting them drift silently reproduces §15a's original defect: no - error at configuration time, only at a consumer's `:kspKotlin` resolution. -3. **Publish with `./gradlew publishAllToMavenLocal`** (this pass), not three separate commands. -4. **Verify against a real external consumer**, per §15a/§15d/§15f's method — the in-repo build - resolves everything by project substitution and cannot catch a coordinate or version mismatch - between components. -5. **Before a non-local release**, clear `~/.m2/repository/io/github/thisisthepy`, run - `publishAllToMavenLocal`, and check the produced version directories against what - `DEFAULT_PROCESSOR_COORDINATES` and the plugin marker's POM declare — the check this section - just ran, repeated. -### 15h. Desktop's stdlib distribution — §15e item 4, closed - -§15c found a plain JVM consumer that compiled, loaded `libpython` off the classpath with no wiring -at all, and then died in `Py_Initialize()` — `Failed to import encodings module` — because nothing -anywhere gives it a standard library. §15e recorded that as the last blocker for a desktop -consumer. This closes it. - -#### First: what is actually in `desktopJar`, because the plan depended on it - -The task this started from assumed the stdlib might already be in the jar, leaving only "how does -a consumer get it out". **It is not.** The published -`python-multiplatform-desktop-3.14.7-alpha01.jar` is 87.7 MB and its `lib/` tree is **14 entries, -every one a shared library**: - - lib/linux-x86_64/libpython3.14.so.1.0 251,884,016 raw <- 64 MB of the jar on its own - lib/macos-aarch64/libpython3.14.dylib 19,394,064 - lib/macos-x86_64/libpython3.14.dylib 20,172,552 - lib/windows-x86_64/python314.dll 6,589,440 (+ python3.dll, 2x vcruntime) - -So the jar carries four platforms' *interpreters* and zero platforms' *standard library*. Any -option that "unpacks it from the jar" first has to put it there. - -#### What putting it there would cost, measured - -Compressed, per platform, from the trees `downloadAllPythonBuilds` already extracts: - -| platform | full stdlib, zipped | per-platform native part | -|---|---|---| -| macos-aarch64 | 8.12 MB | `lib-dynload` + `config-*` + `_sysconfigdata*`: 0.12 MB | -| macos-x86_64 | 7.12 MB | — | -| linux-x86_64 | 8.23 MB | 1.22 MB | -| windows-x86_64 | 20.11 MB (`Lib` + `DLLs`, `.pdb` excluded) | `DLLs` alone 8.36 MB | -| **all four** | **43.5 MB** | | - -Two facts fell out of measuring rather than reading: - -- **The pure-Python stdlib is platform-independent on desktop.** `diff -rq` of macOS-aarch64's - `lib/python3.14` against Linux-x86_64's reports **28 differences across a 1286-file tree**, and - all of them are accounted for: `__pycache__`, `_sysconfigdata__*`/`_sysconfig_vars__*`, - `config-3.14-*`, `build-details.json`, two `lib-dynload` modules, and pip's own `RECORD` and - `direct_url.json`. De-duplicating on that basis would cut 43.5 MB to roughly 15 MB. -- **The `install_only` distributions already omit CPython's test suite**, so the 38%-of-the-AAR - saving §15d found on Android does not exist here — there is nothing equivalent left to strip. - -Even de-duplicated, that is 15 MB added to an artifact **every desktop consumer resolves, to carry -payload of which each one uses a quarter**. That is precisely the shape of the defect §15d -diagnosed when `allMetadataJar` reached 87.4 MB, and the fix there was to narrow the scope, not to -accept the size. - -#### The deciding constraint is not size, though: a JVM cannot set its own environment - -This is what actually rules out a desktop `PythonBootstrap`, and it is not a preference. - -CPython reads `PYTHONHOME` with `getenv(3)`, from the native process environment. Android's -`PythonBootstrap` (§15f) sets it with `Os.setenv` and that is why the runtime shape works there. -**The JVM has no equivalent.** `System.getenv` is an immutable snapshot taken at start-up; -mutating it by reflection changes the JVM's cached map and does *not* touch the environment CPython -reads. A desktop helper would therefore have to call libc `setenv` through Panama — a different -symbol on Windows (`_putenv_s`) — and would then have set a value that `PythonHomeCheck` (`7bf08ca6`, -which reads `System.getenv`) could no longer see, on exactly the check that exists to catch a bad -`PYTHONHOME` before `Py_Initialize()` aborts the process uncatchably. - -Setting the variable **as the child process is launched** has neither problem: CPython and -`PythonHomeCheck` read the same value from the same place. That is already how this repository's -own `desktopTest` works. - -The C API offers no way round it either: `Py_SetPythonHome` was removed in 3.13, and -`PyConfig.home` needs the struct layout the Stable ABI deliberately does not promise — the reason -`Python3.initialize` uses `Py_Initialize()` rather than `Py_InitializeFromConfig` in the first -place. - -#### The options, and why (B) won - -- **(A) ship the stdlib in `desktopJar` and unpack it at runtime.** Rejected on both counts above: - +15 MB (de-duplicated) or +43.5 MB (not) on a shared artifact, *and* it cannot set `PYTHONHOME` - without the `setenv` route and the `PythonHomeCheck` divergence that comes with it. -- **(B) the Gradle plugin stages a prefix.** Taken. §15d rejected this for Android on the grounds - that the stdlib has to end up inside the APK regardless, so a consumer-side download moves the - size rather than removing it. **Neither half transfers to desktop**: a desktop `PYTHONHOME` may - name any path on the machine, so the prefix is fetched once *per machine* and shared by every - project on it, and nothing is added to any artifact. -- **(C) document the python-build-standalone URL and let the consumer unpack it by hand.** This is - the status quo §15c called a blocker. It also asks the consumer to keep a version, an upstream - release tag and a target triple in step with the `libpython` inside `desktopJar` by hand; a - mismatch pairs a stdlib and an interpreter that disagree about ABI while both calling themselves - 3.14.7. - -#### What it does - -`stagePythonHome` (`python-multiplatform-gradle-plugin`) downloads the same asset -`downloadPython_*` does, verifies it against the release's `SHA256SUMS`, extracts it into a shared -cache under the Gradle user home, and sets `PYTHONHOME` on every `JavaExec` and `Test` task. The -consumer's whole diff is one `plugins { }` line. - -Three details are inherited from §15f rather than rediscovered: - -- **A stamp written after the last byte, deleted before a rewrite.** The root build's own - extraction step skips when `extractDir.list()` is non-empty — the same "probe the result" defect - §15f found in three hand-written Android copies. Verified by observation, not by reading: the - task restages when the stamp is removed, when the stamp is stale, and when the stamp is intact - but `lib/python3.14/os.py` has been deleted underneath it. -- **The staged tree is validated against `PythonHomeCheck`'s own marker**, not a second notion of - "usable prefix". `stdlibMarkerRelativePath` returns the file `PythonHomeCheck.diagnose` probes. -- **The archive is downloaded to `.part` and renamed on completion**, and a cached archive is - re-verified rather than trusted, so an interrupted download cannot present as an intact one. - -Two things it deliberately does **not** do. It never overrides a `PYTHONHOME` the consumer already -set — the task is not even registered in that case — because §15c's entire finding is that setting -it by hand is what consumers have had to do, and silently replacing a working conda or system -prefix on a library upgrade would be worse than the gap. And it reads that variable through -`providers.environmentVariable` rather than `System.getenv`: the latter reads the *daemon's* -environment, so `PYTHONHOME=/their/prefix ./gradlew run` against a daemon started without it would -have been told the user had set nothing, and would have overridden them on the very task they were -configuring. - -#### Verified against §15c's own external consumer - -`/Volumes/macMini/consumer-test`, unchanged except for deleting its hand-set `PYTHONHOME` and -adding the plugin line: - - before Fatal Python error: Failed to import encodings module (the §15c gap, reproduced) - after Result from Python: 56 (no PYTHONHOME set anywhere) - -`56` is `sum([2,3,5,7,11]) * 2` — the same value §15c got only after pointing `PYTHONHOME` at this -repository's own build directory. The staged prefix is 1698 files, 69 MB, from a 26 MB archive. - -A second run with `PYTHONHOME` exported was confirmed to stage nothing, register no task, and -still print `56`. - -#### Cost - -Task time from Gradle's own `--profile`, over two independent cold/warm pairs with the cache -deleted between them: - -| | `stagePythonHome` | -|---|---| -| first build on a machine (download 26 MB, SHA-256 verify, extract 1698 files) | **1.507 s** / **1.939 s** | -| archive cached, prefix rebuilt (verify + extract only) | 0.770 s | -| every later build (stamp + marker check) | **0.014 s** / **0.037 s** | - -Between 40x and 107x depending which pair is read; nothing here rests on the margin. What holds is -the order of magnitude, and that the first build is paid **once per machine, not once per -project** — the cache is keyed by version + upstream release + platform under the Gradle user -home, and the next in-repo consumer to ask for it (`:ksp-fixtures:app:desktopTest`, which applies -this plugin) staged nothing at all. The download half of the cold number is this machine's link to -GitHub and should not be read as portable; the 0.770 s local half is the one that generalises. - -#### What could not be checked from this machine - -Worth stating plainly, because a *distribution* problem is by definition about the hosts you are -not sitting at. - -- **Only `macos-aarch64` was executed.** `linux-x86_64`, `macos-x86_64` and `windows-x86_64` have - their URL, asset name, triple and prefix layout pinned as string assertions in - `PythonHomeStagingTest` — a wrong triple for Windows is a 404 at a consumer's first build, and - asserting the string is the only way to catch it here. -- **Windows is the least covered of the four.** Its prefix has a different shape (`Lib/` and - `DLLs/`, not `lib/python3.14/`), which is handled in `stdlibMarkerRelativePath` and matches the - second layout `PythonHomeCheck` already probes — but no Windows host ran it. -- **`linux-aarch64` is staged but has never been run.** `desktopJar` carries no `libpython` for it; - it can only work because `manager.loadFromSidecar` falls back to loading the interpreter out of - `PYTHONHOME` when the classpath has none for the running platform. That path is real and - deliberate (it is what the GraalVM native image already relies on), but the combination is - untested. -- **Packaged distribution is still open.** This puts a prefix on a *developer's* machine and points - `run`/`test` at it. An application shipped to an end user still needs its packaging step - (`jpackage`, Conveyor, an installer) to carry a prefix and set `PYTHONHOME`; the staged directory - is a reasonable thing for such a step to copy, but nothing here does it. - ---- - -## 16. Bindings from resolved artefacts — the second producer, punched through on one path - -`docs/design/ecosystem.md` §5b settles that bindings are produced at build time by **two** producers, split -by what they look at: - - KSP the consumer's own source — declarations it can see being compiled - artefact walker everything the build resolves — third-party jars, AndroidX included - -Only the first existed. This section is the second, built to the smallest shape that reaches all the -way to a Python `import` rather than to the largest shape that could be written. - -### 16a. What runs today - -`python-multiplatform-gradle-plugin` gained `generatePythonArtifactBindings` -(`python.multiplatform.gradle.artifact`), registered when a consumer sets three properties: - - pythonBindings { - artifactConfiguration.set("desktopCompileClasspath") - artifactSourceSet.set("desktopMain") - artifactIncludePackages.set(listOf("junit.runner", "junit.framework")) - } - -It walks the resolved jars with ASM, emits one `FunctionTableFragment` per contributing artefact into -`python.multiplatform.generated.artifacts`, and an `ArtifactTable` listing them. The verified path, -end to end, is `ksp-fixtures/artifact`: - - junit-4.13.2.jar -> ASM -> ArtifactFragment_junit_junit_4_13_2 -> UpcallTable - -> PythonProxySource -> `from junit.runner.Version import id` -> "4.13.2" - -`"4.13.2"` is JUnit's own version string, compiled into `junit/runner/Version.class`; no generator or -fixture can produce it, which is why it is the value the test reads back. - -`artifactIncludePackages` being empty leaves the whole thing unregistered — no task, no configuration -resolution, no generated directory. A build that only wants KSP pays nothing. - -### 16b. Where the walker's output joins KSP's, and why it is a second list - -Same `UpcallTable`, same `FunctionTableFragment`, same Python surface. **Two aggregators.** -`ArtifactTable` is not merged into KSP's `FunctionTable`, for three reasons: - -1. **`FunctionTable` already means something.** `PackageScanFragmentDiscovery` builds it from - `Resolver.getDeclarationsFromPackage(...generated.fragments)` — "every module in this build graph - compiled with the processor". `ksp-fixtures/app`'s `CommonInstallTest` asserts that set exactly. - Merging would silently change a shipped contract and make a consumer's Python namespace grow when - they added an unrelated dependency. -2. **The ordering would have to be built the wrong way round.** For KSP to *see* a walked fragment, - the walker would have to run before `kspKotlin` and land its source where - `getDeclarationsFromPackage` looks. That API's behaviour for declarations present only in the - source being processed is undocumented, and a fragment it silently missed would vanish from the - table with no build failure anywhere. -3. **Which artefacts to bind is a consumer's decision and should be visible at the install site.** - -So the install site is where they meet: - - UpcallTable.install(FunctionTable.fragments + ArtifactTable.fragments) - -`ArtifactTable.registerInto()` is the additive form, for a caller who has already installed. - -### 16c. The filters, judged against `PyREPL`'s - -`PyREPL`'s generator (`app/build.gradle.kts`, 627 lines) is the reference implementation and its -filters were re-derived rather than copied, because it emits `.pyi` **stubs** and this emits **calls**. -A stub never has to compile. - -| PyREPL | here | why | -|---|---|---| -| not `private`/`protected` | **not enough** — must be `public` | PyREPL keeps package-private members. Generated Kotlin lives in another package and cannot call one. `kotlin.text.StringsKt__IndentKt` is exactly that shape | -| drop `` | kept | a constructor needs a `ReflectedClass` and a receiver handle | -| drop `Companion` | subsumed | only statics are bound, and `Companion` is an instance field | -| drop names containing `-` | **kept, and load-bearing for a different reason** | see below | -| (none) | drop non-`static` | an instance method has nowhere to get a receiver yet | -| (none) | drop unbindable types | see 16d | -| (none) | drop ambiguous overloads | see below | - -**The `-` filter turns out to be load-bearing, and PyREPL's instinct was right.** A hyphen is Kotlin's -value-class mangling suffix. It cannot simply be stripped, and — more importantly — it cannot be -replaced by a type check, because *a value class erases to the type it wraps*: `Duration` is a `long`, -so `getInWholeSeconds-impl(J)J` passes any descriptor-level filter while meaning something entirely -different from `long -> long`. The mangled name is the only place the bytecode still admits it. - -**Overloads are dropped entire, not arbitrated.** This is the one place the walker diverges from -`FragmentScanner.distinctByName`, which keeps the first. That rule is right for Kotlin source, where -the collisions are duplicate *views* of one declaration (an instance and a companion property of the -same name, an `expect`/`actual` pair seen twice). In a jar they are genuinely different functions: -`org.junit.Assert.assertEquals` has eight the walker could bind, and picking one — deterministically or -not — means `assertEquals(3, 3)` from Python calls whichever the sort order happened to put first, -which for that method is the deprecated `(double, double)` that always fails. Ambiguity is counted over -what *would be bound*, so an overload the type filter already declined does not make its sibling -ambiguous (`BaseTestRunner.getFilteredTrace` has two overloads, one taking `Throwable`, and is kept). - -The whole of `junit-4.13.2.jar` — 380-odd classes — yields **seven** bindings under these rules. That -smallness is the design, not a bug, and `ArtifactScannerTest` pins the exact list. - -### 16d. The wall: ASM is not enough for a *Kotlin* jar - -The walker binds only declarations callable from Kotlin **by their JVM shape** — a Java static, or a -Kotlin `@JvmStatic`. A Kotlin top-level function is not one, and this is not a policy choice: - -- `kotlin.text.trimIndent` compiles to a public static on `kotlin/text/StringsKt__IndentKt`, which is - **package-private**. Generated Kotlin in another package cannot call it. -- The public name is the facade `kotlin/text/StringsKt`, which declares no methods of its own (it - inherits them) and which **Kotlin cannot name at all** — there is no `StringsKt` in the Kotlin - namespace, only `kotlin.text.trimIndent`. -- Even given the name, the bytecode does not say that `trimIndent`'s first parameter is an *extension - receiver* rather than an ordinary argument. - -The `@Metadata` *kind* is readable with ASM alone (`kotlin.Metadata` is `RUNTIME`-retained), and -`ArtifactScannerTest` pins it against the real `kotlin-stdlib`: `StringsKt` is `k=4` (multi-file -facade), `StringsKt__IndentKt` is `k=5` (multi-file part), `Regex` is `k=1`, `junit.runner.Version` has -none. The *payload* — names, receivers, property/function, value classes, nullability — is in `d1`/`d2` -and needs **`kotlin-metadata-jvm`**. - -**`androidx.compose.material3` is on the far side of that line.** §5b is right that the artefact walker -is the mechanism that reaches it; what this pass establishes is that reaching it needs a metadata -reader, not more descriptor cases. - -**A `kotlin-metadata-jvm` reader has since been written** (`python-multiplatform-gradle-plugin/.../ -artifact/KotlinMetadata.kt`) and `pythonx.compose.material3` is a real, walked module today — -`ksp-fixtures/compose`'s `M3ProofRenderTest.kt` imports from it and drives components through it, not -through a hand-written wrapper. §16 above is otherwise silent on this, so it is recorded here rather -than left implied: reaching material3 is done, generating **calls** for it (not just names) needed the -reader this section says it needed, and it now has one. - -**Whether those bindings actually draw anything was audited component by component -(`8813d8f1`, `1363b445`, `c6f73da2`, 2026-08-18), against 27 components downstream that had been -zero-byte stubs.** Verified against `M3ProofRenderTest.kt` and `docs/design/pythonx-adapter-design.md` §10 -(both updated by the same round) rather than the commit messages alone: 24 of the 27 render -correctly, in three batches — ten (`horizontalDividerDrawsALine...` through -`tabRowComposesItsTabsAnd...`), ten more (`checkboxRendersItsChecked...` through -`searchBarComposesItsContent`), and four more found on re-examination of the seven originally called -unjudgeable — `DatePicker`, `TimePicker` and `SwipeToDismissBox` have positive assertions in -`stateObjectsArePassedToComposables` (each driven through the `remember_*_state` factory already -present in the walked table under its snake-cased name); the fourth, `BottomSheet`, is not a distinct -assertion in that test file but is named explicitly in §10 item 2 of the design doc as bound the same -way, which is where the "four" the commit message counts is actually enumerated. Three remain -genuinely unreachable, each pinned by its own test rather than inferred: `DropdownMenu` composes into -a Popup layer `ImageComposeScene` never captures (`popupLayersAreNotCapturedByImageComposeScene` — 0 -pixels, observed); the dynamic `ColorScheme` factory is Android-only and not in the desktop jar at -all, and the class name itself resolves to a proxy type whose constructor wants a handle rather than -colour arguments, which is the error actually raised -(`colorSchemeResolvesToItsProxyTypeRatherThanAnythingCallable`); `Typography` and the shape scheme are -not declined by any filter — they are simply absent from the walked table, because `ArtifactScanner` -does not collect class constructors, which is a scanner gap rather than a property of either type. - -**Of the three Compose modifiers declined for an unspellable generic type parameter -(`docs/design/pythonx-adapter-design.md` §9.4: `swipeable`, `modifierLocalProvider`, `anchoredDraggable` — -`SwipeableState`, `ProvidableModifierLocal`, `AnchoredDraggableState`), two are now bound.** -Verified by reading the wrappers, not just the design doc: `fixture.compose -.pythonAnchoredDraggableString` (`ksp-fixtures/compose/.../PythonAnchoredDraggable.kt`, `8d4fd6a4`) and -`fixture.compose.pythonSwipeableString` (`.../PythonSwipeable.kt`, `f3b680fa`) each fix the generic -parameter to a concrete `String`, exactly the move the identifier wrapper made for `layoutId`'s -`kotlin.Any` — one hand-written wrapper per concrete type, not a fix to the underlying generic -surface, which both files' own KDoc says outright. `modifierLocalProvider` is the one of the three -still declined; §9.5 of the design doc says so and nothing in this codebase contradicts it. - -Also unbound today, and for the same "no Kotlin type name to cast to" reason (`boundaryTypeOf` returns -`null` rather than falling back to `TypeTag.OBJECT`): every parameter or return that is not a -primitive, `String`, `ByteArray` or `void`. `Ljava/util/List;` has no Kotlin spelling, a -`Ljava/lang/Object;` cast checks nothing, and a value-class descriptor lies. Instance methods, -constructors and fields are unbound too — those need a `ReflectedClass` and a receiver handle, which is -`PythonProxySource`'s existing class-rendering path and a separate step. - -### 16e. klib — implemented and wired, and this heading was stale for longer than the work took - -§5b left this open: "On iOS, androidNative and wasm the artefacts are klibs, and whether the same walk -is possible there — and what a Kotlin declaration from a klib can be bound to at runtime with no JVM -underneath — is the open question." - -Both halves were investigated against this repository's own klibs. Observed, not inferred: - -**Reading is solved, by an API that already ships with the Kotlin this build uses.** A `.klib` is a zip -whose `default/linkdata/package_/*.knm` entries are protobuf-serialised Kotlin metadata — the -package inventory is readable from the directory names alone, and the declarations need a decoder. -`org.jetbrains.kotlin.library.abi.LibraryAbiReader` in `kotlin-compiler-embeddable` (the same -`2.4.20-Beta2` this build pins; the API carries `@ExperimentalLibraryAbiReader`) is that decoder. Run -against `python-multiplatform-iosX64Main-3.14.7-alpha01.klib` it returned the manifest -(`platform=NATIVE, platformTargets=[Native(name=ios_x64)]`) and **441 top-level declarations** with -proper Kotlin qualified names — `python.multiplatform.ffi/withGIL`, `python.multiplatform.ffi/PyObject` -— already sorted into `AbiClass` and `AbiFunction`. - -**And it answers exactly what ASM could not.** `AbiFunction` exposes `isSuspend`, -`hasExtensionReceiverParameter`, `isInline`, `isConstructor`, `valueParameters: List` -and `returnType: AbiType` — Kotlin types, not erased JVM descriptors. There is no facade problem -because there is no facade: a klib records the Kotlin declaration. - -**The call side is easier too.** A generated fragment is Kotlin source compiled into the consumer's own -binary, and the klib is on its compile classpath, so calling a klib declaration is an ordinary Kotlin -call — no reflection, nothing that a closed world or a missing JVM would break. The JVM path is the -awkward one, not the Native path. - -**Both of the reasons this section gave for not doing it were solved by the way it was eventually -done, and the section was never updated.** Checked against the code rather than against this -document: - -- `KlibScanner` (`python-multiplatform-gradle-plugin/.../artifact/KlibScanner.kt`) does the walk, - `@OptIn(ExperimentalLibraryAbiReader::class)` and all. -- It is **not** on the plugin's own classpath at run time. `PythonArtifactBindingsTask` and - `PythonStubsTask` both reach it through `WorkerExecutor.scanKlibIsolated`, which runs - `KlibScanWorkAction` in an isolated worker classloader whose classpath (`klibReaderClasspath`) - is supplied per build. `PythonArtifactBindingsTask`'s KDoc records why: calling `KlibScanner` - in the task's own classloader throws `NoSuchMethodError` in a real build. That worker boundary - is the interface seam this section asked for, and it is stronger than one — an interface would - still have put the 60 MB artefact on one classpath, and this does not. -- `KlibScannerTest` covers it against two real klibs: `kotlin-stdlib`'s, out of the pinned - Kotlin/Native distribution, and `kotlinx-coroutines-core-androidnativearm64:1.8.1`, a real - third-party download. - -What is genuinely still open is narrower than what was written here, and worth stating exactly: - -- **Nothing distinguishes "read the klib and found nothing bindable" from "failed to read it".** - This is the sharp form of the item below, and it was found while trying to close it. - `:ksp-fixtures:klib-artifact` *is* the product-path exercise — it walks - `androidNativeArm64CompileKlibraries` through `PythonBindingsPlugin`, therefore through - `WorkerExecutor.scanKlibIsolated` and its per-build reader classpath, against klibs this build - produced with Kotlin `2.4.20-Beta2`. `:generatePythonArtifactBindings` and `:generatePythonStubs` - both succeed on it (`EXIT=0`, 2026-08-23). - - But the output is an **empty** `ArtifactTable.fragments`, and empty is the *correct* answer for - the namespace it walks: `WalkedKlibArtifactTableTest` pins zero deliberately, because - `kotlinx.coroutines`' public surface is `CoroutineScope`/`Job`/`Flow`-typed extensions plus - `@PublishedApi internal` declarations, none of them bindable. The stub task emits nothing either. - **A successful run and a silently failed read therefore produce identical artefacts**, and the - green build is not the assertion it looks like. - - **(a) blocking it:** nothing. **(b) next step, and it is cheap:** give that fixture one namespace - that yields at least one binding, or assert that the scan's *decline* list is non-empty. Either - separates the two cases, and neither needs the TestKit machinery the next item describes. - -- **The in-process test path is pinned to one Kotlin version.** `KlibScannerTest` reads klibs from - `kotlin-native-prebuilt-macos-aarch64-2.0.20`, and calls `KlibScanner` directly rather than - through a worker, because a test JVM has no worker to hand a classpath to. Reading a klib - produced by a *newer* Kotlin than the one the plugin was compiled against fails there with - `org.jetbrains.kotlin.protobuf.InvalidProtocolBufferException` — observed while attempting this - section, against this repository's own `2.4.20-Beta2` klibs. **This is a property of the test - path, not of the product path**, which is exactly what `klibReaderClasspath` exists to avoid; - but nothing currently asserts that the product path survives a version skew the test path does - not. - - Driving the worker path from JUnit is not the route to that assertion: `ProjectBuilder`'s - `WorkerExecutor` is a stub whose `submit()` throws `UnsupportedOperationException`, so it would - take `gradleTestKit()`, a temporary project directory and a `GradleRunner` run — infrastructure - this plugin has none of, for one assertion. The fixture route above exercises the same code for - a fraction of that. -- A Native consumer needs the per-platform `_pm_resolve`/`_pm_invoke` bootstrap — **closed**, see - 16f item 2 and `UpcallBootstrap`. - -### 16f. What is verified, and what is not - -Verified: one target (`desktop`/JVM), one configuration (`desktopCompileClasspath`), one source set -(`desktopMain`), one artefact (`junit:junit:4.13.2`), on this host. - -Not verified, and each is a real next step rather than a caveat: - -1. ~~**More than one target.** `artifactConfiguration`/`artifactSourceSet` are single-valued and named by - hand.~~ **The wiring half is done; what is left is not the wiring.** `artifactTargets` - (`MapProperty`, configuration name → source set name) makes the answer plural. - The singular pair still works and, when it is the only pair, still gets the unsuffixed - `generatePythonArtifactBindings`/`generatePythonStubs` names an existing build already invokes; - every additional pair gets a task named after its source set, which is what already distinguishes - the output directories. Setting both folds the singular pair into the map rather than registering - it twice. `ArtifactTargetsWiringTest` pins all three properties; neutering the map read fails - exactly the two multi-pair cases and leaves the back-compat one green. - - Deriving the map is still refused, for the reason this entry gave and one more measured since: - this plugin carries no KGP types (the constraint that made `TEST_WORD` a name matcher), and - Android's variant-aware configuration names do not follow the `CompileClasspath` → - `Main` convention a derivation would have to assume. The caller writes the map. - - **(a) what is still blocking a second target end-to-end:** not this wiring. `ksp-fixtures/artifact` - has one target on purpose — "the walker reads jars, and jars are what a JVM target resolves" - (that module's `build.gradle.kts`). A second *JVM* target would exercise the plural wiring and - prove nothing about the walk; a second *real* target is a Kotlin/Native one, and walking its - `.klib` is 16e -- which turned out to be implemented and wired through an isolated worker - already, as that section now records. **(b) next step:** a fixture module with a Kotlin/Native - target and a klib on its compile classpath, not more scanner work. -2. ~~**`PythonProxySource` needs `_pm_resolve`/`_pm_invoke` bound**, which is per-platform and, in - this repository, exists only in `python-multiplatform`'s own **test** source - (`UpcallEntryBridge.desktop.kt`). `ksp-fixtures/artifact` rebuilds the two `ctypes.CFUNCTYPE`s by - hand from the public `UpcallStub` addresses. A consumer has no supported route to this today; that - is the gap, not the fixture's workaround.~~ **Closed for desktop, and the same route now compiles - for every other in-repo target.** `python.multiplatform.ffi.upcall.UpcallBootstrap` (`commonMain`, - `expect object { fun publishToGlobals(): Boolean }`) is the consumer-facing entry point that did - not exist before. `WalkedArtifactPythonImportTest` calls it and the hand-rolled - `ctypes.CFUNCTYPE`/`UpcallStub` workaround is gone from that file — `git log -p` on this commit is - the proof, and `UpcallBootstrapTest` (`commonTest`) pins the contract (`_pm_resolve`, `_pm_invoke`, - `_pm_release`, `_pm_cancel` all present after one call; a second call rebinds cleanly rather than - refusing or failing). - - What each `actual` does was mostly already sitting in production code, unused by anything a - consumer could reach: - - | target | `actual` | what it calls | - |---|---|---| - | desktop | new (`UpcallBootstrap.desktop.kt`) | `ctypes.CFUNCTYPE` over `UpcallStub`'s Panama stubs — the same four addresses the removed workaround used, just no longer hand-assembled per call site | - | androidNative, iOS (`nativeMain`) | new, one line | `python.native.ffi.UpcallEntry.publish` — a complete `PyMethodDef` bootstrap that has existed in `nativeMain` production code all along; nothing outside this module's own tests ever called it | - | Android/ART (`androidMain`) | new, one line | the same `UpcallEntry.publish`, backed by the JNI shim already built into `jni_onload.def` — ships inside the library's own `.so`, no extra consumer wiring | - | wasmJs | new, one line | `python.native.ffi.UpcallEntry.publish` (`wasmJsMain`) over the one `@WasmExport`ed dispatcher | - - **wasmJs has one real remaining requirement, and it is the consumer's, not the library's.** - `@WasmExport` is honoured only in the compilation that produces the final `.wasm`, so the library - cannot declare `pmp_invoke` on a consumer's behalf — every wasmJs consumer must still write the - three-line delegating export `wasmJsTest/UpcallExports.kt` shows, or `UpcallEntry.publish` throws - `IllegalStateException` at the `pmpRegisterUpcall` call inside `publishToGlobals()`. This is - structural (`docs/design/upcall-async-design.md` §12.4's finding still holds for the export itself, even - though the *bootstrap* it blocked is no longer the gap) and not something a future revision of - this entry can close without changing what `@WasmExport` means. - - No entry point published by any target other than desktop's is a stub: every `actual` above - delegates to code this repository already built, tested from its own suite, and shipped — only the - one-line public wrapper and (for wasmJs) the `commonMain` `expect`/desktop `actual` pair were - missing. `androidMain`'s and `nativeMain`'s `UpcallBootstrap.*.kt` files are ~310 bytes because - `UpcallEntry.publish` already does the real work; that is the same shape `UpcallBootstrap.kt` - itself is, not a sign either is unfinished. -3. **The generated source directory is wired reflectively.** `kotlin.sourceSets.getByName(name).kotlin - .srcDir(task)` goes through `Class.getMethod`, for the same reason `setKspArg` does. Both ends of - the chain are Gradle types (`NamedDomainObjectContainer`, `SourceDirectorySet`); only `getKotlin()` - is reflected. A KGP change there fails loudly at configuration time. -4. ~~**No `aar`, no klib, no project classes directory.** Anything that is not a `.jar` is skipped - silently.~~ **The klib half closed by 16e.** `PythonArtifactBindingsTask` now routes `.klib` - files through `KlibScanner` via the isolated worker (`PythonArtifactBindingsTask.kt:118-131`, - `filter { it.name.endsWith(".jar") || it.name.endsWith(".klib") }`); its own KDoc states the - remaining scope precisely: "A resolved artefact that is not a `.jar` or `.klib` — an `aar`, a - project's classes directory — is skipped silently" (`PythonArtifactBindingsTask.kt:28-29`). - `aar` and project classes directories remain genuinely unhandled. +Yes, by Maven coordinates, on desktop and Android: all three components publish with one task +(`publishAllToMavenLocal`, root `build.gradle.kts`), the KSP processor publishes, the plugin's default +processor coordinates are correct, the Android AAR publishes (23.42 MB, test suite and headers +stripped), the root metadata jar is 382 KB, `PythonBootstrap` stages the Android stdlib +(`PM/androidInstrumentedTest/.../env/PythonBootstrapTest.kt`), and `stagePythonHome` stages the +desktop stdlib (`GP/test/kotlin/python/multiplatform/gradle/PythonHomeStagingTest.kt`, SPEC L-4). +Old §15a, §15b, §15d, §15f, §15g, §15h are closed; §15e item 4 is closed by §15h. + +- **15.1 Nothing is published to a remote repository — open.** No publishing repository or signing is + configured (`signing { }` is commented out in `python-multiplatform/build.gradle.kts`), and + `repo1.maven.org/maven2/io/github/thisisthepy/python-multiplatform/` returns 404 (2026-10-03). + Every consumer verification so far used `mavenLocal()`. 10.1 depends on this or on a release asset. +- **15.2 The KSP processor and plugin versions are hand-copied literals — open.** Root + `build.gradle.kts` `allprojects { version = "3.13.0" }` (inherited by `python-multiplatform-ksp`) and + `python-multiplatform-gradle-plugin/build.gradle.kts` `version = "3.13.0"` agree only by hand, while + the library publishes `3.14.7-alpha01` from `gradle.properties`. The plugin bakes + `DEFAULT_PROCESSOR_COORDINATES` from its own version, so the two literals drifting would reproduce + the "processor coordinate that exists nowhere" defect with no configuration-time error. A check, or + one source for both, is what is left. +- **15.3 `stagePythonHome` has run only on `macos-aarch64` — partial.** Linux-x86_64, macOS-x86_64 and + Windows-x86_64 are pinned as string assertions in `PythonHomeStagingTest`; Windows' `Lib/`+`DLLs/` + layout and `linux-aarch64` (no `libpython` in `desktopJar`, relies on `manager.loadFromSidecar`) have + never executed. +- **15.4 A packaged desktop application carries no CPython prefix — open.** `stagePythonHome` serves + `run`/`test` on a developer machine; nothing makes `jpackage`, Conveyor or an installer ship a + prefix and set `PYTHONHOME` for an end user. +- **15.5 The Android AAR duplicates the pure-Python stdlib per ABI — open (measured option, not + taken).** 790 files / 11.91 MB are byte-identical between `arm64-v8a` and `x86_64`; de-duplicating + saves 3.16 MB compressed but changes the published asset layout `/lib/python` that + `PythonBootstrap` and consumers read. + +## 16. Bindings from resolved artefacts (the artefact walker) + +The walker binds jar declarations — statics, top-level functions, extensions, constructors, value +classes, defaults, callbacks, `@Composable`s — by generating Kotlin source (SPEC B-1, B-3, B-4, B-6; +`GP/test/.../artifact/ArtifactScannerTest.kt`, `ksp-fixtures/artifact/`, `ksp-fixtures/compose/`), +reads Kotlin metadata (`GP/main/.../artifact/KotlinMetadata.kt`), walks klibs through an isolated +worker (`KlibScanner.kt`, `KlibScanWorkAction.kt`), and takes several targets (`artifactTargets`, +`GP/test/.../ArtifactTargetsWiringTest.kt`). Consumers bootstrap `_pm_resolve`/`_pm_invoke` through +`UpcallBootstrap` (`PM/commonTest/.../ffi/upcall/UpcallBootstrapTest.kt`). Old §16a–§16d are +description, §16e/§16f's open remainder is below. + +- **16.1 klib declarations bound at run time — partial.** SPEC B-2/N-3. `KlibScannerTest` reads real + klibs and declines with reasons; `ksp-fixtures/klib-artifact` walks + `androidNativeArm64CompileKlibraries` through the plugin, but the namespace it walks + (`kotlinx.coroutines`) yields **zero** bindings by design, so a successful read and a silently + failed one still produce the same empty `ArtifactTable` + (`ksp-fixtures/klib-artifact/src/androidNativeArm64Test/.../WalkedKlibArtifactTableTest.kt`). And no + Gradle task runs that fixture's tests — they are compiled and linked, and run only by pushing + `test.kexe` by hand. Next: a namespace that yields at least one binding, and a run task. +- **16.2 The product klib path under a Kotlin version skew — open.** `KlibScannerTest` reads klibs + from Kotlin/Native 2.0.20 in-process; reading this build's 2.4.20-Beta2 klibs in-process fails with + `InvalidProtocolBufferException`. The worker path with a per-build reader classpath exists to avoid + that, and nothing asserts it does. +- **16.3 `aar` files and project class directories are skipped silently — open.** Only `.jar` and + `.klib` are walked (`GP/main/.../artifact/PythonArtifactBindingsTask.kt` KDoc and filter). +- **16.4 Member (instance) functions of jar classes are not bound — open.** The scanner drops + non-`static` methods: "an instance method needs a receiver" (`ArtifactScanner.kt` KDoc table). +- **16.5 A value-returning function slot (`() -> Float`) does not accept a Python callable — open.** + SPEC B-5; pinned red-by-design by `M3ProofRenderTest.aValueReturningFunctionSlotDoesNotYetAcceptAPythonCallable`. +- **16.6 `.pyi` stubs for handle-returning functions are not wrapped — open.** SPEC B-7. +- **16.7 Compose through Python on Android, iOS and wasm — open.** SPEC B-6/N-2: the render proofs + (`ksp-fixtures/compose/src/desktopTest/`) are desktop only. +- **16.8 Two material3 components not reachable from Python — open.** The dynamic `ColorScheme` + factory is Android-only and the class name resolves to a proxy whose constructor wants a handle + (`M3ProofRenderTest.colorSchemeResolvesToItsProxyTypeRatherThanAnythingCallable`). `DropdownMenu` + renders into a popup layer `ImageComposeScene` does not capture, so it is unverified rather than + known broken (`M3ProofRenderTest.popupLayersAreNotCapturedByImageComposeScene`). + +## 17. Owed to the ecosystem, and decisions waiting on the maintainer + +From `docs/design/ecosystem.md` §5, "What each repository owes — PythonMultiplatform". Of its gaps, +the `@Composable` callable shape, the `sys.meta_path` finder (`_Finder` in +`PM/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapter.kt`, tested in +`PythonxAdapterTest.kt`), the `Composer` hand-off and Python-callable lifetime across recompositions +(`ksp-fixtures/compose/src/desktopTest/.../ComposableRenderTest.kt`, `RecompositionAccumulationTest.kt`) +are done on desktop; the other platforms are 16.7. The production-bundler question is 10.4. + +- **17.1 Retire `stagePythonHome` in favour of pypackpack's Python distribution — open (cross-repo).** + Two acquisition paths disagree on source and destination (`ecosystem.md` §4 item 4). The work lands + in pypackpack first; this repository then removes `PythonHomeStaging.kt`. Needs the maintainer's go + (AGENTS.md §7). +- **17.2 Hand `stageWasmBrowserRuntime` to toolchain — open (cross-repo).** Same shape as 17.1, for the + wasm runtime staging (`WasmBrowserRuntimeStaging.kt`). +- **17.3 Where the generic `pythonx` adapter machinery belongs — decision.** It lives in + `PM/commonMain/kotlin/python/multiplatform/ffi/pythonx/`, while INTENT §2.3 puts `pythonx` in + pythonx-compose (SPEC "Outside intent" item 4). It no longer synthesises a `pythonx` module + (it installs `python_multiplatform.binding`); whether it stays here as a service is unstated. +- **17.4 The committed CPython 3.13 trees — decision.** `python-multiplatform/src/nativeInterop/cinterop/lib` + (15 555 tracked files, Android/desktop/iOS) and `binary/` (3.13.0 archives) are read only by the + `-PpythonVersion=3.13.0` desktop branch; the Android 3.13 stdlib trees by nothing. INTENT §3 says the + interpreter is not vendored. Removing them is a repository-size decision (`ecosystem.md` §5). diff --git a/ksp-fixtures/app/build.gradle.kts b/ksp-fixtures/app/build.gradle.kts index be403215d..f2503a2fb 100644 --- a/ksp-fixtures/app/build.gradle.kts +++ b/ksp-fixtures/app/build.gradle.kts @@ -18,7 +18,7 @@ kotlin { // (build.gradle.kts) -- this module's test binary needs the same CPython symbols // resolved at final link time, since python-multiplatform's klib only declares them via // cinterop and does not itself embed a static libpython. This is a throwaway measurement - // wiring: only linkDebugTestAndroidNativeArm64 needs it (docs/design/upcall-table-design.md + // wiring: only linkDebugTestAndroidNativeArm64 needs it (docs/design/upcall.md // §11.1's tree-shaking measurement), not the ordinary JVM-only fixture test path. val downloadDir = project(":python-multiplatform").layout.buildDirectory.dir("python-standalone").get().asFile val pyVersion = project.findProperty("pythonVersion")?.toString() ?: project.rootProject.version.toString() diff --git a/ksp-fixtures/app/src/androidNativeArm64Test/kotlin/fixture/app/NativeSmokeTest.kt b/ksp-fixtures/app/src/androidNativeArm64Test/kotlin/fixture/app/NativeSmokeTest.kt index 0ac921da2..a8e8a7e9a 100644 --- a/ksp-fixtures/app/src/androidNativeArm64Test/kotlin/fixture/app/NativeSmokeTest.kt +++ b/ksp-fixtures/app/src/androidNativeArm64Test/kotlin/fixture/app/NativeSmokeTest.kt @@ -6,10 +6,10 @@ import kotlin.test.Test import kotlin.test.assertTrue /** - * Compile+link-only smoke test on a real Kotlin/Native target (docs/design/upcall-table-design.md §8's + * Compile+link-only smoke test on a real Kotlin/Native target (docs/design/upcall.md §2.9's * "Verify .klib discovery" open question). Not run here -- androidNativeArm64 targets a * device/emulator this workspace does not execute tests against -- but linking this binary is - * what `docs/design/upcall-table-design.md` §11.1's tree-shaking measurement diffs the size of. + * what `docs/design/upcall.md` §2.6's tree-shaking measurement diffs the size of. */ class NativeSmokeTest { @Test diff --git a/ksp-fixtures/app/src/commonMain/kotlin/fixture/app/AppExtras.kt b/ksp-fixtures/app/src/commonMain/kotlin/fixture/app/AppExtras.kt index 4c4aefe7f..12792a5b0 100644 --- a/ksp-fixtures/app/src/commonMain/kotlin/fixture/app/AppExtras.kt +++ b/ksp-fixtures/app/src/commonMain/kotlin/fixture/app/AppExtras.kt @@ -5,7 +5,7 @@ package fixture.app * * One module emits one fragment however many files it has, so this proves the scan is per-module * rather than per-file -- and it is what makes the incremental measurement in - * `docs/design/upcall-table-design.md` §11.5 meaningful: with a single-file module every dirty-set + * `docs/design/upcall.md` §2.7 meaningful: with a single-file module every dirty-set * measurement is trivially 100%. */ fun triple(x: Long): Long = x * 3 diff --git a/ksp-fixtures/app/src/desktopTest/kotlin/fixture/app/GeneratedSuspendTest.kt b/ksp-fixtures/app/src/desktopTest/kotlin/fixture/app/GeneratedSuspendTest.kt index 0a2039646..49ba2299f 100644 --- a/ksp-fixtures/app/src/desktopTest/kotlin/fixture/app/GeneratedSuspendTest.kt +++ b/ksp-fixtures/app/src/desktopTest/kotlin/fixture/app/GeneratedSuspendTest.kt @@ -26,7 +26,7 @@ import kotlin.test.assertTrue * The answer used to be **silently skipped** -- `BindingPolicy.isExposedFunctionShape` dropped any * declaration carrying `Modifier.SUSPEND`, so nothing was emitted and nothing was logged, and a * user exposing a `suspend fun` got an `AttributeError` from Python with no indication that the - * function had been seen and rejected. `docs/design/upcall-async-design.md` §6 recorded that as + * function had been seen and rejected. `docs/design/upcall.md` §5.4 recorded that as * deliberate: the silence was held in place so that the day a convention landed, the change would * show up as these tests failing rather than as one silence quietly becoming another. * @@ -179,7 +179,7 @@ class GeneratedSuspendTest { @Test fun theGeneratedBodyStartsTheCoroutineAndParksItsOutcomeInAPendingCall() { // `PendingCall.start { suspendingTopLevel(args[0] as Long) }` is what the generator emits. - // Nothing in that fixture reaches a suspension point, so `docs/design/upcall-async-design.md` §5's + // Nothing in that fixture reaches a suspension point, so `docs/design/upcall.md` §5.2's // fast path applies: the call is already complete before `start` returned, and the boundary // can hand Python a real `int` with no Future and no event loop. val call = startCall("fixture.library.suspendingTopLevel", 21L) @@ -218,7 +218,7 @@ class GeneratedSuspendTest { // (`args[0] as kotlin.coroutines.SuspendFunction1<...>`) compiled and checkcast cleanly // anyway -- a classifier with no source declaration in this compilation and no `invoke` entry // in the table, so what crossed was a handle Python could hold and hand back and nothing - // else. `docs/design/upcall-async-design.md` §2.1 measured that as deliberately left open. + // else. `docs/design/upcall.md` §5.3 measured that as deliberately left open. // // It is closed now: `BindingPolicy.hasExposableTypes` rejects any property, parameter or // return type where `KSType.isSuspendFunctionType` is true (recursing into type arguments and @@ -346,7 +346,7 @@ class GeneratedSuspendTest { fun theRejectedClassifierHasNoInvokeEntryEitherWayConsistentWithTheOldMeasurement() { // Even where a suspending-type handle *did* used to cross (the old `suspendingHandler` // path this file no longer exercises), nothing ever exposed `SuspendFunction1.invoke` -- - // `docs/design/upcall-async-design.md` §2.1's point stands regardless of which side of this fix + // `docs/design/upcall.md` §5.3's point stands regardless of which side of this fix // it is read from. assertFalse(UpcallTable.resolve("kotlin.coroutines.SuspendFunction1.invoke").isValid) } diff --git a/ksp-fixtures/artifact/build.gradle.kts b/ksp-fixtures/artifact/build.gradle.kts index 38803fd02..0b37dedd8 100644 --- a/ksp-fixtures/artifact/build.gradle.kts +++ b/ksp-fixtures/artifact/build.gradle.kts @@ -58,7 +58,7 @@ kotlin { // all: `padding` is an ordinary function returning an ordinary object, so // `WalkedArtifactComposeModifierTest` can call it in a plain JVM test with no // `Composer` anywhere. Composables are a different problem - // (`docs/design/pythonx-adapter-design.md` §5) and are not touched here. + // (`docs/archive/pythonx-adapter-design.md` §5) and are not touched here. // // Version pinned to the catalog's `compose-plugin`, which is what the rest of this // build resolves, so no second Compose version enters the cache. diff --git a/ksp-fixtures/artifact/src/desktopMain/kotlin/fixture/artifact/ComposeSeed.kt b/ksp-fixtures/artifact/src/desktopMain/kotlin/fixture/artifact/ComposeSeed.kt index 2404aa415..ef0651b0c 100644 --- a/ksp-fixtures/artifact/src/desktopMain/kotlin/fixture/artifact/ComposeSeed.kt +++ b/ksp-fixtures/artifact/src/desktopMain/kotlin/fixture/artifact/ComposeSeed.kt @@ -63,7 +63,7 @@ fun equalsPaddingThenSize(modifier: Modifier, pad: Double, size: Double): Boolea * | comparison | the default it reaches | why Python could not have faked it | * |---|---|---| * | [equalsFillMaxWidth] | `fraction = 1f` | a body that passed `0f` into the omitted slot builds a different `FillElement`, and [equalsFillMaxWidthFraction] is the control that says so | - * | [equalsWrapContentSize] | `align = Alignment.Center` | `Alignment` crosses as a `TypeTag.OBJECT` handle and **no bound declaration anywhere produces one**, so if the default were not reached there would be nothing to put in the slot at all. This is `docs/design/pythonx-adapter-design.md` §4.5's argument in a single function | + * | [equalsWrapContentSize] | `align = Alignment.Center` | `Alignment` crosses as a `TypeTag.OBJECT` handle and **no bound declaration anywhere produces one**, so if the default were not reached there would be nothing to put in the slot at all. This is `docs/archive/pythonx-adapter-design.md` §4.5's argument in a single function | * | [equalsVerticalPadding] | `start`, `top` and `end` | the argument that *is* written sits in the middle of the list, which §4.5's arity-prefix candidate cannot express | */ fun equalsFillMaxWidth(modifier: Modifier): Boolean = modifier == Modifier.fillMaxWidth() diff --git a/ksp-fixtures/artifact/src/desktopTest/kotlin/fixture/artifact/WalkedArtifactComposeModifierTest.kt b/ksp-fixtures/artifact/src/desktopTest/kotlin/fixture/artifact/WalkedArtifactComposeModifierTest.kt index 1a4f76eb2..ff927a99f 100644 --- a/ksp-fixtures/artifact/src/desktopTest/kotlin/fixture/artifact/WalkedArtifactComposeModifierTest.kt +++ b/ksp-fixtures/artifact/src/desktopTest/kotlin/fixture/artifact/WalkedArtifactComposeModifierTest.kt @@ -34,7 +34,7 @@ import kotlin.test.assertTrue * A `Modifier` extension is not a `@Composable` -- §2.6 counted exactly one function that is both, * out of 500 -- so `padding` is an ordinary function returning an ordinary object and needs no * composition to run. Composables are a separate problem with a separate blocker - * (`docs/design/pythonx-adapter-design.md` §5.3) and nothing here touches them. This module does not even + * (`docs/archive/pythonx-adapter-design.md` §5.3) and nothing here touches them. This module does not even * apply the Compose compiler plugin. * * ### What Python actually holds @@ -169,7 +169,7 @@ class WalkedArtifactComposeModifierTest { /** * The entry carries what a Python adapter needs to build a keyword-argument surface over it -- - * `docs/design/pythonx-adapter-design.md` §2.4's table, which recorded every one of these as missing. + * `docs/archive/pythonx-adapter-design.md` §2.4's table, which recorded every one of these as missing. * * `padding__Dp` is a good witness for all of them at once: it is an extension (so slot 0 is a * receiver, not a first parameter), its declared parameter type is `Dp` while its `TypeTag` is diff --git a/ksp-fixtures/artifact/src/desktopTest/kotlin/fixture/artifact/WalkedArtifactDefaultOmissionTest.kt b/ksp-fixtures/artifact/src/desktopTest/kotlin/fixture/artifact/WalkedArtifactDefaultOmissionTest.kt index 949060991..25adbc2cd 100644 --- a/ksp-fixtures/artifact/src/desktopTest/kotlin/fixture/artifact/WalkedArtifactDefaultOmissionTest.kt +++ b/ksp-fixtures/artifact/src/desktopTest/kotlin/fixture/artifact/WalkedArtifactDefaultOmissionTest.kt @@ -13,7 +13,7 @@ import kotlin.test.Test import kotlin.test.assertEquals /** - * `docs/design/pythonx-adapter-design.md` §4.5, closed against the real jars: **Compose's own functions, + * `docs/archive/pythonx-adapter-design.md` §4.5, closed against the real jars: **Compose's own functions, * called from Python with their required arguments and nothing else.** * * ### What was wrong diff --git a/ksp-fixtures/compose/build.gradle.kts b/ksp-fixtures/compose/build.gradle.kts index 6e52aeba9..4a8581fc9 100644 --- a/ksp-fixtures/compose/build.gradle.kts +++ b/ksp-fixtures/compose/build.gradle.kts @@ -2,7 +2,7 @@ import org.jetbrains.kotlin.gradle.ExperimentalKotlinGradlePluginApi import org.jetbrains.kotlin.gradle.dsl.JvmTarget /** - * The other end of `docs/design/pythonx-adapter-design.md` §5: a real `@Composable`, called from Python, + * The other end of `docs/archive/pythonx-adapter-design.md` §5: a real `@Composable`, called from Python, * inside a real composition, drawing a real pixel. * * ### Why this is not `:ksp-fixtures:artifact` diff --git a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonAnchoredDraggable.kt b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonAnchoredDraggable.kt index 189ed0b49..61807b766 100644 --- a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonAnchoredDraggable.kt +++ b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonAnchoredDraggable.kt @@ -10,7 +10,7 @@ import python.multiplatform.ffi.types.basic.PyString /** * `Modifier.anchoredDraggable`, declined for having a generic type parameter `T` that Python cannot spell - * (`docs/design/pythonx-adapter-design.md` §9.2). The fix is to provide a wrapper for a concrete type. + * (`docs/archive/pythonx-adapter-design.md` §9.2). The fix is to provide a wrapper for a concrete type. * * We choose `String` as the concrete type for `T` because swipe UI states are typically discrete * identifiers like "start", "end", "settled", which map naturally to strings. diff --git a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonComposed.kt b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonComposed.kt index c836b6ae6..e01ca4587 100644 --- a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonComposed.kt +++ b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonComposed.kt @@ -11,7 +11,7 @@ import python.multiplatform.reflection.HandleTable import python.native.ffi.PyLong_AsLongLong /** - * `Modifier.composed` -- `docs/design/pythonx-adapter-design.md` §9.2/§9.4's open question, now answered: + * `Modifier.composed` -- `docs/archive/pythonx-adapter-design.md` §9.2/§9.4's open question, now answered: * **a composer is admitted.** `factory`'s compiled shape is `Function3` (confirmed by `javap -p` on `androidx.compose.ui.ComposedModifierKt`, matching the * lowered shape `PythonCallables` already crosses for every composable `content=` slot), and unlike diff --git a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonComposition.kt b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonComposition.kt index 94f3023f0..bb6ab978f 100644 --- a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonComposition.kt +++ b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonComposition.kt @@ -17,7 +17,7 @@ import python.multiplatform.reflection.HandleTable * ### Why exactly one, and not one per component * * The 2024 `pythonx-compose` wrote a Kotlin wrapper per widget -- 37 files, 28 of them empty -- - * and `docs/design/pythonx-adapter-design.md` §1 measures what that cost: `padding()` composed nothing and + * and `docs/archive/pythonx-adapter-design.md` §1 measures what that cost: `padding()` composed nothing and * `fillMaxSize()` returned `self`, because a per-declaration wrapper is written once and then * never again. This is O(1) in the number of composables and stays O(1) by construction: it names no * composable, takes no composable-specific parameter, and knows nothing about the declaration Python @@ -43,7 +43,7 @@ import python.multiplatform.reflection.HandleTable * * Recomposition. [source] is `exec`ed on every composition pass, so a Python body that is expensive * pays for it every frame, and a `@Composable` reached this way can never be *skipped* the way one - * with stable parameters is. `docs/design/pythonx-adapter-design.md` §5.4 item 4 names this as a property to + * with stable parameters is. `docs/archive/pythonx-adapter-design.md` §5.4 item 4 names this as a property to * measure before the shape is adopted for anything but a proof, and nothing here has measured it. */ @Composable @@ -72,7 +72,7 @@ fun PythonComposition(source: String) { /** * Who holds a Python callable, and the one hook that says when to let go. * - * `docs/design/pythonx-adapter-design.md` §6 item 1: *"When Compose drops the slot, does anything tell the + * `docs/archive/pythonx-adapter-design.md` §6 item 1: *"When Compose drops the slot, does anything tell the * Kotlin holder? `RememberObserver.onForgotten` is the only hook that reports it… Item 1 is the one * that fails silently and should be tested first."* This is that hook, wired to the one thing that * can act on it. diff --git a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonContextMenuOpenDetector.kt b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonContextMenuOpenDetector.kt index e69e3a6ed..ce8cb4072 100644 --- a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonContextMenuOpenDetector.kt +++ b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonContextMenuOpenDetector.kt @@ -6,7 +6,7 @@ import python.multiplatform.ffi.PyObject import python.multiplatform.ffi.types.basic.PyFloat /** - * `Modifier.contextMenuOpenDetector` -- `docs/design/pythonx-adapter-design.md` §9's final declined declaration. + * `Modifier.contextMenuOpenDetector` -- `docs/archive/pythonx-adapter-design.md` §9's final declined declaration. * * It declined because its `key` parameter is typed `Any?`, and `resolveKotlinType` has no boundary type * for a bare `kotlin.Any` parameter (no hint of which `TypeTag` an arbitrary Python value crossing there diff --git a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonDraggable.kt b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonDraggable.kt index 9f4564b74..b07b50ebc 100644 --- a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonDraggable.kt +++ b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonDraggable.kt @@ -9,7 +9,7 @@ import python.multiplatform.ffi.types.basic.PyFloat /** * `Modifier.draggable`, reached the same way `pythonPointerInput` reaches `Modifier.pointerInput` - * (`docs/design/pythonx-adapter-design.md` §9.1) -- `draggable`'s three callback slots decline for the same + * (`docs/archive/pythonx-adapter-design.md` §9.1) -- `draggable`'s three callback slots decline for the same * reason `pointerInput`'s does (`ArtifactScanner`: a suspend function-typed parameter), and the same * fix applies: nothing about Python has to suspend, because the suspension is Kotlin's, hand-written, * ordinarily compiled, and Python is only ever asked a synchronous question. @@ -37,7 +37,7 @@ import python.multiplatform.ffi.types.basic.PyFloat * whole lifetime: `onDelta` is held by the `DraggableState` this function builds and is called by * Compose's internal drag node for as long as that node stays attached, which this function has no * hook into (no `RememberObserver` route without wrapping this in a `@Composable`, which would be a - * different fix -- see `docs/design/pythonx-adapter-design.md` §9.2's note on `composed`). [onDelta], + * different fix -- see `docs/archive/pythonx-adapter-design.md` §9.2's note on `composed`). [onDelta], * [onDragStarted] and [onDragStopped] are therefore **never closed here**: each call to this function * takes a fresh reference to all three and leaks it. Measured consequence: `DraggableRenderTest` * composes this exactly once per scene, so the leak is bounded (three references) for that test and diff --git a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonLayoutId.kt b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonLayoutId.kt index 48a80408e..55a7446f0 100644 --- a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonLayoutId.kt +++ b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonLayoutId.kt @@ -5,7 +5,7 @@ import androidx.compose.ui.layout.layoutId /** * `Modifier.layoutId`, declined for a reason unrelated to suspend - * (`docs/design/pythonx-adapter-design.md` §9's table): its one non-`Modifier` parameter is `kotlin.Any`, and + * (`docs/archive/pythonx-adapter-design.md` §9's table): its one non-`Modifier` parameter is `kotlin.Any`, and * `resolveKotlinType` has no [python.multiplatform.reflection.TypeTag] for "whatever tag the caller * meant" -- `layoutId` is real-world called with a bare `String` or `Int`, not a Kotlin object handle. * One wrapper per concrete tag fixes it, the same way every other concretely-typed hand-written diff --git a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonModifierLocal.kt b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonModifierLocal.kt index 58b9abc77..e6f0524e2 100644 --- a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonModifierLocal.kt +++ b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonModifierLocal.kt @@ -10,7 +10,7 @@ import python.multiplatform.ffi.types.basic.PyString /** * `Modifier.modifierLocalProvider`, declined for having a generic type parameter `T` that Python cannot spell - * (`docs/design/pythonx-adapter-design.md` §9.2). The fix is to provide a wrapper for a concrete type. + * (`docs/archive/pythonx-adapter-design.md` §9.2). The fix is to provide a wrapper for a concrete type. * * We choose `String` as the concrete type for `T`. * diff --git a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonPullRefresh.kt b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonPullRefresh.kt index 16db3255b..d7f042280 100644 --- a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonPullRefresh.kt +++ b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonPullRefresh.kt @@ -20,7 +20,7 @@ import python.multiplatform.ffi.types.basic.PyFloat * `androidx.compose.material.pullrefresh.PullRefreshKt.pullRefresh(Modifier, * onPull: (Float) -> Float, onRelease: suspend (Float) -> Float, enabled: Boolean)`. * - * `docs/design/pythonx-adapter-design.md` §9.4 identifies this overload as requiring a **real nested-scroll + * `docs/archive/pythonx-adapter-design.md` §9.4 identifies this overload as requiring a **real nested-scroll * parent/child tree**: `onPull`/`onRelease` are fired by `NestedScrollConnection` callbacks from a * scrollable descendant's pre-scroll/pre-fling reports, not by pointer events the modifier's own * body awaits directly. This function is that harness. diff --git a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonSwipeable.kt b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonSwipeable.kt index 7a4ba68e1..6656cd981 100644 --- a/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonSwipeable.kt +++ b/ksp-fixtures/compose/src/desktopMain/kotlin/fixture/compose/PythonSwipeable.kt @@ -12,7 +12,7 @@ import python.multiplatform.ffi.types.basic.PyString /** * `Modifier.swipeable` (`androidx.compose.material.SwipeableKt.swipeable-pPrIpRY`), declined for the same - * reason `anchoredDraggable` is (`docs/design/pythonx-adapter-design.md` §9.2, §9.5): its type parameter `T` -- + * reason `anchoredDraggable` is (`docs/archive/pythonx-adapter-design.md` §9.2, §9.5): its type parameter `T` -- * here on both `SwipeableState` and the `anchors: Map` parameter -- has no name a cast could * spell. Fixed the same way: one wrapper per concrete `T`. `String` was chosen for the same reason * `anchoredDraggable`'s wrapper chose it -- the states a swipe moves between are discrete identifiers a diff --git a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/ComposableRenderTest.kt b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/ComposableRenderTest.kt index 298768ecb..9266fb005 100644 --- a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/ComposableRenderTest.kt +++ b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/ComposableRenderTest.kt @@ -343,7 +343,7 @@ class ComposableRenderTest { } /** - * **`docs/design/pythonx-adapter-design.md` §6 item 1, executed:** when Compose drops the slot, the + * **`docs/archive/pythonx-adapter-design.md` §6 item 1, executed:** when Compose drops the slot, the * Python callable comes back. * * `sys.getrefcount` is the measurement and the callable is held in a Python global, so the only diff --git a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/ComposedRenderTest.kt b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/ComposedRenderTest.kt index 6ced81ad3..66a925678 100644 --- a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/ComposedRenderTest.kt +++ b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/ComposedRenderTest.kt @@ -19,7 +19,7 @@ import kotlin.test.assertEquals import kotlin.test.assertTrue /** - * **`Modifier.composed` admits a real composer**, settling `docs/design/pythonx-adapter-design.md` §9.2/§9.4's + * **`Modifier.composed` admits a real composer**, settling `docs/archive/pythonx-adapter-design.md` §9.2/§9.4's * open question: "unverified whether `composed`'s own contract... admits a composer at all." * `fixture.compose.pythonComposed` (`PythonComposed.kt`) is the hand-written wrapper; this proves two * independent things about it, the way every other §9 render test in this module proves both a diff --git a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/DraggableRenderTest.kt b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/DraggableRenderTest.kt index ee9ae3ec9..8a75c834f 100644 --- a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/DraggableRenderTest.kt +++ b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/DraggableRenderTest.kt @@ -20,7 +20,7 @@ import kotlin.test.assertTrue /** * **A real drag gesture reaches Python through `Modifier.draggable`'s suspend slots** -- one of the - * three `docs/design/pythonx-adapter-design.md` §9.2 judges "the same suspend-lambda shape as `pointerInput`" + * three `docs/archive/pythonx-adapter-design.md` §9.2 judges "the same suspend-lambda shape as `pointerInput`" * without attempting. `fixture.compose.pythonDraggable` (`PythonDraggable.kt`) is the same technique * §9.1 already proved for `pointerInput`, applied to `draggable`'s three callback slots: a real * `DraggableState` and two real `suspend CoroutineScope.(...) -> Unit` lambdas, hand-written and diff --git a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/LayoutIdRenderTest.kt b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/LayoutIdRenderTest.kt index ff20706a0..246260421 100644 --- a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/LayoutIdRenderTest.kt +++ b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/LayoutIdRenderTest.kt @@ -19,7 +19,7 @@ import kotlin.test.assertTrue /** * **A real string value, tagged from Python, reaches `Measurable.layoutId`** -- `layoutId` declines - * for a reason unrelated to suspend (`docs/design/pythonx-adapter-design.md` §9's table: its one non-`Modifier` + * for a reason unrelated to suspend (`docs/archive/pythonx-adapter-design.md` §9's table: its one non-`Modifier` * parameter is `kotlin.Any`, which `resolveKotlinType` has no `TypeTag` for). `fixture.compose * .pythonLayoutIdString` (`PythonLayoutId.kt`) fixes the parameter's tag the same way every other * concretely-typed hand-written wrapper in this module does. diff --git a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/ModifierSeededRenderTest.kt b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/ModifierSeededRenderTest.kt index 21c858dce..263996e41 100644 --- a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/ModifierSeededRenderTest.kt +++ b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/ModifierSeededRenderTest.kt @@ -37,7 +37,7 @@ import kotlin.test.assertTrue * ### Why this needs its own `@BeforeTest` rather than `ComposableRenderTest`'s * * `emptyModifier` and `androidx.compose.foundation.layout.size__Dp` are not composables and are not - * reached through `pythonx` (`docs/design/pythonx-adapter-design.md`'s dynamic loader is for the + * reached through `pythonx` (`docs/archive/pythonx-adapter-design.md`'s dynamic loader is for the * composable half of the surface). They come from the *other* producer, * `PythonProxySource` + `FunctionTable`, the route `WalkedArtifactComposeModifierTest` uses for the * identical seed in `ksp-fixtures/artifact`. Both producers run in the same test here -- diff --git a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/PullRefreshRenderTest.kt b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/PullRefreshRenderTest.kt index ba0a9faa6..8f77187aa 100644 --- a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/PullRefreshRenderTest.kt +++ b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/PullRefreshRenderTest.kt @@ -25,7 +25,7 @@ import kotlin.test.assertTrue * `pullRefresh`'s callback overload (`androidx.compose.material.pullrefresh.PullRefreshKt * .pullRefresh(Modifier, onPull: (Float) -> Float, onRelease: suspend (Float) -> Float, enabled: * Boolean)`) is driven by **nested scroll** from a scrollable descendant, not by pointer events - * the modifier itself awaits. `docs/design/pythonx-adapter-design.md` §9.4 identified this as needing + * the modifier itself awaits. `docs/archive/pythonx-adapter-design.md` §9.4 identified this as needing * a larger harness — a real scrollable descendant composed under the `pullRefresh` parent, with * an actual drag that makes the scrollable child's nested-scroll callbacks flow upward. * diff --git a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/RecompositionAccumulationTest.kt b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/RecompositionAccumulationTest.kt index ac63a8885..ba8535e8a 100644 --- a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/RecompositionAccumulationTest.kt +++ b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/RecompositionAccumulationTest.kt @@ -44,7 +44,7 @@ import kotlin.test.assertTrue * * ### What counts as the same callable * - * Not `is`. A `lambda:` written inline is a **new object on every pass** (`docs/design/pythonx-adapter-design.md` + * Not `is`. A `lambda:` written inline is a **new object on every pass** (`docs/archive/pythonx-adapter-design.md` * records this), so an identity test on the callable interns nothing at all for the spelling that * appears in every example in this module. The key is structural and is described where it is * computed, in `PythonxAdapter`'s `_intern_key`; what this file asserts is the consequence: diff --git a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/SwipeableRenderTest.kt b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/SwipeableRenderTest.kt index b7ec11fcb..4272f043f 100644 --- a/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/SwipeableRenderTest.kt +++ b/ksp-fixtures/compose/src/desktopTest/kotlin/fixture/compose/SwipeableRenderTest.kt @@ -19,7 +19,7 @@ import kotlin.test.assertTrue /** * `Modifier.swipeable` (`androidx.compose.material.SwipeableKt.swipeable-pPrIpRY`), declined for the same - * unspellable-`T` reason as `anchoredDraggable` (`docs/design/pythonx-adapter-design.md` §9.2) -- fixed here to + * unspellable-`T` reason as `anchoredDraggable` (`docs/archive/pythonx-adapter-design.md` §9.2) -- fixed here to * `String` the same way `AnchoredDraggableRenderTest` fixes `anchoredDraggable`. See * `fixture.compose.pythonSwipeableString` for the wrapper under test and, specifically, for why * `thresholds: (T, T) -> ThresholdConfig` is not crossed to Python at all. diff --git a/ksp-fixtures/library/build.gradle.kts b/ksp-fixtures/library/build.gradle.kts index d003223a2..4b52c23bd 100644 --- a/ksp-fixtures/library/build.gradle.kts +++ b/ksp-fixtures/library/build.gradle.kts @@ -17,7 +17,7 @@ kotlin { } } // The Native target that actually verifies `.klib` fragment discovery - // (docs/design/upcall-table-design.md §8's open question). Compile-only here: androidNativeArm64 + // (docs/design/upcall.md §2.9's open question). Compile-only here: androidNativeArm64 // targets a device/emulator this workspace does not run tests against. androidNativeArm64() // `:ksp-fixtures:app` grew a second Native leaf so that it would have an intermediate diff --git a/ksp-fixtures/library/src/commonMain/kotlin/fixture/library/Suspending.kt b/ksp-fixtures/library/src/commonMain/kotlin/fixture/library/Suspending.kt index b02bddd8d..d07cb8b33 100644 --- a/ksp-fixtures/library/src/commonMain/kotlin/fixture/library/Suspending.kt +++ b/ksp-fixtures/library/src/commonMain/kotlin/fixture/library/Suspending.kt @@ -6,7 +6,7 @@ import kotlin.jvm.JvmInline * Every shape a `suspend` function can take in a scanned module, so that what the generator does * with it is a *measured* fact rather than a reading of [python.multiplatform.ksp.BindingPolicy]. * - * `docs/design/upcall-async-design.md` explains why none of these can be in the table as they stand: a + * `docs/design/upcall.md` explains why none of these can be in the table as they stand: a * CPython callback slot is a C frame that must hand back a `PyObject *` before it returns, and a * suspension has nothing to hand back. The fixtures exist so that the day an async convention * lands, the change in behaviour shows up as these tests failing rather than as silence. @@ -83,7 +83,7 @@ object SuspendingRegistry { // `hasRenderableSignature` in `BindingPolicy` as its own predicate // (`BindingPolicy.hasExposableTypes`) rather than folded into the declaration-shape checks. // -// `docs/design/upcall-async-design.md` §2.1 measured the base case: the generated cast +// `docs/design/upcall.md` §5.3 measured the base case: the generated cast // (`args[0] as kotlin.coroutines.SuspendFunction1<...>`) compiles and the runtime checkcast // passes, because that classifier is real to the Kotlin compiler even though nothing in this // compilation ever declares it in source -- there is no `invoke` entry for Python to call through, diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt index 765cde230..bcc2bcad4 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt @@ -75,7 +75,7 @@ internal fun isBindingKspConfiguration(name: String, isMultiplatform: Boolean): /** * The module name the generated fragment object is named after. * - * Every fragment in every artifact lands in one package (`docs/design/upcall-table-design.md` §1), so + * Every fragment in every artifact lands in one package (`docs/design/upcall.md` §2.1), so * the name has to be unique across artifacts, not just within a build -- two independent * libraries both called `:core` would otherwise emit the same object into the same package. The * Maven group is what already carries that uniqueness, so it goes in front. diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactRendering.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactRendering.kt index b31c7683e..e5326a644 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactRendering.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactRendering.kt @@ -44,7 +44,7 @@ internal data class ArtifactCallable( * * Carried rather than recomputed because it is not recoverable from anything else here: the * receiver is [ArtifactCallable.paramTags]'s slot 0, and a tag says how a value is marshalled, - * not what it is. `docs/design/pythonx-adapter-design.md` §2.4 row 3 records this exact loss -- + * not what it is. `docs/archive/pythonx-adapter-design.md` §2.4 row 3 records this exact loss -- * "`ArtifactScanner` has both at the moment it constructs the call expression and discards them * one line later" -- and §4.2 records that `docs/design/kotlin-extensions-in-python.md` §4.1 * (an extension becomes a method on its receiver's proxy) has no input without it. @@ -141,7 +141,7 @@ internal fun artifactFragmentObjectName(coordinates: String): String = * * **A `@Composable` omits by a second mechanism and so has a second piece of evidence.** Its body has * no sentinel test and never will: the `$default` bitmask is a *declared trailing parameter* of the - * JVM method (`docs/design/pythonx-adapter-design.md` §5.2), so `pythonx` computes an integer instead of + * JVM method (`docs/archive/pythonx-adapter-design.md` §5.2), so `pythonx` computes an integer instead of * writing `None` into a slot, and there is no branch for the body to have. Carrying a [ThunkSpec] is * that evidence -- it is set by exactly one path, the one that emits the mask parameter -- so the * rule stays "derived from the body", with two bodies to derive from rather than one. diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactScanner.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactScanner.kt index b25e3cf0c..7d3e81d05 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactScanner.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ArtifactScanner.kt @@ -433,7 +433,7 @@ internal object ArtifactScanner { * enforces exactly, so one name reaches one signature by construction; and * `PythonProxySource.renderOne` publishes an entry by `setattr`ing its *leaf* name onto a module, * so two entries sharing a leaf would silently overwrite each other. A dispatcher therefore lives - * in `pythonx` (`docs/design/pythonx-adapter-design.md` §4.1) and selects among these names -- which is + * in `pythonx` (`docs/archive/pythonx-adapter-design.md` §4.1) and selects among these names -- which is * why they have to exist and be distinguishable, and why `ExposedCallable` now carries * `paramNames` and `paramTypeNames` for it to select on. This layer's job is to make the choice * *possible*, not to make it. @@ -801,7 +801,7 @@ internal object ArtifactScanner { * because `androidx.compose.runtime.Composable` is declared * `@Retention(AnnotationRetention.BINARY)`. `RuntimeInvisible*` is exactly what ASM puts in * `invisibleAnnotations`, so the old predicate answered `false` for **every real composable**, - * and `DeclarationModel.isComposable` -- which drives `docs/design/pyi-generation-design.md` §3.6's + * and `DeclarationModel.isComposable` -- which drives `docs/archive/pyi-generation-pythonic-stubs.md` §3.6's * PascalCase rule -- was dead. Nothing caught it because the only composables reaching it were * being declined anyway, and a declined entry's flag is not asserted anywhere. * @@ -829,7 +829,7 @@ internal object ArtifactScanner { * * ### Why the synthetic parameters are exposed rather than hidden * - * `docs/design/pythonx-adapter-design.md` §4.5 rejected every way of *hiding* them, and each rejection + * `docs/archive/pythonx-adapter-design.md` §4.5 rejected every way of *hiding* them, and each rejection * still stands: an arity-prefix entry cannot express "pass `text`, skip `modifier`", presence * branching costs 2^15 call expressions for `Text`, `Text$default` does not exist, and a * generated wrapper cannot restate defaults metadata never carries. What none of those noticed is @@ -848,7 +848,7 @@ internal object ArtifactScanner { * This used to decline one outright, on the stated grounds that "Compose's `$changed` slots count * receivers and its `$default` bits are assigned over value parameters, so a receiver shifts one * numbering and not the other. Nothing here has measured which." It is measured now, out of the - * callee, the way `docs/design/pythonx-adapter-design.md` §5.2 measured the mask in the first place -- + * callee, the way `docs/archive/pythonx-adapter-design.md` §5.2 measured the mask in the first place -- * `javap -c androidx/compose/material3/NavigationBarKt`, whose `NavigationBarItem` is * `RowScope.NavigationBarItem(selected, onClick, icon, modifier = …, …)`: * @@ -1397,7 +1397,7 @@ internal object ArtifactScanner { * The declared shape of one Kotlin function, independent of whether the boundary can carry it. * * `null` when some part of the signature has no name a stub could write -- a type *parameter*, a - * flexible type. `docs/design/pyi-generation-design.md` §3.1's last row and §7: `BindingPolicy` rejects + * flexible type. `docs/archive/pyi-generation-pythonic-stubs.md` §3.1's last row and §7: `BindingPolicy` rejects * generic declarations, and stubbing what cannot be called would be a lie. */ private fun declarationModelOf( @@ -1455,7 +1455,7 @@ internal object ArtifactScanner { val returnType = resolveKotlinType(function.returnType, classpath, BoundaryDirection.RETURN) ?: return declined("no boundary type for return ${kotlinClassifierNameOf(function.returnType) ?: function.returnType.classifier}") // Declared, not marshalled: a `Dp` parameter's tag is FLOAT and its declared name is - // `androidx.compose.ui.unit.Dp`. `docs/design/pythonx-adapter-design.md` §2.4 row 4. A function slot + // `androidx.compose.ui.unit.Dp`. `docs/archive/pythonx-adapter-design.md` §2.4 row 4. A function slot // is the one exception and is the reason [functionSlotTypeName] exists: its declared name // alone (`kotlin.Function1`) says neither what the lambda is invoked with nor what it must // give back, and `pythonx` needs both to build a wrapper at all. @@ -1577,7 +1577,7 @@ internal object ArtifactScanner { * and six others), so 64 is well clear of the corpus while bounding an artefact nobody has * measured. * - * `docs/design/pythonx-adapter-design.md` §4.5 rejects presence branching outright on "2^15 branches for + * `docs/archive/pythonx-adapter-design.md` §4.5 rejects presence branching outright on "2^15 branches for * `Text`". That objection is to an unbounded version of it and, separately, to a case that is not * in the table: `Text` is a `@Composable`, and the arity check in [kotlinCandidates] declines * every composable for the synthetic `$composer`/`$changed` parameters its JVM signature carries. diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ComposableThunks.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ComposableThunks.kt index 9a7c4be42..2e30f0976 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ComposableThunks.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/ComposableThunks.kt @@ -20,7 +20,7 @@ import kotlin.math.ceil * * 1. **The synthetic parameters are unreachable from source.** `androidx.compose.material3.Text` * declares 16 Kotlin parameters and its JVM method takes 20 -- `$composer`, two `$changed` - * bitmasks and one `$default` (`docs/design/pythonx-adapter-design.md` §5.2, and + * bitmasks and one `$default` (`docs/archive/pythonx-adapter-design.md` §5.2, and * `ComposableBindingTest.everyComposableJvmSignatureIsKotlinParamsThenComposerThenInts` over * every composable in three jars). The Compose plugin adds them during IR lowering, so at source * level they are not parameters of anything and cannot be named or passed. Whatever the Compose @@ -239,7 +239,7 @@ internal fun generateThunkClass(fragmentObjectName: String, specs: List f(t: T)`), a dynamic or * error type. That is [ArtifactScanner.declarationModelOf]'s own rule and - * `docs/design/pyi-generation-design.md` §3.1's last row: `BindingPolicy` rejects generic declarations + * `docs/archive/pyi-generation-pythonic-stubs.md` §3.1's last row: `BindingPolicy` rejects generic declarations * and stubbing what cannot even be spelled would be a lie, so there is nothing to record. * * `internal`, not `private`: no real klib available to this module's tests binds a declaration diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/KotlinMetadata.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/KotlinMetadata.kt index bd1850ebf..fe3d1d958 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/KotlinMetadata.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/artifact/KotlinMetadata.kt @@ -522,7 +522,7 @@ internal data class ResolvedFunction( /** * Carried rather than filtered out at the source, because the two consumers of this want * different things from it: the binder declines a `suspend` declaration outright, and the stub - * model records it as declined-because-suspend (`docs/design/pyi-generation-design.md` §3.1 -- "declined + * model records it as declined-because-suspend (`docs/archive/pyi-generation-pythonic-stubs.md` §3.1 -- "declined * by both producers; must not be stubbed"). Dropping it here would make the second indistinguishable * from a declaration that was never declared. */ diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/ArtifactFragmentRenderingTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/ArtifactFragmentRenderingTest.kt index d42a21966..9672e3fd1 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/ArtifactFragmentRenderingTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/ArtifactFragmentRenderingTest.kt @@ -35,7 +35,7 @@ class ArtifactFragmentRenderingTest { ) /** An extension, so that the rendered `isExtension`/`receiverTypeName` pair -- the fields - * `docs/design/pythonx-adapter-design.md` §4.2 needs and §2.4 records as lost -- appears in the pinned + * `docs/archive/pythonx-adapter-design.md` §4.2 needs and §2.4 records as lost -- appears in the pinned * text rather than only in a scanner unit test. */ private val trimIndent = ArtifactCallable( name = "kotlin.text.trimIndent", diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/ArtifactScannerTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/ArtifactScannerTest.kt index 8026384f6..35967f208 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/ArtifactScannerTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/ArtifactScannerTest.kt @@ -377,7 +377,7 @@ class ArtifactScannerTest { "{ args -> (fixture.artifactvalueclass.ropeLength((args[0] as fixture.artifactvalueclass.Rope))) }", ropeLength.lambdaBody, ) - // The declared type survives beside the tag: `docs/design/pythonx-adapter-design.md` §2.4 row 4. + // The declared type survives beside the tag: `docs/archive/pythonx-adapter-design.md` §2.4 row 4. assertEquals(listOf("fixture.artifactvalueclass.Rope"), ropeLength.paramTypeNames) assertEquals(listOf("rope"), ropeLength.paramNames) assertEquals(listOf(false), ropeLength.paramHasDefault) @@ -413,7 +413,7 @@ class ArtifactScannerTest { * A class's own public primary constructor, bound the same way a top-level function is: * `Rope(Double): Rope`'s shape, and [Rope]'s own KDoc records why this used to be entirely * absent -- `ArtifactScanner.kotlinCandidates`'s `ACC_STATIC` filter drops `` along with - * every instance method, and nothing else ever picked it back up. `docs/design/pythonx-adapter-design.md` + * every instance method, and nothing else ever picked it back up. `docs/archive/pythonx-adapter-design.md` * §10 names `Typography`/`Shapes` as the real-world casualty: not declined, simply never scanned. */ @Test @@ -668,7 +668,7 @@ class ArtifactScannerTest { } /** - * `docs/design/pythonx-adapter-design.md` §10's `Typography`/`Shapes` casualty, closed, and pinned + * `docs/archive/pythonx-adapter-design.md` §10's `Typography`/`Shapes` casualty, closed, and pinned * against the structural rule that closed it rather than against the two hard-coded class names * an earlier attempt used. * diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/ComposableBindingTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/ComposableBindingTest.kt index b880419a7..bb3abb503 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/ComposableBindingTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/ComposableBindingTest.kt @@ -8,7 +8,7 @@ import kotlin.test.assertNotNull import kotlin.test.assertTrue /** - * `docs/design/pythonx-adapter-design.md` §5, measured against real Compose Multiplatform jars. + * `docs/archive/pythonx-adapter-design.md` §5, measured against real Compose Multiplatform jars. * * ### What this test exists to pin * diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/DefaultOmissionTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/DefaultOmissionTest.kt index d7b820a8b..b74cc18fc 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/DefaultOmissionTest.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/DefaultOmissionTest.kt @@ -7,7 +7,7 @@ import kotlin.test.assertFalse import kotlin.test.assertTrue /** - * `docs/design/pythonx-adapter-design.md` §4.5 closed for everything the walker actually binds: a Kotlin + * `docs/archive/pythonx-adapter-design.md` §4.5 closed for everything the walker actually binds: a Kotlin * default value is reached by **not writing the argument**, in generated Kotlin *source*. * * ### What the problem was diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/defaultsfixture/DefaultFixtures.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/defaultsfixture/DefaultFixtures.kt index 9aca1cb31..80dc338d4 100644 --- a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/defaultsfixture/DefaultFixtures.kt +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/artifact/defaultsfixture/DefaultFixtures.kt @@ -21,7 +21,7 @@ fun greet(name: String, punctuation: String = "!"): String = name + punctuation /** * Two defaulted parameters, so that the *middle* of the list can be the one omitted. - * `docs/design/pythonx-adapter-design.md` §4.5 names exactly this as what an arity-prefix scheme cannot + * `docs/archive/pythonx-adapter-design.md` §4.5 names exactly this as what an arity-prefix scheme cannot * express: "pass `text` and `font_size`, skip `modifier` and `color`". */ fun label(bag: Bag, prefix: String = "[", suffix: String = "]"): String = prefix + bag.text + suffix diff --git a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/BindingPolicy.kt b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/BindingPolicy.kt index 119c0f7e9..04995ed70 100644 --- a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/BindingPolicy.kt +++ b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/BindingPolicy.kt @@ -28,7 +28,7 @@ object BindingPolicy { // Cheap check first (`shortName`, no resolution); only resolve the annotation type // when the name plausibly matches, since resolving every annotation on every // declaration would be the expensive path `getSymbolsWithAnnotation` was rejected - // for avoiding (docs/design/upcall-table-design.md §2, "Discovery key"). + // for avoiding (docs/design/upcall.md §2.3, "Discovery key"). shortName == "PythonInternal" && annotation.annotationType.resolve().declaration.qualifiedName?.asString() == PYTHON_INTERNAL_ANNOTATION } @@ -157,7 +157,7 @@ object BindingPolicy { /** * Whether this declaration needs the asynchronous calling convention - * (`docs/design/upcall-async-design.md`). + * (`docs/design/upcall.md`). * * `suspend` used to be a rejection here, and silently: a C callback slot has to hand back a * `PyObject *` before it returns and a suspension has nothing to hand back, so until there was @@ -233,7 +233,7 @@ object BindingPolicy { * [FragmentScanner] can tell "not exposed because of shape" (private, `@PythonInternal`, ...) * apart from "not exposed because a type in the signature has no callable entry" and warn only * on the latter -- the former is ordinary and silent by design, the latter is the failure mode - * `docs/design/upcall-async-design.md` §2.1 measured as a classifier that compiles and checkcasts but + * `docs/design/upcall.md` §5.3 measured as a classifier that compiles and checkcasts but * has nothing behind it. */ fun hasExposableTypes(function: KSFunctionDeclaration): Boolean = diff --git a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/Constants.kt b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/Constants.kt index 2b6e45be8..e9540882c 100644 --- a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/Constants.kt +++ b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/Constants.kt @@ -2,7 +2,7 @@ package python.multiplatform.ksp /** * The well-known package fragments are emitted into and the aggregator scans. - * `docs/design/upcall-table-design.md` §1/§2 -- not load-bearing for discovery (the aggregator scans + * `docs/design/upcall.md` §2.1/§2 -- not load-bearing for discovery (the aggregator scans * the whole package), but fixed so generated code from different modules never collides. */ const val FRAGMENTS_PACKAGE = "python.multiplatform.generated.fragments" @@ -35,12 +35,12 @@ const val OPTION_EXCLUDE_PACKAGES = "python.multiplatform.excludePackages" const val ROLE_LIBRARY = "library" const val ROLE_APP = "app" -/** [docs/design/upcall-table-design.md] §11.4: an entry point Python has no use for and cannot call. */ +/** [docs/design/upcall.md] §11.4: an entry point Python has no use for and cannot call. */ fun isMainFunction(name: String, hasNoParamsOrArgsArray: Boolean): Boolean = name == "main" && hasNoParamsOrArgsArray /** Fragment object name for a given (already sanitised) module name. */ fun fragmentObjectName(moduleName: String): String = "$FRAGMENT_PREFIX$moduleName" -/** Periods and hyphens collide with Kotlin identifier syntax; `docs/design/upcall-table-design.md` §1. */ +/** Periods and hyphens collide with Kotlin identifier syntax; `docs/design/upcall.md` §2.1. */ fun sanitiseModuleName(raw: String): String = raw.replace('.', '_').replace('-', '_') diff --git a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/FragmentDiscovery.kt b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/FragmentDiscovery.kt index 6258a4f8d..9aeb73ff6 100644 --- a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/FragmentDiscovery.kt +++ b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/FragmentDiscovery.kt @@ -6,7 +6,7 @@ import com.google.devtools.ksp.symbol.KSClassDeclaration /** * How the app-role processor finds every `Fragment_*` object -- its own dependencies' and its - * own. `docs/design/upcall-table-design.md` §3: the only working implementation today goes through + * own. `docs/design/upcall.md` §2.3: the only working implementation today goes through * `Resolver.getDeclarationsFromPackage`, which is `@KspExperimental`. This interface is the * seam the doc asks for: if that API changes, only [PackageScanFragmentDiscovery] below needs * to change, not [python.multiplatform.ksp.AppProcessor] or anything downstream of it. @@ -19,7 +19,7 @@ interface FragmentDiscovery { /** * The only implementation today. Queries the well-known package on the classpath (JAR on JVM, - * `.klib` metadata on Kotlin/Native -- `docs/design/upcall-table-design.md` §2's "Platform uniformity" + * `.klib` metadata on Kotlin/Native -- `docs/design/upcall.md` §2.3's "Platform uniformity" * table), which is why this discovers fragments compiled by other modules without those modules * publishing anything beyond their ordinary build artifact. */ diff --git a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/FragmentScanner.kt b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/FragmentScanner.kt index e727cfb82..bdc559d52 100644 --- a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/FragmentScanner.kt +++ b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/FragmentScanner.kt @@ -30,7 +30,7 @@ private const val PENDING_CALL = "python.multiplatform.ffi.upcall.PendingCall" * continuation there to give it -- so its call goes inside `PendingCall.start { }`, which is itself * a `suspend` lambda and needs nothing but `kotlin-stdlib` to build. The entry then evaluates to a * `PendingCall` and carries `isSuspend = true`, which is what tells the trampoline to unwrap it. - * See `docs/design/upcall-async-design.md` §5 and [BindingPolicy.isSuspending]. + * See `docs/design/upcall.md` §5.2 and [BindingPolicy.isSuspending]. * * [resultExpr] has already been through [wrapReturnExpression], so a `suspend fun` returning `Int` * widens inside the coroutine and the boundary still sees the `Long` its `INT` tag promises. @@ -124,14 +124,14 @@ class FragmentScanner(private val excludePackages: List, private val log * ([BindingPolicy.isExposedFunctionShape]-family) but names a type * ([BindingPolicy.hasExposableTypes]) with no callable entry of its own is not an ordinary, * expected exclusion the way `private`/`internal` visibility is -- it is the exact silent - * failure mode `docs/design/upcall-async-design.md` §2.1 measured for `suspend` function types, and + * failure mode `docs/design/upcall.md` §5.3 measured for `suspend` function types, and * the whole point of this task is that it should not go unlogged a second time. */ private fun warnUnexposableType(name: String, declaration: KSNode) { logger.warn( "python-multiplatform-ksp: $name has a type with no exposable classifier (a `suspend` " + "function type, or one nested in a generic argument) and is not exposed to Python -- " + - "see docs/design/upcall-async-design.md §2.1.", + "see docs/design/upcall.md §5.3.", declaration, ) } diff --git a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/KsAdapters.kt b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/KsAdapters.kt index 4f3293a89..c3c330baf 100644 --- a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/KsAdapters.kt +++ b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/KsAdapters.kt @@ -49,7 +49,7 @@ fun KSTypeReference.toShape(): TypeShape = resolve().toShape() * (`kotlin.coroutines.SuspendFunctionN`) is compiler-synthesized, not something this * compilation's KSP scan ever visits as source. It gets no `ReflectedClass`, so `invoke` has no * table entry -- the generated cast (`args[0] as kotlin.coroutines.SuspendFunction1<...>`) - * compiles and the runtime checkcast passes regardless (`docs/design/upcall-async-design.md` §2.1, + * compiles and the runtime checkcast passes regardless (`docs/design/upcall.md` §5.3, * `GeneratedSuspendTest`), so nothing downstream catches this. What would cross is a handle * Python can hold and hand back and nothing else. * diff --git a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/PythonBindingProcessorProvider.kt b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/PythonBindingProcessorProvider.kt index 66758ddf9..b1e68e557 100644 --- a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/PythonBindingProcessorProvider.kt +++ b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/PythonBindingProcessorProvider.kt @@ -10,7 +10,7 @@ import java.io.OutputStreamWriter /** * The single entry point KSP loads (`META-INF/services/...SymbolProcessorProvider`). - * `docs/design/upcall-table-design.md` §6: one processor module, two roles selected by + * `docs/design/upcall.md` §2.5: one processor module, two roles selected by * [OPTION_ROLE] -- a library emits a fragment, an app emits a fragment for itself and then * aggregates every fragment it can see. */ @@ -18,7 +18,7 @@ class PythonBindingProcessorProvider : SymbolProcessorProvider { override fun create(environment: SymbolProcessorEnvironment): SymbolProcessor { val moduleName = sanitiseModuleName( environment.options[OPTION_MODULE_NAME] - ?: error("$OPTION_MODULE_NAME is required (see docs/design/upcall-table-design.md §6)"), + ?: error("$OPTION_MODULE_NAME is required (see docs/design/upcall.md §2.5)"), ) val excludePackages = environment.options[OPTION_EXCLUDE_PACKAGES] ?.split(",") @@ -126,7 +126,7 @@ class LibraryProcessor( * `python.multiplatform.role = app`: round 1 does what [LibraryProcessor] does for the app's own * sources; round 2 discovers every `Fragment_*` visible on the classpath (its own, just * generated, plus every dependency's) and emits the aggregator. See - * `docs/design/upcall-table-design.md` §2/§6 for why this needs two rounds -- round 1's own fragment + * `docs/design/upcall.md` §2.3/§6 for why this needs two rounds -- round 1's own fragment * has to exist as a compiled/generated symbol before [discovery] can see it alongside the * others. * diff --git a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/SourceRendering.kt b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/SourceRendering.kt index 472133a8c..c2e953ec8 100644 --- a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/SourceRendering.kt +++ b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/SourceRendering.kt @@ -67,7 +67,7 @@ private fun renderClass(cls: ClassModel): String { /** * Renders one `Fragment_.kt` file body -- what a library-role KSP invocation emits. * - * The object is `public` rather than `internal` as `docs/design/upcall-table-design.md`'s sketch shows + * The object is `public` rather than `internal` as `docs/design/upcall.md`'s sketch shows * it: `internal` is enforced per Kotlin *module*, and the app module compiling generated source * that references a library's fragment is a different module by that rule even inside the same * Gradle build. An `internal` fragment fails to compile from the app side with "cannot access @@ -110,7 +110,7 @@ fun renderFragmentSource(model: FragmentModel): String { /** * Renders `FunctionTable.kt` -- the app-role aggregator. [fragmentQualifiedNames] is the result - * of fragment discovery (`docs/design/upcall-table-design.md` §2): every `Fragment_*` object this app + * of fragment discovery (`docs/design/upcall.md` §2.3): every `Fragment_*` object this app * module and its dependencies emitted, found via * [FragmentDiscovery][python.multiplatform.ksp.FragmentDiscovery] and referenced here * explicitly. That explicit reference is what keeps every fragment reachable to the Kotlin/Native diff --git a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/TypeShape.kt b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/TypeShape.kt index 6e7cd1480..73e8c2af9 100644 --- a/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/TypeShape.kt +++ b/python-multiplatform-ksp/src/main/kotlin/python/multiplatform/ksp/TypeShape.kt @@ -1,7 +1,7 @@ package python.multiplatform.ksp /** - * The boundary's five-plus-two marshalling categories (`docs/design/upcall-table-design.md` §1, + * The boundary's five-plus-two marshalling categories (`docs/design/upcall.md` §2.1, * `python.multiplatform.reflection.TypeTag`). Kept as a processor-local copy rather than a * dependency on the runtime module's enum so this file has no KSP or runtime import and its * mapping logic is plain-JVM-testable. diff --git a/python-multiplatform-ksp/src/test/kotlin/python/multiplatform/ksp/SourceRenderingTest.kt b/python-multiplatform-ksp/src/test/kotlin/python/multiplatform/ksp/SourceRenderingTest.kt index 21ffbad44..e6961790b 100644 --- a/python-multiplatform-ksp/src/test/kotlin/python/multiplatform/ksp/SourceRenderingTest.kt +++ b/python-multiplatform-ksp/src/test/kotlin/python/multiplatform/ksp/SourceRenderingTest.kt @@ -287,7 +287,7 @@ class SourceRenderingTest { @Test fun aSuspendingEntryCarriesTheFlagTheTrampolineBranchesOn() { - // `docs/design/upcall-async-design.md` §5: the trampoline has to know, before it looks at the + // `docs/design/upcall.md` §5.2: the trampoline has to know, before it looks at the // result, whether what came back is the value or a PendingCall. The flag is separate from // `kind` because the two are orthogonal -- a suspending *method* still has its receiver in // args[0], and doubling CallableKind would have made every hasReceiver decision restate it. diff --git a/python-multiplatform/build.gradle.kts b/python-multiplatform/build.gradle.kts index a422f53bb..eb734d1dd 100644 --- a/python-multiplatform/build.gradle.kts +++ b/python-multiplatform/build.gradle.kts @@ -1340,7 +1340,7 @@ tasks.withType` and a Kotlin * result back into a `PyObject *` is [UpcallTrampoline], which is `commonMain` and shared with - * every other platform -- `docs/design/upcall-design.md`'s "what each platform still owes" is only the + * every other platform -- `docs/design/upcall.md` §3.7 is only the * address-publishing step, and on Android that step is the C shim plus these three methods. * * ### Rules these must obey @@ -62,7 +62,7 @@ import python.multiplatform.reflection.UpcallTable * It is not marked with [HighOverheadNativeCall]. That marker is `@RequiresOptIn`, and its job is * to make a Kotlin *caller* acknowledge a cost; nothing in Kotlin calls these — C does, from * CPython — so applying it here would be inert. The cost is recorded instead where the callers - * that can act on it will look: here, in `pmp_upcall_invoke_meth`, and in `docs/design/upcall-design.md`. + * that can act on it will look: here, in `pmp_upcall_invoke_meth`, and in `docs/design/upcall.md`. */ object UpcallCallbacks { diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonCallables.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonCallables.kt index 5fa04b378..4f51d4ff1 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonCallables.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonCallables.kt @@ -15,7 +15,7 @@ import python.multiplatform.reflection.TypeTag /** * The other direction: **a Python callable arriving at a Kotlin function-typed parameter.** * - * `docs/design/pythonx-adapter-design.md` §6 opens by saying the mechanism is half present already -- + * `docs/archive/pythonx-adapter-design.md` §6 opens by saying the mechanism is half present already -- * `UpcallTrampoline.toKotlinObject` wraps anything Python sends that is not an integer handle as a * `PyObject`, so a `lambda:` does reach Kotlin with a reference of its own. What it reaches is a * `PyObject`, and every container composable's `content` slot wants a `kotlin.jvm.functions.FunctionN`. @@ -64,7 +64,7 @@ import python.multiplatform.reflection.TypeTag * * So this file owns the first three quarters of that -- creating, holding and releasing -- and the * one Compose type involved, `RememberObserver`, stays in the module that has Compose - * (`:ksp-fixtures:compose`'s `PythonComposition`). `docs/design/pythonx-adapter-design.md` §6 item 1 names + * (`:ksp-fixtures:compose`'s `PythonComposition`). `docs/archive/pythonx-adapter-design.md` §6 item 1 names * `onForgotten` as "the one that fails silently and should be tested first"; the seam is here and the * test is there. * diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapter.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapter.kt index 1dc86bcbb..5fb2db384 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapter.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapter.kt @@ -17,7 +17,7 @@ import python.multiplatform.reflection.UpcallTable * Pythonic API on top of this one through `python_multiplatform.describe` and `inspect.signature` * ([KotlinSurface]). * - * `docs/design/pythonx-adapter-design.md` §7 draws one line through this whole area: **if it differs per + * `docs/archive/pythonx-adapter-design.md` §7 draws one line through this whole area: **if it differs per * Kotlin declaration it is generated or resolved at run time; if it is the same rule for every * declaration it is `pythonx` Python source.** This object is where the two meet, and it keeps them * apart on purpose: @@ -28,7 +28,7 @@ import python.multiplatform.reflection.UpcallTable * | [renderTable] | one row per entry in [UpcallTable] | this function, from the table | * * The 2024 `pythonx-compose` put both on the wrong side of that line -- a Python file per Compose - * component, 37 of them, 28 empty -- and `docs/design/pythonx-adapter-design.md` §1 measures what it cost: + * component, 37 of them, 28 empty -- and `docs/archive/pythonx-adapter-design.md` §1 measures what it cost: * `padding()` composed nothing and `fill_max_size()` returned `self`, because a per-declaration * wrapper is written once and then never again. Nothing here is per-declaration. * @@ -86,7 +86,7 @@ object PythonxAdapter { * Read it as a file -- it is one, and the indentation the Kotlin literal adds is removed by * `trimIndent`. What it contains, in the order the design asks for it: * - * | | `docs/design/pythonx-adapter-design.md` | + * | | `docs/archive/pythonx-adapter-design.md` | * |---|---| * | `_Finder` / `_Loader` | §2.3, the hook a module `__getattr__` cannot replace | * | the module `__getattr__` the loader installs | §4.1, adapted once and then a dict hit | @@ -105,7 +105,7 @@ object PythonxAdapter { # Pythonic is the job of a real Python package built on top of it (pythonx-compose), which # reads `python_multiplatform.describe` and `inspect.signature` to do it by rule. # - # `docs/design/pythonx-adapter-design.md` §7 draws the line this file lives on: *if it differs per Kotlin + # `docs/archive/pythonx-adapter-design.md` §7 draws the line this file lives on: *if it differs per Kotlin # declaration it is generated or resolved at run time; if it is the same rule for every declaration # it is `pythonx` Python source.* Nothing here mentions a Kotlin declaration by name. What arrives # per declaration is the table `PythonxAdapter.renderTable` emits into `_register_table`, and the @@ -920,7 +920,7 @@ object PythonxAdapter { class _Hybrid: '''`Modifier.padding(16)` and `m.padding(16)`, from one descriptor. - `docs/design/pyi-generation-design.md` §4.3 measured the metaclass alternative failing at run time: a + `docs/archive/pyi-generation-pythonic-stubs.md` §4.3 measured the metaclass alternative failing at run time: a plain `def` on a metaclass is a *non-data* descriptor, so `type.__getattribute__` searches the class's own MRO first and `Modifier.padding(16)` binds `16` to `self`. A descriptor in the class body is found for both spellings and is told which one it is by `obj`. @@ -1343,7 +1343,7 @@ object PythonxAdapter { # `None` is the whole mechanism, and it is not a value being passed: the generated # Kotlin body tests `args[i] == null` and takes a branch whose call expression does # not mention this parameter at all, so the *compiler* supplies the default. - # `docs/design/pythonx-adapter-design.md` §4.5 -- metadata carries the flag and never the + # `docs/archive/pythonx-adapter-design.md` §4.5 -- metadata carries the flag and never the # expression, so this is the only place the default value can come from. # # It costs nothing that was previously possible, and the reason is `_coerce` rather @@ -1621,7 +1621,7 @@ object PythonxAdapter { `import androidx.compose.material3` fails *before* any attribute is touched, so laziness inside a module is not enough to make the module lazy. A finder answers the import, and the `__getattr__` the loader installs answers the names inside it -- two hooks, - `docs/design/pythonx-adapter-design.md` §2.3. + `docs/archive/pythonx-adapter-design.md` §2.3. ''' def find_spec(self, fullname, path=None, target=None): @@ -1704,7 +1704,7 @@ object PythonxAdapter { /** * The generated half: one row per [ExposedCallable], and nothing else. * - * Every field here is one `docs/design/pythonx-adapter-design.md` §2.4 recorded as *missing* from the + * Every field here is one `docs/archive/pythonx-adapter-design.md` §2.4 recorded as *missing* from the * boundary -- parameter names, whether a slot is an extension receiver, the receiver's type, the * **declared** type of a parameter as opposed to its marshalling tag, and whether a parameter * has a default. They are on [ExposedCallable] now, and this is what carries them the last step, diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/AsyncUpcall.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/AsyncUpcall.kt index 70fe68656..481069cc2 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/AsyncUpcall.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/AsyncUpcall.kt @@ -16,7 +16,7 @@ import python.native.ffi.toNativePointer import python.native.ffi.toRawValue /** - * The delivery half of `docs/design/upcall-async-design.md`: how the outcome of a Kotlin coroutine that + * The delivery half of `docs/design/upcall.md`: how the outcome of a Kotlin coroutine that * could not finish inside the C frame reaches the Python code waiting for it. * * [PendingCall] is the half that starts the coroutine and parks its outcome; this is the half that @@ -146,7 +146,7 @@ internal object AsyncUpcall { /** * Gives Python a way to tell [call] it has been abandoned, and returns the handle that costs. * - * `docs/design/upcall-async-design.md` §9.3 could only offer cancellation observed *at completion*, + * `docs/design/upcall.md` §5.6 could only offer cancellation observed *at completion*, * because nothing carried the identity of a running [PendingCall] across to Python. This is the * thing it said was missing: a [HandleTable] handle rides on the `Future` -- captured by the * done callback `_pm_watch` attaches, not stored as an attribute, so nothing depends on whether diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/PendingCall.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/PendingCall.kt index 8fa286f0a..799344f1e 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/PendingCall.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/PendingCall.kt @@ -10,7 +10,7 @@ import kotlin.coroutines.startCoroutine * One Kotlin coroutine started from a frame that had to return without it, and the place its * outcome waits until something asks for it. * - * This is the half of `docs/design/upcall-async-design.md` that all three candidate conventions share. + * This is the half of `docs/design/upcall.md` that all three candidate conventions share. * The problem the doc states is structural: CPython calls a C function pointer, that C frame has * to hand back a `PyObject *` before it returns, and a suspension has nothing to hand back. * `runBlocking` "solves" it by re-seizing the very thread that would have been freed -- on Android @@ -117,7 +117,7 @@ class PendingCall private constructor() : CoroutineContext.Element { * * Resuming a continuation twice is undefined, so there is no back door either. Forcible * cancellation is what a `Job` tree is *for*, and building one is `kotlinx.coroutines`' - * job; `docs/design/upcall-async-design.md` §6 states the no-dependency rule this obeys. + * job; `docs/design/upcall.md` §5.4 states the no-dependency rule this obeys. * * ### What it is * diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/PythonProxySource.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/PythonProxySource.kt index 4ad6badae..d9bae659d 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/PythonProxySource.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/PythonProxySource.kt @@ -12,7 +12,7 @@ import python.multiplatform.reflection.TypeTag import python.multiplatform.reflection.UpcallTable /** - * The Python half of `docs/design/upcall-async-design.md` §8.6's second gap: the `async def` proxy that + * The Python half of `docs/design/upcall.md` §5.5's second gap: the `async def` proxy that * makes `await kotlin_fn(x)` read the same whether the Kotlin body suspended or not. * * Until this existed, `AsyncUpcallDeliveryTest` wrote that proxy by hand (`_await_kotlin`) and the @@ -275,7 +275,7 @@ object PythonProxySource { * duplicated -- the handle `_pm_watch` carries is an ordinary object handle. */ internal const val RELEASE_ENTRY_POINT = "_pm_release" - /** Where names with no package of their own land; `docs/design/upcall-table-design.md` §Runtime. */ + /** Where names with no package of their own land; `docs/design/upcall.md` §Runtime. */ const val DEFAULT_ROOT_MODULE: String = "kotlin" /** diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/UpcallTrampoline.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/UpcallTrampoline.kt index 43c05034a..a786b3bbe 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/UpcallTrampoline.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/ffi/upcall/UpcallTrampoline.kt @@ -59,8 +59,8 @@ import python.native.ffi.toRawValue * * The shape is deliberately the one CPython's own `PyCFunction` slot uses * (`PyObject *(PyObject *self, PyObject *args)`): when the generated proxy type lands, `self` - * takes the handle's place and no new stub is needed. `docs/design/upcall-design.md`'s "왜 트램폴린이 - * 적어도 되는가" is exactly this argument, and the remaining CPython slot shapes + * takes the handle's place and no new stub is needed. `docs/design/upcall.md` §1.3 (formerly "왜 트램폴린이 + * 적어도 되는가") is exactly this argument, and the remaining CPython slot shapes * (`(long,long,long) -> int` for `initproc`/`setter`/`traverse`, `(long) -> int` for `clear`, * `(long) -> void` for `destructor`) are already in `Panama`'s vocabulary. * @@ -122,7 +122,7 @@ object UpcallTrampoline { // A suspending entry's body hands back a PendingCall rather than the value, because // this frame has to return before the coroutine can finish. `AsyncUpcall` either // unwraps an already-complete one -- the common case, and free -- or builds the - // `asyncio.Future` that will carry the answer. See docs/design/upcall-async-design.md §5. + // `asyncio.Future` that will carry the answer. See docs/design/upcall.md §5.2. if (entry.isSuspend) AsyncUpcall.deliver(entry, result) else marshalResult(entry.returnType, result) } catch (t: Throwable) { @@ -171,7 +171,7 @@ object UpcallTrampoline { /** * Tells the [PendingCall] behind [callHandle] that the Python side has stopped waiting for it. * - * The Python end of `docs/design/upcall-async-design.md` §9.3's missing half. Without this, the only + * The Python end of `docs/design/upcall.md` §5.6's missing half. Without this, the only * moment Kotlin could learn that a `Future` had been cancelled was when the coroutine finished * and [AsyncUpcall] found the `Future` already settled -- which is the moment the news becomes * useless. `PythonProxySource`'s `_pm_watch` hangs a `Future.add_done_callback` on the `Future` diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/ExposedCallable.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/ExposedCallable.kt index 34039639f..a40edcdc3 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/ExposedCallable.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/ExposedCallable.kt @@ -7,7 +7,7 @@ import kotlin.jvm.JvmInline * How a value is marshalled across the boundary. The trampoline reads these, never the Kotlin * type, because there is no runtime type to read on Kotlin/Native. * - * `docs/design/upcall-table-design.md` lists five (`INT`, `FLOAT`, `STRING`, `BYTES`, `OBJECT`); + * `docs/design/upcall.md` lists five (`INT`, `FLOAT`, `STRING`, `BYTES`, `OBJECT`); * [BOOLEAN] and [UNIT] are here because Python's `bool` is a distinct type and because a * function returning `Unit` must marshal to `None` rather than to an object handle. */ @@ -30,7 +30,7 @@ enum class TypeTag { * Every CPython callback convention passes the relevant object as an argument -- `self` for * methods, `type` for `tp_new`, the closure for getters -- which is why a handful of shared * trampolines can serve an unbounded number of Kotlin targets and no runtime code generation is - * needed. See `docs/design/upcall-design.md`. + * needed. See `docs/design/upcall.md`. */ enum class CallableKind(val hasReceiver: Boolean) { /** Top-level or companion function. `args` is exactly the declared parameters. */ @@ -98,7 +98,7 @@ class ExposedCallable( * [paramTypes] and [returnType] describe the declaration as written -- the continuation * parameter the Kotlin compiler adds is not here, and [returnType] is the value the coroutine * will eventually produce, not the `PendingCall`. That is what lets - * `docs/design/upcall-async-design.md` §5's fast path marshal a body that never suspended exactly as + * `docs/design/upcall.md` §5.2's fast path marshal a body that never suspended exactly as * a synchronous entry would. */ val isSuspend: Boolean = false, @@ -106,7 +106,7 @@ class ExposedCallable( * The Kotlin declaration's parameter names, in [paramTypes] order, or empty when the producer * did not supply them. * - * `docs/design/pythonx-adapter-design.md` §2.4 is the reason this exists: mapping a Python keyword + * `docs/archive/pythonx-adapter-design.md` §2.4 is the reason this exists: mapping a Python keyword * argument onto a positional slot needs names, and neither side of the boundary had them -- * "this is not a `pythonx` design choice, it is arithmetic." Both producers can read them * (`KmValueParameter.name` for the walker, KSP's own parameter list for the processor), so the @@ -127,7 +127,7 @@ class ExposedCallable( * * [paramTypes] says how a value is *marshalled*, which is deliberately lossy: a `Dp` parameter is * a [TypeTag.FLOAT] because the generated Kotlin wraps the raw number back up into a `Dp` on the - * way in (`docs/design/pythonx-adapter-design.md` §4.4). That is correct for calling and useless for + * way in (`docs/archive/pythonx-adapter-design.md` §4.4). That is correct for calling and useless for * describing -- a Python adapter cannot tell `Dp` from `Float`, and a `.pyi` generator cannot * name the type it is meant to stub. This is the declaration as written. */ @@ -155,7 +155,7 @@ class ExposedCallable( * * **A flag, and nothing acts on it yet.** Every entry this repository generates passes every * argument explicitly; there is no route from here to a call that omits one. - * `docs/design/pythonx-adapter-design.md` §4.5 names four candidates and a measured reason each may not + * `docs/archive/pythonx-adapter-design.md` §4.5 names four candidates and a measured reason each may not * work -- notably that Compose carries a composable's `$default` mask as a *declared* trailing * parameter and emits no `fn$default` bridge at all, so "call the synthetic" is not available. * The flag is carried because metadata has it and because a stub that marks 254 of `Modifier`'s @@ -197,7 +197,7 @@ class ExposedCallable( * A resolved entry in [UpcallTable] -- the thing the Python proxy caches so that no call after * the first one passes a string. * - * `docs/design/upcall-design.md` takes this from ObjC: a selector is fast because it is an interned + * `docs/design/upcall.md` takes this from ObjC: a selector is fast because it is an interned * pointer, not because a table exists. The equivalent here is that the name is resolved once * and this integer is what crosses afterwards. * diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/InstallsUpcallTable.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/InstallsUpcallTable.kt index 883a35000..358421489 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/InstallsUpcallTable.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/InstallsUpcallTable.kt @@ -31,7 +31,7 @@ package python.multiplatform.reflection * installGeneratedUpcallTable() * * `expect`/`actual` rather than an interface plus a runtime registry because the reference chain - * has to stay *static*: `docs/design/upcall-table-design.md` keeps every fragment reachable to the + * has to stay *static*: `docs/design/upcall.md` keeps every fragment reachable to the * Kotlin/Native linker by referencing it explicitly, with no `ServiceLoader` and no * `@EagerInitialization`. A generated object that nothing names is dead code the linker is free to * drop. The generated `actual` is that name, and it is generated rather than written by hand diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/PythonInternal.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/PythonInternal.kt index a239d345d..106c57d54 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/PythonInternal.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/PythonInternal.kt @@ -6,7 +6,7 @@ package python.multiplatform.reflection * library author needs is a way to say "not this one". * * Applies to a top-level function, a class (excluding the whole class and its members), or a - * member function/property. The KSP fragment generator (`docs/design/upcall-table-design.md`) is the + * member function/property. The KSP fragment generator (`docs/design/upcall.md`) is the * only reader of this annotation; it has no effect at runtime. */ @Target(AnnotationTarget.FUNCTION, AnnotationTarget.PROPERTY, AnnotationTarget.CLASS) diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/ReflectedClass.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/ReflectedClass.kt index f38f681aa..5265b967d 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/ReflectedClass.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/ReflectedClass.kt @@ -34,7 +34,7 @@ enum class ReflectedClassKind { * cycle collection. * * KSP emits one of these per exposed class alongside the [ExposedCallable] entries for its - * constructor, methods and accessors -- see `docs/design/upcall-table-design.md`, and for why + * constructor, methods and accessors -- see `docs/design/upcall.md`, and for why * [traverse] exists at all, `docs/design/object-lifetime.md`'s "Cycle collection is part of the * table's job". */ diff --git a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/UpcallTable.kt b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/UpcallTable.kt index 05b81a856..481f97ae4 100644 --- a/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/UpcallTable.kt +++ b/python-multiplatform/src/commonMain/kotlin/python/multiplatform/reflection/UpcallTable.kt @@ -5,7 +5,7 @@ package python.multiplatform.reflection * What a KSP-generated fragment implements: one object per module, holding the calls and * classes that module exposes. * - * `docs/design/upcall-table-design.md` names the generated objects `Fragment_`; this interface + * `docs/design/upcall.md` names the generated objects `Fragment_`; this interface * is the shape they are emitted into and the shape [UpcallTable] registers. `moduleName` is what * makes [UpcallTable.register] idempotent -- an aggregator that references a fragment already * installed manually must not double its entries. @@ -22,11 +22,11 @@ interface FunctionTableFragment { /** - * The runtime half of the build-time function table (ROADMAP §7, `docs/design/upcall-design.md`). + * The runtime half of the build-time function table (ROADMAP §7, `docs/design/upcall.md`). * * A name is resolved to a [CallableHandle] once; every call after that carries only the handle -- * a table index plus an epoch that rejects handles from a table that has since been reinstalled. - * `docs/design/upcall-design.md` takes this from ObjC: the win is not having a table, it is never + * `docs/design/upcall.md` takes this from ObjC: the win is not having a table, it is never * hashing and comparing a string more than once per name. * * [ClassLookup] is the companion registry for the [ReflectedClass] half of a fragment; it is diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/ComposableShapedFragment.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/ComposableShapedFragment.kt index 0c6e9f86c..79e746bac 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/ComposableShapedFragment.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/ComposableShapedFragment.kt @@ -24,7 +24,7 @@ import python.multiplatform.reflection.TypeTag * * `Text(text, modifier = ..., color = ..., fontSize = ...)` reduced to four declared parameters and * the three synthetic groups a composable's JVM signature really carries - * (`docs/design/pythonx-adapter-design.md` §5.2): + * (`docs/archive/pythonx-adapter-design.md` §5.2): * * | slot | name | tag | why | * |---|---|---|---| diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapterTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapterTest.kt index 40cf9104f..81fadf8f8 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapterTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxAdapterTest.kt @@ -15,7 +15,7 @@ import kotlin.test.assertTrue /** * `pythonx` -- the hand-written adaptation layer -- doing the four things - * `docs/design/pythonx-adapter-design.md` asks of it, against a table shaped like the walked Compose one. + * `docs/archive/pythonx-adapter-design.md` asks of it, against a table shaped like the walked Compose one. * * The four are, in the order the design puts them: * @@ -246,7 +246,7 @@ class PythonxAdapterTest { /** * The dispatcher on keyword **names**, which is the half that needs `ExposedCallable.paramNames` - * -- `docs/design/pythonx-adapter-design.md` §2.4 called the absence of those names "arithmetic", and + * -- `docs/archive/pythonx-adapter-design.md` §2.4 called the absence of those names "arithmetic", and * this is the arithmetic working. * * `horizontal=`/`vertical=` selects the two-`Dp` overload even though `padding(m, 8, 4)` would @@ -321,7 +321,7 @@ class PythonxAdapterTest { * ordinary Python method chaining with no combinator machinery. * * Both spellings of `Modifier` are exercised. The class object works through the hybrid - * descriptor `docs/design/pyi-generation-design.md` §4.3 measured (a metaclass `def` loses to the + * descriptor `docs/archive/pyi-generation-pythonic-stubs.md` §4.3 measured (a metaclass `def` loses to the * class's own MRO); the instance spelling is the ordinary one. */ @Test diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxCallableTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxCallableTest.kt index 08b6d4561..7c7492c30 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxCallableTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxCallableTest.kt @@ -17,7 +17,7 @@ import kotlin.test.assertTrue * The direction the composable work left open: **a Python callable reaching a Kotlin function-typed * parameter.** * - * `docs/design/pythonx-adapter-design.md` §6 states the starting position exactly -- "a Python callable can + * `docs/archive/pythonx-adapter-design.md` §6 states the starting position exactly -- "a Python callable can * already cross into Kotlin", as a `PyObject`, because `UpcallTrampoline.toKotlinObject` wraps * anything that is not an integer handle. What it *cannot* do is arrive as a `Function0` or a * `Function3`, which is what every container composable's `content` slot is after the Compose plugin diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxComposableTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxComposableTest.kt index fd1a7fc20..2ff8dddee 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxComposableTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxComposableTest.kt @@ -14,7 +14,7 @@ import kotlin.test.assertFails import kotlin.test.assertTrue /** - * `docs/design/pythonx-adapter-design.md` §5 from the Python end: **the `$default` mask is arithmetic + * `docs/archive/pythonx-adapter-design.md` §5 from the Python end: **the `$default` mask is arithmetic * `pythonx` does, not a branch the generator emits.** * * ### Why the mask has to live here and not in the walker diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxDefaultsTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxDefaultsTest.kt index 0fd88913f..80c0c550d 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxDefaultsTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxDefaultsTest.kt @@ -14,7 +14,7 @@ import kotlin.test.assertFails import kotlin.test.assertTrue /** - * `docs/design/pythonx-adapter-design.md` §4.5 from the Python end: **`f(x)`, not `f(x, None, None, ...)`.** + * `docs/archive/pythonx-adapter-design.md` §4.5 from the Python end: **`f(x)`, not `f(x, None, None, ...)`.** * * ### What the two halves are * @@ -84,7 +84,7 @@ class PythonxDefaultsTest { } /** - * The property `docs/design/pythonx-adapter-design.md` §4.5 says an arity-prefix scheme cannot have: + * The property `docs/archive/pythonx-adapter-design.md` §4.5 says an arity-prefix scheme cannot have: * the argument that is left out is **not** the last one. * * `padding__Dp_Dp_Dp_Dp(bottom=4)` writes one of four parameters and skips three, one of which diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxTableSourceTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxTableSourceTest.kt index ebdeef42b..c3a66fa62 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxTableSourceTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/pythonx/PythonxTableSourceTest.kt @@ -40,7 +40,7 @@ class PythonxTableSourceTest { kind = CallableKind.FUNCTION, ) { 7L } - /** One row per entry, and every field `docs/design/pythonx-adapter-design.md` §2.4 asked for. */ + /** One row per entry, and every field `docs/archive/pythonx-adapter-design.md` §2.4 asked for. */ @Test fun aWalkedExtensionRendersAsOneRowCarryingItsWholeDeclaration() { assertEquals( @@ -109,7 +109,7 @@ class PythonxTableSourceTest { /** * The layer's whole claim about itself, as a grep: **it names no Kotlin declaration.** * - * `docs/design/pythonx-adapter-design.md` §7 says the count of per-component files is the symptom to + * `docs/archive/pythonx-adapter-design.md` §7 says the count of per-component files is the symptom to * watch for, and this is that check one level finer -- a `Text`, a `Button` or a `padding` in * the hand-written source would mean a rule had been written as a special case. The two * package prefixes in the seeded map are the deliberate exception and are asserted to be the diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallPortabilityTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallPortabilityTest.kt index f9a297d56..7fd27fbbe 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallPortabilityTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallPortabilityTest.kt @@ -25,7 +25,7 @@ import kotlin.test.assertTrue * * `AsyncUpcallDeliveryTest` is `desktopTest` because delivery needs a second thread: the upcall has * to arrive from inside a coroutine while a Kotlin thread resolves the `Future`. That shape is not - * portable. What is portable is the half of `docs/design/upcall-async-design.md` §5 that the doc calls the + * portable. What is portable is the half of `docs/design/upcall.md` §5.2 that the doc calls the * fast path -- `suspend` is a signature, not a promise to suspend, and a body that never reaches a * suspension point must cross **without asyncio being involved at all**. * diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/GeneratedProxyCostTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/GeneratedProxyCostTest.kt index a3e633918..776d614a2 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/GeneratedProxyCostTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/GeneratedProxyCostTest.kt @@ -412,7 +412,7 @@ class GeneratedProxyCostTest { println( "\n--- Generated proxy cost: ${currentPlatform.name} --- " + "no proxies are installable on this target, so there is nothing to measure; " + - "see docs/design/upcall-async-design.md 12.4\n", + "see docs/design/upcall.md §3.7\n", ) return@withInterpreter } diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/OwnedResultLifetimeTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/OwnedResultLifetimeTest.kt index 989e79bd5..f2c0bd1fa 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/OwnedResultLifetimeTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/OwnedResultLifetimeTest.kt @@ -362,7 +362,7 @@ class OwnedResultLifetimeTest { println( "\n--- Owned object results: ${currentPlatform.name} --- " + "no proxies are installable on this target, so nothing owns a result here; " + - "see docs/design/upcall-async-design.md 12.4\n", + "see docs/design/upcall.md §3.7\n", ) return@withInterpreter } diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/PendingCallTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/PendingCallTest.kt index a03b207a9..676d161ca 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/PendingCallTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/PendingCallTest.kt @@ -19,7 +19,7 @@ import kotlin.test.assertSame import kotlin.test.assertTrue /** - * The half of `docs/design/upcall-async-design.md` that every candidate shares: starting a Kotlin + * The half of `docs/design/upcall.md` that every candidate shares: starting a Kotlin * coroutine from a frame that must return synchronously, and parking its outcome somewhere the * boundary can address later. * diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/ProxyBootstrap.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/ProxyBootstrap.kt index 714b859df..5fcd1362f 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/ProxyBootstrap.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/ProxyBootstrap.kt @@ -70,7 +70,7 @@ expect val publishesProxyEntryPoints: Boolean * `true` everywhere but wasmJs, where `import asyncio` **traps the wasm instance** -- killing the * Node process rather than raising a Python exception, so neither Kotlin nor Python can catch it. * `AsyncUpcallPortabilityTest` records how that was found (a `commonTest` case took the whole suite - * down with it) and `docs/design/upcall-async-design.md` §9.5 has the import-by-import measurement. + * down with it) and `docs/design/upcall.md` §5.8 has the import-by-import measurement. * * Separate from [publishesProxyEntryPoints] because the two answers came apart the moment wasmJs * grew a bootstrap. Every *synchronous* generated proxy -- constructor, method, property, static -- diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/ProxyHandleLifetimeTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/ProxyHandleLifetimeTest.kt index 92b4ee06b..ccfb41d41 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/ProxyHandleLifetimeTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/ProxyHandleLifetimeTest.kt @@ -235,7 +235,7 @@ class ProxyHandleLifetimeTest { println( "\n--- Proxy handle lifetime: ${currentPlatform.name} --- " + "no proxies are installable on this target, so no proxy lifetime exists to " + - "assert; see docs/design/upcall-async-design.md 12.4\n", + "assert; see docs/design/upcall.md §3.7\n", ) return@withInterpreter } diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/PythonProxyInstallTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/PythonProxyInstallTest.kt index 33a3ba4d0..93a23d724 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/PythonProxyInstallTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/PythonProxyInstallTest.kt @@ -24,7 +24,7 @@ import kotlin.test.assertNull import kotlin.test.assertTrue /** - * The generated Python proxy actually running -- `docs/design/upcall-async-design.md` §8.6's second gap. + * The generated Python proxy actually running -- `docs/design/upcall.md` §5.5's second gap. * * `PythonProxySourceTest` pins what [PythonProxySource.render] emits, as text, with no interpreter. * This is the other half: CPython executes that text and the proxies do what the text says. diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/PythonProxySourceTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/PythonProxySourceTest.kt index d6c9d0820..bb22a1dce 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/PythonProxySourceTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/ffi/upcall/PythonProxySourceTest.kt @@ -71,7 +71,7 @@ class PythonProxySourceTest { assertContains(source, "if hasattr(_pm_r, '__await__'):") assertContains(source, "return await _pm_r") // The whole point of the conditional await: an already-complete call must reach the caller - // as its value, not wrapped in anything. `docs/design/upcall-async-design.md` §5. + // as its value, not wrapped in anything. `docs/design/upcall.md` §5.2. assertContains(source, "return _pm_r") } diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/reflection/TestFragments.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/reflection/TestFragments.kt index 0c565e28a..4ac437839 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/reflection/TestFragments.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/reflection/TestFragments.kt @@ -8,7 +8,7 @@ package python.multiplatform.reflection * an object per module in a well-known package, entries built from lambdas rather than * `KFunction` references, classes carrying member names and a traverse function -- so that * whatever compiles against this API here will compile against generated code later. See - * `docs/design/upcall-table-design.md` §1 and `ksp-experiment/`. + * `docs/design/upcall.md` §2.1 and `ksp-experiment/`. */ /** A Kotlin class the way an exposed user class behaves: identity, state, mutation. */ diff --git a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/reflection/UpcallOverheadTest.kt b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/reflection/UpcallOverheadTest.kt index 55c2ea79a..fe674f034 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/multiplatform/reflection/UpcallOverheadTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/multiplatform/reflection/UpcallOverheadTest.kt @@ -14,7 +14,7 @@ private const val TABLE_ENTRIES = 200 * What the upcall path costs, and specifically the comparison the design rests on: a name * resolved on every call against a handle resolved once and cached. * - * `docs/design/upcall-design.md` puts per-call name passing at hundreds of nanoseconds and a cached + * `docs/design/upcall.md` puts per-call name passing at hundreds of nanoseconds and a cached * handle at 10--20 ns. **These numbers are a lower bound on the gap, not the gap.** Everything * measured here is inside Kotlin; the expensive part of the name path is what happens before * it -- `PyUnicode` to UTF-8 (an allocation) and the FFI boundary crossing, both measured diff --git a/python-multiplatform/src/commonTest/kotlin/python/native/ffi/UpcallBoundaryCostTest.kt b/python-multiplatform/src/commonTest/kotlin/python/native/ffi/UpcallBoundaryCostTest.kt index 6ccf17888..ba49d4624 100644 --- a/python-multiplatform/src/commonTest/kotlin/python/native/ffi/UpcallBoundaryCostTest.kt +++ b/python-multiplatform/src/commonTest/kotlin/python/native/ffi/UpcallBoundaryCostTest.kt @@ -125,7 +125,7 @@ class UpcallBoundaryCostTest { * Convergence needed 40 000 calls cold and 70 000 warm on desktop, and 70 000 on wasm; the * warm side is slower to settle, so it sets the requirement. 100 000 is that 70 000 with * margin, and it is checked rather than assumed -- see the equality this file's figures are - * required to hold in `docs/design/upcall-design.md`, where the same rows are quoted from a full + * required to hold in `docs/design/upcall.md`, where the same rows are quoted from a full * suite, from a suite with the proxy cost test short-circuited, and from a `--tests`-filtered * run of this class alone. * diff --git a/python-multiplatform/src/desktopMain/kotlin/python/native/ffi/UpcallStub.kt b/python-multiplatform/src/desktopMain/kotlin/python/native/ffi/UpcallStub.kt index 83914d36b..0bc5320cf 100644 --- a/python-multiplatform/src/desktopMain/kotlin/python/native/ffi/UpcallStub.kt +++ b/python-multiplatform/src/desktopMain/kotlin/python/native/ffi/UpcallStub.kt @@ -52,7 +52,7 @@ object UpcallTarget { * All of the work is in [UpcallTrampoline], which is `commonMain` -- there is nothing * desktop-specific about turning a tuple into an `Array`. What is desktop-specific is * only how the address of this method reaches Python, i.e. `Panama.createUpcallStubII_L`. - * See `docs/design/upcall-design.md` for what the other platforms need instead. + * See `docs/design/upcall.md` for what the other platforms need instead. */ @JvmStatic fun upcallInvokeWithArgs(handleRaw: Long, argsTuple: Long): Long = diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncCompletionProbeTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncCompletionProbeTest.kt index 04bebff99..74f2e5386 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncCompletionProbeTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncCompletionProbeTest.kt @@ -11,7 +11,7 @@ import kotlin.test.assertNotEquals import kotlin.test.assertTrue /** - * The measurement `docs/design/upcall-async-design.md` is built on, and the only one of its three + * The measurement `docs/design/upcall.md` is built on, and the only one of its three * candidates that can be tested without writing the mechanism first. * * The design question every async upcall convention has to answer is not "can Kotlin start work" diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallCancellationTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallCancellationTest.kt index 773b90d51..37df1dd6c 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallCancellationTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallCancellationTest.kt @@ -17,7 +17,7 @@ import kotlin.test.assertTrue /** * What happens when the Python side gives up on a call that Kotlin is still running -- - * `docs/design/upcall-async-design.md` §8.6's first open question, which that section deliberately refused + * `docs/design/upcall.md` §5.5's first open question, which that section deliberately refused * to answer because it had not been measured. * * The shape of the problem is that the two halves of a suspended call are owned by different diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallDeliveryTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallDeliveryTest.kt index 1a8bc7ed6..7d11ff04b 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallDeliveryTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallDeliveryTest.kt @@ -27,7 +27,7 @@ import kotlin.test.assertNull import kotlin.test.assertTrue /** - * The delivery half of `docs/design/upcall-async-design.md` §5: candidate (C) as the surface Python sees, + * The delivery half of `docs/design/upcall.md` §5.2: candidate (C) as the surface Python sees, * candidate (B) as the mechanism underneath it. * * `PendingCallTest` already pins the half that starts the coroutine, and `AsyncCompletionProbeTest` @@ -56,7 +56,7 @@ import kotlin.test.assertTrue * the shape the generated Python proxy has to have, and it is what makes the fast path free: * `await kotlin_fn(x)` costs a `Future` only when the Kotlin body actually suspended. It is written * out here rather than generated because Python-side module generation does not exist yet - * (`docs/design/upcall-async-design.md` §6, "Python 쪽 프록시가 `await` 를 이해하는 것"). + * (`docs/design/upcall.md` §5.4, "Python 쪽 프록시가 `await` 를 이해하는 것"). */ class AsyncUpcallDeliveryTest { @@ -92,7 +92,7 @@ class AsyncUpcallDeliveryTest { PythonTestFixture.withInterpreter { // No event loop is running on this thread, and that is the point: if the boundary // reached `asyncio.get_running_loop()` for a body that had already finished, this - // would raise instead of answering. `docs/design/upcall-async-design.md` §5 -- `suspend` is a + // would raise instead of answering. `docs/design/upcall.md` §5.2 -- `suspend` is a // signature, not a promise to suspend, and the common case must not pay for one. val raw = UpcallTrampoline.invoke(UpcallTable.resolve("async.doubleNow").raw, tuple("(21,)")) assertNotEquals(0L, raw, "the fast path returned NULL; a Python error was set instead of a value") diff --git a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallEarlyCancellationTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallEarlyCancellationTest.kt index 7ab447b96..e16c20ef0 100644 --- a/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallEarlyCancellationTest.kt +++ b/python-multiplatform/src/desktopTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallEarlyCancellationTest.kt @@ -17,7 +17,7 @@ import kotlin.test.assertNull import kotlin.test.assertTrue /** - * The half of cancellation `docs/design/upcall-async-design.md` §9.3 left undone: **early notice**. + * The half of cancellation `docs/design/upcall.md` §5.6 left undone: **early notice**. * * §9.3 established what cooperative cancellation can and cannot be here. `PendingCall.cancel()` is * a flag a body reaches through `ensureActive()`; it cannot stop a coroutine, because the diff --git a/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/ffi/upcall/AsyncioAvailabilityProbeTest.kt b/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/ffi/upcall/AsyncioAvailabilityProbeTest.kt index b5abd9a37..b5a990b46 100644 --- a/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/ffi/upcall/AsyncioAvailabilityProbeTest.kt +++ b/python-multiplatform/src/iosSimulatorArm64Test/kotlin/python/multiplatform/ffi/upcall/AsyncioAvailabilityProbeTest.kt @@ -6,8 +6,8 @@ import kotlin.test.Test import kotlin.test.assertEquals /** - * `docs/design/upcall-async-design.md` §10.6: whether `asyncio` exists on the iOS distribution has never - * been measured, only assumed absent by analogy with `_ctypes` (`docs/design/upcall-design.md` §"Can + * `docs/design/upcall.md` §8: whether `asyncio` exists on the iOS distribution has never + * been measured, only assumed absent by analogy with `_ctypes` (`docs/design/upcall.md` §"Can * Python call an address at all?"). That analogy does not hold on inspection -- the BeeWare * archive `extractIosSimulatorStdlib` unpacks *does* carry `lib-dynload/_asyncio*.so`, * `lib-dynload/_socket*.so` and `lib-dynload/select*.so`, unlike this project's `Python.xcframework` diff --git a/python-multiplatform/src/nativeMain/README.md b/python-multiplatform/src/nativeMain/README.md index c54e00d81..489b3c0d8 100644 --- a/python-multiplatform/src/nativeMain/README.md +++ b/python-multiplatform/src/nativeMain/README.md @@ -24,7 +24,7 @@ call, not a runtime transition. The upcall cost that shapes the JVM design does `ctypes.CDLL(None)` route the design doc predicted, because neither half of that survives on iOS: the `Python.framework` here has no `_ctypes` (no `lib-dynload` at all), and a `@CName` alias is never emitted into a Kotlin/Native framework or test executable — only into the androidNative -`.so`. Measurements are in `docs/design/upcall-design.md`. +`.so`. Measurements are in `docs/design/upcall.md`. So an entry point that Python must be able to call needs a `PyMethodDef` with a `staticCFunction` in `ml_meth`. Two rules come with that, and both have crashed this repo before: diff --git a/python-multiplatform/src/nativeMain/kotlin/python/native/ffi/UpcallEntry.kt b/python-multiplatform/src/nativeMain/kotlin/python/native/ffi/UpcallEntry.kt index dcf40163d..d7c232196 100644 --- a/python-multiplatform/src/nativeMain/kotlin/python/native/ffi/UpcallEntry.kt +++ b/python-multiplatform/src/nativeMain/kotlin/python/native/ffi/UpcallEntry.kt @@ -38,7 +38,7 @@ private const val METH_O = 0x0008 * `PyObject *pm_upcall_invoke(long callableHandle, PyObject *args)` -- the one C shape argument * passing needs, and the same one `PyCFunction` has once `self` takes the handle's place. * - * This is the symbol `docs/design/upcall-design.md` expects a host to reach with `dlopen(NULL)` / + * This is the symbol `docs/design/upcall.md` expects a host to reach with `dlopen(NULL)` / * `ctypes.CDLL(None)`. Whether it is *findable* that way depends entirely on which binary the * Kotlin/Native code ends up in, and that turned out to vary; see [UpcallEntry] for the three * measurements. [UpcallEntry.invokeAddress] is the route that does not depend on the link at all. @@ -167,7 +167,7 @@ private fun raiseTypeError(message: String) { * This is the shape [pmUpcallInvoke] was written to converge on, reached the way CPython reaches * any built-in function. It exists as well as the raw symbol because **the iOS distribution has * no `_ctypes`** (`Python.framework` exports `PyInit__abc` ... `PyInit_time` and nothing else, - * and carries no `lib-dynload`), so `ctypes.CDLL(None)` -- the route `docs/design/upcall-design.md` + * and carries no `lib-dynload`), so `ctypes.CDLL(None)` -- the route `docs/design/upcall.md` * names for this platform -- cannot be used there at all. A `PyMethodDef` is the only way to hand * Python a callable that lands on a C function pointer on that target, and it is also what the * generated proxy type will install, so nothing here is scaffolding for the test alone. @@ -317,7 +317,7 @@ private fun bindHandle(raw: Long): CPointer? { * * ### Two routes, because the documented one only half exists * - * `docs/design/upcall-design.md` named one route for this platform -- a `@CName` symbol picked up by + * `docs/design/upcall.md` named one route for this platform -- a `@CName` symbol picked up by * `ctypes.CDLL(None)`, on the grounds that Python and Kotlin/Native are one binary so no C glue * is needed. Checked, and it holds in one of the three binaries this project produces: * diff --git a/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallNativeDeliveryTest.kt b/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallNativeDeliveryTest.kt index 28d9243bc..0530e89bd 100644 --- a/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallNativeDeliveryTest.kt +++ b/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallNativeDeliveryTest.kt @@ -22,7 +22,7 @@ import kotlin.test.assertTrue /** * `AsyncUpcallDeliveryTest` (`desktopTest`) pinned that candidate (C)/(B) delivery works, but its * completing thread is a `java.util.concurrent`/`java.lang.Thread` -- unavailable outside the JVM - * targets, which is why that file never moved. `docs/design/upcall-async-design.md` §10.6 records the + * targets, which is why that file never moved. `docs/design/upcall.md` §8 records the * gap this closes: the delivery path itself is `commonMain` with no `expect`/`actual`, so every * target compiles the same source, but only desktop had ever been measured to *run* it. * diff --git a/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallNativeEarlyCancellationTest.kt b/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallNativeEarlyCancellationTest.kt index 6ca31edfd..bee2fa086 100644 --- a/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallNativeEarlyCancellationTest.kt +++ b/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ffi/upcall/AsyncUpcallNativeEarlyCancellationTest.kt @@ -25,7 +25,7 @@ import kotlin.test.assertTrue * `Future.cancel()` reaching `ensureActive()` before the call completes, through `_pm_cancel` -- * but its observer thread is a `java.lang.Thread`. `_pm_cancel` itself has no JVM dependency: it * is a `PyMethodDef` `UpcallEntry.publish` installs identically on every native target - * (`docs/design/upcall-async-design.md` §10.2's table lists `nativeMain` once, "iOS·androidNative + * (`docs/design/upcall.md` §5.6's table lists `nativeMain` once, "iOS·androidNative * 공용"). This is that path exercised with [NativeThread] standing in for the JVM thread, so the * `desktop` cell in §10.6's "확인하지 않은 것" list stops being the only one that says "measured". * diff --git a/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ffi/upcall/NativeThread.kt b/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ffi/upcall/NativeThread.kt index 7cab68c60..ac2ac57a5 100644 --- a/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ffi/upcall/NativeThread.kt +++ b/python-multiplatform/src/nativeTest/kotlin/python/multiplatform/ffi/upcall/NativeThread.kt @@ -18,7 +18,7 @@ import platform.posix.pthread_tVar /** * A bare `pthread_create`, not anything Kotlin-level (`Worker`, a dispatcher, ...). * - * That is deliberate. `docs/design/upcall-async-design.md` §7.1 and §10.6 leave one question open on + * That is deliberate. `docs/design/upcall.md` §8 and §10.6 leave one question open on * every target but desktop: does async delivery work when the completing thread is one the * Kotlin/Native runtime has never attached to on its own? A `Worker` launders that question -- * it is Kotlin/Native's own thread machinery, already known to interoperate with the runtime. A diff --git a/python-multiplatform/src/wasmJsMain/README.md b/python-multiplatform/src/wasmJsMain/README.md index 1e83eaa3c..f3d136f40 100644 --- a/python-multiplatform/src/wasmJsMain/README.md +++ b/python-multiplatform/src/wasmJsMain/README.md @@ -6,7 +6,8 @@ the data path. Everything below is measured. The standalone `wasm-experiment/` that first took the measurements was retired into this source set and `tools/wasm/` (it remains in git history: `git log -- -wasm-experiment`); `docs/platforms/wasm-design.md` records how they were arrived at, including the several +wasm-experiment`); `docs/platforms/wasm-design.md` holds the current decisions and numbers, and +`docs/archive/wasm-design-experiment-log.md` records how they were arrived at, including the several conclusions that were wrong before they were run. Everything the experiment proved that this file stands on is now a test in `wasmJsTest/.../emscripten/`, run against the real interpreter on every `wasmJsNodeTest`: @@ -71,7 +72,7 @@ hits `Py_FatalError: thread state ... must be current when releasing` → `abort `unreachable` opcode. What you see reported is `RuntimeError: unreachable`, three hops from the cause, and the wasm suite dies as *"process exited unexpectedly"* rather than as a red test. -It cost this repo two sections of `docs/design/upcall-async-design.md` (§9.5, §14.4) to misdiagnose as +It cost this repo two sections of `docs/design/upcall.md` (§9.5, §14.4) to misdiagnose as "wasm cannot do `asyncio`". It could not do `asyncio` because `selectors.py` calls `select.poll().poll(0)` at import time to pick its `DefaultSelector` — which is also why `import select` was fine and `import selectors` was not. §15 has the measurement. diff --git a/python-multiplatform/src/wasmJsMain/kotlin/python/native/ffi/UpcallEntry.kt b/python-multiplatform/src/wasmJsMain/kotlin/python/native/ffi/UpcallEntry.kt index 375087fea..9be4f5d3d 100644 --- a/python-multiplatform/src/wasmJsMain/kotlin/python/native/ffi/UpcallEntry.kt +++ b/python-multiplatform/src/wasmJsMain/kotlin/python/native/ffi/UpcallEntry.kt @@ -19,7 +19,7 @@ private const val METHODDEF_SIZE = 16 /** * The wasmJs half of the upcall boundary: [invokeMethod] is what CPython calls, reached the way - * `docs/design/upcall-design.md` names for this target and ROADMAP §11 already measured -- + * `docs/design/upcall.md` names for this target and ROADMAP §11 already measured -- * `@WasmExport` plus `Table.set`, 3.1 ns/call through `call_indirect`. Everything past `self`/`args` * is [UpcallTrampoline], which is `commonMain` and knows nothing about wasm. * @@ -31,7 +31,7 @@ private const val METHODDEF_SIZE = 16 * ### One export is enough for the whole bootstrap * * This used to publish nothing into Python at all: only a *bound* callable crossed, and - * `docs/design/upcall-async-design.md` §12.4 recorded that `_pm_resolve`/`_pm_invoke`/`_pm_release`/ + * `docs/design/upcall.md` §3.7 recorded that `_pm_resolve`/`_pm_invoke`/`_pm_release`/ * `_pm_cancel` would each need a **new `@WasmExport`**, which -- being honoured only in the * application's own compilation -- the library could never add. The consequence was that * `PythonProxySource.install()` refused on this target and no generated proxy existed here. @@ -52,7 +52,7 @@ private const val METHODDEF_SIZE = 16 * before. * * **What this does not buy** is the async half. `import asyncio` traps this wasm instance rather - * than raising (`AsyncUpcallPortabilityTest`, `docs/design/upcall-async-design.md` §9.5), so a proxy whose + * than raising (`AsyncUpcallPortabilityTest`, `docs/design/upcall.md` §5.8), so a proxy whose * Kotlin body genuinely suspends still cannot be awaited here. The synchronous surface -- * constructors, methods, properties, statics -- is what became reachable. * @@ -108,7 +108,7 @@ object UpcallEntry { * `_pm_resolve(name) -> handle`, `METH_VARARGS`, accepting **`str` or `bytes`**. * * Both spellings, for the reason `nativeMain`'s `pmResolveMethod` records and the ART shim - * learned the hard way (`docs/design/upcall-async-design.md` §13.2): `PythonProxySource`'s `_pm_lookup` + * learned the hard way (`docs/design/upcall.md` §3.7): `PythonProxySource`'s `_pm_lookup` * has to send `bytes` because desktop reaches its resolver through * `ctypes.CFUNCTYPE(c_long, c_char_p)`, which refuses a `str` outright -- while a `PyMethodDef` * host called by hand writes `str`. Costs one failed `PyUnicode_AsUTF8` per name, at install diff --git a/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/ffi/WasmSelectorsImportTest.kt b/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/ffi/WasmSelectorsImportTest.kt index c296bcbd5..f96f1b57c 100644 --- a/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/ffi/WasmSelectorsImportTest.kt +++ b/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/ffi/WasmSelectorsImportTest.kt @@ -6,7 +6,7 @@ import kotlin.test.assertTrue /** * `import selectors` used to take the whole Node process down on this target, and `import asyncio` - * with it. docs/design/upcall-async-design.md 9.5 recorded that as `RuntimeError: unreachable` and left + * with it. docs/design/upcall.md 9.5 recorded that as `RuntimeError: unreachable` and left * the cause open; 15 has it now, and it was never about `selectors` or about wasm. * * Emscripten feature-detects JSPI at runtime and wraps only `main` in `WebAssembly.promising`. diff --git a/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/ffi/upcall/ProxyBootstrap.wasmJs.kt b/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/ffi/upcall/ProxyBootstrap.wasmJs.kt index 77a536784..69a830a1a 100644 --- a/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/ffi/upcall/ProxyBootstrap.wasmJs.kt +++ b/python-multiplatform/src/wasmJsTest/kotlin/python/multiplatform/ffi/upcall/ProxyBootstrap.wasmJs.kt @@ -13,7 +13,7 @@ actual val publishesProxyEntryPoints: Boolean = true /** * Still `false`, but **no longer for the reason §9.5 gave**. `import asyncio` does not trap any - * more: `docs/design/upcall-async-design.md` §15 found the trap was Emscripten's *runtime* JSPI detection + * more: `docs/design/upcall.md` §5.8 found the trap was Emscripten's *runtime* JSPI detection * meeting a boot that never calls `main`, fixed it in `cpython.mjs`, and `WasmSelectorsImportTest` * now watches `selectors` and `asyncio` import cleanly on this target. * diff --git a/sample/src/commonMain/kotlin/org/thisisthepy/python/multiplatform/demo/ui/App.kt b/sample/src/commonMain/kotlin/org/thisisthepy/python/multiplatform/demo/ui/App.kt index 80ce6e8c9..a920e8a48 100644 --- a/sample/src/commonMain/kotlin/org/thisisthepy/python/multiplatform/demo/ui/App.kt +++ b/sample/src/commonMain/kotlin/org/thisisthepy/python/multiplatform/demo/ui/App.kt @@ -175,7 +175,7 @@ private fun AwaitSection() { DemoCard("7 — await over a suspend fun") { // Neither button below may run on wasmJs: PythonDemo.awaitFastPath() and // PythonDemo.awaitSuspending() both reach `import asyncio`, which traps this wasm - // instance rather than raising (docs/design/upcall-async-design.md §9.5) -- there would be no + // instance rather than raising (docs/design/upcall.md §5.8) -- there would be no // process left to show an error in. Gated here, before either is ever called, rather than // inside them: this is the one section the boundary itself cannot recover from. if (currentPlatform.platformType == PlatformType.Wasm) { diff --git a/sample/src/desktopMain/kotlin/org/thisisthepy/python/multiplatform/demo/NativeImageMain.kt b/sample/src/desktopMain/kotlin/org/thisisthepy/python/multiplatform/demo/NativeImageMain.kt index e3328610d..2cdb694e6 100644 --- a/sample/src/desktopMain/kotlin/org/thisisthepy/python/multiplatform/demo/NativeImageMain.kt +++ b/sample/src/desktopMain/kotlin/org/thisisthepy/python/multiplatform/demo/NativeImageMain.kt @@ -55,7 +55,7 @@ fun main() { println("KOTLIN: args stub @ 0x${invokeWithArgsAddr.toString(16)}") // Python resolves the entry to a handle by name exactly once, then calls back through that - // handle -- the ObjC-selector-cache shape docs/design/upcall-design.md argues for, reached here via + // handle -- the ObjC-selector-cache shape docs/design/upcall.md argues for, reached here via // ctypes function pointers instead of a generated proxy type. // Not a tty under Gradle/native-image, so sys.stdout is block-buffered: without an explicit // flush before the process exits (there is no Py_Finalize call on this path), every print() diff --git a/sample/src/desktopMain/kotlin/org/thisisthepy/python/multiplatform/demo/bindings/UpcallDemo.desktop.kt b/sample/src/desktopMain/kotlin/org/thisisthepy/python/multiplatform/demo/bindings/UpcallDemo.desktop.kt index eea35b80b..41d6cf60e 100644 --- a/sample/src/desktopMain/kotlin/org/thisisthepy/python/multiplatform/demo/bindings/UpcallDemo.desktop.kt +++ b/sample/src/desktopMain/kotlin/org/thisisthepy/python/multiplatform/demo/bindings/UpcallDemo.desktop.kt @@ -11,7 +11,7 @@ private const val PY_EVAL_INPUT: Int = 258 /** * Resolve once, invoke many. Both handles are computed here and live on in `__main__`, so every * call afterwards passes an integer and never a string again -- which is the whole argument of - * `docs/design/upcall-design.md` (a selector is fast because it is interned, not because a table + * `docs/design/upcall.md` (a selector is fast because it is interned, not because a table * exists). * * `ctypes` stands in for the generated proxy type a finished binder would install. Two stub diff --git a/sample/src/wasmJsMain/kotlin/org/thisisthepy/python/multiplatform/demo/bindings/ProxyDemo.wasmJs.kt b/sample/src/wasmJsMain/kotlin/org/thisisthepy/python/multiplatform/demo/bindings/ProxyDemo.wasmJs.kt index 53f5ea409..faad0d13f 100644 --- a/sample/src/wasmJsMain/kotlin/org/thisisthepy/python/multiplatform/demo/bindings/ProxyDemo.wasmJs.kt +++ b/sample/src/wasmJsMain/kotlin/org/thisisthepy/python/multiplatform/demo/bindings/ProxyDemo.wasmJs.kt @@ -37,7 +37,7 @@ actual fun installPythonProxies(): String = try { * Not wired on wasmJs, and not fixable by starting a thread the way desktop's and Android's do. * * `import asyncio` traps this wasm instance rather than raising - * (`AsyncUpcallPortabilityTest`, `docs/design/upcall-async-design.md` §9.5, and + * (`AsyncUpcallPortabilityTest`, `docs/design/upcall.md` §5.8, and * `ProxyBootstrap.wasmJs.kt`'s `proxyBootstrapSupportsAsyncio = false`) -- the process goes down * with it, not just the call. So this function must never run `import asyncio`, and neither may * [awaitFastPathDemo] (shared, `ProxyDemo.kt`) be reached from this target: `ui/App.kt` gates @@ -46,5 +46,5 @@ actual fun installPythonProxies(): String = try { */ actual fun awaitSuspendingDemo(): String = "not available on wasmJs: `import asyncio` traps this wasm instance instead of raising " + - "(docs/design/upcall-async-design.md §9.5), so this demo never calls it. The synchronous surface " + + "(docs/design/upcall.md §5.8), so this demo never calls it. The synchronous surface " + "above -- sections 5 and 6 -- is what `4472f83a` bought back on this target; `await` is not."