diff --git a/ksp-fixtures/app/build.gradle.kts b/ksp-fixtures/app/build.gradle.kts index f2503a2fb..fd3257b5f 100644 --- a/ksp-fixtures/app/build.gradle.kts +++ b/ksp-fixtures/app/build.gradle.kts @@ -57,3 +57,112 @@ pythonBindings { role.set("app") processor.set(projects.pythonMultiplatformKsp) } + +// --------------------------------------------------------------------------------------------- +// TypedPython: compiled code calling Kotlin through the binder (issue #45, `TypedPythonKotlinCallTest`) +// --------------------------------------------------------------------------------------------- +// +// `buildTypedPythonBridge` compiles `src/desktopTest/typedpython/tp_kotlin_bridge.py` with the +// TypedPython pipeline (`python-multiplatform-ksp/src/main/python/typedpython`) into an extension +// for the CPython this fixture *embeds*, not the host's: +// +// - headers: `python-multiplatform`'s `cpythonIncludeDirectories` provider +// (`docs/platforms/python-version-acquisition.md` §7), for this host's macOS target and the +// flavour `-PpythonFreeThreaded` selects. The provider carries the `downloadPython_*` task. +// - extension suffix: `typedpython/build_bridge.py` reads `EXT_SUFFIX` from the `_sysconfigdata_*.py` +// of that same extracted tree and cross-checks `Py_GIL_DISABLED` against its `pyconfig.h`. +// +// The pipeline itself runs on a *host* Python that has Pyrefly 1.3.2 (the gate needs it): +// `-PtypedpythonPython=`, else `python-multiplatform-ksp/.venv/bin/python` when that +// exists. Without one -- or on a host other than macOS, where the JVM does not load libpython with +// global symbols and a `dynamic_lookup` extension could not resolve the C API -- the task is skipped +// and the test reports itself skipped with the reason. When the task does run, any failure fails the +// build: a gate error, a function left interpreted, a C compiler error. +// +// The provider is reached by name and reflection, not `getByType()`: this +// project and `:python-multiplatform` each apply the bindings plugin in their own `plugins {}` block, +// so each build script may see its own copy of the plugin's classes, and a typed lookup across the +// two can fail with "extension of type ... does not exist" even though it is registered. +evaluationDependsOn(":python-multiplatform") + +val typedPythonHost: File? = (findProperty("typedpythonPython") as String?)?.let { File(it) } + ?: rootDir.resolve("python-multiplatform-ksp/.venv/bin/python").takeIf { it.isFile } + +val typedPythonMacTarget: String? = + if (!System.getProperty("os.name").orEmpty().startsWith("Mac")) null + else when (System.getProperty("os.arch")) { + "aarch64", "arm64" -> "macos-aarch64" + "x86_64", "amd64" -> "macos-x86_64" + else -> null + } + +val typedPythonSkipReason: String? = when { + typedPythonMacTarget == null -> + "host ${System.getProperty("os.name")}/${System.getProperty("os.arch")} is not a macOS desktop target" + typedPythonHost == null -> + "no host Python with pyrefly 1.3.2: pass -PtypedpythonPython= or create " + + "python-multiplatform-ksp/.venv (pyproject.toml there pins pyrefly)" + typedPythonHost?.isFile != true -> "-PtypedpythonPython=$typedPythonHost does not exist" + else -> null +} + +@Suppress("UNCHECKED_CAST") +val typedPythonIncludeDir: Provider? = typedPythonMacTarget?.let { target -> + val includes = project(":python-multiplatform").extensions.getByName("cpythonIncludeDirectories") + val flavourClass = includes.javaClass.classLoader.loadClass("python.multiplatform.gradle.CPythonFlavour") + val freeThreaded = (findProperty("pythonFreeThreaded") as String?)?.toBoolean() ?: false + val flavour = flavourClass.enumConstants.single { (it as Enum<*>).name == if (freeThreaded) "FREE_THREADED" else "GIL" } + includes.javaClass.getMethod("includeDir", String::class.java, flavourClass) + .invoke(includes, target, flavour) as Provider +} + +val typedPythonBridgeDir = layout.buildDirectory.dir("typedpython/bridge") + +val buildTypedPythonBridge by tasks.registering(Exec::class) { + group = "verification" + description = "Compiles tp_kotlin_bridge.py with TypedPython against the embedded CPython's headers (#45)." + + val compilerDir = rootDir.resolve("python-multiplatform-ksp/src/main/python") + val sourceDir = file("src/desktopTest/typedpython") + val script = file("typedpython/build_bridge.py") + val outDir = typedPythonBridgeDir + val tmpDir = layout.buildDirectory.dir("typedpython/tmp") + val include = typedPythonIncludeDir + val skip = typedPythonSkipReason + + onlyIf("TypedPython bridge: ${skip ?: "enabled"}") { skip == null } + inputs.files(fileTree(compilerDir) { exclude("**/__pycache__/**") }).withPropertyName("compiler") + inputs.files(fileTree(sourceDir) { exclude("**/__pycache__/**") }).withPropertyName("sources") + inputs.file(script).withPropertyName("script") + if (include != null && skip == null) inputs.dir(include).withPropertyName("cpythonInclude") + inputs.property("host", typedPythonHost?.absolutePath ?: "") + outputs.dir(outDir) + + executable = typedPythonHost?.absolutePath ?: "python3" + // Pyrefly and the gate write temporary files; keep them inside this build directory (AGENTS.md §2). + environment("TMPDIR", tmpDir.get().asFile.absolutePath) + environment("PYTHONDONTWRITEBYTECODE", "1") + argumentProviders.add(CommandLineArgumentProvider { + listOf( + script.absolutePath, + compilerDir.absolutePath, + File(sourceDir, "tp_kotlin_bridge.py").absolutePath, + include!!.get().asFile.absolutePath, + outDir.get().asFile.absolutePath, + ) + }) + doFirst { tmpDir.get().asFile.mkdirs() } +} + +tasks.named("desktopTest") { + dependsOn(buildTypedPythonBridge) + if (typedPythonSkipReason == null) { + inputs.dir(typedPythonBridgeDir).withPropertyName("typedPythonBridge") + systemProperty( + "typedpython.bridge.properties", + typedPythonBridgeDir.get().file("bridge.properties").asFile.absolutePath, + ) + } else { + systemProperty("typedpython.bridge.skip", typedPythonSkipReason) + } +} diff --git a/ksp-fixtures/app/src/commonMain/kotlin/fixture/app/TypedPythonBridge.kt b/ksp-fixtures/app/src/commonMain/kotlin/fixture/app/TypedPythonBridge.kt new file mode 100644 index 000000000..1a856b27d --- /dev/null +++ b/ksp-fixtures/app/src/commonMain/kotlin/fixture/app/TypedPythonBridge.kt @@ -0,0 +1,90 @@ +package fixture.app + +import python.multiplatform.ffi.PyObject +import python.multiplatform.reflection.PythonInternal + +/* + * The Kotlin surface compiled TypedPython code calls through the binder (issue #45). The Python + * side is `src/desktopTest/typedpython/tp_kotlin_bridge.py`; the test is + * `TypedPythonKotlinCallTest`. Every call records itself in [TpBridgeLedger], so the test can + * compare what Kotlin saw from the interpreted run with what it saw from the compiled one. + */ + +/** + * A Kotlin object Python creates through its constructor, receives as a typed `TpSample` proxy, + * calls a method on, and hands back ([tpAccumulate]). + * + * Created by calling the class, not through a factory function: a Kotlin function returning a + * Kotlin object reaches Python as a generic `_PmObject` owner, not as the class's rendered proxy, + * so its methods are not reachable (`PythonProxySource`'s "What is still not rendered" table; the + * first run of this test hit it). The rendered class's own constructor is the path that gives a + * typed proxy today. + */ +class TpSample(val index: Long, val weight: Double) { + init { + TpBridgeLedger.samples += 1 + } + + fun weigh(factor: Double): Double { + TpBridgeLedger.weighs += 1 + return weight * factor + index + } +} + +/** Takes the proxy back (a parameter declared as a Kotlin class, so the binder unwraps it to the object). */ +fun tpAccumulate(sample: TpSample, amount: Double): Double { + TpBridgeLedger.accumulates += 1 + TpBridgeLedger.received += amount + TpBridgeLedger.indexSum += sample.index + return amount / (1.0 + sample.weight) +} + +/** Throws once [value] passes [limit]; the binder turns the throw into a Python `RuntimeError(message)`. */ +fun tpCheck(value: Double, limit: Double): Double { + TpBridgeLedger.checks += 1 + if (value > limit) throw IllegalStateException("tpCheck: $value exceeds $limit") + return value +} + +/** + * Receives a Python object as a [PyObject]. The trampoline wraps it with a reference of its own + * (`borrowed = true`), which the wrapper's cleaner gives back once the Kotlin collector finds the + * wrapper unreachable -- so the token's `sys.getrefcount` is only meaningful after a JVM collection + * and the cleaner have run (`TypedPythonKotlinCallTest.settledCounts`). + * + * Deliberately **not** closed here: closing a `PyObject` argument inside an upcall releases its + * reference twice and crashed the test JVM (`Py_XDECREF` on the cleaner thread) -- binder issue #98. + */ +fun tpTouch(token: PyObject?): Long { + TpBridgeLedger.touches += 1 + return if (token == null) 0L else 1L +} + +/** What Kotlin observed. Not exposed to Python: only the test reads and resets it. */ +@PythonInternal +object TpBridgeLedger { + var samples: Long = 0 + var weighs: Long = 0 + var accumulates: Long = 0 + var checks: Long = 0 + var touches: Long = 0 + + /** Sum of every `amount` [tpAccumulate] received -- a Kotlin-side effect that depends on Python's float math. */ + var received: Double = 0.0 + var indexSum: Long = 0 + + fun reset() { + samples = 0 + weighs = 0 + accumulates = 0 + checks = 0 + touches = 0 + received = 0.0 + indexSum = 0 + } + + /** Every field, `received` as its exact bits, in one comparable string. */ + fun snapshot(): String = + "samples=$samples weighs=$weighs accumulates=$accumulates checks=$checks touches=$touches " + + "received=$received (bits ${received.toRawBits().toULong().toString(16)}) indexSum=$indexSum" +} diff --git a/ksp-fixtures/app/src/desktopTest/kotlin/fixture/app/TypedPythonKotlinCallTest.kt b/ksp-fixtures/app/src/desktopTest/kotlin/fixture/app/TypedPythonKotlinCallTest.kt new file mode 100644 index 000000000..a1d4eec96 --- /dev/null +++ b/ksp-fixtures/app/src/desktopTest/kotlin/fixture/app/TypedPythonKotlinCallTest.kt @@ -0,0 +1,356 @@ +package fixture.app + +import org.junit.Assume.assumeTrue +import python.multiplatform.ffi.Python3 +import python.multiplatform.ffi.upcall.PythonProxySource +import python.multiplatform.ffi.upcall.UpcallBootstrap +import python.multiplatform.generated.FunctionTable +import python.multiplatform.reflection.HandleTable +import python.multiplatform.reflection.UpcallTable +import java.io.File +import java.util.Properties +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * Issue #45: compiled TypedPython code calls Kotlin through the binder, inside the embedded + * runtime, and behaves exactly as the same code interpreted. + * + * `src/desktopTest/typedpython/tp_kotlin_bridge.py` is loaded twice into this JVM's interpreter: + * as source (`_tpi`, interpreted) and as the extension `buildTypedPythonBridge` compiled from it + * against the embedded CPython's headers and extension suffix (`_tpc`, compiled). Its functions call + * the Kotlin in `commonMain/.../TypedPythonBridge.kt` through the KSP-generated binder -- the Kotlin + * constructor (a typed, owned `TpSample` proxy), a method on that proxy, the proxy passed back to a + * Kotlin function, a Python object passed to a `PyObject` parameter, and a Kotlin throw. + * + * The four done criteria, one test each, plus a premise test that the comparison is not vacuous + * (the compiled module really is compiled, for this interpreter): + * + * | criterion | test | + * |---|---| + * | equal results | [compiledAndInterpretedRunsReturnTheSameFloatAndKotlinSeesTheSameCalls] | + * | equal Kotlin-side call counts | the same test, through [TpBridgeLedger] | + * | a Kotlin exception has the same Python type and message | [aKotlinExceptionReachesTheCompiledCallerAsTheSamePythonException] | + * | no reference leaks across 10,000 calls | [tenThousandCompiledCallsLeakNoReferenceAndNoKotlinHandle] | + * + * **Skipped, not failed,** when the build could not compile the extension: no host Python with + * Pyrefly 1.3.2 (`-PtypedpythonPython=`, default `python-multiplatform-ksp/.venv/bin/python`) + * or a host other than macOS. The skip reason names which. When the build *did* try and failed, the + * build fails -- `buildTypedPythonBridge` is a dependency of `desktopTest`. + */ +class TypedPythonKotlinCallTest { + + private lateinit var bridge: Properties + + @BeforeTest + fun install() { + val skip = System.getProperty("typedpython.bridge.skip") + assumeTrue("TypedPython bridge extension not built: $skip", skip == null) + val propertiesPath = System.getProperty("typedpython.bridge.properties") + assumeTrue( + "TypedPython bridge extension not built: system property typedpython.bridge.properties is unset " + + "(run through :ksp-fixtures:app:desktopTest)", + propertiesPath != null, + ) + val file = File(propertiesPath!!) + // Present-but-missing is a build defect, not a skip: the task ran and wrote nothing. + assertTrue(file.isFile, "buildTypedPythonBridge ran but $file does not exist") + bridge = Properties().apply { file.inputStream().use { load(it) } } + + Python3.initialize(silent = true) + UpcallTable.clear() + HandleTable.releaseAll() + UpcallTable.install(FunctionTable.fragments) + check(UpcallBootstrap.publishToGlobals()) { "UpcallBootstrap.publishToGlobals() failed" } + PythonProxySource.install() + TpBridgeLedger.reset() + + // Both copies are loaded after the binder is installed: the module body runs + // `from fixture.app import ...`, which must find the proxies this install just rendered. + // A fresh copy per test, because `UpcallTable.clear()` above invalidates the handles the + // previous test's proxies were rendered with. + Python3.exec( + """ + import importlib.machinery as _tp_machinery + import importlib.util as _tp_util + + def _tp_load_pair(source, extension, name): + spec = _tp_util.spec_from_file_location(name + '_interpreted', source) + interpreted = _tp_util.module_from_spec(spec) + spec.loader.exec_module(interpreted) + # The extension's PyInit_ is named after the source stem, so it is loaded under it. + loader = _tp_machinery.ExtensionFileLoader(name, extension) + spec = _tp_util.spec_from_file_location(name, extension, loader=loader) + compiled = _tp_util.module_from_spec(spec) + loader.exec_module(compiled) + return interpreted, compiled + + _tpi, _tpc = _tp_load_pair(${py(prop("source"))}, ${py(prop("extension"))}, ${py(prop("module"))}) + del _tp_load_pair, _tp_machinery, _tp_util + """.trimIndent(), + ) + } + + @AfterTest + fun cleanup() { + if (!::bridge.isInitialized) return + Python3.exec( + "import gc\n" + + "for _n in ('_tpi', '_tpc', '_tp_token', '_tp_r_i', '_tp_r_c', '_tp_e_i', '_tp_e_c'):\n" + + " globals().pop(_n, None)\n" + + "gc.collect()", + ) + UpcallTable.clear() + HandleTable.releaseAll() + TpBridgeLedger.reset() + } + + /** + * Premise for everything below. If any line here fails, the other tests would be comparing the + * interpreter with itself, or loading an extension built for a different CPython. + */ + @Test + fun theCompiledModuleIsCompiledForTheInterpreterItRunsIn() { + Python3.exec( + """ + import importlib.machinery, sys, sysconfig, types + _v = '%d.%d.%d' % sys.version_info[:3] + assert _v == ${py(prop("pythonVersion"))}, ( + 'built against headers of ' + ${py(prop("pythonVersion"))} + ' but running in ' + _v) + assert ${py(prop("extSuffix"))} in importlib.machinery.EXTENSION_SUFFIXES, ( + 'extension suffix ' + ${py(prop("extSuffix"))} + ' is not one this interpreter imports: ' + + repr(importlib.machinery.EXTENSION_SUFFIXES)) + _gil_disabled = bool(sysconfig.get_config_var('Py_GIL_DISABLED') or 0) + assert _gil_disabled == ${pyBool(prop("gilDisabled"))}, ( + 'flavour mismatch: extension built with Py_GIL_DISABLED=' + ${py(prop("gilDisabled"))} + + ', interpreter has ' + repr(_gil_disabled)) + _names = ${py(prop("compiledFunctions"))}.split(',') + assert sorted(_names) == ['checked', 'drive'], _names + for _n in _names: + assert type(getattr(_tpc, _n)) is types.BuiltinFunctionType, (_n, type(getattr(_tpc, _n))) + assert type(getattr(_tpi, _n)) is types.FunctionType, (_n, type(getattr(_tpi, _n))) + assert sorted(_tpc.__typedpython_interpreted__) == ['checked', 'drive'], _tpc.__typedpython_interpreted__ + assert _tpc.__typedpython_deopts__ == 0 + # Calling the Kotlin class gives its typed proxy -- with the method the module calls -- + # not a bare handle integer or a generic owner. + from fixture.app import TpSample + _s = TpSample(3, 0.5) + assert type(_s) is TpSample, type(_s) + assert _s.index == 3 and _s.weight == 0.5 and _s.weigh(2.0) == 4.0 + del _s, _n, _names, _v, _gil_disabled, TpSample + """.trimIndent(), + ) + } + + @Test + fun compiledAndInterpretedRunsReturnTheSameFloatAndKotlinSeesTheSameCalls() { + Python3.exec("_tp_token = object()\n_tp_r_i = _tpi.drive(200, 0.25, _tp_token)") + val interpreted = TpBridgeLedger.snapshot() + // Not vacuous: every Kotlin entry point was reached, once per step. + assertEquals(200L, TpBridgeLedger.samples, interpreted) + assertEquals(200L, TpBridgeLedger.weighs, interpreted) + assertEquals(200L, TpBridgeLedger.accumulates, interpreted) + assertEquals(200L, TpBridgeLedger.touches, interpreted) + assertEquals((0L until 200L).sum(), TpBridgeLedger.indexSum, interpreted) + + TpBridgeLedger.reset() + Python3.exec("_tp_r_c = _tpc.drive(200, 0.25, _tp_token)") + val compiled = TpBridgeLedger.snapshot() + + assertEquals(interpreted, compiled, "Kotlin saw different calls or values from the compiled run") + Python3.exec( + """ + assert type(_tp_r_c) is float and type(_tp_r_i) is float, (type(_tp_r_i), type(_tp_r_c)) + assert _tp_r_c.hex() == _tp_r_i.hex(), ( + 'compiled ' + _tp_r_c.hex() + ' != interpreted ' + _tp_r_i.hex()) + assert _tpc.__typedpython_deopts__ == 0, _tpc.__typedpython_deopts__ + """.trimIndent(), + ) + } + + @Test + fun aKotlinExceptionReachesTheCompiledCallerAsTheSamePythonException() { + Python3.exec( + """ + def _tp_catch(f): + try: + f(100, 40.0) + except BaseException as e: + return type(e), str(e) + raise AssertionError('tpCheck never threw: the limit is never passed') + _tp_e_i = _tp_catch(_tpi.checked) + """.trimIndent(), + ) + val interpreted = TpBridgeLedger.snapshot() + assertTrue(TpBridgeLedger.checks > 1, "the throw must come after some successful calls: $interpreted") + + TpBridgeLedger.reset() + Python3.exec("_tp_e_c = _tp_catch(_tpc.checked)\ndel _tp_catch") + val compiled = TpBridgeLedger.snapshot() + + assertEquals(interpreted, compiled, "the compiled caller stopped at a different call than the interpreted one") + Python3.exec( + """ + assert _tp_e_i[0] is RuntimeError, _tp_e_i + assert _tp_e_i[1].startswith('tpCheck: '), _tp_e_i + assert _tp_e_c == _tp_e_i, 'compiled raised ' + repr(_tp_e_c) + ', interpreted raised ' + repr(_tp_e_i) + # And the path that does not throw agrees too. + assert _tpc.checked(5, 1e9).hex() == _tpi.checked(5, 1e9).hex() + assert _tpc.__typedpython_deopts__ == 0, _tpc.__typedpython_deopts__ + """.trimIndent(), + ) + } + + /** + * 10,000 calls of each compiled function -- `drive` (3 Kotlin objects created and released per + * call) and `checked` on its throwing path -- must leave every reference count and the Kotlin + * handle table where they were. + * + * Watched: the token passed to Kotlin on every step, the binder proxies the compiled code reads + * as globals on every step (a missed release of a `Global` read shows here), and + * [HandleTable.liveCount] (a proxy never deallocated keeps its Kotlin object rooted). The same + * loop runs interpreted first as a control, so a leak in the binder itself is reported as such and + * not blamed on the compiled code. + * + * **The token's count is read only after the Kotlin side has given its references back.** The + * binder wraps the token in a `PyObject` that takes a reference of its own, and that reference + * returns only when the JVM collects the wrapper and its cleaner runs (`tpTouch` cannot close it: + * #98). Read straight after the loop, the interpreted control showed `{'token': (10, 17350)}`. + * [settledCounts] therefore drives `System.gc()` / `System.runFinalization()` between reads -- + * with the GIL released, so the cleaner thread can take it -- until the counts reach the baseline + * or stop moving, within a bounded time. Same idea as `CycleCollectionTest.settleJvmFinalisation`; + * the library's own `ReleaseCounter` is `internal` and not reachable from this module. + * + * Free-threaded CPython defers reference counting of functions and classes, so `sys.getrefcount` + * on those says nothing there; on that flavour only the token and the handle table are compared. + */ + @Test + fun tenThousandCompiledCallsLeakNoReferenceAndNoKotlinHandle() { + // Blocked by python-multiplatform#98: when the cleaner releases the wrappers that 10,000 + // upcalls leave behind, the binder releases one reference twice and the JVM dies (SIGSEGV in + // Py_XDECREF on Cleaner-0) -- on the interpreted control as much as on the compiled loop. + // Re-enable with -Dtypedpython.leakTest=true once #98 is fixed. + assumeTrue( + "blocked by python-multiplatform#98 (binder double release on the cleaner path)", + System.getProperty("typedpython.leakTest") == "true", + ) + Python3.exec( + """ + import gc, sys, sysconfig + import fixture.app as _tp_app + _tp_token = object() + _tp_watched = {'token': _tp_token} + if not (sysconfig.get_config_var('Py_GIL_DISABLED') or 0): + for _n in ('TpSample', 'tpAccumulate', 'tpCheck', 'tpTouch'): + _tp_watched[_n] = getattr(_tp_app, _n) + _tp_watched['drive'] = _tpc.drive + _tp_watched['checked'] = _tpc.checked + del _tp_app + + def _tp_counts(): + gc.collect() + return repr(sorted((k, sys.getrefcount(v)) for k, v in _tp_watched.items())) + + def _tp_loop(module, n): + for _ in range(n): + module.drive(3, 0.25, _tp_token) + try: + module.checked(100, 40.0) + except RuntimeError: + pass + + _tp_loop(_tpi, 1) + _tp_loop(_tpc, 1) + """.trimIndent(), + ) + // The baseline has no target to reach, so it is "stopped moving" -- with a floor of rounds so + // the warm-up's pending wrappers are collected before it is taken. + val before = settledCounts(target = null) + val baseline = HandleTable.liveCount + TpBridgeLedger.reset() + + Python3.exec("_tp_loop(_tpi, 10000)") + assertEquals( + before, settledCounts(target = before), + "control: the INTERPRETED loop leaks references even after the Kotlin side has released, " + + "so the binder does (sorted (name, sys.getrefcount) pairs)", + ) + assertEquals(baseline, HandleTable.liveCount, "control: the interpreted loop left Kotlin objects rooted") + val interpreted = TpBridgeLedger.snapshot() + assertEquals(30_000L, TpBridgeLedger.samples, interpreted) + assertEquals(10_000L * 3, TpBridgeLedger.touches, interpreted) + + TpBridgeLedger.reset() + Python3.exec("_tp_loop(_tpc, 10000)\nassert _tpc.__typedpython_deopts__ == 0, _tpc.__typedpython_deopts__") + assertEquals( + before, settledCounts(target = before), + "the COMPILED loop leaks references (sorted (name, sys.getrefcount) pairs; the interpreted " + + "control above settled back to the baseline)", + ) + assertEquals(baseline, HandleTable.liveCount, "the compiled loop left Kotlin objects rooted") + assertEquals(interpreted, TpBridgeLedger.snapshot(), "Kotlin saw different calls from the compiled loop") + + Python3.exec( + "for _n in ('_tp_watched', '_tp_counts', '_tp_loop', '_n'):\n" + + " globals().pop(_n, None)", + ) + } + + /** + * The watched reference counts once the JVM has collected the `PyObject` wrappers the binder made + * and their cleaners have handed the references back. + * + * Called with this thread **not** holding the GIL (between `Python3.exec` calls), so the cleaner + * thread can take it. Each round: `System.gc()`, `System.runFinalization()`, a short sleep, then one + * read. Returns as soon as the reading equals [target]; with no target, once it has not moved for + * [STABLE_ROUNDS] reads after at least [MIN_ROUNDS]. Gives up after [SETTLE_TIMEOUT_MS] and returns + * the last reading, so the caller's assertion shows where it stopped. + */ + private fun settledCounts(target: String?): String { + val deadline = System.currentTimeMillis() + SETTLE_TIMEOUT_MS + var last = readCounts() + var stable = 0 + var rounds = 0 + while (System.currentTimeMillis() < deadline) { + if (target != null && last == target) return last + System.gc() + System.runFinalization() + Thread.sleep(20) + val now = readCounts() + rounds++ + if (now == last) stable++ else stable = 0 + last = now + if (target == null && rounds >= MIN_ROUNDS && stable >= STABLE_ROUNDS) return last + } + return last + } + + private fun readCounts(): String = + Python3.import("__main__").getAttr("__dict__").let { globals -> + Python3.eval("_tp_counts()", PY_EVAL_INPUT, globals, globals).toString() + } + + private fun prop(key: String): String = + bridge.getProperty(key) ?: error("bridge.properties has no '$key': $bridge") + + /** A Python string literal for [s]. */ + private fun py(s: String): String = + "'" + s.replace("\\", "\\\\").replace("'", "\\'").replace("\n", "\\n") + "'" + + private fun pyBool(s: String): String = when (s) { + "true" -> "True" + "false" -> "False" + else -> error("not a boolean: $s") + } + + private companion object { + /** `Py_eval_input`: [Python3.eval]'s start symbol for a single expression. */ + const val PY_EVAL_INPUT = 258 + const val SETTLE_TIMEOUT_MS = 30_000L + const val MIN_ROUNDS = 10 + const val STABLE_ROUNDS = 5 + } +} diff --git a/ksp-fixtures/app/src/desktopTest/typedpython/fixture/__init__.pyi b/ksp-fixtures/app/src/desktopTest/typedpython/fixture/__init__.pyi new file mode 100644 index 000000000..e69de29bb diff --git a/ksp-fixtures/app/src/desktopTest/typedpython/fixture/app/__init__.pyi b/ksp-fixtures/app/src/desktopTest/typedpython/fixture/app/__init__.pyi new file mode 100644 index 000000000..267595828 --- /dev/null +++ b/ksp-fixtures/app/src/desktopTest/typedpython/fixture/app/__init__.pyi @@ -0,0 +1,16 @@ +# Type-checking stub for the TypedPython gate only (issue #45). It mirrors the part of +# `ksp-fixtures/app/src/commonMain/kotlin/fixture/app/TypedPythonBridge.kt` that +# `tp_kotlin_bridge.py` uses. Kotlin `Long` is `int`, `Double` is `float`, `PyObject?` is `object`. +# If this drifts from the Kotlin, the gate still passes and the run-time test fails at the call. + +class TpSample: + def __init__(self, index: int, weight: float) -> None: ... + @property + def index(self) -> int: ... + @property + def weight(self) -> float: ... + def weigh(self, factor: float) -> float: ... + +def tpAccumulate(sample: TpSample, amount: float) -> float: ... +def tpCheck(value: float, limit: float) -> float: ... +def tpTouch(token: object) -> int: ... diff --git a/ksp-fixtures/app/src/desktopTest/typedpython/tp_kotlin_bridge.py b/ksp-fixtures/app/src/desktopTest/typedpython/tp_kotlin_bridge.py new file mode 100644 index 000000000..7677eca9c --- /dev/null +++ b/ksp-fixtures/app/src/desktopTest/typedpython/tp_kotlin_bridge.py @@ -0,0 +1,44 @@ +# typedpython: compiled +"""Issue #45: compiled TypedPython code calling Kotlin through the binder. + +Every name imported below is a binder proxy (`fixture/app/TypedPythonBridge.kt`, published by +`PythonProxySource`). In the IR they are opaque objects: `Global` reads them, `GetAttr` reaches the +proxy's method, `CallObject` calls them. The proxy comes from calling the Kotlin class itself +(`TpSample(...)`): a Kotlin *function* returning a Kotlin object hands Python a generic owner +without the class's methods (see `TypedPythonBridge.kt`). That makes both functions impure, so nothing in them may +deopt -- the floats and the `range()` indices around the calls stay native, everything crossing to +Kotlin is boxed. + +`TypedPythonKotlinCallTest` imports this file twice in the embedded interpreter: as source +(interpreted) and as the extension `build_bridge.py` built from it against the embedded CPython's +headers (compiled), and compares the two. + +The gate type-checks the imports against `fixture/app/__init__.pyi` beside this file, a hand-written +stub of the same Kotlin surface (the bindings plugin generates `.pyi` only for walked artefacts, not +for a KSP-scanned module). Nothing at run time reads that stub: the binder's modules are in +`sys.modules` before this file is loaded. +""" +from fixture.app import TpSample, tpAccumulate, tpCheck, tpTouch + + +def drive(n: int, scale: float, token: object) -> float: + """Per step: a Kotlin constructor call that returns a typed proxy, a method call on that proxy, + the proxy handed back to a Kotlin function, and a PyObject handed to Kotlin -- inside native + float math.""" + total: float = 0.0 + for i in range(n): + x: float = float(i) * scale + 0.5 + sample: TpSample = TpSample(i, x) + w: float = float(sample.weigh(scale)) + total = total * 0.75 + float(tpAccumulate(sample, w * w + 1.0)) + tpTouch(token) + return total + + +def checked(n: int, limit: float) -> float: + """`tpCheck` throws in Kotlin once `acc` passes `limit`; the caller sees what Python sees.""" + acc: float = 0.0 + for i in range(n): + acc = acc + float(i) * 1.5 + acc = float(tpCheck(acc, limit)) + return acc diff --git a/ksp-fixtures/app/typedpython/build_bridge.py b/ksp-fixtures/app/typedpython/build_bridge.py new file mode 100644 index 000000000..cd754b3e6 --- /dev/null +++ b/ksp-fixtures/app/typedpython/build_bridge.py @@ -0,0 +1,104 @@ +"""Builds `tp_kotlin_bridge.py` into a TypedPython extension for the CPython this fixture embeds (#45). + +Run by the `buildTypedPythonBridge` task in `ksp-fixtures/app/build.gradle.kts` with a *host* +interpreter that has Pyrefly 1.3.2 (the gate needs it; `-PtypedpythonPython`, default +`python-multiplatform-ksp/.venv/bin/python`). The host interpreter only runs the compiler; the +extension is built for the embedded one: + +- headers: `include_dir`, from `python-multiplatform`'s `cpythonIncludeDirectories` provider + (`docs/platforms/python-version-acquisition.md` §7) -- the directory holding `Python.h`; +- extension suffix and flavour: read from that same tree's `_sysconfigdata_*.py`, which sits at + `/../../lib//` in a python-build-standalone prefix + (`include/python3.14[t]` beside `lib/python3.14[t]`). It is the file the embedded interpreter's own + `sysconfig` reads, so `EXT_SUFFIX` there is the suffix its import system accepts. `Py_GIL_DISABLED` + is cross-checked against `pyconfig.h` in `include_dir`, so headers and suffix cannot come from + different flavours. + +Writes `/` and `/bridge.properties`, which the test reads to +check that the interpreter it runs in is the one the extension was built for. + +Fails (exit 1) when the module does not compile in full: a function left interpreted would make +the test compare the interpreter with itself. + +usage: build_bridge.py +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + + +def sysconfig_vars(include_dir: Path) -> tuple[Path, dict[str, object]]: + lib = include_dir.parent.parent / "lib" / include_dir.name + found = sorted(lib.glob("_sysconfigdata_*.py")) + if len(found) != 1: + raise SystemExit(f"build_bridge: expected one _sysconfigdata_*.py in {lib}, found {found}") + namespace: dict[str, object] = {} + exec(compile(found[0].read_text(encoding="utf-8"), str(found[0]), "exec"), namespace) + data = namespace.get("build_time_vars") + if not isinstance(data, dict): + raise SystemExit(f"build_bridge: {found[0]} has no build_time_vars dict") + return found[0], data + + +def header_gil_disabled(include_dir: Path) -> bool: + text = (include_dir / "pyconfig.h").read_text(encoding="utf-8", errors="replace") + return re.search(r"^\s*#\s*define\s+Py_GIL_DISABLED\s+1\b", text, re.MULTILINE) is not None + + +def header_version(include_dir: Path) -> str: + text = (include_dir / "patchlevel.h").read_text(encoding="utf-8", errors="replace") + m = re.search(r'^\s*#\s*define\s+PY_VERSION\s+"([^"]+)"', text, re.MULTILINE) + if m is None: + raise SystemExit(f"build_bridge: no PY_VERSION in {include_dir / 'patchlevel.h'}") + return m.group(1) + + +def main(argv: list[str]) -> int: + if len(argv) != 5: + print(__doc__, file=sys.stderr) + return 2 + package_parent, module, include_dir, out_dir = (Path(a).resolve() for a in argv[1:]) + sys.path.insert(0, str(package_parent)) + from typedpython import pipeline # noqa: E402 (needs the path above) + + if not (include_dir / "Python.h").is_file(): + raise SystemExit(f"build_bridge: no Python.h in {include_dir}") + data_file, data = sysconfig_vars(include_dir) + suffix = data.get("EXT_SUFFIX") + if not isinstance(suffix, str) or not suffix: + raise SystemExit(f"build_bridge: no EXT_SUFFIX in {data_file}") + gil_disabled = bool(data.get("Py_GIL_DISABLED") or 0) + if gil_disabled != header_gil_disabled(include_dir): + raise SystemExit(f"build_bridge: Py_GIL_DISABLED differs between {data_file} ({gil_disabled}) " + f"and {include_dir / 'pyconfig.h'}") + + out_dir.mkdir(parents=True, exist_ok=True) + for stale in out_dir.glob(module.stem + ".*"): + stale.unlink() + result = pipeline.compile_module(module, out_dir, python_include=include_dir, ext_suffix=suffix) + if result.extension is None or result.skipped or result.diagnostics: + raise SystemExit(f"build_bridge: {module.name} did not compile in full: extension=" + f"{result.extension}, skipped={result.skipped}, diagnostics={result.diagnostics}") + + compiled = ",".join(f.name for f in result.module.functions) + (out_dir / "bridge.properties").write_text( + "\n".join([ + f"extension={result.extension}", + f"module={module.stem}", + f"source={module}", + f"extSuffix={suffix}", + f"gilDisabled={str(gil_disabled).lower()}", + f"pythonVersion={header_version(include_dir)}", + f"includeDir={include_dir}", + f"compiledFunctions={compiled}", + ]) + "\n", + encoding="utf-8", + ) + print(f"build_bridge: {result.extension} ({compiled}; {suffix}; gilDisabled={gil_disabled})") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv))