diff --git a/docs/SPEC.md b/docs/SPEC.md index 71fc24df0..d54621613 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -253,6 +253,10 @@ All in `PM/commonTest`, so they run wherever the interpreter loads. 3.14t runs the whole desktop suite with `-PpythonFreeThreaded=true` (236 tests, 0 failures; ROADMAP §9). Only desktop has free-threaded prebuilts; Android and iOS do not. `Py_LIMITED_API` is not defined, so `abi3t` is not a blocker. +- **T-3** `python-multiplatform` publishes per-target, per-flavour CPython include directories as + `Provider` carrying the extraction task (`cpythonIncludeDirectories` extension; see + `docs/platforms/python-version-acquisition.md` §7). `Status: partial` (only the configured flavour is + extracted) — `GP/CPythonIncludeDirectoriesTest.kt`. ## 8. Measurement diff --git a/docs/platforms/python-version-acquisition.md b/docs/platforms/python-version-acquisition.md index ddb77bfbb..a3b01c798 100644 --- a/docs/platforms/python-version-acquisition.md +++ b/docs/platforms/python-version-acquisition.md @@ -59,3 +59,34 @@ CPython 3.14.2 for `wasm32-emscripten` (Emscripten 5.0.3, matched to the `pyemsc PEP 783) under the git-ignored `.caches/`, and `python-multiplatform-wasm-runtime` is the published zip of the result. This is deliberately a different patch release from the native `pythonVersion`; see [`wasm-design.md`](wasm-design.md). + +## 7. Include directories as a build output + +Compiling a C extension against the embedded CPython needs that CPython's headers, not the host's. +`python-multiplatform` exposes them as a project extension named `cpythonIncludeDirectories` +(`python.multiplatform.gradle.CPythonIncludeDirectories`, in the Gradle plugin module). + +* **Key:** `(target, flavour)`. Targets: `macos-aarch64`, `macos-x86_64`, `linux-x86_64`, + `windows-x86_64`, `android-aarch64`, `android-x86_64`, `ios-arm64`, `ios-arm64_x86_64-simulator` + (the iOS names are XCFramework slices). Flavour: `CPythonFlavour.GIL` (default) or `FREE_THREADED` + (desktop only). +* **Value:** `Provider` of the directory that directly contains `Python.h`, i.e. the `-I` + directory: `/[-freethreaded]/python/include/python3.14[t]` on desktop (Windows has no + `python3.14` level), `/android-*/prefix/include/python3.14`, + `/ios/Python.xcframework//include/python3.14`. +* **Task dependency:** the provider carries the matching `downloadPython_*` task; using it as a task + input makes Gradle download and extract first. +* **Only the configured flavour is extracted.** `-PpythonFreeThreaded=true` selects the free-threaded + desktop tree; asking for the other flavour throws a `GradleException` naming that property + (`isAvailable` tests it without throwing). + +```kotlin +// consumer build.gradle.kts +evaluationDependsOn(":python-multiplatform") +val includes = project(":python-multiplatform").extensions.getByType() +tasks.register("compileExt") { + val inc = includes.includeDir("macos-aarch64") // or ("linux-x86_64", CPythonFlavour.FREE_THREADED) + inputs.dir(inc) + commandLine("cc", "-I${inc.get().asFile}", "-c", "ext.c") +} +``` diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt new file mode 100644 index 000000000..6dabefb47 --- /dev/null +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/CPythonIncludeDirectories.kt @@ -0,0 +1,99 @@ +package python.multiplatform.gradle + +import org.gradle.api.GradleException +import org.gradle.api.file.Directory +import org.gradle.api.provider.Provider +import org.gradle.api.tasks.TaskProvider + +/** Which CPython build a header directory belongs to. The two are different trees with different `Py_GIL_DISABLED` layouts. */ +enum class CPythonFlavour { GIL, FREE_THREADED } + +/** + * Where, inside the build-time acquisition tree (`python-multiplatform/build/python-standalone/ + * extracted//`), the headers of each target's CPython live. Pure functions, so the + * layout is pinned by tests without downloading anything. + * + * The value is the directory that **directly contains `Python.h`** -- the one a compiler wants as + * `-I`. That is one level below the `include/` directory on every target except Windows + * (`include/python3.14/`, `include/python3.14t/` for free-threaded), which is why the issue's + * table (`.../include`) and this differ by that last segment. + */ +object CPythonIncludeLayout { + val desktopTargets = listOf("macos-aarch64", "macos-x86_64", "linux-x86_64", "windows-x86_64") + val androidTargets = listOf("android-aarch64", "android-x86_64") + + /** XCFramework slice directory names; the simulator slice is a fat arm64+x86_64 one. */ + val iosTargets = listOf("ios-arm64", "ios-arm64_x86_64-simulator") + + val allTargets: List = desktopTargets + androidTargets + iosTargets + + fun supports(target: String, flavour: CPythonFlavour): Boolean = + target in allTargets && (flavour == CPythonFlavour.GIL || target in desktopTargets) + + /** The extraction task that produces [target] -- the name `python-multiplatform/build.gradle.kts` registers. */ + fun downloadTaskName(target: String): String = when (target) { + in iosTargets -> "downloadPython_ios" + in desktopTargets, in androidTargets -> "downloadPython_${target.replace("-", "_")}" + else -> throw IllegalArgumentException("unknown CPython target '$target'") + } + + /** Path of the include directory relative to the version-keyed extraction root. */ + fun relativePath(target: String, flavour: CPythonFlavour, pythonVersion: String): String { + require(supports(target, flavour)) { "no $flavour CPython for target '$target'" } + val parts = pythonVersion.split('.') + require(parts.size >= 2) { "python version '$pythonVersion' has no major.minor" } + val tag = "${parts[0]}.${parts[1]}" + if (flavour == CPythonFlavour.FREE_THREADED) "t" else "" + return when (target) { + "windows-x86_64" -> + (if (flavour == CPythonFlavour.FREE_THREADED) "windows-x86_64-freethreaded" else "windows-x86_64") + + "/python/include" + in desktopTargets -> { + val dir = if (flavour == CPythonFlavour.FREE_THREADED) "$target-freethreaded" else target + "$dir/python/include/python$tag" + } + in androidTargets -> "$target/prefix/include/python$tag" + else -> "ios/Python.xcframework/$target/include/python$tag" + } + } +} + +/** + * The public output: one `Provider` per (target, flavour), each carrying the extraction + * task as its build dependency, so anything that wires it as an input (`-I`, an `@InputDirectory`) + * triggers the download and unpack. + * + * Only the flavour the build was configured for (`-PpythonFreeThreaded`) has an extraction task; + * asking for the other one fails with a message naming the property, rather than returning a path + * nothing will ever create. + * + * @param extractedRoot the version-keyed extraction directory + * @param extractionTask the task that fills [extractedRoot] for a target+flavour, or null when this build does not extract it + */ +class CPythonIncludeDirectories( + private val extractedRoot: Directory, + private val pythonVersion: String, + private val extractionTask: (target: String, flavour: CPythonFlavour) -> TaskProvider<*>?, +) { + val targets: List get() = CPythonIncludeLayout.allTargets + + /** Whether [includeDir] would succeed for this pair in this build. */ + fun isAvailable(target: String, flavour: CPythonFlavour = CPythonFlavour.GIL): Boolean = + CPythonIncludeLayout.supports(target, flavour) && extractionTask(target, flavour) != null + + /** The directory holding `Python.h` for [target]; resolving it makes Gradle run the extraction task first. */ + fun includeDir(target: String, flavour: CPythonFlavour = CPythonFlavour.GIL): Provider { + if (!CPythonIncludeLayout.supports(target, flavour)) { + throw GradleException( + "No $flavour CPython include directory for '$target'. Targets: ${CPythonIncludeLayout.allTargets}; " + + "free-threaded exists for desktop targets only.", + ) + } + val task = extractionTask(target, flavour) ?: throw GradleException( + "The $flavour CPython for '$target' is not extracted by this build. " + + "Only the flavour selected with -PpythonFreeThreaded is downloaded.", + ) + val relative = CPythonIncludeLayout.relativePath(target, flavour, pythonVersion) + // `TaskProvider.map` keeps the task as the producer, so the dependency travels with the value. + return task.map { extractedRoot.dir(relative) } + } +} diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt new file mode 100644 index 000000000..2c3cfe6fc --- /dev/null +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/CPythonIncludeDirectoriesTest.kt @@ -0,0 +1,111 @@ +package python.multiplatform.gradle + +import org.gradle.api.GradleException +import org.gradle.testfixtures.ProjectBuilder +import java.io.File +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNotEquals +import kotlin.test.assertTrue + +/** Issue #46: per-target CPython include directories as public providers. */ +class CPythonIncludeDirectoriesTest { + + private val version = "3.14.7" + + @Test + fun layoutMatchesTheObservedAcquisitionTree() { + val l = CPythonIncludeLayout + assertEquals("macos-aarch64/python/include/python3.14", l.relativePath("macos-aarch64", CPythonFlavour.GIL, version)) + assertEquals("linux-x86_64-freethreaded/python/include/python3.14t", l.relativePath("linux-x86_64", CPythonFlavour.FREE_THREADED, version)) + assertEquals("windows-x86_64/python/include", l.relativePath("windows-x86_64", CPythonFlavour.GIL, version)) + assertEquals("windows-x86_64-freethreaded/python/include", l.relativePath("windows-x86_64", CPythonFlavour.FREE_THREADED, version)) + assertEquals("android-aarch64/prefix/include/python3.14", l.relativePath("android-aarch64", CPythonFlavour.GIL, version)) + assertEquals("ios/Python.xcframework/ios-arm64/include/python3.14", l.relativePath("ios-arm64", CPythonFlavour.GIL, version)) + } + + @Test + fun freeThreadedIsDistinctFromDefaultOnEveryDesktopTarget() { + for (t in CPythonIncludeLayout.desktopTargets) { + assertNotEquals( + CPythonIncludeLayout.relativePath(t, CPythonFlavour.GIL, version), + CPythonIncludeLayout.relativePath(t, CPythonFlavour.FREE_THREADED, version), + ) + } + } + + @Test + fun freeThreadedDoesNotExistOffDesktop() { + assertFalse(CPythonIncludeLayout.supports("android-aarch64", CPythonFlavour.FREE_THREADED)) + assertFalse(CPythonIncludeLayout.supports("ios-arm64", CPythonFlavour.FREE_THREADED)) + assertFailsWith { + CPythonIncludeLayout.relativePath("ios-arm64", CPythonFlavour.FREE_THREADED, version) + } + } + + @Test + fun downloadTaskNamesMatchTheRootBuild() { + assertEquals("downloadPython_macos_aarch64", CPythonIncludeLayout.downloadTaskName("macos-aarch64")) + assertEquals("downloadPython_android_x86_64", CPythonIncludeLayout.downloadTaskName("android-x86_64")) + assertEquals("downloadPython_ios", CPythonIncludeLayout.downloadTaskName("ios-arm64_x86_64-simulator")) + } + + /** Registry over fake extraction tasks that really create `Python.h`; only [configured] desktop flavour is extracted. */ + private fun fixture(configured: CPythonFlavour): Pair { + val project = ProjectBuilder.builder().build() + val root = project.layout.projectDirectory.dir("extracted/$version") + val tasks = CPythonIncludeLayout.allTargets.associateWith { target -> + // Android and iOS are always GIL; desktop follows the configured flavour. + val flavour = if (target in CPythonIncludeLayout.desktopTargets) configured else CPythonFlavour.GIL + project.tasks.register("fake_${target.replace('-', '_')}") { + doLast { + root.dir(CPythonIncludeLayout.relativePath(target, flavour, version)).asFile + .also { it.mkdirs() }.resolve("Python.h").writeText("/* $target */") + } + } + } + val dirs = CPythonIncludeDirectories(root, version) { target, flavour -> + val extracted = if (target in CPythonIncludeLayout.desktopTargets) configured else CPythonFlavour.GIL + if (flavour == extracted) tasks[target] else null + } + return project to dirs + } + + @Test + fun everyTargetResolvesToADistinctDirectoryHoldingPythonH() { + val (project, dirs) = fixture(CPythonFlavour.GIL) + val seen = mutableSetOf() + for (t in dirs.targets) { + val producer = project.tasks.getByName("fake_${t.replace('-', '_')}") + val provider = dirs.includeDir(t) + // A consumer that takes the provider as an input must depend on the extraction task. + val consumer = project.tasks.register("use_${t.replace('-', '_')}") { inputs.dir(provider) }.get() + assertTrue( + producer in consumer.taskDependencies.getDependencies(consumer), + "$t: provider does not carry its extraction task", + ) + producer.actions.forEach { it.execute(producer) } + val dir = provider.get().asFile + assertTrue(File(dir, "Python.h").isFile, "$t: Python.h missing in $dir") + assertTrue(seen.add(dir), "$t: duplicate directory $dir") + } + } + + @Test + fun theOtherFlavourFailsLoudlyNamingTheProperty() { + val (_, dirs) = fixture(CPythonFlavour.GIL) + assertFalse(dirs.isAvailable("macos-aarch64", CPythonFlavour.FREE_THREADED)) + val e = assertFailsWith { dirs.includeDir("macos-aarch64", CPythonFlavour.FREE_THREADED) } + assertTrue("pythonFreeThreaded" in e.message.orEmpty()) + assertFailsWith { dirs.includeDir("ios-arm64", CPythonFlavour.FREE_THREADED) } + } + + @Test + fun configuredFreeThreadedExposesFreeThreadedKey() { + val (_, dirs) = fixture(CPythonFlavour.FREE_THREADED) + assertTrue(dirs.isAvailable("linux-x86_64", CPythonFlavour.FREE_THREADED)) + assertTrue(dirs.includeDir("linux-x86_64", CPythonFlavour.FREE_THREADED).get().asFile.path.endsWith("python3.14t")) + } +} diff --git a/python-multiplatform/build.gradle.kts b/python-multiplatform/build.gradle.kts index eb734d1dd..38f55e8a9 100644 --- a/python-multiplatform/build.gradle.kts +++ b/python-multiplatform/build.gradle.kts @@ -18,6 +18,9 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinNativeLink import org.jetbrains.kotlin.konan.target.Family import org.jetbrains.kotlin.konan.target.KonanTarget.* import org.jetbrains.kotlin.konan.target.linker +import python.multiplatform.gradle.CPythonFlavour +import python.multiplatform.gradle.CPythonIncludeDirectories +import python.multiplatform.gradle.CPythonIncludeLayout import python.multiplatform.gradle.GenerateWasmProxyExportsTask @@ -614,6 +617,27 @@ val downloadAllPythonBuilds by tasks.registering { dependsOn(downloadPython_ios) } +/** + * Public build output (issue #46): the include directory of the CPython this build acquires, per + * target and flavour, each carrying its extraction task. Consumers (pypackpack's compile slot, + * TypedPython's C build) read it with + * `project(":python-multiplatform").extensions.getByType().includeDir("macos-aarch64")` + * after `evaluationDependsOn(":python-multiplatform")`. Only the desktop flavour selected with + * `-PpythonFreeThreaded` has a task; Android and iOS are always GIL. See + * `docs/platforms/python-version-acquisition.md` §7. + */ +extensions.add( + "cpythonIncludeDirectories", + CPythonIncludeDirectories( + layout.projectDirectory.dir(extractedDir.absolutePath), + configuredPythonVersion, + ) { target, flavour -> + val desktop = target in CPythonIncludeLayout.desktopTargets + val extractedFlavour = if (desktop && pythonFreeThreaded) CPythonFlavour.FREE_THREADED else CPythonFlavour.GIL + if (flavour == extractedFlavour) tasks.named(CPythonIncludeLayout.downloadTaskName(target)) else null + }, +) + val androidBuildDir = "$projectDir/build/android" // =================================================================================================