From 738955a70ee54f1a717d7e9cf6b30b2f015745b7 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:53:59 +0900 Subject: [PATCH 1/5] Docs: Desktop jar packs real libraries and extracts into a cache (SPEC L-9a, #74) --- docs/SPEC.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/SPEC.md b/docs/SPEC.md index 19ac3529f..f796b5208 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -71,6 +71,15 @@ fail, then implement. extracts it. `Status: partial` — exercised by the shared suite on the simulator; no iOS-specific lifecycle test beyond `PM/iosSimulatorArm64Test/.../AsyncioAvailabilityProbeTest.kt`. - **L-8** Initialisation sets `builtins.compiled` to an identity decorator (an existing one is kept), so `@compiled` needs no import on any platform (N-7, issue #42). `Status: implemented` — `PM/commonTest/.../ffi/BuiltinCompiledTest.kt`. +- **L-9a** (issue #74) Desktop packaging and loading of `libpython`: every `lib/**` entry of the desktop jar is a real + file (never 0 bytes -- Gradle extracts the python-build-standalone symlink `libpython3.14.so` as an empty file, so + symlinks are resolved and the real library is packed under the `System.mapLibraryName` name, no `.so.1.0`); linux + libraries are stripped of debug info when an ELF-capable `strip` exists on the build host (best-effort). The + classpath library is extracted to `/python-multiplatform///` (macOS + `~/Library/Caches`, Windows `%LOCALAPPDATA%`, else `$XDG_CACHE_HOME` or `~/.cache`; fallback + `java.io.tmpdir/python-multiplatform-`; override `-Dpython.multiplatform.cache`), never the working + directory, and reused when its size matches. `Status: implemented` -- + `PM/desktopTest/.../ffi/DesktopJarLibrariesTest.kt`, `ExtractLibraryCacheTest.kt`. ## 2. Low-level C API (downcall surface) From 826de4ac245807d4c4b300a4efa196f67f0d7b6b Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:53:59 +0900 Subject: [PATCH 2/5] Test: No empty library in the desktop jar; extraction stays out of the cwd (#74) --- .../native/ffi/DesktopJarLibrariesTest.kt | 43 +++++++++++ .../native/ffi/ExtractLibraryCacheTest.kt | 76 +++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 python-multiplatform/src/desktopTest/kotlin/python/native/ffi/DesktopJarLibrariesTest.kt create mode 100644 python-multiplatform/src/desktopTest/kotlin/python/native/ffi/ExtractLibraryCacheTest.kt diff --git a/python-multiplatform/src/desktopTest/kotlin/python/native/ffi/DesktopJarLibrariesTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/native/ffi/DesktopJarLibrariesTest.kt new file mode 100644 index 000000000..a43e8d562 --- /dev/null +++ b/python-multiplatform/src/desktopTest/kotlin/python/native/ffi/DesktopJarLibrariesTest.kt @@ -0,0 +1,43 @@ +package python.native.ffi + +import java.io.File +import java.util.zip.ZipFile +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +/** + * Issue #74. The desktop jar once carried a 0-byte `lib/linux-x86_64/libpython3.14.so` (a symlink in + * the python-build-standalone archive, extracted by Gradle as an empty file) next to the real + * library under a name nothing asked for. Expected red before the fix: the zero-length entry. + */ +class DesktopJarLibrariesTest { + private fun jar(): ZipFile { + val path = System.getProperty("pm.desktopJar") + assertNotNull(path, "pm.desktopJar is not set; desktopTest must depend on desktopJar") + assertTrue(File(path).isFile, "desktop jar not built: $path") + return ZipFile(path) + } + + @Test + fun noLibraryEntryIsEmpty() { + jar().use { zip -> + val libs = zip.entries().asSequence().filter { !it.isDirectory && it.name.startsWith("lib/") }.toList() + assertTrue(libs.isNotEmpty(), "the desktop jar has no lib/** entries") + val empty = libs.filter { it.size == 0L }.map { it.name } + assertTrue(empty.isEmpty(), "0-byte library entries (symlink stubs?): $empty") + } + } + + @Test + fun linuxCarriesTheRealLibraryUnderTheLoaderName() { + jar().use { zip -> + val names = zip.entries().asSequence().map { it.name }.toList() + if (names.none { it.startsWith("lib/linux-x86_64/") }) return + val py = names.filter { it.startsWith("lib/linux-x86_64/libpython3.") && it.endsWith(".so") && !it.endsWith("libpython3.so") } + assertEquals(1, py.size, "expected exactly one libpython3.[t].so, got $py") + assertTrue(names.none { Regex("""\.so\.\d""").containsMatchIn(it) }, "versioned .so.N.M names must not ship: $names") + } + } +} diff --git a/python-multiplatform/src/desktopTest/kotlin/python/native/ffi/ExtractLibraryCacheTest.kt b/python-multiplatform/src/desktopTest/kotlin/python/native/ffi/ExtractLibraryCacheTest.kt new file mode 100644 index 000000000..3de2e071d --- /dev/null +++ b/python-multiplatform/src/desktopTest/kotlin/python/native/ffi/ExtractLibraryCacheTest.kt @@ -0,0 +1,76 @@ +package python.native.ffi + +import python.multiplatform.OSType +import python.multiplatform.Versions +import python.multiplatform.currentPlatform +import java.io.File +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotEquals +import kotlin.test.assertTrue + +/** + * Issue #74. `extractLibrary` used to write `File(".", name)`: into the user's working directory. + * Expected red before the fix: the file appears in the cwd and nothing is cached (extractLibrary + * also had a different signature, so this fails to compile until the cache parameter exists). + */ +class ExtractLibraryCacheTest { + private fun libName(): String { + val tagged = Versions.currentVersion.taggedVersionString + return if (currentPlatform.os == OSType.Windows) "python" + tagged.replace(".", "") else "python$tagged" + } + + @Test + fun extractsIntoTheCacheNotTheWorkingDirectory() { + val cache = Files.createTempDirectory("pm-cache").toFile() + val cwdCopy = File(".", System.mapLibraryName(libName())) + val cwdBefore = cwdCopy.exists() + try { + val out = manager.extractLibrary(libName(), cacheRoot = cache) + assertTrue(out.isFile && out.length() > 0, "extracted file missing or empty: $out") + assertTrue(out.canonicalPath.startsWith(cache.canonicalPath), "$out is outside the cache $cache") + assertTrue(out.canonicalPath.contains(Versions.currentVersion.versionString), "no per-version directory: $out") + assertEquals(cwdBefore, cwdCopy.exists(), "extractLibrary touched the working directory") + } finally { + cache.deleteRecursively() + } + } + + @Test + fun reusesAnIdenticalCachedCopy() { + val cache = Files.createTempDirectory("pm-cache").toFile() + try { + val first = manager.extractLibrary(libName(), cacheRoot = cache) + val stamp = first.lastModified() - 100_000 + assertTrue(first.setLastModified(stamp)) + val second = manager.extractLibrary(libName(), cacheRoot = cache) + assertEquals(first.canonicalPath, second.canonicalPath) + assertEquals(stamp, second.lastModified(), "an identical cached library was rewritten") + } finally { + cache.deleteRecursively() + } + } + + @Test + fun replacesACachedCopyOfTheWrongSize() { + val cache = Files.createTempDirectory("pm-cache").toFile() + try { + val first = manager.extractLibrary(libName(), cacheRoot = cache) + val size = first.length() + first.writeBytes(ByteArray(7)) + val second = manager.extractLibrary(libName(), cacheRoot = cache) + assertEquals(size, second.length()) + assertNotEquals(7L, second.length()) + } finally { + cache.deleteRecursively() + } + } + + @Test + fun cacheRootIsNotTheWorkingDirectory() { + val root = manager.cacheRoot().canonicalFile + assertNotEquals(File(".").canonicalFile, root) + assertTrue(root.name.startsWith("python-multiplatform"), root.toString()) + } +} From 0e27bb696cb2ecfe6afc67b1c22e8973942453fd Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:53:59 +0900 Subject: [PATCH 3/5] Fix: Pack resolved, best-effort stripped libpython and extract into a per-version cache (#74) --- python-multiplatform/build.gradle.kts | 105 +++++++++++------- .../kotlin/python/native/ffi/manager.kt | 76 ++++++++++--- 2 files changed, 123 insertions(+), 58 deletions(-) diff --git a/python-multiplatform/build.gradle.kts b/python-multiplatform/build.gradle.kts index 6f8a0ac0d..fea2cd972 100644 --- a/python-multiplatform/build.gradle.kts +++ b/python-multiplatform/build.gradle.kts @@ -1097,6 +1097,56 @@ kotlin { into("META-INF/LICENSE") } } + // Issue #74. python-build-standalone ships `libpython3.14.so -> libpython3.14.so.1.0` as a + // symlink. Gradle's `tarTree` extracts a symlink as an empty regular file, so the jar used + // to carry a 0-byte `libpython3.14.so` (the name the loader asks for) beside the real + // 252 MB `libpython3.14.so.1.0` that nothing asked for. Staging here: + // 1. drops every zero-length library (a symlink stub is never a library), + // 2. renames `libX.so.1.0` to `libX.so`, so the jar carries the real file under the name + // `System.mapLibraryName` produces, + // 3. strips debug info from linux libraries, best-effort (see below). + val stageDesktopLibraries = tasks.register("stageDesktopLibraries") { + dependsOn(downloadAllPythonBuilds) + val stageDir = layout.buildDirectory.dir("desktop-libs") + into(stageDir) + from("$extractedDir") { + include("macos-*$desktopFlavourSuffix/python/lib/libpython*.dylib") + include("linux-*$desktopFlavourSuffix/python/lib/libpython*.so*") + include("windows-*$desktopFlavourSuffix/python/python*.dll") + include("windows-*$desktopFlavourSuffix/python/vcruntime*.dll") + // `macos-*` matches `macos-aarch64-freethreaded` too. + if (!pythonFreeThreaded) exclude("*-freethreaded/**") + eachFile { + if (file.length() == 0L) { exclude(); return@eachFile } + val platform = relativePath.segments[0].removeSuffix(desktopFlavourSuffix) + val filename = name.replace(Regex("""(\.so)(\.\d+)+$"""), "$1") + path = "lib/$platform/$filename" + } + includeEmptyDirs = false + } + doLast { + // Best-effort: GNU strip / llvm-strip are not on every build host (Apple's strip + // cannot read ELF). When none works the unstripped library ships, and says so. + val candidates = listOf("llvm-strip", "x86_64-linux-gnu-strip", "strip") + stageDir.get().asFile.walkTopDown() + .filter { it.isFile && it.parentFile.name.startsWith("linux-") && it.name.endsWith(".so") } + .forEach { lib -> + val before = lib.length() + val done = candidates.any { tool -> + try { + val proc = ProcessBuilder(tool, "--strip-debug", lib.absolutePath) + .redirectErrorStream(true).start() + proc.inputStream.readBytes() + proc.waitFor() == 0 + } catch (_: java.io.IOException) { false } + } + logger.lifecycle( + if (done) "stripped ${lib.name}: $before -> ${lib.length()} bytes" + else "no ELF-capable strip found; ${lib.name} ships unstripped ($before bytes)" + ) + } + } + } tasks.withType().matching { it.name == "desktopJar" }.configureEach { if (configuredPythonVersion == "3.13.0" && !pythonFreeThreaded) { from(libPathForDesktop) { @@ -1106,34 +1156,11 @@ kotlin { into("lib") } } else { - dependsOn(downloadAllPythonBuilds) - from("$extractedDir") { - include("macos-*$desktopFlavourSuffix/python/lib/libpython*.dylib") - include("linux-*$desktopFlavourSuffix/python/lib/libpython*.so*") - include("windows-*$desktopFlavourSuffix/python/python*.dll") - include("windows-*$desktopFlavourSuffix/python/vcruntime*.dll") - // `macos-*` matches `macos-aarch64-freethreaded` too, so a default build whose - // build directory has ever seen `-PpythonFreeThreaded=true` would otherwise - // pack both flavours' libraries into the same platform directory. - if (!pythonFreeThreaded) exclude("*-freethreaded/**") - eachFile { - // `path` here is already destination-relative -- `into("lib")` below is - // applied before eachFile sees the file, so the leading segment is "lib", - // not the platform directory from the `from(...)` source tree. Indexing - // parts[0] silently dropped the platform and collapsed every platform's - // library onto the same jar entry (DuplicatesStrategy.WARN then kept only - // the last one copied, breaking every platform but that one). - val parts = path.split("/") - // The flavour suffix exists only to keep the two extraction trees apart on - // disk; the jar layout is what `manager.platformDirectory()` looks up, and - // it names platforms alone. A free-threaded jar carries the same directory - // names with a differently-named library inside. - val platform = parts[1].removeSuffix(desktopFlavourSuffix) - val filename = parts.last() - path = "lib/$platform/$filename" - } - into("lib") - includeEmptyDirs = false + // Issue #74: the libraries are packed from the staged tree, not straight from the + // extraction, so that symlinks are resolved and the linux library is stripped. + dependsOn(stageDesktopLibraries) + from(stageDesktopLibraries.map { it.destinationDir }) { + include("lib/**") } } } @@ -2676,26 +2703,18 @@ tasks.withType().configureEac } val copyDesktopPythonBinariesForTests by tasks.registering(Copy::class) { - dependsOn(downloadAllPythonBuilds) - from("$extractedDir") { - include("macos-*$desktopFlavourSuffix/python/lib/libpython*.dylib") - include("linux-*$desktopFlavourSuffix/python/lib/libpython*.so*") - include("windows-*$desktopFlavourSuffix/python/python*.dll") - include("windows-*$desktopFlavourSuffix/python/vcruntime*.dll") - if (!pythonFreeThreaded) exclude("*-freethreaded/**") - eachFile { - val parts = path.split("/") - val platform = parts[0].removeSuffix(desktopFlavourSuffix) - val filename = parts.last() - path = "lib/$platform/$filename" - } - includeEmptyDirs = false - } + // Same resolved/stripped tree the desktop jar packs (issue #74). + val stage = tasks.named("stageDesktopLibraries") + dependsOn(stage) + from(stage.map { it.destinationDir }) into(layout.buildDirectory.dir("desktop-test-binaries")) } tasks.named("desktopTest") { dependsOn(copyDesktopPythonBinariesForTests) + // DesktopJarLibrariesTest opens the produced jar. + dependsOn("desktopJar") + systemProperty("pm.desktopJar", tasks.named("desktopJar").get().archiveFile.get().asFile.absolutePath) classpath += files(layout.buildDirectory.dir("desktop-test-binaries")) javaLauncher.set( diff --git a/python-multiplatform/src/jvmMain/kotlin/python/native/ffi/manager.kt b/python-multiplatform/src/jvmMain/kotlin/python/native/ffi/manager.kt index a04082fcb..d53cb7c74 100644 --- a/python-multiplatform/src/jvmMain/kotlin/python/native/ffi/manager.kt +++ b/python-multiplatform/src/jvmMain/kotlin/python/native/ffi/manager.kt @@ -46,8 +46,13 @@ internal object manager { } } - System.loadLibrary(libName) - libList.forEach { it.delete() } + if (libList.isEmpty()) { + System.loadLibrary(libName) + } else { + // Absolute path: the cache directory is not on `java.library.path`, and the extracted + // files are the cache, so they are not deleted after loading (issue #74). + libList.forEach { System.load(it.absolutePath) } + } } /** @@ -128,22 +133,63 @@ internal object manager { currentPlatform.os.name.lowercase(Locale.getDefault()) + (if (currentPlatform.isArm) "-aarch64" else "-x86_64") + /** + * Root of the extraction cache. Never the working directory (issue #74). + * + * macOS `~/Library/Caches/python-multiplatform`, Windows `%LOCALAPPDATA%\\python-multiplatform`, + * elsewhere `$XDG_CACHE_HOME` or `~/.cache` + `/python-multiplatform`. When none of those is + * usable the fallback is `java.io.tmpdir/python-multiplatform-`. Overridable with the + * `python.multiplatform.cache` system property (tests use it). + */ + internal fun cacheRoot(): File { + System.getProperty("python.multiplatform.cache")?.takeIf { it.isNotBlank() }?.let { return File(it) } + val home = System.getProperty("user.home")?.takeIf { it.isNotBlank() } + val base = when (currentPlatform.os) { + OSType.MacOS -> home?.let { File(it, "Library/Caches") } + OSType.Windows -> System.getenv("LOCALAPPDATA")?.takeIf { it.isNotBlank() }?.let { File(it) } + else -> System.getenv("XDG_CACHE_HOME")?.takeIf { it.isNotBlank() }?.let { File(it) } + ?: home?.let { File(it, ".cache") } + } + val root = base?.let { File(it, "python-multiplatform") } + if (root != null && (root.isDirectory || root.mkdirs()) && root.canWrite()) return root + val user = System.getProperty("user.name") ?: "user" + return File(System.getProperty("java.io.tmpdir"), "python-multiplatform-$user") + } + + /** + * Copies the bundled library into `///` and returns it, + * reusing a previous copy whose size equals the resource's. The write goes to a temporary + * sibling and is moved into place, so a concurrent process never loads a half-written file. + */ @Synchronized - private fun extractLibrary(libraryName: String, location: String = "/lib"): File { - var libraryFileName = System.mapLibraryName(libraryName) - val tempFile = File(".", libraryFileName) + internal fun extractLibrary( + libraryName: String, + location: String = "/lib", + cacheRoot: File = cacheRoot(), + ): File { + val fileName = System.mapLibraryName(libraryName) + val resource = "$location/${platformDirectory()}/$fileName" + val url = javaClass.getResource(resource) + ?: throw UnsatisfiedLinkError("Library ${platformDirectory()}/$fileName not found in JAR") - libraryFileName = platformDirectory() + "/" + libraryFileName + val dir = File(cacheRoot, Versions.currentVersion.versionString + "/" + platformDirectory()) + dir.mkdirs() + val target = File(dir, fileName) - javaClass.getResourceAsStream("$location/$libraryFileName").use { inputStream -> - if (inputStream == null) { - throw UnsatisfiedLinkError("Library $libraryFileName not found in JAR") - } - FileOutputStream(tempFile).use { outputStream -> - inputStream.copyTo(outputStream) - } - } + val expected = url.openConnection().let { c -> c.contentLengthLong.also { (c as? java.net.HttpURLConnection)?.disconnect() } } + if (target.isFile && expected > 0 && target.length() == expected) return target - return tempFile + val tmp = File.createTempFile("$fileName.", ".part", dir) + try { + url.openStream().use { input -> FileOutputStream(tmp).use { input.copyTo(it) } } + if (tmp.length() == 0L) throw UnsatisfiedLinkError("Library $resource is empty in the JAR") + java.nio.file.Files.move( + tmp.toPath(), target.toPath(), + java.nio.file.StandardCopyOption.REPLACE_EXISTING, + ) + } finally { + tmp.delete() + } + return target } } From e42b01bc691d03385c864f245d49707bd88d439f Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 15:15:33 +0900 Subject: [PATCH 4/5] Test: Staged prefix materialises archive symlinks; a packaged home with an empty libpython fails (#74) --- docs/SPEC.md | 8 +- .../gradle/StagedPrefixLinksTest.kt | 89 +++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/StagedPrefixLinksTest.kt diff --git a/docs/SPEC.md b/docs/SPEC.md index f1dd5c526..6682a48bd 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -96,8 +96,12 @@ fail, then implement. classpath library is extracted to `/python-multiplatform///` (macOS `~/Library/Caches`, Windows `%LOCALAPPDATA%`, else `$XDG_CACHE_HOME` or `~/.cache`; fallback `java.io.tmpdir/python-multiplatform-`; override `-Dpython.multiplatform.cache`), never the working - directory, and reused when its size matches. `Status: implemented` -- - `PM/desktopTest/.../ffi/DesktopJarLibrariesTest.kt`, `ExtractLibraryCacheTest.kt`. + directory, and reused when its size matches. The same holds for the plugin's staged prefix and the packaged + `python-multiplatform-home`: `stagePythonHome` extracts the archive itself and materialises every symlink/hard + link as a copy of its in-archive target (Gradle's `tarTree` yields 0-byte files), the staging stamp changed so + old prefixes are re-extracted, and the packaged copy fails the build on any 0-byte `libpython*`. + `Status: implemented` -- `PM/desktopTest/.../ffi/DesktopJarLibrariesTest.kt`, `ExtractLibraryCacheTest.kt`, + `GP/StagedPrefixLinksTest.kt`. ## 2. Low-level C API (downcall surface) diff --git a/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/StagedPrefixLinksTest.kt b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/StagedPrefixLinksTest.kt new file mode 100644 index 000000000..b3ea0e044 --- /dev/null +++ b/python-multiplatform-gradle-plugin/src/test/kotlin/python/multiplatform/gradle/StagedPrefixLinksTest.kt @@ -0,0 +1,89 @@ +package python.multiplatform.gradle + +import org.gradle.api.GradleException +import java.io.ByteArrayOutputStream +import java.io.File +import java.nio.file.Files +import java.util.zip.GZIPOutputStream +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * Issue #74. Gradle's `tarTree` extracts the archive's `lib/libpython3.14.so -> libpython3.14.so.1.0` + * as a 0-byte file, which the staged prefix, the packaged `python-multiplatform-home` and the + * desktop jar then all carried. Expected red before the fix: the extractor did not exist; with + * `tarTree` the staged link is empty. + */ +class StagedPrefixLinksTest { + + private fun entry(out: ByteArrayOutputStream, name: String, type: Char, data: ByteArray = ByteArray(0), link: String = "") { + val h = ByteArray(512) + fun put(off: Int, s: String) = s.toByteArray().copyInto(h, off) + put(0, name) + put(100, "0000755\u0000") + put(124, String.format("%011o\u0000", data.size)) + h[156] = type.code.toByte() + put(157, link) + put(257, "ustar\u000000") + for (i in 148 until 156) h[i] = ' '.code.toByte() + put(148, String.format("%06o\u0000 ", h.sumOf { it.toInt() and 0xff })) + out.write(h) + out.write(data) + out.write(ByteArray((512 - data.size % 512) % 512)) + } + + private fun fakeArchive(dir: File, payload: ByteArray): File { + val tar = ByteArrayOutputStream() + entry(tar, "python/", '5') + entry(tar, "python/lib/", '5') + entry(tar, "python/lib/libpython3.14.so.1.0", '0', payload) + entry(tar, "python/lib/libpython3.14.so", '2', link = "libpython3.14.so.1.0") + entry(tar, "python/lib/libpython3.so", '2', link = "libpython3.14.so") // link to a link + tar.write(ByteArray(1024)) + val f = File(dir, "fake.tar.gz") + GZIPOutputStream(f.outputStream()).use { it.write(tar.toByteArray()) } + return f + } + + @Test + fun symlinksBecomeCopiesOfTheirTargets() { + val work = Files.createTempDirectory(File("../.tmp").apply { mkdirs() }.toPath(), "links").toFile() + try { + val payload = ByteArray(5000) { (it % 251).toByte() } + val root = File(work, "out") + extractTarGzMaterialisingLinks(fakeArchive(work, payload), root) + + val real = File(root, "python/lib/libpython3.14.so.1.0") + val link = File(root, "python/lib/libpython3.14.so") + val chained = File(root, "python/lib/libpython3.so") + assertContentEquals(payload, real.readBytes()) + assertTrue(link.length() > 0, "staged symlink is empty") + assertContentEquals(real.readBytes(), link.readBytes()) + assertContentEquals(real.readBytes(), chained.readBytes()) + assertTrue(!Files.isSymbolicLink(link.toPath())) + } finally { + work.deleteRecursively() + } + } + + @Test + fun theStampChangedSoOldPrefixesAreReExtracted() { + assertTrue("links-materialised" in stagingStamp("3.14.7", "20260807", "linux-x86_64", false)) + } + + @Test + fun aPackagedHomeWithAnEmptyLibraryFailsTheBuild() { + val work = Files.createTempDirectory(File("../.tmp").apply { mkdirs() }.toPath(), "packaged").toFile() + try { + val lib = File(work, "lib").apply { mkdirs() } + File(lib, "libpython3.14.so.1.0").writeBytes(ByteArray(10)) + requireNoEmptyLibraries(work) // clean tree passes + File(lib, "libpython3.14.so").writeBytes(ByteArray(0)) + assertFailsWith { requireNoEmptyLibraries(work) } + } finally { + work.deleteRecursively() + } + } +} From f65bb48b2e40cdcfab5c13c9dee3b7d410f79213 Mon Sep 17 00:00:00 2001 From: BREW <81044362+b-re-w@users.noreply.github.com> Date: Sat, 3 Oct 2026 15:15:33 +0900 Subject: [PATCH 5/5] Fix: Extract the CPython archive with symlinks materialised and guard the packaged home (#74) --- .../gradle/PythonBindingsPlugin.kt | 3 + .../multiplatform/gradle/PythonHomeStaging.kt | 150 +++++++++++++++++- 2 files changed, 148 insertions(+), 5 deletions(-) diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt index b51ae2379..1194975dd 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonBindingsPlugin.kt @@ -661,6 +661,9 @@ class PythonBindingsPlugin : Plugin { exclude(PACKAGED_PREFIX_EXCLUDES) into(PACKAGED_HOME_DIRECTORY) } + doLast { + requireNoEmptyLibraries(destinationDir.resolve(PACKAGED_HOME_DIRECTORY)) + } } } diff --git a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonHomeStaging.kt b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonHomeStaging.kt index f9f1e8dc4..0dfb4ca81 100644 --- a/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonHomeStaging.kt +++ b/python-multiplatform-gradle-plugin/src/main/kotlin/python/multiplatform/gradle/PythonHomeStaging.kt @@ -202,7 +202,149 @@ internal fun stagingStamp( platform: String, freeThreaded: Boolean, ): String = "python-multiplatform $pythonVersion+$pbsRelease $platform " + - (if (freeThreaded) "freethreaded" else "default") + (if (freeThreaded) "freethreaded" else "default") + + // Bumped when the extraction itself changes. "links-materialised" (issue #74): Gradle's + // `tarTree` turned the archive's `libpython3.14.so -> libpython3.14.so.1.0` into a 0-byte file, + // so every prefix staged before this is wrong and must be extracted again. + " links-materialised" + +/** + * Extracts a python-build-standalone `.tar.gz` into [root], turning every symbolic and hard link + * into a copy of its in-archive target (issue #74). + * + * Gradle's `tarTree` extracts a symlink as an empty regular file, so `lib/libpython3.14.so` -- a + * link to `libpython3.14.so.1.0` -- came out 0 bytes and `System.load` failed with "file too + * short". A copy rather than a link also survives being copied into a jar or an app image, which + * flatten links. Dependency-free: `GZIPInputStream` plus a minimal ustar/GNU/pax header reader. + * A link whose target is not inside the archive, or that escapes [root], is skipped. + */ +internal fun extractTarGzMaterialisingLinks(archive: File, root: File) { + val rootPath = root.canonicalFile.toPath() + fun inside(rel: String): java.nio.file.Path? { + val p = rootPath.resolve(rel).normalize() + return if (p.startsWith(rootPath) && p != rootPath) p else null + } + class Link(val path: java.nio.file.Path, val target: String, val hard: Boolean) + val links = mutableListOf() + + java.util.zip.GZIPInputStream(archive.inputStream().buffered(1 shl 16)).use { input -> + val header = ByteArray(512) + var longName: String? = null + var longLink: String? = null + var paxPath: String? = null + var paxLink: String? = null + + fun readFully(buf: ByteArray, len: Int): Boolean { + var n = 0 + while (n < len) { val r = input.read(buf, n, len - n); if (r < 0) return false; n += r } + return true + } + fun str(off: Int, len: Int): String { + var end = off + while (end < off + len && header[end] != 0.toByte()) end++ + return String(header, off, end - off, Charsets.UTF_8) + } + fun octal(off: Int, len: Int): Long = str(off, len).trim().ifEmpty { "0" }.toLong(8) + fun readData(size: Long): ByteArray { + val padded = (size + 511) / 512 * 512 + val buf = ByteArray(padded.toInt()) + check(readFully(buf, buf.size)) { "truncated tar entry in $archive" } + return buf.copyOf(size.toInt()) + } + + while (true) { + if (!readFully(header, 512)) break + if (header.all { it == 0.toByte() }) break + var name = str(0, 100) + val prefix = str(345, 155) + if (prefix.isNotEmpty() && str(257, 5) == "ustar") name = "$prefix/$name" + val mode = octal(100, 8) + val size = octal(124, 12) + val type = header[156].toInt().toChar() + var linkName = str(157, 100) + when (type) { + 'L' -> { longName = String(readData(size), Charsets.UTF_8).trimEnd('\u0000'); continue } + 'K' -> { longLink = String(readData(size), Charsets.UTF_8).trimEnd('\u0000'); continue } + 'x' -> { + String(readData(size), Charsets.UTF_8).lineSequence().forEach { rec -> + val kv = rec.substringAfter(' ', "") + when { + kv.startsWith("path=") -> paxPath = kv.removePrefix("path=") + kv.startsWith("linkpath=") -> paxLink = kv.removePrefix("linkpath=") + } + } + continue + } + 'g' -> { readData(size); continue } + } + (paxPath ?: longName)?.let { name = it } + (paxLink ?: longLink)?.let { linkName = it } + longName = null; longLink = null; paxPath = null; paxLink = null + + val dest = inside(name.trimEnd('/')) ?: run { + if (size > 0) readData(size) + null + } ?: continue + when (type) { + '5' -> java.nio.file.Files.createDirectories(dest) + '2' -> links += Link(dest, linkName, hard = false) + '1' -> links += Link(dest, linkName, hard = true) + '0', '\u0000', '7' -> { + java.nio.file.Files.createDirectories(dest.parent) + dest.toFile().outputStream().use { out -> + var remaining = size + val buf = ByteArray(1 shl 16) + while (remaining > 0) { + val r = input.read(buf, 0, minOf(buf.size.toLong(), remaining).toInt()) + check(r >= 0) { "truncated tar entry $name in $archive" } + out.write(buf, 0, r); remaining -= r + } + } + var pad = (512 - size % 512) % 512 + while (pad > 0) { val r = input.skip(pad); if (r <= 0) { input.read(); pad-- } else pad -= r } + if (mode and 0b001_001_001L != 0L) dest.toFile().setExecutable(true, false) + } + else -> if (size > 0) readData(size) + } + } + } + + // Links last, repeated until stable, so a link to a link (or to a later entry) resolves. + var pending: List = links + while (pending.isNotEmpty()) { + val unresolved = mutableListOf() + for (link in pending) { + val source = if (link.hard) inside(link.target) + else link.path.parent.resolve(link.target).normalize().takeIf { it.startsWith(rootPath) } + val file = source?.toFile() + if (file == null || !file.exists() || (file.isFile && pending.any { it.path == source })) { + unresolved += link; continue + } + if (file.isDirectory) file.copyRecursively(link.path.toFile(), overwrite = true) + else { + java.nio.file.Files.createDirectories(link.path.parent) + file.copyTo(link.path.toFile(), overwrite = true) + link.path.toFile().setExecutable(file.canExecute(), false) + } + } + if (unresolved.size == pending.size) break // remaining targets are outside the archive + pending = unresolved + } +} + +/** + * Fails when a packaged prefix carries a zero-length shared library -- a symlink stub is never a + * library, and shipping one fails at the user's first launch with "file too short" (issue #74). + */ +internal fun requireNoEmptyLibraries(dir: File) { + if (!dir.isDirectory) return + val empty = dir.walkTopDown() + .filter { it.isFile && it.length() == 0L && it.name.startsWith("libpython") } + .map { it.relativeTo(dir).path }.toList() + if (empty.isNotEmpty()) { + throw GradleException("packaged CPython prefix contains zero-length libraries: $empty (symlink stubs?)") + } +} /** * The directory a packaged desktop application carries its CPython prefix in, inside Compose @@ -334,10 +476,8 @@ abstract class StagePythonHomeTask : DefaultTask() { } logger.lifecycle("Staging CPython $version ($target) into $prefix") - fs.copy { - from(archives.tarTree(archives.gzip(archive))) - into(root) - } + // Not `tarTree`: it extracts the archive's symlinks as empty files (issue #74). + extractTarGzMaterialisingLinks(archive, root) if (!marker.isFile) { throw GradleException(