diff --git a/README.md b/README.md index ebc6c6d..57a5082 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,43 @@ The API is designed to be as JavaScript-standard as possible, so `XzReadableStre If you're using this to decompress content from a file or other source, rather than an HTTP response body, you'll need to get a `ReadableStream` (a web stream) for the file's data to replace the `compressedResponse.body` stream above. For example, in Node.js you could use `Readable.toWeb(fs.createReadStream(filename))` to stream from the disk, or `new Blob([buffer]).stream()` if you already have the data in a `Buffer`. +## Cloudflare Workers & other restricted runtimes + +Some runtimes disallow dynamic wasm compilation from bytes at runtime (for example, Cloudflare Workers). + +To support these environments, you can provide a pre-compiled `WebAssembly.Module` with `XzReadableStream.setWasmModule(...)` once at startup, before creating any `XzReadableStream` instances. + +### Cloudflare Workers example + +```js +import { XzReadableStream } from 'xz-decompress'; +import xzWasmModule from 'xz-decompress/dist/native/xz-decompress.wasm'; + +// Call once during startup/module initialization: +XzReadableStream.setWasmModule(xzWasmModule); + +export default { + async fetch(request) { + const compressedResponse = await fetch('https://example.com/somefile.xz'); + return new Response(new XzReadableStream(compressedResponse.body)); + } +}; +``` + +### Generic runtime pattern + +If your runtime/bundler gives you a `WebAssembly.Module` from a static `.wasm` import, pass it to `setWasmModule` in the same way. + +If your runtime only gives you bytes or a URL, you may need to compile yourself: + +```js +const wasmBytes = await fetch(wasmUrl).then((r) => r.arrayBuffer()); +const wasmModule = await WebAssembly.compile(wasmBytes); +XzReadableStream.setWasmModule(wasmModule); +``` + +Note: this fallback only works where `WebAssembly.compile(...)` is allowed by the runtime. + ## What about `.tar.xz` files? Since the `.xz` format only represents one file, it's common for people to bundle up a collection of files as `.tar`, and then compress this to `.tar.xz`. diff --git a/package-lock.json b/package-lock.json index bdc2a39..53e2742 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2004,15 +2004,15 @@ } }, "node_modules/terser-webpack-plugin": { - "version": "5.3.16", - "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.16.tgz", - "integrity": "sha512-h9oBFCWrq78NyWWVcSwZarJkZ01c2AyGrzs1crmHZO3QUg9D61Wu4NPjBy69n7JqylFF5y+CsUZYmYEIZ3mR+Q==", + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.6.1.tgz", + "integrity": "sha512-201R5j+sJpK8nFWwKVyNfZot8FaJbLZDq5evriVzbV1wDtSXDjRUDRfJzHpAaxFDMEhsZL1QkeqM61wgsS3KaQ==", "dev": true, + "license": "MIT", "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", "jest-worker": "^27.4.5", "schema-utils": "^4.3.0", - "serialize-javascript": "^6.0.2", "terser": "^5.31.1" }, "engines": { @@ -2026,12 +2026,39 @@ "webpack": "^5.1.0" }, "peerDependenciesMeta": { + "@minify-html/node": { + "optional": true + }, "@swc/core": { "optional": true }, + "@swc/css": { + "optional": true + }, + "@swc/html": { + "optional": true + }, + "clean-css": { + "optional": true + }, + "cssnano": { + "optional": true + }, + "csso": { + "optional": true + }, "esbuild": { "optional": true }, + "html-minifier-terser": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "postcss": { + "optional": true + }, "uglify-js": { "optional": true } @@ -3873,15 +3900,14 @@ } }, "terser-webpack-plugin": { - "version": "5.3.16", - "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.16.tgz", - "integrity": "sha512-h9oBFCWrq78NyWWVcSwZarJkZ01c2AyGrzs1crmHZO3QUg9D61Wu4NPjBy69n7JqylFF5y+CsUZYmYEIZ3mR+Q==", + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.6.1.tgz", + "integrity": "sha512-201R5j+sJpK8nFWwKVyNfZot8FaJbLZDq5evriVzbV1wDtSXDjRUDRfJzHpAaxFDMEhsZL1QkeqM61wgsS3KaQ==", "dev": true, "requires": { "@jridgewell/trace-mapping": "^0.3.25", "jest-worker": "^27.4.5", "schema-utils": "^4.3.0", - "serialize-javascript": "^6.0.2", "terser": "^5.31.1" } }, diff --git a/package.json b/package.json index 38160b3..ae9afae 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ }, "files": [ "dist/package/**", + "dist/native/xz-decompress.wasm", "types.d.ts" ] } diff --git a/src/xz-decompress.js b/src/xz-decompress.js index 8b63409..b593625 100644 --- a/src/xz-decompress.js +++ b/src/xz-decompress.js @@ -5,6 +5,15 @@ const ReadableStream = globalThis.ReadableStream // This won't be reached in modern browsers, and bundlers will ignore due to 'browser' field in package.json: || require('stream/web').ReadableStream; +// Try to compile the wasm module eagerly at import time. This fails in runtimes +// like Cloudflare Workers that block WebAssembly.compile entirely; those runtimes +// must call XzReadableStream.setWasmModule() with a pre-compiled module instead. +let _wasmModulePromise = (async () => { + const base64Wasm = xzwasmBytes.replace('data:application/wasm;base64,', ''); + const wasmBytes = Uint8Array.from(atob(base64Wasm), c => c.charCodeAt(0)).buffer; + return WebAssembly.compile(wasmBytes); +})().catch(() => null); + const XZ_OK = 0; const XZ_STREAM_END = 1; @@ -97,35 +106,67 @@ export class XzReadableStream extends ReadableStream { static _moduleInstance; static _contextMutex = new ContextMutex(); + /** + * Provide a pre-compiled WebAssembly.Module for runtimes that block + * dynamic compilation (e.g. Cloudflare Workers). + */ + static setWasmModule(wasmModule) { + _wasmModulePromise = Promise.resolve(wasmModule); + XzReadableStream._moduleInstance = null; + XzReadableStream._moduleInstancePromise = null; + } + static async _getModuleInstance() { - const base64Wasm = xzwasmBytes.replace('data:application/wasm;base64,', ''); - const wasmBytes = Uint8Array.from(atob(base64Wasm), c => c.charCodeAt(0)).buffer; - const wasmOptions = {}; - const module = await WebAssembly.instantiate(wasmBytes, wasmOptions); - XzReadableStream._moduleInstance = module.instance; + const compiledModule = await _wasmModulePromise; + if (!compiledModule) { + throw new Error( + 'WebAssembly compilation is not available in this runtime. ' + + 'Call XzReadableStream.setWasmModule(module) with a pre-compiled WebAssembly.Module before use.' + ); + } + XzReadableStream._moduleInstance = await WebAssembly.instantiate(compiledModule); } constructor(compressedStream) { let xzContext; let unconsumedInput = null; + let finalized = false; + let initError = null; const compressedReader = compressedStream.getReader(); - super({ - async start(controller) { - await XzReadableStream._contextMutex.acquire(); + function finalizeOnce() { + if (finalized) return; + finalized = true; + if (xzContext) { + xzContext.dispose(); + xzContext = null; + } + XzReadableStream._contextMutex.release(); + } - try { - if (!XzReadableStream._moduleInstance) { - await (XzReadableStream._moduleInstancePromise || (XzReadableStream._moduleInstancePromise = XzReadableStream._getModuleInstance())); - } - xzContext = new XzContext(XzReadableStream._moduleInstance); - } catch (error) { - XzReadableStream._contextMutex.release(); - throw error; + const initPromise = (async () => { + await XzReadableStream._contextMutex.acquire(); + try { + if (!XzReadableStream._moduleInstance) { + await (XzReadableStream._moduleInstancePromise || (XzReadableStream._moduleInstancePromise = XzReadableStream._getModuleInstance())); } + xzContext = new XzContext(XzReadableStream._moduleInstance); + } catch (error) { + initError = error; + XzReadableStream._contextMutex.release(); + } + })(); + + super({ + async start() { + await initPromise; + if (initError) throw initError; }, async pull(controller) { + await initPromise; + if (initError) throw initError; + try { if (xzContext.needsMoreInput()) { if (unconsumedInput === null || unconsumedInput.byteLength === 0) { @@ -144,26 +185,20 @@ export class XzReadableStream extends ReadableStream { xzContext.resetOutputBuffer(); if (nextOutputResult.finished) { - xzContext.dispose(); - XzReadableStream._contextMutex.release(); + finalizeOnce(); controller.close(); } } catch (error) { - if (xzContext) { - xzContext.dispose(); - } - XzReadableStream._contextMutex.release(); + finalizeOnce(); throw error; } }, - cancel() { + async cancel() { + await initPromise; try { - if (xzContext) { - xzContext.dispose(); - } return compressedReader.cancel(); } finally { - XzReadableStream._contextMutex.release(); + finalizeOnce(); } } }); diff --git a/test/test.spec.ts b/test/test.spec.ts index 875203d..dc8c680 100644 --- a/test/test.spec.ts +++ b/test/test.spec.ts @@ -1,5 +1,7 @@ import { ReadableStream as NodeReadableStream } from 'stream/web'; import { expect } from 'chai'; +import { readFileSync } from 'fs'; +import { resolve } from 'path'; import { XzReadableStream } from ".."; @@ -142,4 +144,187 @@ describe("Streaming XS decompression", () => { const result = await collectOutputString(validStream); expect(result).to.equal('hello world\n'); }); + + it("handles early pull before initialization completes", async () => { + const dataStream = buildStaticDataStream(Buffer.from(HELLO_WORLD_XZ, 'base64')); + const stream = new XzReadableStream(dataStream); + + // Immediately call reader.read() without waiting for start to settle + const reader = stream.getReader(); + const { value, done } = await reader.read(); + + expect(done).to.be.false; + expect(value).to.be.instanceOf(Uint8Array); + + // Read remaining + let result = new TextDecoder().decode(value); + while (true) { + const { value: chunk, done: finished } = await reader.read(); + if (finished) break; + result += new TextDecoder().decode(chunk); + } + expect(result).to.equal('hello world\n'); + }); + + it("handles concurrent reads queued during initialization", async () => { + const dataStream = buildStaticDataStream(Buffer.from(HELLO_WORLD_XZ, 'base64')); + const stream = new XzReadableStream(dataStream); + const reader = stream.getReader(); + + // Queue multiple reads immediately (before init can finish) + const read1 = reader.read(); + const read2 = reader.read(); + const read3 = reader.read(); + + const results = await Promise.all([read1, read2, read3]); + + // At least the first should have data, stream may close within these reads + const hasData = results.some(r => !r.done && r.value && r.value.byteLength > 0); + expect(hasData).to.be.true; + + reader.releaseLock(); + }); + + it("cancel during initialization does not deadlock", async () => { + const dataStream = buildStaticDataStream(Buffer.from(HELLO_WORLD_XZ, 'base64')); + const stream = new XzReadableStream(dataStream); + const reader = stream.getReader(); + + // Cancel immediately without reading + await reader.cancel(); + + // Next stream should still work (mutex released) + const validDataStream = buildStaticDataStream(Buffer.from(HELLO_WORLD_XZ, 'base64')); + const validStream = new XzReadableStream(validDataStream); + const result = await collectOutputString(validStream); + expect(result).to.equal('hello world\n'); + }); + + it("initialization failure propagates to pull, not undefined-context errors", async () => { + // Temporarily break the module instance to force init failure + const originalPromise = (XzReadableStream as any)._moduleInstancePromise; + const originalInstance = (XzReadableStream as any)._moduleInstance; + + (XzReadableStream as any)._moduleInstance = null; + (XzReadableStream as any)._moduleInstancePromise = Promise.reject(new Error('Simulated init failure')); + // Suppress unhandled rejection + (XzReadableStream as any)._moduleInstancePromise.catch(() => {}); + + const dataStream = buildStaticDataStream(Buffer.from(HELLO_WORLD_XZ, 'base64')); + const stream = new XzReadableStream(dataStream); + + try { + await collectOutputString(stream); + expect.fail('Expected stream to throw'); + } catch (error: any) { + expect(error.message).to.equal('Simulated init failure'); + } + + // Restore original state + (XzReadableStream as any)._moduleInstancePromise = originalPromise; + (XzReadableStream as any)._moduleInstance = originalInstance; + + // Subsequent streams should work + const validDataStream = buildStaticDataStream(Buffer.from(HELLO_WORLD_XZ, 'base64')); + const validStream = new XzReadableStream(validDataStream); + const result = await collectOutputString(validStream); + expect(result).to.equal('hello world\n'); + }); +}); + +describe("setWasmModule", () => { + const wasmPath = resolve(__dirname, '../dist/native/xz-decompress.wasm'); + + afterEach(() => { + // Reset to built-in module so other tests aren't affected + (XzReadableStream as any)._moduleInstance = null; + (XzReadableStream as any)._moduleInstancePromise = null; + }); + + it("works with a pre-compiled WebAssembly.Module", async () => { + const wasmBytes = readFileSync(wasmPath); + const wasmModule = await WebAssembly.compile(wasmBytes); + + (XzReadableStream as any)._moduleInstance = null; + (XzReadableStream as any)._moduleInstancePromise = null; + XzReadableStream.setWasmModule(wasmModule); + + const dataStream = buildStaticDataStream(Buffer.from(HELLO_WORLD_XZ, 'base64')); + const stream = new XzReadableStream(dataStream); + const result = await collectOutputString(stream); + expect(result).to.equal('hello world\n'); + }); + + it("works for multiple sequential streams after setWasmModule", async () => { + const wasmBytes = readFileSync(wasmPath); + const wasmModule = await WebAssembly.compile(wasmBytes); + + (XzReadableStream as any)._moduleInstance = null; + (XzReadableStream as any)._moduleInstancePromise = null; + XzReadableStream.setWasmModule(wasmModule); + + for (let i = 0; i < 5; i++) { + const dataStream = buildStaticDataStream(Buffer.from(HELLO_WORLD_XZ, 'base64')); + const stream = new XzReadableStream(dataStream); + const result = await collectOutputString(stream); + expect(result).to.equal('hello world\n'); + } + }); + + it("works for parallel streams after setWasmModule", async () => { + const wasmBytes = readFileSync(wasmPath); + const wasmModule = await WebAssembly.compile(wasmBytes); + + (XzReadableStream as any)._moduleInstance = null; + (XzReadableStream as any)._moduleInstancePromise = null; + XzReadableStream.setWasmModule(wasmModule); + + const promises = Array.from({ length: 10 }, () => { + const dataStream = buildStaticDataStream(Buffer.from(HELLO_WORLD_XZ, 'base64')); + return collectOutputString(new XzReadableStream(dataStream)); + }); + + const results = await Promise.all(promises); + for (const result of results) { + expect(result).to.equal('hello world\n'); + } + }); + + it("throws a clear error when no module is available", async () => { + (XzReadableStream as any)._moduleInstance = null; + (XzReadableStream as any)._moduleInstancePromise = null; + // Simulate a runtime where compile failed (null resolved) + XzReadableStream.setWasmModule(null as any); + + const dataStream = buildStaticDataStream(Buffer.from(HELLO_WORLD_XZ, 'base64')); + const stream = new XzReadableStream(dataStream); + + try { + await collectOutputString(stream); + expect.fail('Expected stream to throw'); + } catch (error: any) { + expect(error.message).to.include('setWasmModule'); + } + }); + + it("overrides a previously cached module instance", async () => { + const wasmBytes = readFileSync(wasmPath); + const wasmModule = await WebAssembly.compile(wasmBytes); + + // First, populate _moduleInstance via setWasmModule + XzReadableStream.setWasmModule(wasmModule); + const dataStream1 = buildStaticDataStream(Buffer.from(HELLO_WORLD_XZ, 'base64')); + await collectOutputString(new XzReadableStream(dataStream1)); + expect((XzReadableStream as any)._moduleInstance).to.not.be.null; + + // Calling setWasmModule again should clear the cached instance + const wasmModule2 = await WebAssembly.compile(wasmBytes); + XzReadableStream.setWasmModule(wasmModule2); + expect((XzReadableStream as any)._moduleInstance).to.be.null; + + // Should still work with the new module + const dataStream2 = buildStaticDataStream(Buffer.from(HELLO_WORLD_XZ, 'base64')); + const result = await collectOutputString(new XzReadableStream(dataStream2)); + expect(result).to.equal('hello world\n'); + }); }); \ No newline at end of file diff --git a/types.d.ts b/types.d.ts index fdb7c3d..c7e54d3 100644 --- a/types.d.ts +++ b/types.d.ts @@ -1,3 +1,8 @@ export class XzReadableStream extends ReadableStream { + /** + * Provide a pre-compiled WebAssembly.Module for runtimes that block + * dynamic compilation (e.g. Cloudflare Workers). + */ + static setWasmModule(wasmModule: WebAssembly.Module): void; constructor(compressedStream: ReadableStream); } \ No newline at end of file