diff --git a/clients/python/src/sqlfs/sandbox.py b/clients/python/src/sqlfs/sandbox.py index cb4acef..0bea987 100644 --- a/clients/python/src/sqlfs/sandbox.py +++ b/clients/python/src/sqlfs/sandbox.py @@ -49,12 +49,57 @@ #: client via ``Client(max_file_size=...)`` or disable with ``max_file_size=0``. DEFAULT_MAX_FILE_SIZE = 64 * 1024 * 1024 +#: Largest file the ``python3`` runtime (CPython compiled to WASM) can read via +#: ``open()``. CPython reads sandbox files through an 8 MiB SharedArrayBuffer +#: IPC bridge; ``open()`` on a larger file fails with an opaque +#: ``FileNotFoundError`` / ``OSError``. The bytes still ingest fine and stay +#: readable from bash (``cat``/``grep``/``awk``) and ``js-exec`` — only +#: ``python3`` cannot read them. +CPYTHON_READ_LIMIT = 8 * 1024 * 1024 + def _content_size(content: Union[str, bytes]) -> int: """Raw byte length of a write payload (``str`` measured as UTF-8).""" return len(content.encode("utf-8")) if isinstance(content, str) else len(content) +def _enforce_cpython_read_limit( + files: Mapping[str, Union[str, bytes]], + *, + allow_oversized: bool, +) -> None: + """Reject files the ``python3`` runtime could not later ``open()``. + + Raises ``ValidationError`` (code ``EFILE_TOO_LARGE_FOR_CPYTHON``) naming + every file above :data:`CPYTHON_READ_LIMIT`. Pass ``allow_oversized=True`` + to ingest anyway — the file is stored and usable from bash/js-exec, but + ``python3`` ``open()`` will fail on it. + """ + if allow_oversized: + return + too_big = [ + (path, size) + for path, content in files.items() + if (size := _content_size(content)) > CPYTHON_READ_LIMIT + ] + if too_big: + details = [ + f"{path} ({size} bytes > {CPYTHON_READ_LIMIT} python3 open() limit)" + for path, size in too_big + ] + raise ValidationError( + "file exceeds the 8 MiB limit that the python3 (CPython WASM) runtime " + "can read via open(): " + + "; ".join(details) + + ". The bytes ingest fine and stay usable from bash (cat/grep/awk) and " + "js-exec — only python3 open() will fail on these files. Pass " + "allow_oversized=True to ingest anyway, or split the file into <8 MiB " + "chunks if you need to read it from python3.", + code="EFILE_TOO_LARGE_FOR_CPYTHON", + details=details, + ) + + def _enforce_max_file_size( files: Mapping[str, Union[str, bytes]], max_file_size: int, @@ -409,17 +454,25 @@ def ingest_files( files: Mapping[str, Union[str, bytes]], *, base_path: str = "/home/user/project", + allow_oversized: bool = False, ) -> Dict[str, Any]: """`POST /ingest-files` — write a JSON manifest of files (auto base64). Keys are paths relative to `base_path`. Values may be `str` (encoded utf-8) or `bytes` (encoded as-is); the SDK base64-encodes them. - Each file is checked against the client's ``max_file_size`` first; an - oversized file raises ``ValidationError`` (code ``EFILE_TOO_LARGE``) - before anything is encoded or sent over the network. + Two client-side checks run before anything is encoded or sent: + + 1. Each file is checked against the client's ``max_file_size``; an + oversized file raises ``ValidationError`` (code ``EFILE_TOO_LARGE``). + 2. Any file larger than 8 MiB (:data:`CPYTHON_READ_LIMIT`) raises + ``ValidationError`` (code ``EFILE_TOO_LARGE_FOR_CPYTHON``) because the + ``python3`` runtime cannot ``open()`` it. Pass ``allow_oversized=True`` + to ingest anyway — the file stays usable from bash/js-exec, but + ``python3`` ``open()`` will fail on it. """ _enforce_max_file_size(files, self._max_file_size) + _enforce_cpython_read_limit(files, allow_oversized=allow_oversized) encoded: Dict[str, str] = {} for path, content in files.items(): data = content.encode("utf-8") if isinstance(content, str) else content diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py index cea6142..980e531 100644 --- a/clients/python/tests/test_client.py +++ b/clients/python/tests/test_client.py @@ -622,6 +622,35 @@ def test_ingest_files_base64_encodes(): assert sent["files"]["b.bin"] == base64.b64encode(b"\x00\x01").decode() +@respx.mock +def test_ingest_files_blocks_oversized_for_cpython(): + route = respx.post(f"{BASE_URL}/v1/sandboxes/sb/ingest-files").mock( + return_value=httpx.Response(200, json={"status": "ok", "fileCount": 1}) + ) + sb = make_client().sandboxes.attach("sb") + big = b"\x00" * (8 * 1024 * 1024 + 1) + with pytest.raises(ValidationError) as exc: + sb.ingest_files({"big.csv": big}) + assert exc.value.code == "EFILE_TOO_LARGE_FOR_CPYTHON" + limit = 8 * 1024 * 1024 + assert exc.value.details == [f"big.csv ({len(big)} bytes > {limit} python3 open() limit)"] + # Nothing sent over the network. + assert route.call_count == 0 + + +@respx.mock +def test_ingest_files_allow_oversized_bypasses_cpython_check(): + route = respx.post(f"{BASE_URL}/v1/sandboxes/sb/ingest-files").mock( + return_value=httpx.Response(200, json={"status": "ok", "fileCount": 1}) + ) + sb = make_client().sandboxes.attach("sb") + big = b"\x00" * (8 * 1024 * 1024 + 1) + sb.ingest_files({"big.csv": big}, allow_oversized=True) + assert route.call_count == 1 + sent = json.loads(route.calls[0].request.content.decode()) + assert list(sent["files"].keys()) == ["big.csv"] + + # ── Error mapping ──────────────────────────────────────────────────────────── @respx.mock def test_400_maps_to_validation_error(): diff --git a/clients/typescript/src/sandbox.ts b/clients/typescript/src/sandbox.ts index 2efd1f3..88f11f0 100644 --- a/clients/typescript/src/sandbox.ts +++ b/clients/typescript/src/sandbox.ts @@ -216,9 +216,11 @@ export class Sandbox { async ingestFiles( files: Record, - options: { basePath?: string } = {}, + options: { basePath?: string; allowOversized?: boolean } = {}, ): Promise> { enforceMaxFileSize(files, this.maxFileSize); + // Files >8 MiB can't be read by python3's open() (CPython WASM IPC bridge cap). + enforceCpythonReadLimit(files, options.allowOversized ?? false); const encoded: Record = Object.create(null) as Record; for (const [path, content] of Object.entries(files)) { encoded[path] = toBase64(content); @@ -288,6 +290,39 @@ function enforceMaxFileSize(files: Record, maxFileSize: num }); } +/** + * Largest file the `python3` runtime (CPython compiled to WASM) can read via + * `open()`. CPython reads sandbox files through an 8 MiB SharedArrayBuffer IPC + * bridge; `open()` on a larger file fails with an opaque error. The bytes still + * ingest fine and stay readable from bash and js-exec — only `python3` cannot + * read them. + */ +export const cpythonReadLimit = 8 * 1024 * 1024; + +/** + * Reject files the `python3` runtime could not later `open()`. Throws + * ValidationError (code `EFILE_TOO_LARGE_FOR_CPYTHON`) unless `allowOversized` + * is set — in which case the file ingests but `python3` open() fails on it. + */ +function enforceCpythonReadLimit(files: Record, allowOversized: boolean): void { + if (allowOversized) { + return; + } + const tooBig = Object.entries(files) + .map(([path, content]) => [path, contentSize(content)] as const) + .filter(([, size]) => size > cpythonReadLimit); + if (tooBig.length === 0) { + return; + } + const details = tooBig.map(([path, size]) => `${path} (${size} bytes > ${cpythonReadLimit} python3 open() limit)`); + throw new ValidationError( + `file exceeds the 8 MiB limit that the python3 (CPython WASM) runtime can read via open(): ${details.join( + "; ", + )}. The bytes ingest fine and stay usable from bash (cat/grep/awk) and js-exec — only python3 open() will fail on these files. Pass allowOversized: true to ingest anyway, or split the file into <8 MiB chunks if you need to read it from python3.`, + { code: "EFILE_TOO_LARGE_FOR_CPYTHON", details }, + ); +} + function toBytes(content: FileContent): Uint8Array { if (typeof content === "string") { return new TextEncoder().encode(content); diff --git a/clients/typescript/tests/files.test.ts b/clients/typescript/tests/files.test.ts index 257aa2d..517718a 100644 --- a/clients/typescript/tests/files.test.ts +++ b/clients/typescript/tests/files.test.ts @@ -63,4 +63,28 @@ describe("TypeScript SQL-FS SDK files", () => { ); expect(blocked.fetchMock).not.toHaveBeenCalled(); }); + + it("blocks ingest of files >8 MiB the python3 runtime can't open()", async () => { + const { fetchMock } = makeFetch([jsonResponse(200, {})]); + const sb = makeClient(fetchMock).sandboxes.attach("sb"); + const big = new Uint8Array(8 * 1024 * 1024 + 1); + + await expect(sb.ingestFiles({ "big.csv": big })).rejects.toMatchObject({ + code: "EFILE_TOO_LARGE_FOR_CPYTHON", + details: [`big.csv (${big.byteLength} bytes > ${8 * 1024 * 1024} python3 open() limit)`], + }); + // Nothing should have been sent over the network. + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("ingests an oversized file when allowOversized is set", async () => { + const { fetchMock, captured } = makeFetch([jsonResponse(200, { count: 1 })]); + const sb = makeClient(fetchMock).sandboxes.attach("sb"); + const big = new Uint8Array(8 * 1024 * 1024 + 1); + + await expect(sb.ingestFiles({ "big.csv": big }, { allowOversized: true })).resolves.toEqual({ count: 1 }); + expect(fetchMock).toHaveBeenCalledTimes(1); + const sentBody = captured[0]?.body as { files: Record }; + expect(Object.keys(sentBody.files)).toEqual(["big.csv"]); + }); }); diff --git a/plugins/sql-fs/.claude-plugin/plugin.json b/plugins/sql-fs/.claude-plugin/plugin.json index 5b302f3..c2cf5cd 100644 --- a/plugins/sql-fs/.claude-plugin/plugin.json +++ b/plugins/sql-fs/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "sql-fs", - "version": "1.1.0", + "version": "1.1.1", "description": "Operator skill for the SQL-FS API — persistent bash sandbox service with SQL-backed filesystem", "author": { "name": "Harry Nguyen", diff --git a/plugins/sql-fs/skills/api/SKILL.md b/plugins/sql-fs/skills/api/SKILL.md index 5dd43f8..7cdcbf7 100644 --- a/plugins/sql-fs/skills/api/SKILL.md +++ b/plugins/sql-fs/skills/api/SKILL.md @@ -14,11 +14,11 @@ Invoke with optional sub-commands: | Invocation | What happens | |---|---| -| `/sqlfs:api` | General assistant — answer questions, generate commands | -| `/sqlfs:api setup` | Walk through auth bootstrap and first sandbox | -| `/sqlfs:api exec