diff --git a/.changeset/fix-ingest-base64-stack-overflow.md b/.changeset/fix-ingest-base64-stack-overflow.md new file mode 100644 index 0000000..aac119b --- /dev/null +++ b/.changeset/fix-ingest-base64-stack-overflow.md @@ -0,0 +1,5 @@ +--- +"sql-fs-api": patch +--- + +Fix `POST /v1/sandboxes/:id/ingest-files` returning 500 (Internal Server Error) for files larger than ~750 KB. `isValidBase64` ran a structural regex whose `(?:[A-Za-z0-9+/]{4})*` quantifier overflowed V8's call stack (`RangeError: Maximum call stack size exceeded`) on base64 strings beyond ~1 MB — failing during request validation, before any database work. The regex is now skipped for strings over 1 MB, relying solely on the canonical round-trip check (`Buffer.from(s, "base64").toString("base64") === s`), which is native and never overflows. Ingesting multi-MB files now succeeds. diff --git a/README.md b/README.md index 7fbda29..9b7ace2 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,10 @@ Key design choices: | `SESSION_IDLE_MS` | No | `600000` | Evict idle Bash instances after this many ms | | `MAX_CONCURRENT_PYTHON` | No | `5` | Cap on concurrent CPython WASM workers (~80 MB each) | | `MAX_CONCURRENT_JS` | No | `5` | Cap on concurrent QuickJS workers (~64 MB each) | +| `MAX_REQUEST_BODY_BYTES` | No | `268435456` | Hard cap on any HTTP request body (256 MB) — file write, bulk write, ingest. Applied before auth/handlers. Since base64 inflates content ~33%, this is usually the binding limit on ingest: ~190 MB of raw file bytes per call. | +| `MAX_INGEST_BYTES` | No | `536870912` | Max total decoded bytes across one `ingest-files` manifest (512 MB). The request-body cap above normally trips first. | +| `MAX_INGEST_FILES` | No | `10000` | Max number of entries (files + paths) in one `ingest-files` manifest. | +| `MAX_INGEST_PATHS_CONCURRENCY` | No | `16` | Max concurrent host-file reads for the MCP `paths` ingest mode (bounds file descriptors / memory). | | `REDIS_URL` | No | — | Redis connection string. Required for multi-replica deployments. Without it, only the in-process mutex protects execution. | | `REDIS_EXEC_LOCK_LEASE_MS` | No | `60000` | Distributed exec lock TTL. Must be > `REDIS_EXEC_LOCK_RENEW_MS`. | | `REDIS_EXEC_LOCK_RENEW_MS` | No | `20000` | Lock heartbeat interval. Must be strictly less than lease. | diff --git a/clients/python/CHANGELOG.md b/clients/python/CHANGELOG.md index c1dd0d1..0879ccf 100644 --- a/clients/python/CHANGELOG.md +++ b/clients/python/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to the SQL-FS Python SDK are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.3.0] - 2026-06-05 + +### Added + +- `Client(max_file_size=...)` — a per-file size ceiling (default 64 MiB) enforced + client-side on every write path (`ingest_files`, `fs.write`, `fs.write_files`) + **before** any content is base64-encoded or sent. Oversized files raise + `ValidationError(code="EFILE_TOO_LARGE")` (with `status=None`) naming each + offending path and size. Set `max_file_size=0` to disable. The limit is threaded + to every `Sandbox` the client creates or attaches. + ## [0.2.5] - 2026-05-27 ### Changed diff --git a/clients/python/README.md b/clients/python/README.md index 7dc2ce5..ad6309a 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -56,6 +56,24 @@ If you already hold a JWT (e.g. minted via `pnpm token:create`), pass `token=` i fs = Client(base_url="...", token="eyJhbGciOi...") ``` +### Per-file size limit + +`Client(max_file_size=...)` (default **64 MiB**) caps individual files on every +write path — `ingest_files`, `fs.write`, `fs.write_files` — and is checked +**client-side before anything is base64-encoded or sent**. An oversized file +raises `ValidationError(code="EFILE_TOO_LARGE")` (with `status=None`) naming each +offending path and size; nothing is transmitted. The limit is threaded to every +`Sandbox` the client creates or attaches. + +```python +fs = Client(base_url="...", auth_secret="...", sub="agent", max_file_size=128 * 1024 * 1024) # raise to 128 MiB +fs = Client(base_url="...", auth_secret="...", sub="agent", max_file_size=0) # disable the check +``` + +> The server also caps the whole request body (`MAX_REQUEST_BODY_BYTES`, default +> 256 MB); after ~33% base64 inflation that's ~190 MB of raw bytes per call across +> all files. The 64 MiB default keeps a single file well inside that. + ## API surface ### `Client` @@ -117,6 +135,10 @@ All exceptions derive from `SQLFSError`. HTTP status codes map to: Each error exposes `.code` (server error code, e.g. `ENOENT`), `.status`, and `.details`. +`ValidationError` is also raised **client-side** with `code="EFILE_TOO_LARGE"` and +`status=None` when a file exceeds `Client(max_file_size=...)` — before any HTTP +request is made. `.details` lists each offending `path (size > limit)`. + ## Performance patterns `exec_batch` is for collapsing many round-trips, not for parallelising diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 2982cbf..7569b6d 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sql-fs-sdk" -version = "0.2.5" +version = "0.3.0" description = "Python SDK for the SQL-FS API — persistent bash sandboxes for AI agents" readme = "README.md" requires-python = ">=3.9" diff --git a/clients/python/src/sqlfs/_version.py b/clients/python/src/sqlfs/_version.py index fe404ae..493f741 100644 --- a/clients/python/src/sqlfs/_version.py +++ b/clients/python/src/sqlfs/_version.py @@ -1 +1 @@ -__version__ = "0.2.5" +__version__ = "0.3.0" diff --git a/clients/python/src/sqlfs/client.py b/clients/python/src/sqlfs/client.py index f239313..091615e 100644 --- a/clients/python/src/sqlfs/client.py +++ b/clients/python/src/sqlfs/client.py @@ -9,7 +9,7 @@ from ._http import Transport from .models import SandboxInfo, SandboxRecord -from .sandbox import Sandbox +from .sandbox import DEFAULT_MAX_FILE_SIZE, Sandbox class Client: @@ -28,6 +28,11 @@ class Client: The client is safe to keep around for the lifetime of your process. Use `with Client(...) as c:` to ensure HTTP connections are released. + + `max_file_size` (bytes, default 64 MiB) caps individual files on every + write path (`ingest_files`, `fs.write`, `fs.write_files`). Oversized files + raise `ValidationError` (code `EFILE_TOO_LARGE`) client-side, before any + content is base64-encoded or sent. Set to 0 to disable the check. """ def __init__( @@ -44,6 +49,7 @@ def __init__( max_retries: int = 3, user_agent: Optional[str] = None, http_client: Optional[httpx.Client] = None, + max_file_size: int = DEFAULT_MAX_FILE_SIZE, ) -> None: self._transport = Transport( base_url=base_url, @@ -58,7 +64,7 @@ def __init__( user_agent=user_agent, http_client=http_client, ) - self.sandboxes = SandboxesResource(self._transport) + self.sandboxes = SandboxesResource(self._transport, max_file_size=max_file_size) @property def token(self) -> str: @@ -78,8 +84,14 @@ def __exit__(self, *_exc: Any) -> None: class SandboxesResource: """`client.sandboxes.*` — sandbox CRUD.""" - def __init__(self, transport: Transport) -> None: + def __init__( + self, + transport: Transport, + *, + max_file_size: int = DEFAULT_MAX_FILE_SIZE, + ) -> None: self._t = transport + self._max_file_size = max_file_size def list(self) -> List[SandboxRecord]: """`GET /v1/sandboxes` — list sandboxes owned by the caller.""" @@ -131,7 +143,7 @@ def create( resp = self._t.request("POST", "/sandboxes", json_body=body or None) record = SandboxRecord.from_api(resp.json()) - return Sandbox(self._t, record.id, record=record) + return Sandbox(self._t, record.id, record=record, max_file_size=self._max_file_size) def get(self, sandbox_id: str) -> SandboxInfo: """`GET /v1/sandboxes/{id}` — fetch sandbox metadata.""" @@ -144,7 +156,7 @@ def attach(self, sandbox_id: str) -> Sandbox: Does not hit the network — use `.get(id)` first if you want to verify the sandbox exists / is accessible to the current token. """ - return Sandbox(self._t, sandbox_id) + return Sandbox(self._t, sandbox_id, max_file_size=self._max_file_size) def delete(self, sandbox_id: str) -> None: """`DELETE /v1/sandboxes/{id}` — destroy the sandbox and all blobs.""" diff --git a/clients/python/src/sqlfs/sandbox.py b/clients/python/src/sqlfs/sandbox.py index db38570..cb4acef 100644 --- a/clients/python/src/sqlfs/sandbox.py +++ b/clients/python/src/sqlfs/sandbox.py @@ -44,13 +44,54 @@ TreeEntry, ) +#: Default per-file size ceiling (bytes) enforced client-side before any +#: content is base64-encoded or sent over the network. 64 MiB. Override per +#: client via ``Client(max_file_size=...)`` or disable with ``max_file_size=0``. +DEFAULT_MAX_FILE_SIZE = 64 * 1024 * 1024 + + +def _content_size(content: Union[str, bytes]) -> int: + """Raw byte length of a write payload (``str`` measured as UTF-8).""" + return len(content.encode("utf-8")) if isinstance(content, str) else len(content) + + +def _enforce_max_file_size( + files: Mapping[str, Union[str, bytes]], + max_file_size: int, +) -> None: + """Reject oversized files *before* anything is encoded or sent. + + Raises ``ValidationError`` (code ``EFILE_TOO_LARGE``) naming every offending + path and its size. A ``max_file_size`` of 0 (or negative) disables the check. + """ + if max_file_size <= 0: + return + too_big = [ + (path, size) + for path, content in files.items() + if (size := _content_size(content)) > max_file_size + ] + if too_big: + details = [f"{path} ({size} bytes > {max_file_size} limit)" for path, size in too_big] + raise ValidationError( + "file exceeds max_file_size: " + "; ".join(details), + code="EFILE_TOO_LARGE", + details=details, + ) + class FilesAPI: """`sandbox.fs.*` — file operations on a single sandbox.""" - def __init__(self, transport: Transport, sandbox_id: str) -> None: + def __init__( + self, + transport: Transport, + sandbox_id: str, + max_file_size: int = DEFAULT_MAX_FILE_SIZE, + ) -> None: self._t = transport self._id = sandbox_id + self._max_file_size = max_file_size def read(self, path: str) -> ReadResult: """`GET /files/{path}` — read raw bytes + parsed `X-FS-Stat` header.""" @@ -74,6 +115,7 @@ def read_text(self, path: str, encoding: str = "utf-8") -> str: def write(self, path: str, content: Union[str, bytes]) -> None: """`PUT /files/{path}` — write raw bytes; parents auto-created.""" + _enforce_max_file_size({path: content}, self._max_file_size) body = content.encode("utf-8") if isinstance(content, str) else content self._t.request( "PUT", @@ -89,6 +131,7 @@ def write_files(self, files: Mapping[str, str]) -> None: Keys are absolute paths inside the sandbox, values are file contents (text). For binary content, prefer `ingest_files()` (base64). """ + _enforce_max_file_size(files, self._max_file_size) self._t.request( "POST", f"/sandboxes/{self._id}/writeFiles", @@ -146,11 +189,13 @@ def __init__( sandbox_id: str, *, record: Optional[SandboxRecord] = None, + max_file_size: int = DEFAULT_MAX_FILE_SIZE, ) -> None: self._t = transport self._id = sandbox_id self._record = record - self.fs = FilesAPI(transport, sandbox_id) + self._max_file_size = max_file_size + self.fs = FilesAPI(transport, sandbox_id, max_file_size) @property def id(self) -> str: @@ -369,7 +414,12 @@ def ingest_files( Keys are paths relative to `base_path`. Values may be `str` (encoded utf-8) or `bytes` (encoded as-is); the SDK base64-encodes them. + + Each file is checked against the client's ``max_file_size`` first; an + oversized file raises ``ValidationError`` (code ``EFILE_TOO_LARGE``) + before anything is encoded or sent over the network. """ + _enforce_max_file_size(files, self._max_file_size) encoded: Dict[str, str] = {} for path, content in files.items(): data = content.encode("utf-8") if isinstance(content, str) else content diff --git a/package.json b/package.json index 8f1ff41..884121c 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "hono": "^4.7.0", "ioredis": "^5.4.0", "jose": "^6.0.0", - "just-bash": "^2.14.5", + "just-bash": "^3.0.1", "lru-cache": "^11.0.0", "mssql": "^11.0.0", "mysql2": "^3.12.0", diff --git a/plugins/sql-fs/skills/api/ref/endpoints.md b/plugins/sql-fs/skills/api/ref/endpoints.md index 06a843f..d37fa29 100644 --- a/plugins/sql-fs/skills/api/ref/endpoints.md +++ b/plugins/sql-fs/skills/api/ref/endpoints.md @@ -447,6 +447,18 @@ Request body: The previous "≤25 files per batch" rule no longer applies — the dialect now uses a single bulk INSERT. Practical caps are HTTP body size and the 240 s ACA gateway window. +**Size limits:** the only ceiling is the request body — `MAX_REQUEST_BODY_BYTES` +(default 256 MB). Since base64 inflates content ~33%, that's ~190 MB of raw bytes +per call, across all files. Multi-MB files ingest fine and store byte-exact (a +single 128 MB file has been verified end-to-end). Individual files are buffered + +base64-decoded in memory, so very large files are memory- and time-heavy — prefer +splitting big payloads across several calls. (Clients such as the Python SDK also +enforce their own per-file ceiling — default 64 MB — *before* sending.) + +> Fixed in this release: base64 validation previously overflowed V8's regex stack +> on strings beyond ~1 MB, so files larger than ~750 KB returned +> `500 INTERNAL_ERROR` during validation. Large-file ingest now succeeds. + --- ## Admin Maintenance diff --git a/plugins/sql-fs/skills/py-sdk/SKILL.md b/plugins/sql-fs/skills/py-sdk/SKILL.md index c76015c..19c8b0c 100644 --- a/plugins/sql-fs/skills/py-sdk/SKILL.md +++ b/plugins/sql-fs/skills/py-sdk/SKILL.md @@ -143,6 +143,12 @@ methods are banned for agent use. a local folder into a fresh sandbox (single HTTP round-trip, base64-safe for binary). After ingest, all further interaction must be via `sb.exec / exec_batch / exec_stream`. +**Per-file size limit:** every write path (`ingest_files`, `fs.write`, `fs.write_files`) +enforces `Client(max_file_size=...)` — default **64 MiB** — **before** anything is +encoded or sent. An oversized file raises `ValidationError(code="EFILE_TOO_LARGE")` +client-side (no network round-trip). Raise it via `Client(max_file_size=...)` or +disable with `max_file_size=0`. See `ref/client.md`. + When a user asks for an `sb.fs.*` snippet, **decline politely and produce the exec equivalent instead**, citing this policy. If the workflow truly cannot be expressed via exec (e.g. streaming a 100 MB binary download to disk), surface that as a diff --git a/plugins/sql-fs/skills/py-sdk/ref/client.md b/plugins/sql-fs/skills/py-sdk/ref/client.md index e97265e..abddcbf 100644 --- a/plugins/sql-fs/skills/py-sdk/ref/client.md +++ b/plugins/sql-fs/skills/py-sdk/ref/client.md @@ -26,6 +26,7 @@ Client( max_retries: int = 3, # 5xx / 429 retries with jitter user_agent: str | None = None, # defaults to "sqlfs-python/" http_client: httpx.Client | None = None, # bring-your-own (e.g. for mocks) + max_file_size: int = 64 * 1024 * 1024, # per-file ceiling (bytes); 0 disables ) ``` @@ -36,6 +37,24 @@ Client( the JWT is minted on the first request that needs it (or when you read `client.token`). +**`max_file_size` (bytes, default 64 MiB).** A per-file ceiling enforced +**client-side**, before any content is base64-encoded or sent over the network. +It applies to every write path — `sb.ingest_files(...)`, `sb.fs.write(...)`, and +`sb.fs.write_files(...)`. A file larger than the limit raises +`ValidationError(code="EFILE_TOO_LARGE")` naming each offending path and its +size; nothing is transmitted. The limit is threaded down to every `Sandbox` the +client creates or attaches. Set `max_file_size=0` to disable the check entirely. + +```python +fs = Client(base_url=..., auth_secret=..., sub="agent", max_file_size=128 * 1024 * 1024) # raise to 128 MiB +fs = Client(base_url=..., auth_secret=..., sub="agent", max_file_size=0) # disable +``` + +> Sizing note: the server caps the whole HTTP request body (default 256 MB) and +> base64 inflates content ~33%, so the practical per-request ceiling is ~190 MB +> of raw bytes regardless of `max_file_size`. The default 64 MiB keeps a single +> file well inside that, with margin for batching several files in one ingest. + --- ## Context manager diff --git a/plugins/sql-fs/skills/py-sdk/ref/errors.md b/plugins/sql-fs/skills/py-sdk/ref/errors.md index 0dc232b..6ac8bcb 100644 --- a/plugins/sql-fs/skills/py-sdk/ref/errors.md +++ b/plugins/sql-fs/skills/py-sdk/ref/errors.md @@ -26,7 +26,7 @@ SQLFSError base — never raised directly by the SDK ├── AuthError 401 / 403 ├── NotFoundError 404 ├── ConflictError 409 (EEXIST, ENOTEMPTY) -├── ValidationError 400 (INVALID_INPUT, EISDIR, ENOTDIR, EINVAL, ELOOP) +├── ValidationError 400 (INVALID_INPUT, EISDIR, ENOTDIR, EINVAL, ELOOP); also client-side EFILE_TOO_LARGE ├── ExecTimeoutError 408 (script ran longer than timeout_ms) ├── RateLimitError 429 ├── ServerError 5xx after retries exhausted @@ -54,6 +54,30 @@ with exponential jitter. The exception you see is the **final** failure. --- +## Client-side validation (raised before any HTTP request) + +Not every `ValidationError` comes from the server. The SDK enforces the +`Client(max_file_size=...)` per-file ceiling (default 64 MiB) locally, so an +oversized file fails **before** anything is base64-encoded or sent: + +| `code` | Raised by | Cause | +|---|---|---| +| `EFILE_TOO_LARGE` | `ingest_files`, `fs.write`, `fs.write_files` | A file exceeds the client's `max_file_size`. `e.status` is `None` (no HTTP round-trip happened); `e.details` lists each offending `path (size > limit)`. | + +```python +from sqlfs import ValidationError + +try: + sb.ingest_files({"huge.bin": payload}) +except ValidationError as e: + if e.code == "EFILE_TOO_LARGE": + print("too big, never sent:", e.details) # e.status is None +``` + +Set `max_file_size=0` on the `Client` to disable this check. + +--- + ## Common attributes Every exception exposes: diff --git a/plugins/sql-fs/skills/py-sdk/ref/sandbox.md b/plugins/sql-fs/skills/py-sdk/ref/sandbox.md index ecb50bc..2b9d40a 100644 --- a/plugins/sql-fs/skills/py-sdk/ref/sandbox.md +++ b/plugins/sql-fs/skills/py-sdk/ref/sandbox.md @@ -165,10 +165,17 @@ sb.ingest_files( Returns the server's response dict, e.g. `{"status": "ok", "fileCount": 125}`. +**Per-file size check (client-side).** Before encoding anything, each file is +checked against the client's `max_file_size` (default 64 MiB — see +`ref/client.md`). A file over the limit raises +`ValidationError(code="EFILE_TOO_LARGE")` and **nothing is sent**. Raise or +disable it with `Client(max_file_size=...)` / `max_file_size=0`. + **Hard limits to keep in mind:** -- All file bytes are buffered into one HTTP request body. Practical ceiling - before things get slow: ~10 MB total, ~500 files. For larger payloads, split - into multiple `ingest_files` calls. +- All file bytes are buffered into one HTTP request body. The server caps the + whole body (default 256 MB); after ~33% base64 inflation that's ~190 MB of raw + bytes per call. Individual large files (tens of MB) ingest fine and store + byte-exact — split very large payloads across multiple `ingest_files` calls. --- diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fe99344..363da3b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -36,8 +36,8 @@ importers: specifier: ^6.0.0 version: 6.2.2 just-bash: - specifier: ^2.14.5 - version: 2.14.5 + specifier: ^3.0.1 + version: 3.0.1 lru-cache: specifier: ^11.0.0 version: 11.3.5 @@ -1968,8 +1968,8 @@ packages: resolution: {integrity: sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==} engines: {node: '>=12', npm: '>=6'} - just-bash@2.14.5: - resolution: {integrity: sha512-MCBGnRlDeZ/MM7mcw+ZuSGFMBsggajrmKz6e/hrOAN7syvVZkjiY+Vh2wyCwN/CdcnAX5SxbiQB51n5nrQuX+g==} + just-bash@3.0.1: + resolution: {integrity: sha512-YVyzCN08fKarUnwqy7rKOAcX+2MLYLnYInuowmUXn3mqhrtd4ieZNBuzdQG+qYV9DqnIWuv9Whiph0WRIWsBtw==} hasBin: true jwa@2.0.1: @@ -4494,7 +4494,7 @@ snapshots: ms: 2.1.3 semver: 7.7.4 - just-bash@2.14.5: + just-bash@3.0.1: dependencies: diff: 8.0.4 fast-xml-parser: 5.7.3 diff --git a/src/api/ingest-validation.ts b/src/api/ingest-validation.ts index a8263d2..e189646 100644 --- a/src/api/ingest-validation.ts +++ b/src/api/ingest-validation.ts @@ -50,8 +50,13 @@ export function isValidRelativePath(p: string): boolean { * bytes whose checksum no longer matches the value the caller sent. */ const BASE64_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/; +// V8's regex engine overflows the call stack on strings beyond ~1 MB due to +// recursion in the `*` quantifier path for very long inputs. Skip the regex +// for large strings and rely solely on the canonical round-trip check, which +// is native code and never overflows. +const BASE64_RE_MAX_LEN = 1_000_000; export function isValidBase64(s: string): boolean { - if (!BASE64_RE.test(s)) return false; + if (s.length <= BASE64_RE_MAX_LEN && !BASE64_RE.test(s)) return false; return Buffer.from(s, "base64").toString("base64") === s; }