From be17c4328ce4665650e981241b4f7cb541c68b14 Mon Sep 17 00:00:00 2001 From: Team AppThreat Date: Sun, 13 Sep 2026 11:12:56 +0100 Subject: [PATCH 1/3] Implement the 9.1 roadmap: sync-path UDFs, JS virtual tables, changeset rebasing, ergonomics parity, tag store, compat shim and query spans --- README.md | 272 ++++- binding.gyp | 3 +- contrib/check-jsdoc.js | 4 +- deps/sqlite3.gyp | 5 + docs/install.md | 11 +- docs/performance.md | 34 +- examples/kysely-dialect.mjs | 95 ++ lib/augment.d.ts | 277 +++++ lib/compat.d.ts | 157 +++ lib/compat.js | 618 ++++++++++ lib/migrate.d.ts | 28 + lib/migrate.js | 157 +++ lib/native.d.ts | 153 +++ lib/promises.js | 97 +- lib/sqlite3.d.ts | 104 ++ lib/sqlite3.js | 1941 ++++++++++++++++++++++++++++--- package.json | 14 +- plans/01-project-assessment.md | 139 +++ plans/02-competitor-research.md | 307 +++++ plans/03-feature-gap-matrix.md | 139 +++ plans/04-roadmap.md | 256 ++++ plans/README.md | 48 + src/database.cc | 188 +++ src/database.h | 91 +- src/function.cc | 199 +++- src/node_sqlite3.cc | 51 + src/session.cc | 233 +++- src/session.h | 22 + src/statement.cc | 315 ++++- src/statement.h | 80 +- src/vtab.cc | 1260 ++++++++++++++++++++ src/vtab.h | 113 ++ test/aggregate.test.js | 29 +- test/compat.test.js | 178 +++ test/diagnostics.test.js | 176 +++ test/ergonomics.test.js | 583 ++++++++++ test/function.test.js | 149 ++- test/migrate.test.js | 134 +++ test/session_rebase.test.js | 202 ++++ test/sync.test.js | 4 +- test/tagstore.test.js | 149 +++ test/vtab.test.js | 508 ++++++++ tools/gen-types.js | 43 +- 43 files changed, 9221 insertions(+), 345 deletions(-) create mode 100644 examples/kysely-dialect.mjs create mode 100644 lib/compat.d.ts create mode 100644 lib/compat.js create mode 100644 lib/migrate.d.ts create mode 100644 lib/migrate.js create mode 100644 plans/01-project-assessment.md create mode 100644 plans/02-competitor-research.md create mode 100644 plans/03-feature-gap-matrix.md create mode 100644 plans/04-roadmap.md create mode 100644 plans/README.md create mode 100644 src/vtab.cc create mode 100644 src/vtab.h create mode 100644 test/compat.test.js create mode 100644 test/diagnostics.test.js create mode 100644 test/ergonomics.test.js create mode 100644 test/migrate.test.js create mode 100644 test/session_rebase.test.js create mode 100644 test/tagstore.test.js create mode 100644 test/vtab.test.js diff --git a/README.md b/README.md index 165025a..8f31031 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ relying on a ❌ below. | Asynchronous API (event loop stays free) | ✅ callbacks + promises | ❌ synchronous only | | `async` iteration (`for await`), streams | ✅ `iterate`, `stream` | ❌ (sync `iterate` only) | | Worker-thread connection pool | ✅ `pool()` | ❌ | -| Transaction helper with savepoints | ✅ `transaction()` | ❌ hand-rolled `BEGIN`/`COMMIT` | +| Transaction helper with savepoints | ✅ `transaction()`, reusable `createTransaction()` | ❌ hand-rolled `BEGIN`/`COMMIT` | | Statement cache | ✅ `cacheStatements()`, implicit on sync | ❌ prepare per call | | Custom collations | ✅ `collation()` / `removeCollation()` | ❌ | | Incremental blob I/O | ✅ `openBlob()` | ❌ read/write whole values | @@ -50,44 +50,63 @@ relying on a ❌ below. | Query cancellation | ✅ `cancellationToken()`, connection-wide | ❌ | | WAL checkpoint control | ✅ `checkpoint()` | ❌ | | Schema introspection | ✅ `tableInfo()`, `columns()`, `parameterNames` | ⚠️ `columns()` only | -| Changeset utilities | ✅ concat / invert / iterate | ⚠️ apply + create only | +| Changeset utilities | ✅ concat / invert / iterate / **rebase** / `session.diff()` | ⚠️ apply + create only | +| JavaScript virtual tables | ✅ `db.table()` generator tables + `db.values()` array tables | ❌ | +| Atomic `batch()` | ✅ multi-statement, libsql-style modes | ❌ | +| `pragma()` / `explain()` helpers | ✅ parsed rows, `EXPLAIN QUERY PLAN` | ❌ | +| SQL text dump (`.dump`) | ✅ `db.dump()`, streaming `iterdump()` | ❌ | +| Error token byte offset | ✅ `err.offset` on failed prepares | ❌ | +| `diagnostics_channel` query spans | ✅ `subscribeQueries()` + the `sqlite.db.query` mirror | ⚠️ `sqlite.db.query` only | | Backup control | ✅ handle you step yourself (`remaining`, `idle`, retry policy) | ⚠️ one-shot promise (`rate`, `progress`) | -| Integer read modes | ✅ `number` / `mixed` / `bigint`, per connection | ⚠️ `setReadBigInts()` per statement | +| Integer read modes | ✅ `number` / `mixed` / `bigint`, per connection or statement | ⚠️ `setReadBigInts()` per statement | | Electron support | ✅ tested in CI, main + utility process | ⚠️ works, untested by us | ## What both have User-defined functions and aggregates (including window functions via -`inverse`), sessions and changesets, the authorizer, extension loading, +`inverse`) — on the async paths here through a worker round trip, and on +the synchronous fast path through a direct re-entrant call (see below) — +sessions and changesets, the authorizer, extension loading, `serialize`/`deserialize`, incremental online backup with progress reporting, `readOnly` and busy-timeout connection options, array row -mode, bare and unknown named-parameter control, and extended result -codes on errors. Both also **refuse to truncate** an INTEGER outside the -safe range rather than silently losing precision — they differ only in -how you opt into `BigInt`. +mode, bare and unknown named-parameter control, tagged-template queries +(`createTagStore()` here is promise-native and carries the +`raw`/`join`/`identifier` composition helpers), `inTransaction`, and +extended result codes on errors. Both also **refuse to truncate** an +INTEGER outside the safe range rather than silently losing precision — +they differ only in how you opt into `BigInt`. ## What only `node:sqlite` has -| Capability | Why it matters | -| ---------------------------------------------------------------------- | ------------------------------------------------------------- | -| **Zero install** — built in, no compiler, no prebuild, no supply chain | Usually the deciding factor | -| **UDFs callable from synchronous queries** | See below — a real architectural difference, not an oversight | -| Tagged-template queries (`createTagStore`) | Ergonomic SQL literals | -| `enableDefensive()` | Hardening for untrusted SQL | - -The UDF difference is worth understanding before choosing. `node:sqlite` -runs SQLite on the main thread, so a JavaScript callback can run inline -while a query is stepping. This package runs asynchronous queries on a -worker, and a JS callback fires safely there — but calling a UDF from -the _synchronous_ fast path would mean SQLite blocking the JS thread -that has to run the callback, which deadlocks. It refuses instead, with -an error saying so. So: **UDFs, aggregates and window functions work on -the async API here, not on `getSync`/`allSync`/`runSync`.** If you need -custom functions inside otherwise-synchronous code, `node:sqlite` is the -better fit. +| Capability | Why it matters | +| ---------------------------------------------------------------------- | -------------------------------- | +| **Zero install** — built in, no compiler, no prebuild, no supply chain | Usually the deciding factor | +| `enableDefensive()` as a method | ⚠️ this package has `dbConfig()` | + +### The UDF story + +`node:sqlite` runs SQLite on the main thread, so a JavaScript callback +runs inline while a query is stepping. This package runs asynchronous +queries on a worker, and a JS callback fires safely there — one blocking +round trip to the JS thread per call (a few microseconds; fine for +bounded-row logic, wrong for per-row bulk predicates). Since 9.1 the +**synchronous fast path runs UDFs directly**: on `getSync`/`allSync`/ +`runSync` the JS thread is the one executing SQL, so the callback is +invoked re-entrantly on that same thread — exactly like `node:sqlite` — +with the one hard rule preserved (a UDF cannot drive *its own* statement +re-entrantly; other statements on the connection work). A throwing +callback surfaces through the step error with the thrown value as +`cause`, and JS collations/progress callbacks still refuse on the sync +path (they have no error channel). + +The cost model in one line: **sync-path UDFs are direct calls; async-path +UDFs pay one worker→JS round trip each.** Migrating from `node:sqlite` is mostly mechanical — `DatabaseSync` maps -to `Database` plus the `*Sync` methods. See +to `Database` plus the `*Sync` methods — and since 9.1 there is a +drop-in shim: `import { DatabaseSync } from '@appthreat/sqlite3/compat'` +maps the `node:sqlite` surface onto this package's sync fast path (with +the documented divergences where a synchronous form cannot exist). See [MIGRATING-TO-V9.md](MIGRATING-TO-V9.md) for the value-marshalling differences, which are where the surprises live. @@ -101,10 +120,12 @@ npm install @appthreat/sqlite3 pnpm add @appthreat/sqlite3 # or yarn add @appthreat/sqlite3 -# or -bun add @appthreat/sqlite3 ``` +On the Bun runtime use Bun's built-in `bun:sqlite` — this package's +addon needs N-API behaviour Bun 1.4 does not provide (see +[docs/install.md](docs/install.md#bun-install-works-runtime-does-not-yet)). + - GitHub's `master` branch: `npm install https://github.com/AppThreat/node-sqlite3/tarball/master` Requires Node.js >= 24. See [docs/install.md](docs/install.md) for the full @@ -437,9 +458,17 @@ fetch-and-filter overtakes a UDF predicate, measured at ~60× on a real Two deliberate restrictions follow from the threading model: - A JS function reached from a **synchronous method** - (`getSync`/`runSync`/`allSync`/`prepareSync`) fails with an explicit - error instead of deadlocking: the JS thread is the one blocked inside - SQLite there and cannot run the callback. Use the async API. + (`getSync`/`runSync`/`allSync`/`prepareSync` steps) runs **directly**: + the JS thread is the one executing SQL there, so the callback is + invoked re-entrantly on that same thread — same-thread, no round + trip, like `node:sqlite`. (Since 9.1; before that this refused.) The + one rule: such a callback cannot drive *its own* statement + re-entrantly — use another statement or the async API. Nor can it do + anything that flushes the statement cache mid-step, since the cache + holds the executing statement: registering or removing a function, + aggregate, collation, virtual table or authorizer policy, clearing a + tag store, or closing the connection all refuse with a message saying + so. Do those before or after the query. - While a JS **collation** is registered, the synchronous methods refuse to run entirely (remove it with `removeCollation()` or use the async API): a comparison would need the blocked JS thread, and unlike @@ -605,6 +634,32 @@ const inverse = sqlite3.invertChangeset(changeset); // undoes the apply const both = sqlite3.concatChangeset(a, b); // a then b ``` +### Rebasing (9.1): the fork-free sync loop + +Apply with `{ rebase: true }` to harvest a **rebase buffer** — the +record of which conflicting changes were omitted or replaced — then +rebase later local changesets against it before applying them remotely. +Together with `session.diff()` (a changeset of the differences between +an attached database's table and this one, without recording anything) +this is a complete offline-first sync toolkit on stock SQLite; no other +JS driver has rebasing (rusqlite is the only binding anywhere): + +```js +// one round of sync: apply the server's changes, remember the conflicts +const rebase = await serverDb.applyChangeset(serverChangeset, { + conflict: "omit", + rebase: true, // resolves the rebase buffer (null if no conflicts) +}); +// later: rebase the client's new work against those resolutions and push +const rebased = sqlite3.rebaseChangeset(clientChangeset, rebase); +await serverDb.applyChangeset(rebased); +``` + +`session.diff('t', 'other')` records the changes that transform the +attached database `other`'s table into this connection's — the +"what changed between these two databases" primitive for verification +and sync tooling. + `applyChangeset` wraps the apply in one savepoint: either every change lands or the whole apply rolls back. `conflict` decides what happens on a collision — `'abort'` (the default) rolls back, `'omit'` skips the @@ -682,6 +737,135 @@ cannot grow through the handle: size the column first (e.g. through a blob handle surfaces as a `'preupdate'` delete event (the new values are not yet available inside `sqlite3_blob_write`). +## Ergonomics, virtual tables and migrations (9.1) + +The sync-first drivers make a set of small things trivial; 9.1 adds the +whole bundle, promise-native: + +```js +// pragmas with parsed results (the recommended way to run them) +await db.pragma("journal_mode = WAL"); +const version = await db.pragma("user_version", { simple: true }); + +// the query planner's own account, without executing +const plan = await db.explain("SELECT * FROM users WHERE id = ?"); + +// atomic multi-statement batches (migrations, seeding) +await db.batch([ + "CREATE TABLE t (a)", + { sql: "INSERT INTO t VALUES (?)", args: 1 }, +]); + +// reusable transactions with begin-mode variants +const move = db.createTransaction((tx, from, to, n) => /* ... */); +await move.immediate(1, 2, 50); + +// .dump-style SQL export (streaming form: sqlite3.iterdump(db)) — +// AUTOINCREMENT counters, user_version and virtual-table content included +const sqlText = await db.dump(); + +// live state +db.inTransaction; // true inside BEGIN +db.txnState; // 'none' | 'read' | 'write' +db.limits; // the run-time limits +db.status("cacheHit"); // { current, highwater } +db.location(); // the attached file's path + +// array/pluck row modes on the async paths too +const ids = await db.all("SELECT id FROM t", { rowMode: "pluck" }); + +// failed prepares carry the failing token's byte offset +try { db.prepareSync("SELECT * FRUM t"); } +catch (err) { err.offset; } // 9 + +// per-statement integer mode +const stmt = db.prepareSync(sql, { integerMode: "bigint" }); +``` + +**JavaScript virtual tables** — generator-computed, read-only, working +from both the async paths (rows are pulled in batches through the worker +round trip) and the sync methods (direct re-entrant calls): + +```js +db.table("sequence", { + columns: ["value", "count"], + parameters: ["count"], // HIDDEN → a table-valued function's argument + rows: function* (count) { + for (let i = 0; i < count; i++) yield [i, count]; + }, +}); +await db.all("SELECT value FROM sequence(5)"); +``` + +Rows are pulled as the query consumes them (64 at first, growing to +1024), so an **unbounded generator is fine** — `SELECT … LIMIT 3` over an +infinite sequence stops after the first batch, and a table larger than +memory streams. A scan that stops early leaves the generator suspended +and never resumes it, so generator `finally` blocks are not a place to +release resources. An unconstrained HIDDEN parameter reaches the +generator as `undefined`; one the query constrained is also reported as +that column's value, so `SELECT count FROM sequence(5)` works without the +generator echoing it. + +`db.values(array)` exposes any JS array as a queryable table — the +rusqlite `rarray()` ergonomics no JS driver had: `JOIN` against +in-memory data instead of building IN-lists. `drop()` the handle when you +are done (anonymous registrations are capped at 32 per connection, the +oldest being dropped to make room). + +**Tagged templates** — node:sqlite's tag store, promise-native, with +the composition helpers ORMs need (the ones Bun notably lacks): + +```js +const store = db.createTagStore(); +const table = store.identifier("users"); +await store.all`SELECT * FROM ${table} WHERE id = ${id}`; +await store.all`SELECT * FROM ${table} WHERE id IN (${store.join(ids)})`; +``` + +Every interpolated value binds as a parameter unless it came from +`raw`/`identifier`/`identifierPath`/`join`/`empty` — a look-alike object +(`{ text, params }` out of `JSON.parse`) binds like anything else rather +than becoming SQL. `join()` takes values as well as fragments, which is +what an IN-list of user data needs. Creating a store turns the connection +statement cache on if it was off, since reusing statements is the point. + +**Migrations** — `PRAGMA user_version`-based, sequential, each in one +transaction; from a directory of `NNN-name.sql` files or a list: + +```js +await sqlite3.migrate(db, "migrations/"); +``` + +**Observability** — finished-statement spans on diagnostics channels, +armed only while subscribed: + +```js +const unsubscribe = sqlite3.subscribeQueries(({ sql, durationMs }) => { + apm.record(sql, durationMs); // also mirrored to node's sqlite.db.query +}); +``` + +**node:sqlite drop-in** — code written against the built-in module can +switch without rewriting: + +```js +import { DatabaseSync } from "@appthreat/sqlite3/compat"; +const db = new DatabaseSync(":memory:"); // opens synchronously +db.exec("CREATE TABLE t (a)"); +const row = db.prepare("SELECT * FROM t WHERE a = ?").get(1); +``` + +The shim maps onto the sync fast path (re-entrant UDFs included) and +exposes the full async surface through `db.native`. The places a +synchronous form cannot exist keep this package's async signatures and +are listed at the top of [lib/compat.js](lib/compat.js): sessions, +`serialize`, and `close()`, which starts the close rather than completing +it (`await using`, or `await db.native.close()`, when a caller must know +the file is free — deleting or reopening it on Windows). +`StatementSync.iterate()` materialises its rows, because the sync path has +no mid-cursor suspension. + ## Worker threads and the connection pool (v9) The addon is context-aware: it loads cleanly in every `worker_threads` @@ -735,6 +919,32 @@ reads. See [docs/concurrency.md](docs/concurrency.md) for the full picture: `serialize()`/`parallelize()` semantics, WAL, busy timeouts, and when to use one connection, several, or the pool. +## Using with Kysely and Drizzle + +Both major ORMs need only what this package already has — no separate +dialect package. For **Kysely** (async-native, the natural fit) a +working dialect is ~40 lines over one connection: +`acquireConnection`/`releaseConnection` hand out the database, the +transaction verbs are raw `BEGIN`/`COMMIT`/`ROLLBACK`, and Kysely's own +`SqliteAdapter`/`SqliteIntrospector`/`SqliteQueryCompiler` fill the +rest — +[examples/kysely-dialect.mjs](examples/kysely-dialect.mjs) is a +zero-dependency template: + +```js +import { kyselyFor } from "./examples/kysely-dialect.mjs"; +const kysely = kyselyFor(db, { + onCreateConnection: (c) => c.pragma("journal_mode = WAL"), +}); +const rows = await kysely.selectFrom("users").selectAll().execute(); +``` + +For **Drizzle** (whose SQLite dialect is synchronous), map onto the +sync fast path: `prepare` → `db.prepareSync`, `run`/`all`/`get` → the +`*Sync` methods, `transaction` → `db.transaction(fn, { mode: "immediate" })`. +Drizzle's better-sqlite3 driver is ~100 lines; the deltas are those +three substitutions plus reading `lastInsertRowid` from the run result. + ## Source install To skip searching for pre-compiled binaries, and force a build from source, use diff --git a/binding.gyp b/binding.gyp index 11fd9bf..6beb5e2 100644 --- a/binding.gyp +++ b/binding.gyp @@ -68,7 +68,8 @@ "src/function.cc", "src/node_sqlite3.cc", "src/session.cc", - "src/statement.cc" + "src/statement.cc", + "src/vtab.cc" ], "defines": [ "NAPI_VERSION=<(napi_build_version)", "NAPI_DISABLE_CPP_EXCEPTIONS=1" ] } diff --git a/contrib/check-jsdoc.js b/contrib/check-jsdoc.js index 51501b8..4ab0edb 100644 --- a/contrib/check-jsdoc.js +++ b/contrib/check-jsdoc.js @@ -86,10 +86,12 @@ function signatureFrom(lines, i) { // Count top-level commas in a balanced parameter list (angles included so // generic parameter defaults with `` do not confuse depth). function countParams(list) { + // A trailing comma is formatting, not a parameter. + const trimmed = list.replace(/,\s*$/, ''); let depth = 0; let count = 0; let seen = false; - for (const c of list) { + for (const c of trimmed) { if (c === '(' || c === '<' || c === '[' || c === '{') depth++; else if (c === ')' || c === '>' || c === ']' || c === '}') { depth--; diff --git a/deps/sqlite3.gyp b/deps/sqlite3.gyp index 2b04ab8..e8ad170 100755 --- a/deps/sqlite3.gyp +++ b/deps/sqlite3.gyp @@ -90,6 +90,9 @@ # ~30 KB of extra amalgamation code is accepted in exchange for # column metadata (stmt.columns) and db.tableInfo(). 'SQLITE_ENABLE_COLUMN_METADATA', + # Phase 6 (observability): sqlite3_normalized_sql compiles only + # with this define. + 'SQLITE_ENABLE_NORMALIZE', 'SQLITE_DEFAULT_MEMSTATUS=0' ], }, @@ -113,6 +116,8 @@ 'SQLITE_ENABLE_STAT4', # See the direct_dependent_settings copy above (Deliverable 07). 'SQLITE_ENABLE_COLUMN_METADATA', + # See the direct_dependent_settings copy above (Phase 6). + 'SQLITE_ENABLE_NORMALIZE', 'SQLITE_DEFAULT_MEMSTATUS=0' ], 'conditions': [ diff --git a/docs/install.md b/docs/install.md index fbc4442..7783f06 100644 --- a/docs/install.md +++ b/docs/install.md @@ -7,9 +7,18 @@ This is the complete installation guide. Requirements: **Node.js >= 24** npm install @appthreat/sqlite3 pnpm add @appthreat/sqlite3 yarn add @appthreat/sqlite3 -bun add @appthreat/sqlite3 ``` +### Bun: install works, runtime does not (yet) + +The package installs under Bun and the binding loads — but Bun 1.4's +N-API implementation cannot run this addon's row delivery: synchronous +reads and async completions fail with `Invalid argument` where every +supported Node works. This is **not specific to recent versions** — +v9.0.2 fails identically — and it is tracked as a Bun N-API gap, not a +package bug. On the Bun runtime, use Bun's own built-in `bun:sqlite`; +this package is for Node (>= 24) and Electron (>= 35). + Nothing is downloaded at install time and nothing is compiled at install time on the platforms below — the prebuilt binaries ship inside the npm tarball itself. diff --git a/docs/performance.md b/docs/performance.md index f606b08..ceb456c 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -380,20 +380,26 @@ The same arithmetic applies to aggregates (`step` is one round trip per row) and collations (O(n log n) round trips for a sort — sorting in JS after `all()` is faster for anything but small or one-off sorts). -### Future direction: UDFs on the synchronous fast path - -The refusal of JS functions on the sync methods is policy, not the -structural limit above. On `getSync`/`runSync`/`allSync` the JS thread -is already the one executing SQL, so a callback could be invoked -re-entrantly — a direct call with no cross-thread round trip, the way -`node:sqlite` runs UDFs inline — and unlike collations, a function has -an error channel (`sqlite3_result_error`), so failures are reportable -mid-query. The current build refuses instead with an explicit error -(`src/function.cc`, `SyncRefusalMessage`). Landing inline sync-path -invocation would make the natural `... AND vers_compare(?, vers)` -shape genuinely fast, closing the gap the README's comparison table -attributes to `node:sqlite`. Deliberate follow-up work, out of scope -for this release. +### UDFs on the synchronous fast path (shipped in 9.1) + +The former refusal of JS functions on the sync methods was policy, not a +structural limit, and 9.1 removed it: on `getSync`/`runSync`/`allSync` +the JS thread is already the one executing SQL, so the callback is +invoked re-entrantly — a direct call with no cross-thread round trip, +the way `node:sqlite` runs UDFs inline — and unlike collations, a +function has an error channel (`sqlite3_result_error`), so failures are +reported mid-query with the thrown value attached as `cause`. The +natural `... AND vers_compare(?, vers)` shape is now a plain function +call per row on the sync path. + +Two guardrails came with it: a UDF cannot drive *its own* statement +re-entrantly (the one hard rule SQLite has; other statements on the +connection work, the connection mutex being recursive), and functions +cannot be registered or removed from inside a sync-invoked callback +(the registration handlers dispatch inline, and swapping an +implementation sqlite is mid-stepping on is not a supported sqlite +operation). JS collations and the JS progress callback still refuse on +the sync path, as before: they have no error channel. ## Where this package loses diff --git a/examples/kysely-dialect.mjs b/examples/kysely-dialect.mjs new file mode 100644 index 0000000..cdacef2 --- /dev/null +++ b/examples/kysely-dialect.mjs @@ -0,0 +1,95 @@ +// A zero-dependency Kysely dialect for @appthreat/sqlite3 (Phase 6). +// Kysely's SqliteDialect is written against better-sqlite3's sync +// surface; this adapter maps the same contract onto this package's +// async-first connection — prepared statements carry the `readonly` +// flag Kysely's driver checks, and iterate() provides the streaming +// reads its SELECTs want. +// +// Requires kysely as a peer: `npm install kysely`. + +import { + Kysely, + SqliteAdapter, + SqliteIntrospector, + SqliteQueryCompiler, +} from 'kysely'; + +/** + * Builds a Kysely instance over one @appthreat/sqlite3 connection. + * + * @param {import('@appthreat/sqlite3').Database} db the connection. + * @param {{ onCreateConnection?: (db: unknown) => Promise | void }} [hooks] + * `onCreateConnection` runs once per connection (where pragmas go). + */ +export function kyselyFor(db, hooks = {}) { + const dialect = { + createAdapter: () => new SqliteAdapter(), + createQueryCompiler: () => new SqliteQueryCompiler(), + createIntrospector: (database) => + new SqliteIntrospector(database, dialect), + createDriver: () => ({ + async init() { + if (hooks.onCreateConnection) { + await hooks.onCreateConnection(db); + } + // Kysely prepares one statement per compiled query; the + // cache keeps re-issued queries (its identity columns + // lookups, its selects) off the prepare path. + db.cacheStatements(); + }, + async acquireConnection() { + return { + async executeQuery(compiledQuery) { + const { sql, parameters } = compiledQuery; + const stmt = await db.prepare(sql); + try { + if (stmt.readonly) { + return { rows: await stmt.all(parameters) }; + } + const run = await stmt.run(parameters); + return { + rows: [], + insertId: + run.lastID !== undefined && + run.lastID !== null + ? run.lastID + : undefined, + numUpdatedOrDeletedRows: + run.changes !== undefined && + run.changes !== null + ? BigInt(run.changes) + : undefined, + }; + } finally { + await stmt.finalize(); + } + }, + }; + }, + async releaseConnection() { + /* one long-lived connection */ + }, + async beginTransaction() { + await db.exec('BEGIN'); + }, + async commitTransaction() { + await db.exec('COMMIT'); + }, + async rollbackTransaction() { + await db.exec('ROLLBACK'); + }, + async destroy() { + // The caller owns the connection (it passed it in); + // close it yourself after kysely.destroy(). + }, + }), + }; + return new Kysely({ dialect }); +} + +// Usage: +// const db = await sqlite3.open('app.db'); +// const kysely = kyselyFor(db, { +// onCreateConnection: (c) => c.pragma('journal_mode = WAL'), +// }); +// const rows = await kysely.selectFrom('users').selectAll().execute(); diff --git a/lib/augment.d.ts b/lib/augment.d.ts index c3c059e..50f3a6f 100644 --- a/lib/augment.d.ts +++ b/lib/augment.d.ts @@ -38,6 +38,7 @@ import type { CheckpointMode, CheckpointOptions, CheckpointResult, + Database, FunctionOptions, OpenBlobOptions, Row, @@ -56,6 +57,109 @@ import type { TransactionOptions, } from './promises.js'; +/** + * The tagged-template statement store `db.createTagStore()` returns: an + * LRU of prepared statements driven by template literals, with the + * composition helpers ORMs need. + * + * @since 9.1.0 + */ +export interface TagStore { + /** + * Template tag resolving the first row. + * @param templates the template strings. + * @param values the interpolated values. + * @returns the first row, or undefined. + */ + get( + templates: TemplateStringsArray, + ...values: unknown[] + ): Promise; + /** + * Template tag resolving every row. + * @param templates the template strings. + * @param values the interpolated values. + * @returns the rows. + */ + all(templates: TemplateStringsArray, ...values: unknown[]): Promise; + /** + * Template tag returning the backpressured async iterator. + * @param templates the template strings. + * @param values the interpolated values. + * @returns the async iterator. + */ + iterate( + templates: TemplateStringsArray, + ...values: unknown[] + ): AsyncIterableIterator; + /** + * Template tag running the statement. + * @param templates the template strings. + * @param values the interpolated values. + * @returns the run result. + */ + run( + templates: TemplateStringsArray, + ...values: unknown[] + ): Promise; + /** + * Drops every cached SQL key, and the connection statements they + * name. + * @returns nothing. + */ + clear(): void; + /** The number of composed SQL keys the store is holding. */ + readonly size: number; + /** The maximum number of SQL keys the store holds. */ + readonly capacity: number; + /** The connection. */ + readonly db: Database; + /** + * Builds a raw-SQL fragment. + * @param text the SQL text. + * @returns the fragment. + */ + raw(text: string): SqlFragment; + /** + * Joins fragments and/or plain values with a separator (IN-lists and + * friends). A fragment contributes its SQL text; anything else binds + * as a parameter, which is what an IN-list of user data needs. + * @param items the fragments or values. + * @param separator the joining text. + * @returns the joined fragment. + */ + join(items: (SqlFragment | unknown)[], separator?: string): SqlFragment; + /** + * Quotes one SQL identifier for safe interpolation. + * @param name the identifier. + * @returns the quoted fragment. + */ + identifier(name: string): SqlFragment; + /** + * Quotes a dotted identifier path part by part. + * @param dotted the dot-separated path. + * @returns the quoted fragment. + */ + identifierPath(dotted: string): SqlFragment; + /** + * Builds the empty fragment. + * @returns the fragment. + */ + empty(): SqlFragment; +} + +/** + * One composed piece of SQL: literal text plus bind parameters. + * + * @since 9.1.0 + */ +export interface SqlFragment { + /** The SQL text. */ + text: string; + /** The bind parameters, in text order. */ + params: unknown[]; +} + declare module './native.js' { interface Database { // ---- Promise mode (v9): a call whose last argument is not a @@ -679,6 +783,158 @@ declare module './native.js' { * supports the getSync/runSync/allSync fast path. */ prepareSync(sql: string): Statement; + /** + * Prepares synchronously with a per-statement integer-mode + * override (node:sqlite's `readBigInts`, better-sqlite3's + * `safeIntegers`, as a one-shot option). + * @since 9.1.0 + */ + prepareSync( + sql: string, + options: { integerMode?: 'number' | 'bigint' | 'mixed' }, + ): Statement; + + /** + * Runs a `PRAGMA` and resolves its parsed rows; `{ simple: true }` + * resolves the first column of the first row. + * @since 9.1.0 + */ + pragma( + source: string, + options?: { simple?: boolean }, + ): Promise[] | unknown>; + /** + * Resolves the `EXPLAIN QUERY PLAN` rows (or, with + * `{ full: true }`, the VDBE program) without executing the + * statement. + * @since 9.1.0 + */ + explain( + sql: string, + options?: { full?: boolean }, + ): Promise[]>; + /** + * Runs an array of statements atomically in one transaction; + * read-shaped statements resolve their rows, others their run + * result. + * @since 9.1.0 + */ + batch( + statements: Array< + | string + | { sql: string; args?: BindParams } + | [string, ...BindValue[]] + >, + options?: { + mode?: 'write' | 'read' | 'deferred' | 'exclusive'; + }, + ): Promise; + /** + * Serializes the database to `.dump`-style SQL text. + * @since 9.1.0 + */ + dump(): Promise; + /** + * Reads one `sqlite3_db_status` counter, by friendly name + * (`'cacheHit'`) or DBSTATUS_* constant. + * @since 9.1.0 + */ + status( + op: string | number, + options?: { reset?: boolean }, + ): { current: number; highwater: number }; + /** + * Releases non-essential page-cache memory; returns the bytes + * freed. + * @since 9.1.0 + */ + releaseMemory(): number; + /** + * The current run-time limits, by friendly name. + * @since 9.1.0 + */ + readonly limits: Record; + /** + * The filesystem path of an attached database (empty for + * in-memory/temp schemas). + * @since 9.1.0 + */ + location(dbName?: string): string; + /** + * Registers a read-only virtual table computed by a JavaScript + * generator (eponymous form). `parameters` names the subset of + * `columns` declared HIDDEN — the table-valued function's + * arguments. Rows are pulled in batches as the query consumes + * them, so an unbounded generator works with `LIMIT`; a scan that + * stops early leaves the generator suspended without resuming it. + * @since 9.1.0 + */ + table( + name: string, + definition: { + columns: Array; + parameters?: string[]; + rows: ( + this: undefined, + ...args: unknown[] + ) => Iterable>; + }, + ): this; + /** + * Registers a named virtual-table module instantiated per + * `CREATE VIRTUAL TABLE ... USING name(args)`; the factory + * declares its columns as `factory.columns` and receives the + * DDL argument strings. + * @since 9.1.0 + */ + table( + name: string, + factory: ((...args: string[]) => unknown) & { + columns: Array; + parameters?: string[]; + }, + ): this; + /** + * Removes a virtual-table module registered with `db.table()`. + * @since 9.1.0 + */ + removeTable(name: string): this; + /** + * Exposes one JS array (or iterable) as a queryable table with + * `key`/`value` columns; returns `{ name, drop() }`. Anonymous + * registrations are capped at 32 per connection (the oldest is + * dropped), so `drop()` each handle when done or pass an explicit + * `{ name }`, which opts out of the cap. + * @since 9.1.0 + */ + values( + iterable: Iterable, + options?: { name?: string }, + ): { name: string; drop(): void }; + /** + * Builds a tagged-template statement store (an LRU keyed on the + * joined SQL) with `get`/`all`/`iterate`/`run` tags and the + * `raw`/`join`/`identifier`/`identifierPath`/`empty` composition + * helpers. Enables the connection statement cache if it is not + * already on; only a fragment from those helpers is spliced in as + * SQL text, every other interpolated value binds. + * @since 9.1.0 + */ + createTagStore(maxSize?: number): TagStore; + /** + * Builds a reusable transaction wrapper carrying + * `.deferred()`/`.immediate()`/`.exclusive()` begin-mode + * variants. + * @since 9.1.0 + */ + createTransaction( + fn: (tx: Database, ...args: unknown[]) => unknown, + options?: TransactionOptions, + ): ((...args: unknown[]) => Promise) & { + deferred(...args: unknown[]): Promise; + immediate(...args: unknown[]): Promise; + exclusive(...args: unknown[]): Promise; + }; /** Backs the database up to a file, returned synchronously. */ backup( @@ -758,6 +1014,14 @@ declare module './native.js' { _statementForSync(sql: string): Statement; /** Finalizes every cached statement, emptying the cache. @internal */ _drainStatementCache(): void; + /** + * True while the JavaScript thread is inside SQLite on this + * connection — i.e. inside a callback a synchronous method invoked + * re-entrantly. The operations that finalize the executing + * statement (every registration, which flushes the statement + * cache) refuse then. @internal + */ + readonly _inSyncCall: boolean; // ---- Sessions, changesets, serialization and blob I/O // (Deliverable 08). @@ -1020,6 +1284,19 @@ declare module './native.js' { } interface Session { + /** + * Records the differences between `fromDb`'s table and this + * session's table into the session (sqlite3session_diff), + * without either database being written; harvest with + * `changeset()`. @since 9.1.0 + */ + diff(table: string, fromDb: string): Promise; + /** session.diff, callback form. @since 9.1.0 */ + diff( + table: string, + fromDb: string, + callback: (this: Session, err: SqliteError | null) => void, + ): this; /** Harvests the recorded changes as a changeset. @since 9.0.0 */ changeset(): Promise; /** Harvests the recorded changes, callback form. */ diff --git a/lib/compat.d.ts b/lib/compat.d.ts new file mode 100644 index 0000000..85559e3 --- /dev/null +++ b/lib/compat.d.ts @@ -0,0 +1,157 @@ +// Hand-maintained declaration for the node:sqlite compatibility shim +// (lib/compat.js). The documented divergences from node:sqlite's surface +// are listed in the implementation file's header comment. + +import type { + ApplyChangesetOptions, + Database, + Row, + Session, +} from './native.js'; + +/** + * A prepared statement in the node:sqlite StatementSync shape, wrapping + * this package's synchronous fast path. + * + * @since 9.1.0 + */ +export declare class StatementSync { + /** True once finalized (explicitly or through dispose). */ + get finalized(): boolean; + /** The statement's SQL text. */ + get sourceSQL(): string; + /** The SQL with the most recent bound values substituted. */ + get expandedSQL(): string; + /** The statement's result columns. */ + columns(): Array<{ + name: string; + database?: string; + table?: string; + column?: string; + type?: string; + }>; + /** Steps once and returns the first row (or undefined). */ + get(...params: unknown[]): Row | undefined; + /** Steps to completion and returns every row. */ + all(...params: unknown[]): Row[]; + /** Runs the statement, returning `{ changes, lastInsertRowid }`. */ + run(...params: unknown[]): { + changes: number | bigint; + lastInsertRowid: number | bigint; + }; + /** + * A synchronous iterator over the rows. Divergence: the rows are + * materialised first (the sync fast path has no mid-cursor + * suspension). + */ + iterate(...params: unknown[]): IterableIterator; + /** + * Sets whether integers read as BigInt, re-preparing the statement + * (the integer mode belongs to the prepared statement here). + */ + setReadBigInts(value: boolean): void; + /** Sets whether rows are arrays. */ + setReturnArrays(value: boolean): void; + /** Finalizes the statement. */ + close(): void; + /** `using` support. */ + [Symbol.dispose](): void; +} + +/** + * A connection in the node:sqlite DatabaseSync shape, mapping onto this + * package's synchronous fast path. + * + * @since 9.1.0 + */ +export declare class DatabaseSync { + /** + * Opens a connection (synchronously — the connection is usable the + * moment the constructor returns). + */ + constructor( + location: string, + options?: { + readOnly?: boolean; + open?: boolean; + enableForeignKeyConstraints?: boolean; + enableDoubleQuotedStringLiterals?: boolean; + allowExtension?: boolean; + timeout?: number; + }, + ); + /** The underlying connection (the full async surface). */ + get native(): Database; + /** True while open. */ + get open(): boolean; + /** Alias of `open` (node:sqlite's name). */ + get isOpen(): boolean; + /** True inside an explicit transaction. */ + get isTransaction(): boolean; + /** Runs a SQL script synchronously. */ + exec(sql: string): undefined; + /** Prepares a statement synchronously. */ + prepare( + sql: string, + options?: { + readBigInts?: boolean; + returnArrays?: boolean; + allowBareNamedParameters?: boolean; + allowUnknownNamedParameters?: boolean; + persistent?: boolean; + }, + ): StatementSync; + /** Registers a scalar SQL function. */ + function( + name: string, + options?: + | ((...args: unknown[]) => unknown) + | { + deterministic?: boolean; + directOnly?: boolean; + varargs?: boolean; + }, + fn?: (...args: unknown[]) => unknown, + ): void; + /** Registers an aggregate (or window) SQL function. */ + aggregate( + name: string, + spec: { + start: () => unknown; + step: (acc: unknown, ...args: unknown[]) => unknown; + result: (acc: unknown) => unknown; + inverse?: (acc: unknown, ...args: unknown[]) => unknown; + deterministic?: boolean; + varargs?: boolean; + }, + ): void; + /** Loads a SQLite extension (gated by allowExtension). */ + loadExtension(path: string, entryPoint?: string): void; + /** The allowExtension gate. */ + enableLoadExtension(allow: boolean): void; + /** Toggles SQLite defensive mode. */ + enableDefensive(active: boolean): void; + /** + * The filesystem path of an attached database, or null for an + * in-memory or temporary one (node:sqlite's shape). + */ + location(dbName?: string): string | null; + /** + * Not available: this package's authorizer is declarative by design. + * Use `db.native.authorizer(policy)`. + */ + setAuthorizer(): never; + /** Creates a changeset-recording session (async divergence). */ + createSession(options?: { table?: string }): Session; + /** Applies a changeset (async divergence). */ + applyChangeset( + changeset: Uint8Array, + options?: ApplyChangesetOptions, + ): Promise; + /** Serializes the database (async divergence). */ + serialize(): Promise; + /** Closes the connection (asynchronously under the hood). */ + close(): void; + /** `await using` support. */ + [Symbol.asyncDispose](): Promise; +} diff --git a/lib/compat.js b/lib/compat.js new file mode 100644 index 0000000..eacfadb --- /dev/null +++ b/lib/compat.js @@ -0,0 +1,618 @@ +// The node:sqlite compatibility shim (Phase 5): +// `import { DatabaseSync } from '@appthreat/sqlite3/compat'`. +// +// A zero-dependency drop-in for code written against Node's built-in +// node:sqlite that outgrew it — when the connection is idle the mapped +// calls run on this package's synchronous fast path (parity with +// node:sqlite's speed), and the moment the application needs pools, +// sessions, streaming or cancellation it can reach through `db.native` +// for the full async surface without changing databases. +// +// What maps 1:1: open/close/isOpen/isTransaction, exec (synchronous +// multi-statement scripts), prepare + StatementSync get/all/run/iterate/ +// columns/sourceSQL/expandedSQL, function/aggregate (the Phase 2 +// re-entrant UDFs make these genuinely synchronous), loadExtension/ +// enableLoadExtension, enableDefensive, location, prepare options +// (readBigInts, returnArrays, allowBareNamedParameters, +// allowUnknownNamedParameters, persistent — accepted, some as no-ops). +// +// Documented divergences (loud, not silent): +// - setAuthorizer() throws: this package's authorizer is a declarative +// C++ rule list by design (no JavaScript on the prepare path); reach +// through `db.native.authorizer(policy)` for the supported form. +// - createSession()/applyChangeset() keep this package's async +// signatures (they return promises) — node:sqlite's sync ones have no +// equivalent on the sync fast path. +// - serialize() returns the bytes asynchronously. +// - StatementSync.close() finalizes asynchronously under the hood; the +// statement is unusable immediately after. DatabaseSync.close() is the +// same shape: it starts the close (the connection is unusable at once) +// and the handle is released on the queue — `await db[Symbol. +// asyncDispose]()`, or `await db.native.close()`, when the close must +// have completed (deleting the file, reopening it on Windows). +// - StatementSync.iterate() materialises the rows before yielding: the +// sync fast path has no mid-cursor suspension. Use +// `db.native.iterate()` for a true streaming cursor. +// - setReadBigInts() after prepare() applies to lastInsertRowid; +// integer *columns* are read in the mode fixed at prepare time, so +// pass `{ readBigInts: true }` to prepare() (or re-prepare) to change +// how rows are read. + +import sqlite3 from './sqlite3.js'; + +/** + * Splits one SQL script into complete statements, using sqlite3_complete + * (via the binding's `complete()`) to find statement boundaries — quotes, +// comments and trigger bodies included. The trailing remainder (an + * incomplete final statement) is kept and will fail loudly on run. + * + * @param {string} sql the script. + * @returns {string[]} the complete statements. + * @private + */ +function splitScript(sql) { + /** @type {string[]} */ + const statements = []; + let current = ''; + let i = 0; + while (i < sql.length) { + const ch = sql[i]; + current += ch; + i++; + if (ch === ';') { + const candidate = current.trim(); + if (candidate.length > 0 && sqlite3.complete(candidate)) { + statements.push(candidate); + current = ''; + } + } + } + const rest = current.trim(); + if (rest.length > 0) { + // An unterminated tail: run it through complete() to fail with + // sqlite's own message rather than dropping it silently. + statements.push(rest); + } + return statements; +} + +/** + * A prepared statement in the node:sqlite shape, wrapping this package's + * synchronous fast path. + * + * @since 9.1.0 + */ +class StatementSync { + /** @type {import('./sqlite3-binding.js').Statement} */ + #stmt; + #readBigInts; + #returnArrays; + /** @type {(mode: 'number' | 'bigint') => import('./sqlite3-binding.js').Statement} */ + #reprepare; + + /** + * @param {import('./sqlite3-binding.js').Statement} stmt the wrapped statement. + * @param {{ readBigInts?: boolean, returnArrays?: boolean }} options + * @param {(mode: 'number' | 'bigint') => import('./sqlite3-binding.js').Statement} [reprepare] + * re-prepares the same SQL under another integer mode, for + * setReadBigInts (the mode belongs to the prepared statement). + */ + constructor(stmt, options, reprepare) { + this.#stmt = stmt; + this.#readBigInts = options?.readBigInts === true; + this.#returnArrays = options?.returnArrays === true; + this.#reprepare = + reprepare ?? + (() => { + throw new Error( + 'setReadBigInts() cannot re-prepare this statement', + ); + }); + } + + /** + * True once finalized (explicitly or through dispose). + * + * @returns {boolean} finalized. + */ + get finalized() { + return this.#stmt.finalized; + } + + /** + * The statement's SQL text. + * + * @returns {string} the SQL. + */ + get sourceSQL() { + return /** @type {string} */ (this.#stmt.sql); + } + + /** + * The SQL with the most recent bound values substituted. + * + * @returns {string} the expanded SQL. + */ + get expandedSQL() { + return /** @type {string} */ (this.#stmt.expandedSQL); + } + + /** + * The statement's result columns. + * + * @returns {{ name: string, database?: string, table?: string, column?: string, type?: string }[]} + * the column descriptors. + */ + columns() { + // node:sqlite names the origin column `column`; this package's + // native accessor calls it `origin`. + const native = /** @type {any[]} */ (this.#stmt.columns) ?? []; + return native.map(({ origin, ...rest }) => + origin === undefined ? rest : { ...rest, column: origin }, + ); + } + + /** + * The bind arguments for one call: node:sqlite re-executes a + * statement called again with no parameters (this package's statement + * cursor semantics otherwise continue), so an explicit empty bind + * forces the reset. + * + * @param {unknown[]} params + * @returns {unknown[]} + */ + #bindArgs(params) { + // Only a zero-parameter statement can be safely re-executed with + // an explicit empty bind (which forces the reset); a parameterful + // statement called without arguments keeps this package's + // re-bind-with-last-values semantics. + if (params.length === 0 && this.#stmt.parameterCount === 0) { + return [[]]; + } + return params; + } + + /** + * @param {unknown[]} params + * @returns {unknown[]} + */ + #rowModeOption(params) { + return this.#returnArrays ? [...params, { rowMode: 'array' }] : params; + } + + /** + * Steps once and returns the first row (or undefined). + * + * @param {...unknown} params bind parameters. + * @returns {any} the row. + */ + get(...params) { + return this.#stmt.getSync( + ...this.#rowModeOption(this.#bindArgs(params)), + ); + } + + /** + * Steps to completion and returns every row. + * + * @param {...unknown} params bind parameters. + * @returns {any[]} the rows. + */ + all(...params) { + return this.#stmt.allSync( + ...this.#rowModeOption(this.#bindArgs(params)), + ); + } + + /** + * Runs the statement, returning `{ changes, lastInsertRowid }`. + * + * @param {...unknown} params bind parameters. + * @returns {{ changes: number | bigint, lastInsertRowid: number | bigint }} the run result. + */ + run(...params) { + this.#stmt.runSync(...this.#bindArgs(params)); + return { + changes: this.#stmt.changes, + lastInsertRowid: this.#readBigInts + ? this.#stmt.lastIDBigInt + : this.#stmt.lastID, + }; + } + + /** + * A synchronous iterator over the rows. Divergence: the rows are + * materialised first (the sync fast path has no mid-cursor + * suspension), so this costs the memory of the whole result — reach + * for `db.native.iterate()` when that matters. + * + * @param {...unknown} params bind parameters. + * @returns {IterableIterator} the rows. + */ + *iterate(...params) { + yield* this.all(...params); + } + + /** + * Sets whether integers read as BigInt (node:sqlite's toggle). The + * integer mode belongs to the prepared statement here, so this + * re-prepares the same SQL under the new mode — column values follow + * the setting, as they do in node:sqlite, rather than only + * `lastInsertRowid`. + * + * @param {boolean} value the new setting. + * @returns {void} + */ + setReadBigInts(value) { + const next = value === true; + if (next === this.#readBigInts) return; + const replacement = this.#reprepare(next ? 'bigint' : 'number'); + const previous = this.#stmt; + this.#stmt = replacement; + this.#readBigInts = next; + if (!previous.finalized) { + previous.finalize(function () { + /* best effort: the replacement is live */ + }); + } + } + + /** + * Sets whether rows are arrays (node:sqlite's toggle). + * + * @param {boolean} value the new setting. + * @returns {void} + */ + setReturnArrays(value) { + this.#returnArrays = value === true; + } + + /** + * Finalizes the statement. Accepted for shape parity; the underlying + * finalize is asynchronous (queued), and the statement is unusable + * immediately. + * + * @returns {void} + */ + close() { + if (!this.#stmt.finalized) { + this.#stmt.finalize(function () { + /* best effort */ + }); + } + } + + /** + * `using` support. + * + * @returns {void} + */ + [Symbol.dispose]() { + this.close(); + } +} + +/** + * A connection in the node:sqlite `DatabaseSync` shape, mapping onto this + * package's synchronous fast path. + * + * @since 9.1.0 + * @example + * import { DatabaseSync } from '@appthreat/sqlite3/compat'; + * const db = new DatabaseSync(':memory:'); + * db.exec('CREATE TABLE t (a)'); + * const stmt = db.prepare('SELECT * FROM t WHERE a = ?'); + * const row = stmt.get(1); + * db.close(); + */ +class DatabaseSync { + /** @type {import('./sqlite3-binding.js').Database} */ + #db; + #allowExtension = false; + + /** + * Opens a connection. Options map onto this package's opens and + * configure() calls: `readOnly`, `open` (default true), + * `enableForeignKeyConstraints` (default true, node:sqlite's own + * default), `enableDoubleQuotedStringLiterals` (refused: this + * package keeps strict SQL), `allowExtension`, `timeout`. + * + * @param {string} location the database filename or `:memory:`. + * @param {{ readOnly?: boolean, open?: boolean, enableForeignKeyConstraints?: boolean, allowExtension?: boolean, timeout?: number }} [options] + */ + constructor(location, options = {}) { + const known = new Set([ + 'readOnly', + 'open', + 'enableForeignKeyConstraints', + 'enableDoubleQuotedStringLiterals', + 'allowExtension', + 'timeout', + ]); + for (const key of Object.keys(options)) { + if (!known.has(key)) { + throw new TypeError( + `DatabaseSync received unknown option '${key}'`, + ); + } + } + if (options.open === false) { + throw new TypeError( + "DatabaseSync option 'open: false' has no equivalent; " + + 'open the connection when ready instead', + ); + } + if (options.enableDoubleQuotedStringLiterals === true) { + throw new TypeError( + 'DatabaseSync cannot enable double-quoted string ' + + 'literals: this package deliberately keeps strict SQL', + ); + } + const mode = + options.readOnly === true + ? sqlite3.OPEN_READONLY + : sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE; + // syncOpen: the native synchronous open (node:sqlite's semantics — + // the connection is usable the moment the constructor returns). + this.#db = new sqlite3.Database(location, { mode, syncOpen: true }); + if (options.timeout !== undefined) { + this.#db.configure('busyTimeout', options.timeout); + } + this.#allowExtension = options.allowExtension === true; + if (options.enableForeignKeyConstraints !== false) { + this.#db.runSync('PRAGMA foreign_keys = ON'); + } + } + + /** + * The underlying connection, for reaching the full async surface + * (pools, sessions, streaming, cancellation) without a second open. + * + * @returns {import('./sqlite3-binding.js').Database} the connection. + */ + get native() { + return this.#db; + } + + /** + * True while open. + * + * @returns {boolean} open. + */ + get open() { + return this.#db.open; + } + + /** + * Alias of {@link DatabaseSync.open} (node:sqlite's name). + * + * @returns {boolean} open. + */ + get isOpen() { + return this.#db.open; + } + + /** + * True inside an explicit transaction. + * + * @returns {boolean} in transaction. + */ + get isTransaction() { + return this.#db.inTransaction; + } + + /** + * Runs a SQL script (possibly several statements) synchronously. + * Returns undefined, like node:sqlite's exec. + * + * @param {string} sql the script. + * @returns {undefined} + */ + exec(sql) { + if (typeof sql !== 'string') { + throw new TypeError('exec() requires a SQL string'); + } + for (const statement of splitScript(sql)) { + this.#db.runSync(statement); + } + return undefined; + } + + /** + * Prepares a statement synchronously. + * + * @param {string} sql the SQL. + * @param {{ readBigInts?: boolean, returnArrays?: boolean, allowBareNamedParameters?: boolean, allowUnknownNamedParameters?: boolean, persistent?: boolean }} [options] + * `readBigInts`/`returnArrays` map onto this package's per-call + * row shapes; the remaining options are accepted for shape parity + * (bare named parameters are always allowed here, unknown named + * parameters always refused — this package's strictness). + * @returns {StatementSync} the statement. + */ + prepare(sql, options = {}) { + const known = new Set([ + 'readBigInts', + 'returnArrays', + 'allowBareNamedParameters', + 'allowUnknownNamedParameters', + 'persistent', + ]); + for (const key of Object.keys(options)) { + if (!known.has(key)) { + throw new TypeError( + `prepare() received unknown option '${key}'`, + ); + } + } + /** @param {'number' | 'bigint'} [mode] */ + const prepare = (mode) => + this.#db.prepareSync(sql, { integerMode: mode }); + const stmt = prepare( + options.readBigInts === true ? 'bigint' : undefined, + ); + return new StatementSync(stmt, options, prepare); + } + + /** + * Registers a scalar SQL function (synchronous, via the re-entrant + * direct-call path). + * + * @param {string} name the SQL name. + * @param {((...args: unknown[]) => unknown) | { deterministic?: boolean, directOnly?: boolean, varargs?: boolean }} [options] + * the options object, or the implementation directly. + * @param {(...args: unknown[]) => unknown} [fn] the implementation. + * @returns {void} + */ + function(name, options, fn) { + /** @type {any} */ (this.#db).function(name, options, fn); + } + + /** + * Registers an aggregate (or window, with `inverse`) SQL function. + * + * @param {string} name the SQL name. + * @param {{ start: () => unknown, step: (acc: unknown, ...args: unknown[]) => unknown, result: (acc: unknown) => unknown, inverse?: (acc: unknown, ...args: unknown[]) => unknown, deterministic?: boolean, varargs?: boolean }} spec + * the implementation. + * @returns {void} + */ + aggregate(name, spec) { + /** @type {any} */ (this.#db).aggregate(name, spec); + } + + /** + * Loads a SQLite extension. + * + * @param {string} path the extension file. + * @param {string} [entryPoint] the optional entry point. + * @returns {void} + */ + loadExtension(path, entryPoint) { + if (!this.#allowExtension) { + throw new Error( + 'DatabaseSync.loadExtension() requires allowExtension: ' + + 'true at construction (node:sqlite has the same gate)', + ); + } + if (entryPoint !== undefined) { + throw new TypeError( + 'loadExtension entryPoint is not supported; the extension ' + + 'must use its default entry point', + ); + } + this.#db.loadExtension(path, function () { + /* sync-shaped: completes on the queue */ + }); + } + + /** + * No-op gate (the allowExtension constructor option is the real + * gate), kept for node:sqlite's shape. + * + * @param {boolean} allow the new setting. + * @returns {void} + */ + enableLoadExtension(allow) { + this.#allowExtension = allow === true; + } + + /** + * Toggles SQLite defensive mode. + * + * @param {boolean} active the new state. + * @returns {void} + */ + enableDefensive(active) { + this.#db.dbConfig(sqlite3.DBCONFIG_DEFENSIVE, active === true); + } + + /** + * The filesystem path of an attached database, or null for an + * in-memory or temporary one (node:sqlite's shape; this package's own + * `location()` returns the empty string there). + * + * @param {string} [dbName] the attached name. + * @returns {string | null} the path. + */ + location(dbName) { + const path = this.#db.location(dbName); + return path === '' ? null : path; + } + + /** + * Not available: this package's authorizer is declarative (a C++ rule + * list — no JavaScript runs on the prepare path, by design). Use + * `db.native.authorizer(policy)`. + * + * @returns {never} + */ + setAuthorizer() { + throw new Error( + 'DatabaseSync.setAuthorizer() is not available: this ' + + "package's authorizer is declarative (a C++ rule list, no " + + 'JavaScript on the prepare path). Reach through ' + + 'db.native.authorizer(policy) for the supported form', + ); + } + + /** + * Creates a changeset-recording session. Divergence: this package's + * sessions are asynchronous, so the returned session's methods return + * promises (node:sqlite's are synchronous). + * + * @param {{ table?: string }} [options] + * @returns {import('./sqlite3-binding.js').Session} the session. + */ + createSession(options = {}) { + return this.#db.session(options); + } + + /** + * Applies a changeset (asynchronous here; see createSession). + * + * @param {Uint8Array} changeset the bytes. + * @param {import('./native.js').ApplyChangesetOptions} [options] + * @returns {Promise} resolves once applied. + */ + applyChangeset(changeset, options) { + return this.#db.applyChangeset(changeset, options); + } + + /** + * Serializes the database (asynchronous here). + * + * @returns {Promise} the bytes. + */ + async serialize() { + return this.#db.serializeToBytes(); + } + + /** + * Closes the connection. Divergence: the close is queued (this + * package's close is asynchronous), so the connection refuses further + * work at once but the file handle is released a turn later — use + * `await using` / `Symbol.asyncDispose` when a caller must know the + * file is free, e.g. before deleting or reopening it on Windows. + * + * @returns {void} + */ + close() { + if (this.#db.open) { + this.#db.close(function () { + /* sync-shaped: completes on the queue */ + }); + } + } + + /** + * `await using` support. + * + * @returns {Promise} resolves once closed. + */ + async [Symbol.asyncDispose]() { + if (this.#db.open) { + await this.#db.close(); + } + } +} + +export { DatabaseSync, StatementSync }; diff --git a/lib/migrate.d.ts b/lib/migrate.d.ts new file mode 100644 index 0000000..9d5e2c7 --- /dev/null +++ b/lib/migrate.d.ts @@ -0,0 +1,28 @@ +// Hand-maintained declaration for lib/migrate.js (sqlite3.migrate). The +// runner itself is JSDoc-documented in the implementation file. + +import type { Database } from './native.js'; + +/** + * One migration: a name, and either a SQL script or a programmatic body. + */ +export interface Migration { + /** The migration's name (for errors and reporting). */ + name: string; + /** The SQL script (or use `up`). */ + sql?: string; + /** A programmatic body; receives the transaction connection. */ + up?(tx: Database): unknown; +} + +/** + * Runs every pending migration against the connection, in order, each in + * one transaction with `PRAGMA user_version` tracking the position. + * Idempotent. + * + * @since 9.1.0 + */ +export declare function migrate( + db: Database, + migrations: string | Migration[], +): Promise<{ applied: string[]; from: number; to: number }>; diff --git a/lib/migrate.js b/lib/migrate.js new file mode 100644 index 0000000..6093499 --- /dev/null +++ b/lib/migrate.js @@ -0,0 +1,157 @@ +// `sqlite3.migrate(db, migrations)` (Phase 5): a dependency-free, +// `PRAGMA user_version`-based sequential migration runner. No driver +// ships one (libsql ships only a CLI); keeping it opt-in and tiny is the +// point. +// +// Migrations are either an array of { name, sql } / { name, up(db) } +// entries, or a directory path of `NNN-name.sql` files (sorted by the +// numeric prefix, falling back to lexicographic). Each pending migration +// runs inside one transaction (a JS `up` body can await; a raw `sql` +// script may contain several statements) and bumps `user_version` inside +// that transaction, so an interrupted run leaves nothing half-applied. + +import { readdirSync, readFileSync } from 'node:fs'; +import path from 'node:path'; + +/** + * One migration. + * + * @typedef {object} Migration + * @property {string} name the migration's name (for errors and reporting). + * @property {string} [sql] the SQL script (or use `up`). + * @property {string} [up_sql] the normalized SQL script (internal). + * @property {(tx: import('./native.js').Database) => unknown | Promise} [up] + * a programmatic body; receives the transaction connection. + */ + +/** + * Normalizes the migrations argument into an ordered name/body list. + * + * @param {string | Migration[]} migrations a directory path or a list. + * @returns {Migration[]} the ordered migrations. + * @throws {TypeError} for a malformed list or unreadable directory. + * @private + */ +function loadMigrations(migrations) { + if (typeof migrations === 'string') { + let entries; + try { + entries = readdirSync(migrations); + } catch (err) { + throw new TypeError( + `migrate() cannot read the migrations directory ${migrations}: ` + + /** @type {Error} */ (err).message, + ); + } + return entries + .filter((entry) => entry.endsWith('.sql')) + .map((entry) => { + const stem = entry.slice(0, -4); + return /** @type {Migration} */ ({ + name: stem, + up_sql: readFileSync(path.join(migrations, entry), 'utf8'), + }); + }) + .sort((a, b) => { + const na = Number.parseInt(a.name ?? '', 10); + const nb = Number.parseInt(b.name ?? '', 10); + if (Number.isInteger(na) && Number.isInteger(nb) && na !== nb) { + return na - nb; + } + return (a.name ?? '').localeCompare(b.name ?? ''); + }); + } + if (!Array.isArray(migrations)) { + throw new TypeError( + 'migrate() requires an array of migrations or a directory path', + ); + } + return migrations.map((entry, i) => { + if ( + entry === null || + typeof entry !== 'object' || + typeof entry.name !== 'string' || + (typeof entry.up !== 'function' && + typeof entry.up_sql !== 'string' && + typeof entry.sql !== 'string') + ) { + throw new TypeError( + `migrate()[${i}] must be { name, sql } or { name, up }`, + ); + } + if (entry.up !== undefined && (entry.sql || entry.up_sql)) { + throw new TypeError( + `migrate()[${i}] (${entry.name}) has both sql and up; pick one`, + ); + } + return /** @type {Migration} */ ({ + name: entry.name, + up: entry.up, + up_sql: entry.up_sql ?? entry.sql, + }); + }); +} + +/** + * Runs every pending migration against the connection, in order, each in + * one transaction with `PRAGMA user_version` tracking the position. + * Idempotent: already-applied migrations (by position — the first + * `user_version` entries) are skipped. + * + * @param {import('./native.js').Database} db the connection. + * @param {string | Migration[]} migrations a directory of `NNN-name.sql` + * files or a list of `{ name, sql }` / `{ name, up }` entries. + * @returns {Promise<{ applied: string[], from: number, to: number }>} the + * applied names and the version range. + * @throws {TypeError} when the migrations are malformed; rejects with the + * failing migration's error (its work rolled back). + * @since 9.1.0 + * @example + * await sqlite3.migrate(db, [ + * { name: '0001-create-users', sql: 'CREATE TABLE users (id INTEGER PRIMARY KEY)' }, + * { name: '0002-seed', up: (tx) => tx.run('INSERT INTO users DEFAULT VALUES') }, + * ]); + */ +export async function migrate(db, migrations) { + const list = loadMigrations(migrations); + const from = + /** @type {number} */ ( + await db.pragma('user_version', { simple: true }) + ) ?? 0; + /** @type {string[]} */ + const applied = []; + for (let i = from; i < list.length; i++) { + const migration = list[i]; + try { + await db.transaction( + async (tx) => { + if (migration.up !== undefined) { + await migration.up(tx); + } else { + await tx.exec(/** @type {string} */ (migration.up_sql)); + } + await tx.run(`PRAGMA user_version = ${i + 1}`); + }, + { mode: 'immediate' }, + ); + applied.push(migration.name); + } catch (err) { + const context = `migration ${migration.name} (version ${ + i + 1 + }) failed`; + // Prefix the failure in place when it is an Error (keeping its + // code/errno/stack), and wrap anything else — a migration that + // rejects with a string must not make this throw a TypeError + // about assigning to `message`. + if (err instanceof Error) { + err.message = `${context}: ${err.message}`; + throw err; + } + throw new Error(`${context}: ${String(err)}`, { cause: err }); + } + } + // `to` is where the database now is, which is `from` when nothing was + // pending — including a database ahead of the list (a rollback of the + // application without a rollback of its schema). + return { applied, from, to: Math.max(from, list.length) }; +} diff --git a/lib/native.d.ts b/lib/native.d.ts index da81824..d2f92aa 100644 --- a/lib/native.d.ts +++ b/lib/native.d.ts @@ -633,6 +633,97 @@ export declare class Database extends EventEmitter { * @since 9.0.0 */ readonly totalChanges: number | bigint; + + /** + * True while an explicit transaction is open on the connection + * (`sqlite3_get_autocommit` == 0). Refuses (throws) while a JS + * callback round trip could hold the connection mutex. + * + * @since 9.1.0 + */ + readonly inTransaction: boolean; + + /** + * The main schema's transaction state (`sqlite3_txn_state`): + * `'none'`, `'read'` or `'write'`. + * + * @since 9.1.0 + */ + readonly txnState: 'none' | 'read' | 'write'; + + /** + * Reads one `sqlite3_db_status` counter. `_dbStatus(op, reset?)` + * resolves `{ current, highwater }`; the friendly-name wrapper is + * `db.status()`. + * + * @param op a DBSTATUS_* constant. + * @param reset zero the counters after reading. + * @returns the counter values. + * @since 9.1.0 + */ + _dbStatus( + op: number, + reset?: boolean, + ): { current: number; highwater: number }; + + /** + * Releases non-essential page-cache memory + * (`sqlite3_db_release_memory`). + * + * @returns the bytes freed. + * @since 9.1.0 + */ + _releaseMemory(): number; + + /** + * Reads one run-time limit (`sqlite3_limit(id, -1)`); the friendly + * wrapper is the `db.limits` getter. + * + * @param id a LIMIT_* constant. + * @returns the current value. + * @since 9.1.0 + */ + _getLimit(id: number): number; + + /** + * The filesystem path of an attached database + * (`sqlite3_db_filename`); empty for in-memory/temp schemas. + * + * @param dbName the attached database name. + * @returns the path. + * @since 9.1.0 + */ + _dbLocation(dbName: string): string; + + /** + * Registers a JavaScript virtual-table module (the wrapped form is + * `db.table()`). + * + * @param name the module name. + * @param columns the declared column names. + * @param params the HIDDEN parameter names (a subset of columns). + * @param factory the factory (named-module form), or null. + * @param rows the row generator (eponymous form), or null. + * @returns this database. + * @since 9.1.0 + */ + _registerVtab( + name: string, + columns: string[], + params: string[], + factory: ((...args: string[]) => unknown) | null, + rows: (() => Iterable) | null, + ): this; + + /** + * Removes a virtual-table module (the wrapped form is + * `db.removeTable()`). + * + * @param name the module name. + * @returns this database. + * @since 9.1.0 + */ + _removeVtab(name: string): this; } /** @@ -931,6 +1022,13 @@ export interface ApplyChangesetOptions { onConflict?: ApplyChangesetOptions['conflict']; /** Receives each affected table name; return false to skip it. */ filter?: (table: string) => boolean; + /** + * Harvest the conflict-resolution rebase buffer alongside the apply + * (sqlite3changeset_apply_v2): promise mode resolves the buffer to + * rebase later changesets with (null when no conflicts occurred). + * @since 9.1.0 + */ + rebase?: boolean; } /** @@ -1320,6 +1418,31 @@ export declare class Statement extends EventEmitter { * const fullscanSteps = stmt.status(sqlite3.STMTSTATUS_FULLSCAN_STEP); */ status(op: number, reset?: boolean): number; + + /** + * The statement's SQL with the most recent bound values substituted + * (`sqlite3_expanded_sql`). + * @since 9.1.0 + */ + readonly expandedSQL: string; + + /** + * The statement's SQL with literals folded to `?` + * (`sqlite3_normalized_sql`; this build compiles with + * SQLITE_ENABLE_NORMALIZE). + * @since 9.1.0 + */ + readonly normalizedSQL: string; + + /** + * Applies a per-statement integer-mode override (the + * `prepare(..., { integerMode })` plumbing). + * + * @param mode the integer mode. + * @returns this statement. + * @since 9.1.0 + */ + _setIntegerMode(mode: 'number' | 'bigint' | 'mixed'): this; } /** @@ -1414,6 +1537,36 @@ declare const binding: { * @since 9.0.0 */ invertChangeset(changeset: ChangesetBytes): Uint8Array; + /** + * Rebases a changeset against the conflict resolutions harvested by + * `applyChangeset(..., { rebase: true })` — the client-server sync + * primitive (sqlite3rebaser_*). + * + * @param changeset the changeset to rebase. + * @param rebase the harvested rebase buffer. + * @returns the rebased changeset bytes. + * @since 9.1.0 + */ + rebaseChangeset( + changeset: ChangesetBytes, + rebase: ChangesetBytes, + ): Uint8Array; + /** + * True when a SQL string is a complete statement + * (`sqlite3_complete`) — the REPL/CLI helper. + * + * @param sql the SQL string. + * @returns whether the statement is complete. + * @since 9.1.0 + */ + complete(sql: string): boolean; + /** + * The build's SQLITE_COMPILE_OPTIONS (sqlite3_compileoption_get). + * + * @returns the compile-time options. + * @since 9.1.0 + */ + compileOptions(): string[]; /** * Installs the generator the addon uses to compile a row builder for diff --git a/lib/promises.js b/lib/promises.js index 709f9dc..0964e9c 100644 --- a/lib/promises.js +++ b/lib/promises.js @@ -1070,16 +1070,14 @@ function install( Backup.prototype.finish = dualMode(backupCores.finish, { void: true }); // Deliverable 08: sessions, changesets, serialization and blob I/O. - // applyChangeset resolves nothing (the database is the outcome); - // serializeToBytes/changeset/patchset resolve the bytes; blob - // read/write resolve the number of bytes transferred. - Database.prototype.applyChangeset = dualMode(dbCores.applyChangeset, { - void: true, - }); + // applyChangeset resolves the (usually absent) rebase buffer: a plain + // apply calls back with no second argument, so the default pick + // resolves undefined; { rebase: true } resolves the harvested bytes. Database.prototype.serializeToBytes = dualMode(dbCores.serializeToBytes); Session.prototype.changeset = dualMode(sessionCores.changeset); Session.prototype.patchset = dualMode(sessionCores.patchset); + Session.prototype.diff = dualMode(sessionCores.diff, { void: true }); Session.prototype.close = dualMode(sessionCores.close, { void: true }); Blob.prototype.read = dualMode(blobCores.read, { @@ -1091,6 +1089,8 @@ function install( Blob.prototype.reopen = dualMode(blobCores.reopen, { void: true }); Blob.prototype.close = dualMode(blobCores.close, { void: true }); + Database.prototype.applyChangeset = dualMode(dbCores.applyChangeset); + /** * Opens a database and resolves once the connection is ready. * @@ -1266,6 +1266,90 @@ function install( ); }; + /** + * The reusable transaction form (Phase 1): returns a wrapped function + * instead of running the body once — better-sqlite3/bun/Deno's shape, + * but async-aware (the body may await, and nesting still becomes a + * savepoint through the same AsyncLocalStorage tracking the inline + * form uses). + * + * The wrapper carries `.deferred()`, `.immediate()` and + * `.exclusive()` variants that override the begin mode per call; the + * default is whichever mode was passed here (`'deferred'` unless + * said otherwise — pass `'immediate'` for write-heavy bodies: it + * takes the write lock up front and avoids the deferred-write + * upgrade failure under concurrency). + * + * @param {(tx: import('./sqlite3-binding.js').Database, ...args: unknown[]) => unknown} fn + * the transaction body; receives `(tx, ...args)`. + * @param {object} [options] + * @param {'deferred' | 'immediate' | 'exclusive'} [options.mode='deferred'] + * @param {boolean} [options.savepoint=false] + * @param {boolean} [options.serialize=false] + * @param {AbortSignal} [options.signal] + * @returns {((...args: unknown[]) => Promise) & { + * deferred: (...args: unknown[]) => Promise, + * immediate: (...args: unknown[]) => Promise, + * exclusive: (...args: unknown[]) => Promise, + * }} the reusable wrapper. + * @throws {TypeError} when `fn` is not a function or `mode` is invalid. + * @since 9.1.0 + * @example + * const move = db.createTransaction((tx, from, to, amount) => + * tx.run('UPDATE accounts SET bal = bal - ? WHERE id = ?', amount, from) + * .then(() => tx.run('UPDATE accounts SET bal = bal + ? WHERE id = ?', amount, to)), + * ); + * await move(1, 2, 50); + * await move.immediate(2, 1, 50); + */ + Database.prototype.createTransaction = function (fn, options = {}) { + if (typeof fn !== 'function') { + throw new TypeError('createTransaction() requires a function body'); + } + const mode = options?.mode ?? 'deferred'; + if (!TRANSACTION_MODES.has(mode)) { + throw new TypeError( + "createTransaction() mode must be 'deferred', 'immediate' or 'exclusive'", + ); + } + const base = { + savepoint: options?.savepoint === true, + serialize: options?.serialize === true, + signal: options?.signal, + }; + /** + * @param {string} beginMode + * @param {unknown[]} args + * @returns {Promise} + */ + const invoke = (beginMode, args) => + runTransaction(this, (tx) => fn(tx, ...args), { + ...base, + mode: beginMode, + }); + /** + * @param {...unknown} args + * @returns {Promise} + */ + const wrapper = (...args) => invoke(mode, args.slice()); + /** + * @param {...unknown} args + * @returns {Promise} + */ + wrapper.deferred = (...args) => invoke('deferred', args.slice()); + /** + * @param {...unknown} args + * @returns {Promise} + */ + wrapper.immediate = (...args) => invoke('immediate', args.slice()); + /** + * @param {...unknown} args + * @returns {Promise} + */ + wrapper.exclusive = (...args) => invoke('exclusive', args.slice()); + return wrapper; + }; + // Dispose support: `await using` closes/finalizes; a double dispose is // a benign no-op rather than a rejection. Errors that only say "this // was already torn down" are swallowed; real errors propagate. @@ -1530,6 +1614,7 @@ export function installPromiseApi(sqlite3) { sessionCores = { changeset: Session.prototype.changeset, patchset: Session.prototype.patchset, + diff: Session.prototype.diff, close: Session.prototype.close, }; blobCores = { diff --git a/lib/sqlite3.d.ts b/lib/sqlite3.d.ts index 6735e0d..9f4fde7 100644 --- a/lib/sqlite3.d.ts +++ b/lib/sqlite3.d.ts @@ -51,6 +51,14 @@ export type sqlite3 = import('./sqlite3-binding.js').NativeBinding & { open: import('./promises.js').OpenFunction; deserializeFromBytes: (bytes: Uint8Array | ArrayBuffer | DataView, options?: import('./native.js').DeserializeOptions) => Promise; pool: typeof import('./pool.js').pool; + iterdump: (db: import('./sqlite3-binding.js').Database) => AsyncGenerator; + migrate: typeof import('./migrate.js').migrate; + subscribeQueries: (onMessage: (message: { + sql: string; + database: import('./sqlite3-binding.js').Database; + duration: bigint; + durationMs: number; + }) => void) => () => void; }; declare const sqlite3: sqlite3; declare const NativeDatabase: typeof import("./native.js").Database & DatabaseConstructor; @@ -126,6 +134,102 @@ declare class DatabaseClass extends NativeDatabase { */ constructor(filename: string, a?: number | OpenOptions | ((this: import('./sqlite3-binding.js').Database, err: import('./native.js').SqliteError | null) => void), b?: ((this: import('./sqlite3-binding.js').Database, err: import('./native.js').SqliteError | null) => void) | OpenOptions); } +export type PragmaOptions = { + /** + * return only the first column of the first + * row (the scalar form better-sqlite3 popularised). + */ + simple?: boolean; +}; +export type BatchStatement = { + /** + * one statement. + */ + sql: string; + /** + * bind parameters: an array of positional + * values, an object of named parameters, or a single positional value. + * The parameters may also follow the SQL in an array entry + * (`[sql, ...params]`). + */ + args?: unknown; +}; +/** + * One composed piece of SQL for the tag store: literal text plus bind + * parameters. `sql.raw`/`identifier`/`join` build these; a plain value in + * a template hole becomes a bind parameter instead. + */ +export type SqlFragment = { + /** + * the SQL text. + */ + text: string; + /** + * the bind parameters, in text order. + */ + params: unknown[]; +}; +export type VtabDefinition = { + /** + * the row generator: invoked once per query with the table-function + * parameter values; yields arrays (in column order) or objects keyed by + * column name. + */ + rows: (this: undefined, ...args: unknown[]) => Iterable>; + /** + * the + * result columns (a `'name TYPE'` string or `{ name, type }`; the type + * is documentation — SQLite virtual tables are typeless). + */ + columns: (string | { + name: string; + type?: string; + })[]; + /** + * a subset of `columns` to declare + * HIDDEN — the table-valued function's arguments + * (`SELECT * FROM name(arg)` passes `arg` to `rows`). + */ + parameters?: string[]; +}; +/** + * The per-connection registry of db.values() tables: registration order + * (for the cap) and the drop handles. + */ +export type ValuesRegistry = { + /** + * the table names, oldest registration first. + */ + order: string[]; + /** + * the handles. + */ + byName: Map void; + }>; +}; +/** + * One finished-statement span, published on the diagnostics channels. + */ +export type QuerySpan = { + /** + * the expanded SQL text. + */ + sql: string; + /** + * the connection. + */ + database: import('./sqlite3-binding.js').Database; + /** + * the measured duration in nanoseconds. + */ + duration: bigint; + /** + * the measured duration in milliseconds. + */ + durationMs: number; +}; export default sqlite3; export { Backup, Blob, Session, Statement } from './sqlite3-binding.js'; export { DatabaseClass as Database }; diff --git a/lib/sqlite3.js b/lib/sqlite3.js index 223ebf9..9256bdf 100644 --- a/lib/sqlite3.js +++ b/lib/sqlite3.js @@ -9,10 +9,12 @@ // this file adds to the public surface is declared in lib/augment.d.ts // and emitted into the generated lib/sqlite3.d.ts. +import diagnostics_channel from 'node:diagnostics_channel'; import { EventEmitter } from 'node:events'; import os from 'node:os'; import path from 'node:path'; +import { migrate } from './migrate.js'; import { pool } from './pool.js'; import { associateStatement, @@ -66,6 +68,9 @@ import { extendTrace } from './trace.js'; * open: import('./promises.js').OpenFunction, * deserializeFromBytes: (bytes: Uint8Array | ArrayBuffer | DataView, options?: import('./native.js').DeserializeOptions) => Promise, * pool: typeof import('./pool.js').pool, + * iterdump: (db: import('./sqlite3-binding.js').Database) => AsyncGenerator, + * migrate: typeof import('./migrate.js').migrate, + * subscribeQueries: (onMessage: (message: { sql: string, database: import('./sqlite3-binding.js').Database, duration: bigint, durationMs: number }) => void) => () => void, * }} sqlite3 */ @@ -727,7 +732,14 @@ function isOpenOptions(value) { const keys = Object.keys(value); return ( keys.length > 0 && - keys.every((key) => key === 'mode' || key === 'untrusted') + keys.every( + (key) => + key === 'mode' || + key === 'untrusted' || + // Internal: the node:sqlite compat shim's synchronous open + // (lib/compat.js); not part of the public surface. + key === 'syncOpen', + ) ); } @@ -796,15 +808,31 @@ class DatabaseClass extends NativeDatabase { 'Database expects a mode number, an options object or a callback as its second argument', ); } + let syncOpen = false; + if ( + isOpenOptions(a) && + /** @type {{ syncOpen?: unknown }} */ (/** @type {unknown} */ (a)) + .syncOpen !== undefined + ) { + const flag = /** @type {{ syncOpen?: unknown }} */ ( + /** @type {unknown} */ (a) + ).syncOpen; + if (typeof flag !== 'boolean') { + throw new TypeError("open option 'syncOpen' must be a boolean"); + } + syncOpen = flag; + } if (b !== undefined && b !== null) { if (typeof b === 'function') { callback = b; } else if (isOpenOptions(b)) { - const opts = /** @type {OpenOptions} */ (b); + const opts = + /** @type {OpenOptions & { syncOpen?: boolean }} */ (b); if (opts.mode !== undefined && mode === undefined) { mode = opts.mode; } if (opts.untrusted === true) untrusted = true; + if (opts.syncOpen === true) syncOpen = true; } } assertOpenPermitted(filename, mode); @@ -812,7 +840,15 @@ class DatabaseClass extends NativeDatabase { filename, ...(mode !== undefined ? [mode] : []), ...(callback !== undefined ? [callback] : []), + ...(syncOpen ? [true] : []), ); + // diagnostics_channel (Phase 6): tracked so a subscriber arriving + // after the connection was opened can arm query-span publication + // on it; armed eagerly here when one is already listening. + trackConnection(this); + if (diagnosticsChannelSubscribers.size > 0) { + armDiagnosticsFor(this); + } if (untrusted) { applyUntrustedHardening(this); } else if (permissionModelActive()) { @@ -1170,11 +1206,12 @@ Database.prototype.applyChangeset = function (changeset, options, callback) { let onConflict; /** @type {((table: string) => boolean) | null | undefined} */ let onFilter; + let wantRebase = false; if (options !== undefined && options !== null) { if (typeof options !== 'object' || Array.isArray(options)) { throw new TypeError('applyChangeset() options must be an object'); } - const known = new Set(['conflict', 'onConflict', 'filter']); + const known = new Set(['conflict', 'onConflict', 'filter', 'rebase']); for (const key of Object.keys(options)) { if (!known.has(key)) { throw new TypeError( @@ -1184,6 +1221,15 @@ Database.prototype.applyChangeset = function (changeset, options, callback) { } onConflict = options.conflict ?? options.onConflict; onFilter = options.filter; + if ( + options.rebase !== undefined && + typeof options.rebase !== 'boolean' + ) { + throw new TypeError( + "applyChangeset() option 'rebase' must be a boolean", + ); + } + wantRebase = options.rebase === true; } if (onConflict === undefined || onConflict === null) { decision = sqlite3.CHANGESET_ABORT; @@ -1217,6 +1263,7 @@ Database.prototype.applyChangeset = function (changeset, options, callback) { typeof onConflict === 'function' ? onConflict : null, typeof onFilter === 'function' ? onFilter : null, callback, + wantRebase, ); return this; }; @@ -1591,6 +1638,9 @@ function prepareAsync(db, sql, bindArgs) { }, ), ); + // A trailing { integerMode } bag is a prepare option, not a bind + // argument; applied to the statement before any work is scheduled. + applyPrepareOptions(statement, bindArgs); if (bindArgs.length > 0) { try { @@ -1667,6 +1717,9 @@ Database.prototype.prepare = function (sql, ...args) { ); associateStatement(this, statement); try { + // A trailing { integerMode } bag is a prepare option, not a bind + // argument. + applyPrepareOptions(statement, args); const bindVariadic = /** @type {(...args: unknown[]) => import('./sqlite3-binding.js').Statement} */ ( /** @type {unknown} */ (nativeStatementBind) @@ -1938,18 +1991,79 @@ Database.prototype.cacheStatements = function (maxEntries) { }; /** - * Prepares synchronously on the main thread. + * True for a trailing `{ integerMode }` prepare-options bag. Like the + * rowMode bag: named bind keys carry a sigil, so a plain object owning + * only `integerMode` could never have been a legal bind argument. + * + * @param {unknown} value the candidate. + * @returns {boolean} whether it is the options bag. + * @private + */ +function isPrepareOptions(value) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + return false; + } + const keys = Object.keys(value); + return keys.length > 0 && keys.every((key) => key === 'integerMode'); +} + +/** + * Applies a prepare-options bag (if the last bind argument is one) to the + * statement. Returns true when a bag was consumed. + * + * @param {import('./sqlite3-binding.js').Statement} statement the fresh statement. + * @param {unknown[]} bindArgs the bind arguments; a bag is popped in place. + * @returns {boolean} whether an options bag was applied. + * @throws {TypeError} for a bad integerMode value (the native setter's). + * @private + */ +function applyPrepareOptions(statement, bindArgs) { + const last = bindArgs[bindArgs.length - 1]; + if (bindArgs.length === 0 || !isPrepareOptions(last)) return false; + bindArgs.pop(); + const mode = + /** @type {{ integerMode?: 'number' | 'bigint' | 'mixed' }} */ (last) + .integerMode; + if (mode !== undefined) { + /** @type {(...args: unknown[]) => unknown} */ ( + /** @type {unknown} */ (statement._setIntegerMode) + )(mode); + } + return true; +} + +/** + * Prepares a statement synchronously on the main thread. * * Throws when the database is not fully idle. The returned statement * also supports the getSync/runSync/allSync fast path. * * @this {import('./sqlite3-binding.js').Database} * @param {string} sql the SQL statement to prepare. + * @param {{ integerMode?: 'number' | 'bigint' | 'mixed' }} [options] a + * per-statement integer-mode override (node:sqlite's `readBigInts`, + * better-sqlite3's `safeIntegers`, as a one-shot option). * @returns {import('./sqlite3-binding.js').Statement} the prepared statement. * @throws {Error} When the database is not fully idle. */ -Database.prototype.prepareSync = function (sql) { - return new Statement(this, sql, undefined, true); +Database.prototype.prepareSync = function (sql, options) { + const statement = new Statement(this, sql, undefined, true); + if (options !== undefined && options !== null) { + if (typeof options !== 'object' || Array.isArray(options)) { + throw new TypeError('prepareSync() options must be an object'); + } + for (const key of Object.keys(options)) { + if (key !== 'integerMode') { + throw new TypeError( + `prepareSync() received unknown option '${key}'`, + ); + } + } + if (options.integerMode !== undefined) { + applyPrepareOptions(statement, [options]); + } + } + return statement; }; /** @@ -2137,6 +2251,40 @@ Database.prototype._statementForSync = function (sql) { /** @type {(...args: unknown[]) => unknown} */ const nativeClose = Database.prototype.close; +/** + * Refuses an operation that would disturb the statement SQLite is + * stepping. `db._inSyncCall` is true exactly while a JavaScript callback + * invoked re-entrantly by a synchronous method (`getSync`/`runSync`/ + * `allSync`, a virtual table's generator, a user function or aggregate) + * is on the stack — the JS thread is inside `sqlite3_step`. Registrations + * flush the statement cache, which finalizes that very statement, so they + * must be refused *before* touching anything: the native entry points + * refuse too, but by then the flush has already freed the live VM. + * + * @param {import('./sqlite3-binding.js').Database} db the connection. + * @param {string} what the refused operation, as a sentence opener. + * @returns {void} + * @throws {Error} when a synchronous method's callback is on the stack. + * @private + */ +function assertNotInSyncCallback(db, what) { + if ( + !( + /** @type {{ _inSyncCall?: boolean }} */ ( + /** @type {unknown} */ (db) + )._inSyncCall + ) + ) { + return; + } + throw new Error( + `${what} from inside a JavaScript callback invoked by a ` + + 'synchronous method on this connection: SQLite is mid-step on ' + + 'this connection and the statement cache holds the executing ' + + 'statement. Do it before or after the query', + ); +} + /** * Finalizes every statement in the statement cache, emptying it. * @@ -2151,6 +2299,11 @@ const nativeClose = Database.prototype.close; * @private */ Database.prototype._drainStatementCache = function () { + // Never from inside a callback a synchronous method invoked: the + // stepping statement is in the sync cache, and finalizing it would + // free the VM sqlite is executing. Every caller checks first (see + // assertNotInSyncCallback); this is the backstop. + assertNotInSyncCallback(this, 'the statement cache cannot be flushed'); // The implicit sync cache is drained on exactly the same events as the // opt-in one: close(), and every user-function registration or removal // (a prepared statement keeps invoking the implementation it was @@ -2185,7 +2338,10 @@ Database.prototype._drainStatementCache = function () { * @returns {any} */ Database.prototype.close = function (...args) { + assertNotInSyncCallback(this, 'the connection cannot be closed'); this._drainStatementCache(); + // See liveConnections (diagnostics_channel). + untrackConnection(this); // Deliberately not deferred. close() is scheduled exclusively and // Work_BeginClose requires pending == 0, so the native queue already // makes it wait for the finalizes above (each either completes inline @@ -2251,6 +2407,7 @@ const MAX_FUNCTION_NAME = 255; * db.all("SELECT name FROM t WHERE name REGEXP '^a'"); */ Database.prototype.function = function (name, options, fn) { + assertNotInSyncCallback(this, 'a user function cannot be registered'); if (typeof options === 'function') { fn = options; options = undefined; @@ -2313,6 +2470,7 @@ Database.prototype.function = function (name, options, fn) { * db.get('SELECT median(salary) FROM employees'); */ Database.prototype.aggregate = function (name, spec) { + assertNotInSyncCallback(this, 'an aggregate cannot be registered'); if (spec === null || typeof spec !== 'object' || Array.isArray(spec)) { throw new TypeError( 'aggregate() requires an implementation object with start, step and result functions', @@ -2376,6 +2534,7 @@ Database.prototype.aggregate = function (name, spec) { * db.all('SELECT name FROM t ORDER BY name COLLATE locale'); */ Database.prototype.collation = function (name, fn) { + assertNotInSyncCallback(this, 'a collation cannot be registered'); if (typeof name !== 'string' || name.length === 0) { throw new TypeError('collation() requires a non-empty name string'); } @@ -2409,6 +2568,7 @@ Database.prototype.collation = function (name, fn) { * @since 9.0.0 */ Database.prototype.removeFunction = function (name) { + assertNotInSyncCallback(this, 'a user function cannot be removed'); if (typeof name !== 'string' || name.length === 0) { throw new TypeError( 'removeFunction() requires a non-empty name string', @@ -2432,6 +2592,7 @@ Database.prototype.removeFunction = function (name) { * @since 9.0.0 */ Database.prototype.removeCollation = function (name) { + assertNotInSyncCallback(this, 'a collation cannot be removed'); if (typeof name !== 'string' || name.length === 0) { throw new TypeError( 'removeCollation() requires a non-empty name string', @@ -2484,197 +2645,1435 @@ Database.prototype.withCollation = function (name, cmp, fn) { })(); }; -// --- Hooks, authorizer, progress, WAL and introspection (Deliverable 07) -- +// --- Ergonomics parity (Phase 1) and virtual tables (Phase 4) --------------- // -// The commit/rollback/wal hooks are installed by the on()/removeListener() -// overrides above: the native sqlite hook exists only while a listener is -// registered, so an installed-but-unused hook costs nothing. +// pragma()/explain()/batch()/dump() are thin, promise-native wrappers over +// the async paths; table()/values() wrap the native vtab registration with +// definition validation. The introspection helpers (status/limits/location) +// map friendly names onto the native entry points. + +// Leading whitespace and SQL comments (line and block), so the keyword +// test below sees the statement's first real token. +const SQL_LEADING_NOISE_RE = + /^(?:\s+|--[^\n]*(?:\n|$)|\/\*[\s\S]*?(?:\*\/|$))+/; +// Statements whose result is rows rather than a change count. +const BATCH_READ_RE = /^(select|pragma|explain|with|values)\b/i; +// ... and the modifying statements that also return rows: RETURNING makes +// an INSERT/UPDATE/DELETE a read as far as the caller is concerned, and +// running it through run() would throw the rows away. +const BATCH_RETURNING_RE = /\bRETURNING\b/i; /** - * Installs (or removes) a declarative authorizer on the connection. + * Whether a batch entry's rows must be collected rather than its change + * count. * - * The policy is evaluated inside SQLite itself, in C++ — there is no - * JavaScript callback on the prepare path, so it is fast and safe from any - * thread that prepares a statement. This is the supported way to sandbox - * user-supplied SQL: with `{ default: 'deny' }` everything is refused - * unless a rule explicitly allows it. + * @param {string} sql one statement. + * @returns {boolean} true when the statement yields rows. + * @private + */ +function batchStatementReadsRows(sql) { + const bare = sql.replace(SQL_LEADING_NOISE_RE, ''); + return BATCH_READ_RE.test(bare) || BATCH_RETURNING_RE.test(bare); +} + +/** + * @typedef {object} PragmaOptions + * @property {boolean} [simple] return only the first column of the first + * row (the scalar form better-sqlite3 popularised). + * @since 9.1.0 + */ + +/** + * Runs a `PRAGMA` and resolves its parsed result rows — the recommended + * way to run pragmas (they are statements, and this keeps them on the + * async queue like every other statement). `PRAGMA ${source}` is + * prepared as-is, so argument forms work too: `db.pragma('table_info(t)')`. * - * `deny` rules are evaluated before `allow` rules, and a denied action - * fails the statement with `SQLITE_AUTH` ("not authorized"). The statement - * cache is flushed on every change: a cached statement was compiled while - * the old policy was in force and would bypass the new one entirely. + * With `{ simple: true }` resolves the first column of the first row + * (`db.pragma('user_version', { simple: true })` → the number), or + * undefined when the pragma returned no rows. * * @this {import('./sqlite3-binding.js').Database} - * @param {import('./native.js').AuthorizerPolicy | null} [policy] the - * policy to install, or null/undefined to remove the authorizer. - * @returns {import('./sqlite3-binding.js').Database} this database, for chaining. - * @throws {TypeError} when the policy or one of its rules is malformed. - * @since 9.0.0 + * @param {string} source the pragma source (without the `PRAGMA` keyword). + * @param {PragmaOptions} [options] + * @returns {Promise[] | unknown>} the rows, or the + * scalar with `{ simple: true }`. + * @throws {TypeError} when the source or options are malformed. + * @since 9.1.0 * @example - * db.authorizer({ - * default: 'deny', - * allow: [ - * { action: sqlite3.SELECT }, - * { action: sqlite3.READ, table: 'users' }, - * ], - * }); + * await db.pragma('journal_mode = WAL'); + * const version = await db.pragma('user_version', { simple: true }); */ -Database.prototype.authorizer = function (policy) { - if (policy === null || policy === undefined) { - this._drainStatementCache(); - /** @type {(...args: unknown[]) => unknown} */ ( - /** @type {unknown} */ (this._setAuthorizer) - )(); - return this; +Database.prototype.pragma = async function pragma(source, options) { + if (typeof source !== 'string' || source.length === 0) { + throw new TypeError('pragma() requires a non-empty source string'); } - if (typeof policy !== 'object' || Array.isArray(policy)) { - throw new TypeError('authorizer() policy must be an object or null'); - } - const known = new Set(['default', 'allow', 'deny', 'ignore']); - for (const key of Object.keys(policy)) { - if (!known.has(key)) { - throw new TypeError( - `authorizer() received unknown option '${key}'`, - ); + let simple = false; + if (options !== undefined && options !== null) { + if (typeof options !== 'object' || Array.isArray(options)) { + throw new TypeError('pragma() options must be an object'); } + for (const key of Object.keys(options)) { + if (key !== 'simple') { + throw new TypeError( + `pragma() received unknown option '${key}'`, + ); + } + } + if ( + options.simple !== undefined && + typeof options.simple !== 'boolean' + ) { + throw new TypeError("pragma() option 'simple' must be a boolean"); + } + simple = options.simple === true; } - const decisions = new Set(['allow', 'deny', 'ignore']); - const decisionOf = /** @type {Record} */ ({ - allow: sqlite3.OK, - deny: sqlite3.DENY, - ignore: sqlite3.IGNORE, - }); - const fallback = policy.default === undefined ? 'allow' : policy.default; - if (!decisions.has(fallback)) { - throw new TypeError( - "authorizer() default must be 'allow', 'deny' or 'ignore'", - ); - } + const rows = /** @type {Record[]} */ ( + await this.all(`PRAGMA ${source}`) + ); + if (!simple) return rows; + if (rows.length === 0) return undefined; + const first = rows[0]; + const key = Object.keys(first)[0]; + return key === undefined ? undefined : first[key]; +}; - /** - * Normalizes one rule list into native rows - * [action, verdict, arg1, arg2, database, trigger]. - * - * @param {unknown} rules the raw rule list. - * @param {string} verdict the list's decision name ('allow' etc). - * @param {string} who the list's name in the policy, for error messages. - * @returns {unknown[][]} the native rule rows. - */ - const normalize = (rules, verdict, who) => { - if (rules === undefined || rules === null) return []; - if (!Array.isArray(rules)) { - throw new TypeError( - `authorizer() '${who}' must be an array of rules`, - ); +/** + * Resolves the `EXPLAIN QUERY PLAN` rows for a statement — the query + * planner's own account of what it will do (index choices, scan order), + * without executing it. Parameters may be left unbound: a plan does not + * run the statement. Pass `{ full: true }` for the raw VDBE program + * (`EXPLAIN`), the low-level opcode listing. + * + * @this {import('./sqlite3-binding.js').Database} + * @param {string} sql the statement to explain. + * @param {{ full?: boolean }} [options] + * @returns {Promise[]>} the plan rows. + * @throws {TypeError} when the sql or options are malformed. + * @since 9.1.0 + * @example + * const plan = await db.explain('SELECT * FROM t WHERE id = ?'); + */ +Database.prototype.explain = async function explain(sql, options) { + if (typeof sql !== 'string' || sql.length === 0) { + throw new TypeError('explain() requires a non-empty SQL string'); + } + let full = false; + if (options !== undefined && options !== null) { + if (typeof options !== 'object' || Array.isArray(options)) { + throw new TypeError('explain() options must be an object'); } - return rules.map((rule, i) => { - if ( - rule === null || - typeof rule !== 'object' || - Array.isArray(rule) - ) { + for (const key of Object.keys(options)) { + if (key !== 'full') { throw new TypeError( - `authorizer() ${who}[${i}] must be a rule object`, + `explain() received unknown option '${key}'`, ); } - const where = `authorizer() ${who}[${i}]`; - // null = match anything; an explicit '' targets an empty - // argument (previously unexpressible — D08 closes the D07 - // finding). - const row = /** @type {(number | string | null)[]} */ ([ - -1, - decisionOf[verdict], - null, - null, - null, - null, - ]); - if (rule.action !== undefined) { - if ( - typeof rule.action !== 'number' || - !Number.isInteger(rule.action) - ) { - throw new TypeError( - `${where} action must be an integer constant`, - ); - } - row[0] = rule.action; - } - const arg1 = rule.arg1 !== undefined ? rule.arg1 : rule.table; - const arg2 = rule.arg2 !== undefined ? rule.arg2 : rule.column; - const parts = [ - [arg1, 'arg1'], - [arg2, 'arg2'], - [rule.database, 'database'], - [rule.trigger, 'trigger'], - ]; - parts.forEach((part, j) => { - const value = part[0]; - const name = /** @type {string} */ (part[1]); - if (value === undefined || value === null) return; - if (typeof value !== 'string') { - throw new TypeError(`${where} ${name} must be a string`); - } - row[2 + j] = value; - }); - return row; - }); - }; - - // Deny first: the sandbox reading — a deny must not be rescuable by a - // later allow, whatever the array order. - const rows = [ - ...normalize(policy.deny, 'deny', 'deny'), - ...normalize(policy.ignore, 'ignore', 'ignore'), - ...normalize(policy.allow, 'allow', 'allow'), - ]; - - this._drainStatementCache(); - /** @type {(...args: unknown[]) => unknown} */ ( - /** @type {unknown} */ (this._setAuthorizer) - )(decisionOf[fallback], rows); - return this; + } + if (options.full !== undefined && typeof options.full !== 'boolean') { + throw new TypeError("explain() option 'full' must be a boolean"); + } + full = options.full === true; + } + return /** @type {Promise[]>} */ ( + this.all(`${full ? 'EXPLAIN' : 'EXPLAIN QUERY PLAN'} ${sql}`) + ); }; /** - * Installs a progress handler. Two forms: - * - * - `db.progress(period, callback)` — a JavaScript callback invoked every - * `period` VM instructions; returning truthy aborts the running - * statement with `SQLITE_INTERRUPT`. Each invocation is a blocking - * round trip to the JS thread from whatever thread is executing SQL, - * so it is the expensive form: fine for progress bars over a handful - * of long queries, wrong for anything per-row. While it is installed, - * the synchronous methods (`getSync`/`runSync`/`allSync` and - * `prepareSync`) refuse to run — the callback could fire on the thread - * that would have to service it. - * - `db.cancellationToken()` — the recommended form; see there. + * @typedef {object} BatchStatement + * @property {string} sql one statement. + * @property {unknown} [args] bind parameters: an array of positional + * values, an object of named parameters, or a single positional value. + * The parameters may also follow the SQL in an array entry + * (`[sql, ...params]`). + * @since 9.1.0 + */ + +/** + * Runs an array of statements atomically, inside one transaction: either + * every statement lands or the whole batch rolls back. Entries are SQL + * strings, `[sql, ...params]` arrays or `{ sql, args }` objects; + * row-returning statements (`SELECT`/`PRAGMA`/`WITH`/`VALUES`/`EXPLAIN`, + * and anything carrying a `RETURNING` clause) resolve their rows into the + * results array, everything else resolves its `{ lastID, changes }`. * - * Calling `db.progress()` with no callback removes the handler. + * `options.mode` maps the libsql batch modes onto BEGIN forms: + * `'write'` (the default) → `BEGIN IMMEDIATE`, `'read'`/`'deferred'` → + * `BEGIN DEFERRED`, `'exclusive'` → `BEGIN EXCLUSIVE`. * * @this {import('./sqlite3-binding.js').Database} - * @param {number | (() => unknown)} [period] VM instructions between - * invocations (default 1000), or the callback directly. - * @param {() => unknown} [callback] called with no arguments; a truthy - * return aborts the statement. - * @returns {import('./sqlite3-binding.js').Database} this database, for chaining. - * @throws {TypeError} when the period or callback has the wrong type. - * @since 9.0.0 + * @param {(string | BatchStatement | [string, ...unknown[]])[]} statements + * @param {{ mode?: 'write' | 'read' | 'deferred' | 'exclusive' }} [options] + * @returns {Promise} one result per statement. + * @throws {TypeError} when the statements or mode are malformed. + * @since 9.1.0 * @example - * db.progress(10000, () => shouldStop); + * await db.batch([ + * 'CREATE TABLE t (a)', + * { sql: 'INSERT INTO t VALUES (?)', args: 1 }, + * ]); */ -Database.prototype.progress = function (period, callback) { - if (typeof period === 'function' && callback === undefined) { - callback = period; - period = 1000; +Database.prototype.batch = async function batch(statements, options) { + if (!Array.isArray(statements)) { + throw new TypeError('batch() requires an array of statements'); } - if (callback === undefined || callback === null) { - // Also the documented removal form: db.progress(). - progressOwner.delete(this); - /** @type {(...args: unknown[]) => unknown} */ ( - /** @type {unknown} */ (this._progressCallback) - )(); - return this; + const mode = options?.mode ?? 'write'; + const begin = + mode === 'write' + ? 'immediate' + : mode === 'read' || mode === 'deferred' + ? 'deferred' + : mode === 'exclusive' + ? 'exclusive' + : null; + if (begin === null) { + throw new TypeError( + "batch() mode must be 'write', 'read', 'deferred' or 'exclusive'", + ); + } + /** @type {{ sql: string, args?: unknown }[]} */ + const entries = statements.map((entry, i) => { + if (typeof entry === 'string') return { sql: entry }; + if (Array.isArray(entry)) { + if (entry.length === 0 || typeof entry[0] !== 'string') { + throw new TypeError( + `batch()[${i}] must be a SQL string or [sql, ...params]`, + ); + } + return { sql: entry[0], args: entry.slice(1) }; + } + if ( + entry === null || + typeof entry !== 'object' || + typeof entry.sql !== 'string' + ) { + throw new TypeError( + `batch()[${i}] must be a SQL string, [sql, ...params] or { sql, args }`, + ); + } + return { sql: entry.sql, args: entry.args }; + }); + return this.transaction( + async (tx) => { + /** @type {unknown[]} */ + const results = []; + for (const { + sql, + args, + } of /** @type {{ sql: string, args?: unknown }[]} */ (entries)) { + // args is a positional array, a named-parameter object + // or a single positional value; all three are documented, + // and spreading the last two used to throw "Spread + // syntax requires ...iterable". + const bindArgs = + args === undefined || args === null + ? [] + : Array.isArray(args) + ? /** @type {any[]} */ (args) + : [args]; + if (batchStatementReadsRows(sql)) { + results.push(await tx.all(sql, ...bindArgs)); + } else { + results.push(await tx.run(sql, ...bindArgs)); + } + } + return results; + }, + { mode: begin }, + ); +}; + +/** + * Serializes one SQL literal for {@link sqlite3.iterdump}: the inverse of + * the bind marshalling (strings quoted, blobs as X'…' hex, BigInt exact). + * + * @param {unknown} value the value to serialize. + * @returns {string} the SQL literal. + * @private + */ +function dumpLiteral(value) { + if (value === null || value === undefined) return 'NULL'; + if (typeof value === 'number') { + // `Infinity`/`NaN` are JavaScript spellings, not SQL: restoring + // them fails with "no such column: Infinity". SQLite's own shell + // writes the overflowing literal, which reloads as ±inf; NaN + // cannot be stored by SQLite at all (it becomes NULL), so this + // arm only matters for the value a REAL column can hold. + if (Number.isNaN(value)) return 'NULL'; + if (value === Number.POSITIVE_INFINITY) return '9.0e+999'; + if (value === Number.NEGATIVE_INFINITY) return '-9.0e+999'; + return String(value); + } + if (typeof value === 'bigint') return value.toString(); + if (typeof value === 'boolean') return value ? '1' : '0'; + if (typeof value === 'string') { + return `'${value.replaceAll("'", "''")}'`; + } + if (Buffer.isBuffer(value) || value instanceof Uint8Array) { + return `X'${Buffer.from( + value.buffer, + value.byteOffset, + value.byteLength, + ).toString('hex')}'`; + } + throw new TypeError( + `iterdump cannot serialize a ${typeof value} column value`, + ); +} + +/** + * Quotes one identifier (schema, table or column name) for dump SQL. + * + * @param {string} name the identifier. + * @returns {string} the quoted identifier. + * @private + */ +function dumpIdent(name) { + return `"${name.replaceAll('"', '""')}"`; +} + +/** + * The columns of one table that a restore can INSERT into: everything + * except generated columns (`PRAGMA table_xinfo` reports those as hidden + * 2/3), which SQLite computes and refuses as INSERT targets. + * + * @param {import('./native.js').Database} db the connection. + * @param {string} table the table name. + * @returns {Promise} the insertable column names, in order. + * @private + */ +async function dumpInsertableColumns(db, table) { + const info = /** @type {{ name: string, hidden: number }[]} */ ( + await db.all(`PRAGMA table_xinfo(${dumpIdent(table)})`) + ); + return info + .filter((column) => Number(column.hidden) === 0) + .map((column) => column.name); +} + +/** + * Streams the database as `.dump`-style SQL: the schema (tables first, + * then their rows, then indexes/views/triggers), the AUTOINCREMENT + * counters and `user_version`, framed by the `BEGIN`/`COMMIT` a restore + * needs. Python `sqlite3`'s `iterdump()` equivalent; nothing else in Node + * has it. + * + * Rows are streamed (`iterate`), so dumping a table larger than memory is + * fine, and the reads run inside a deferred transaction — unless the + * caller already has one open — so the dump is a point-in-time snapshot. + * Abandoning the iterator early rolls that transaction back. + * + * Virtual tables keep their content: the `CREATE VIRTUAL TABLE` statement + * is emitted (which recreates the empty shadow tables), and each shadow + * table's rows follow as `DELETE` + `INSERT` against a + * `CREATE TABLE IF NOT EXISTS`, so an FTS index is restored exactly as it + * was instead of being silently dropped. No `writable_schema` games, so + * the output restores into a defensive-mode connection too. + * + * @param {import('./native.js').Database} db the connection. + * @returns {AsyncGenerator} the SQL statements. + * @since 9.1.0 + * @example + * for await (const statement of sqlite3.iterdump(db)) fs.write(statement); + */ +async function* iterdump(db) { + // A snapshot needs one read transaction for the whole walk. When the + // caller already has a transaction open, theirs is the snapshot. + const ownTransaction = !db.inTransaction; + if (ownTransaction) await db.exec('BEGIN DEFERRED'); + let committed = false; + try { + yield 'PRAGMA foreign_keys=OFF;\nBEGIN TRANSACTION;\n'; + // sqlite_schema's rowid order is creation order — tables before + // the indexes/triggers created for them, which is the restore-safe + // order. + const schema = + /** @type {{ name: string, type: string, sql: string }[]} */ ( + await db.all( + 'SELECT name, type, sql FROM sqlite_schema ' + + "WHERE sql NOT NULL AND name NOT LIKE 'sqlite_%' " + + 'ORDER BY rowid', + ) + ); + // Virtual tables keep their content, which lives in the shadow + // tables CREATE VIRTUAL TABLE itself creates. Those are dumped + // with IF NOT EXISTS + DELETE before their rows, so the restore + // works whether the DDL made them or not — which also makes a + // misidentified `_*` user table harmless. + const virtualTables = [ + ...new Set( + schema + .filter( + (row) => + row.type === 'table' && + /^\s*CREATE\s+VIRTUAL\s+TABLE\b/i.test(row.sql), + ) + .map((row) => row.name), + ), + ]; + /** @param {string} name @returns {boolean} */ + const isShadowTable = (name) => + virtualTables.some((vtab) => name.startsWith(`${vtab}_`)); + + /** @type {{ name: string, sql: string }[]} */ + const deferred = []; + for (const row of schema) { + if (row.type !== 'table') { + deferred.push(row); + continue; + } + const shadow = isShadowTable(row.name); + const target = dumpIdent(row.name); + if (shadow && !/\bIF\s+NOT\s+EXISTS\b/i.test(row.sql)) { + yield `${row.sql.replace( + /^\s*CREATE\s+TABLE\s+/i, + 'CREATE TABLE IF NOT EXISTS ', + )};\n`; + } else { + yield `${row.sql};\n`; + } + if (virtualTables.includes(row.name)) continue; + if (shadow) yield `DELETE FROM ${target};\n`; + const names = await dumpInsertableColumns(db, row.name); + if (names.length === 0) continue; + const columns = names.map(dumpIdent).join(','); + for await (const data of db.iterate( + `SELECT ${columns} FROM ${target}`, + )) { + const values = names + .map((name) => + dumpLiteral( + /** @type {Record} */ (data)[name], + ), + ) + .join(','); + yield `INSERT INTO ${target}(${columns}) VALUES(${values});\n`; + } + } + // AUTOINCREMENT high-water marks: without these a restored + // database reuses rowids the original had already handed out. + const sequences = /** @type {{ name: string, seq: unknown }[]} */ ( + await db + .all( + 'SELECT name, seq FROM sqlite_schema JOIN sqlite_sequence USING (name) ' + + "WHERE sqlite_schema.type = 'table' ORDER BY name", + ) + .catch(() => []) + ); + if (sequences.length > 0) { + yield 'DELETE FROM sqlite_sequence;\n'; + for (const row of sequences) { + yield 'INSERT INTO sqlite_sequence(name,seq) VALUES' + + `(${dumpLiteral(row.name)},${dumpLiteral(row.seq)});\n`; + } + } + for (const row of deferred) { + yield `${row.sql};\n`; + } + // user_version carries the schema position sqlite3.migrate() and + // most migration tools key on; a dump that dropped it would rewind + // the restored database's migration state. + const userVersion = Number( + await db.pragma('user_version', { simple: true }), + ); + if (Number.isInteger(userVersion) && userVersion !== 0) { + yield `PRAGMA user_version = ${userVersion};\n`; + } + yield 'COMMIT;\n'; + if (ownTransaction) { + await db.exec('COMMIT'); + committed = true; + } + } finally { + // Abandoned early (break/throw): the read transaction must not + // stay open on the connection. + if (ownTransaction && !committed) { + try { + await db.exec('ROLLBACK'); + } catch { + // Already resolved by an outer failure; nothing to undo. + } + } + } +} + +/** + * Serializes the whole database to `.dump`-style SQL text (see + * {@link sqlite3.iterdump} for the streaming form). + * + * @this {import('./sqlite3-binding.js').Database} + * @returns {Promise} the SQL text of the dump. + * @since 9.1.0 + * @example + * fs.writeFileSync('backup.sql', await db.dump()); + */ +Database.prototype.dump = async function dump() { + let out = ''; + for await (const statement of iterdump(this)) { + out += statement; + } + return out; +}; + +sqlite3.iterdump = iterdump; +sqlite3.migrate = migrate; + +// Friendly names for the run-time limits db.limits reports, over the +// LIMIT_* constants the binding exports. +const LIMIT_NAMES = /** @type {const} */ ([ + ['length', 'LIMIT_LENGTH'], + ['sqlLength', 'LIMIT_SQL_LENGTH'], + ['column', 'LIMIT_COLUMN'], + ['exprDepth', 'LIMIT_EXPR_DEPTH'], + ['compoundSelect', 'LIMIT_COMPOUND_SELECT'], + ['vdbeOp', 'LIMIT_VDBE_OP'], + ['functionArg', 'LIMIT_FUNCTION_ARG'], + ['attached', 'LIMIT_ATTACHED'], + ['likePatternLength', 'LIMIT_LIKE_PATTERN_LENGTH'], + ['variableNumber', 'LIMIT_VARIABLE_NUMBER'], + ['triggerDepth', 'LIMIT_TRIGGER_DEPTH'], + ['workerThreads', 'LIMIT_WORKER_THREADS'], +]); + +// Friendly names for the db.status() counters, over the DBSTATUS_* +// constants the binding exports. +const DB_STATUS_NAMES = /** @type {const} */ ([ + ['lookasideUsed', 'DBSTATUS_LOOKASIDE_USED'], + ['cacheUsed', 'DBSTATUS_CACHE_USED'], + ['schemaUsed', 'DBSTATUS_SCHEMA_USED'], + ['stmtUsed', 'DBSTATUS_STMT_USED'], + ['lookasideHit', 'DBSTATUS_LOOKASIDE_HIT'], + ['lookasideMissSize', 'DBSTATUS_LOOKASIDE_MISS_SIZE'], + ['lookasideMissFull', 'DBSTATUS_LOOKASIDE_MISS_FULL'], + ['cacheHit', 'DBSTATUS_CACHE_HIT'], + ['cacheMiss', 'DBSTATUS_CACHE_MISS'], + ['cacheWrite', 'DBSTATUS_CACHE_WRITE'], + ['cacheSpill', 'DBSTATUS_CACHE_SPILL'], + ['deferredFks', 'DBSTATUS_DEFERRED_FKS'], +]); + +/** + * Reads one `sqlite3_db_status` counter for the connection — cache + * hits/misses, schema memory, and friends. Pass the counter by friendly + * name (`'cacheHit'`) or by `sqlite3.DBSTATUS_*` constant. Resolves + * `{ current, highwater }`; `{ reset: true }` zeroes the counters after + * reading. + * + * Note: this build compiles with `SQLITE_DEFAULT_MEMSTATUS=0`, so the + * process-wide memory counters report zero; the cache and schema + * counters are the useful ones. + * + * @this {import('./sqlite3-binding.js').Database} + * @param {string | number} op the counter name or constant. + * @param {{ reset?: boolean }} [options] + * @returns {{ current: number, highwater: number }} the counter values. + * @throws {TypeError} when the name is unknown. + * @since 9.1.0 + * @example + * const { current } = db.status('cacheHit'); + */ +Database.prototype.status = function status(op, options) { + const reset = options?.reset === true; + if (typeof op === 'string') { + const entry = DB_STATUS_NAMES.find(([name]) => name === op); + if (entry === undefined) { + throw new TypeError(`db.status() received unknown counter '${op}'`); + } + return this._dbStatus(/** @type {any} */ (sqlite3)[entry[1]], reset); + } + return this._dbStatus(op, reset); +}; + +/** + * Releases non-essential memory held by this connection (page cache + * beyond the working set) — the pool-pressure lever. Returns the number + * of bytes freed (`sqlite3_db_release_memory`). + * + * @this {import('./sqlite3-binding.js').Database} + * @returns {number} bytes freed. + * @since 9.1.0 + */ +Database.prototype.releaseMemory = function releaseMemory() { + return /** @type {number} */ (this._releaseMemory()); +}; + +/** + * The current run-time limits, by friendly name — the read form of + * `configure('limit', sqlite3.LIMIT_*, value)`. + * + * @this {import('./sqlite3-binding.js').Database} + * @name Database#limits + * @type {Record} + * @since 9.1.0 + */ +Object.defineProperty(Database.prototype, 'limits', { + get() { + /** @type {Record} */ + const out = {}; + for (const [name, constant] of LIMIT_NAMES) { + out[name] = /** @type {number} */ ( + this._getLimit(/** @type {any} */ (sqlite3)[constant]) + ); + } + return out; + }, + configurable: true, +}); + +/** + * Resolves the filesystem path of an attached database (`''` for + * in-memory or temporary schemas) — `sqlite3_db_filename`. + * + * @this {import('./sqlite3-binding.js').Database} + * @param {string} [dbName] the attached name (default `'main'`). + * @returns {string} the path. + * @since 9.1.0 + */ +Database.prototype.location = function location(dbName) { + return /** @type {string} */ (this._dbLocation(dbName ?? 'main')); +}; + +// --- Tagged-template queries (Phase 5) ---------------------------------------- + +// The fragment brand. Composition helpers (raw/identifier/identifierPath/ +// join/empty) mint fragments carrying this symbol, and only a value +// carrying it is spliced into the SQL as text. Structural detection ("has +// a `text` property") would turn any attacker-shaped object arriving in a +// template hole — `JSON.parse('{"text":"1 OR 1=1","params":[]}')` — into +// raw SQL. Everything unbranded is a bind parameter, always. +const SQL_FRAGMENT = Symbol('@appthreat/sqlite3.sqlFragment'); + +/** + * One composed piece of SQL for the tag store: literal text plus bind + * parameters. `sql.raw`/`identifier`/`join` build these; a plain value in + * a template hole becomes a bind parameter instead. + * + * @typedef {object} SqlFragment + * @property {string} text the SQL text. + * @property {unknown[]} params the bind parameters, in text order. + * @private + */ + +/** + * Builds a fragment. + * + * @param {string} text the SQL text. + * @param {unknown[]} params the bind parameters. + * @returns {SqlFragment} the fragment. + * @private + */ +function fragment(text, params) { + return Object.freeze({ text, params, [SQL_FRAGMENT]: true }); +} + +/** + * True for a value minted by one of this store's composition helpers. + * + * @param {unknown} value the candidate. + * @returns {boolean} whether it is a branded fragment. + * @private + */ +function isFragment(value) { + return ( + value !== null && + typeof value === 'object' && + /** @type {{ [SQL_FRAGMENT]?: unknown }} */ (value)[SQL_FRAGMENT] === + true + ); +} + +/** + * A SQL identifier quoted for safe interpolation — the Kysely/Sequelize + * helper names Bun notably lacks. Accepts plain identifiers and quoted + * forms (already-`"quoted"` strings pass through). + * + * @param {string} name the identifier. + * @returns {SqlFragment} the quoted fragment. + * @throws {TypeError} for anything that is not a plain identifier. + * @since 9.1.0 + */ +function sqlIdentifier(name) { + if (typeof name !== 'string' || name.length === 0) { + throw new TypeError('sql.identifier() requires a non-empty string'); + } + if (/^[A-Za-z_][A-Za-z0-9_$]*$/.test(name)) { + return fragment(`"${name}"`, []); + } + // Schema-qualified or already-quoted forms pass through. The pattern + // is what makes that safe: every part is wrapped in double quotes and + // may not contain one, so nothing can close the quoting early. (The + // previous guard also rejected every name containing a quote, which + // made this branch unreachable.) + if (!/[\\\0]/.test(name) && /^"[^"]+"(\."[^"]+")*$/.test(name)) { + return fragment(name, []); + } + throw new TypeError( + `sql.identifier() received ${JSON.stringify(name)}, which is not ` + + 'a plain identifier (quote each part yourself for exotic names)', + ); +} + +/** + * A dotted identifier path quoted part by part: `identifierPath('main.t.x')`. + * + * @param {string} dotted the dot-separated path. + * @returns {SqlFragment} the quoted fragment. + * @throws {TypeError} for empty or malformed parts. + * @since 9.1.0 + */ +function sqlIdentifierPath(dotted) { + if (typeof dotted !== 'string' || dotted.length === 0) { + throw new TypeError('sql.identifierPath() requires a non-empty string'); + } + // Dots inside a "quoted part" belong to the name, not to the path: + // identifierPath('"a.b".c') is two parts, not three. + /** @type {string[]} */ + const parts = []; + let current = ''; + let quoted = false; + for (const ch of dotted) { + if (ch === '"') { + quoted = !quoted; + current += ch; + } else if (ch === '.' && !quoted) { + parts.push(current); + current = ''; + } else { + current += ch; + } + } + parts.push(current); + if (quoted) { + throw new TypeError( + 'sql.identifierPath() received an unterminated quoted part', + ); + } + return fragment( + parts + .map((part) => { + if (part.length === 0) { + throw new TypeError( + 'sql.identifierPath() received an empty part', + ); + } + return sqlIdentifier(part).text; + }) + .join('.'), + [], + ); +} + +/** + * Interpolates one template-hole value into the SQL under construction. + * + * @param {string[]} out the accumulated SQL chunks. + * @param {unknown[]} params the accumulated bind parameters. + * @param {unknown} value the hole's value. + * @returns {void} + * @private + */ +function appendHole(out, params, value) { + if (isFragment(value)) { + const frag = /** @type {SqlFragment} */ (value); + out.push(frag.text); + params.push(...frag.params); + return; + } + out.push('?'); + params.push(value); +} + +/** + * Builds the {@link TagStore} — tagged-template queries driven by an LRU + * of composed SQL (`store.all\`SELECT … WHERE id = ${id}\``), the + * node:sqlite `createTagStore` shape plus the composition helpers ORMs + * need: `store.raw`, `store.join`, `store.identifier`, + * `store.identifierPath` and `store.empty`. Interpolated values become + * positional `?` parameters (only a fragment from those helpers is + * spliced in as SQL text); the joined SQL is the cache key. + * + * Statement reuse is the connection's: creating a store enables the + * connection statement cache ({@link Database#cacheStatements}) if it is + * not already on (an existing one is left at its own size), since + * composing the same SQL repeatedly is the whole point of a tag store. + * The store's own LRU holds the composed SQL keys (never the bound + * values, which would pin every buffer the first call bound), and + * `clear()` empties both. + * + * Promise-native (an improvement on node's sync-only store). + * + * @this {import('./sqlite3-binding.js').Database} + * @param {number} [maxSize=200] the cache capacity. + * @returns {import('./augment.js').TagStore} the store. + * @throws {TypeError} for a non-positive capacity. + * @since 9.1.0 + * @example + * const store = db.createTagStore(); + * const rows = await store.all`SELECT * FROM t WHERE id = ${id}`; + */ +Database.prototype.createTagStore = function createTagStore(maxSize = 200) { + if ( + typeof maxSize !== 'number' || + !Number.isInteger(maxSize) || + maxSize < 1 + ) { + throw new TypeError( + 'createTagStore() maxSize must be a positive integer', + ); + } + const db = this; + // The composed-SQL keys this store has seen, most recently used last + // (Map insertion order). Values are never retained: the statements + // themselves live in the connection's cache. + /** @type {Set} */ + const cache = new Set(); + if (!db._stmtCache) db.cacheStatements(); + + /** + * Builds (or reuses) the statement arguments for one template call. + * + * @param {TemplateStringsArray} templates the template strings. + * @param {unknown[]} values the interpolated values. + * @returns {{ sql: string, params: unknown[] }} the statement and bind values. + */ + const build = (templates, values) => { + /** @type {string[]} */ + const chunks = []; + /** @type {unknown[]} */ + const params = []; + chunks.push(templates.raw[0]); + for (let i = 0; i < values.length; i++) { + appendHole(chunks, params, values[i]); + chunks.push(templates.raw[i + 1]); + } + const sql = chunks.join(''); + // Most recently used last; the oldest key is evicted at capacity. + cache.delete(sql); + cache.add(sql); + if (cache.size > maxSize) { + const oldest = cache.values().next().value; + if (oldest !== undefined) cache.delete(oldest); + } + return { sql, params }; + }; + + /** @type {import('./augment.js').TagStore} */ + const store = /** @type {import('./augment.js').TagStore} */ ( + /** @type {unknown} */ ({ + get db() { + return db; + }, + get size() { + return cache.size; + }, + get capacity() { + return maxSize; + }, + clear() { + // The statements themselves live in the connection's + // statement cache (keyed by the same SQL strings); clear + // that too or clear() frees nothing. Refuse first, before + // either half is touched: flushing the cache from inside a + // sync-invoked callback would finalize the statement + // sqlite is stepping. + assertNotInSyncCallback(db, 'the tag store cannot be cleared'); + cache.clear(); + db._drainStatementCache(); + }, + }) + ); + /** + * @param {string} method + * @param {TemplateStringsArray} templates + * @param {unknown[]} values + * @returns {any} + */ + const tag = (method, templates, values) => { + // (typed by the JSDoc above) + if (!Array.isArray(templates) || !Array.isArray(templates.raw)) { + throw new TypeError( + 'the tag store is used as a template tag: store.' + + `${method}\`SELECT …\``, + ); + } + const { sql, params } = build(templates, values); + return /** @type {any} */ (db)[method](sql, ...params); + }; + store.get = (templates, ...values) => tag('get', templates, values); + store.all = (templates, ...values) => tag('all', templates, values); + store.iterate = (templates, ...values) => tag('iterate', templates, values); + store.run = (templates, ...values) => tag('run', templates, values); + store.raw = (text) => { + if (typeof text !== 'string') { + throw new TypeError('sql.raw() requires a string'); + } + return fragment(text, []); + }; + store.join = (items, separator = ', ') => { + if (!Array.isArray(items) || items.length === 0) { + throw new TypeError('sql.join() requires a non-empty array'); + } + if (typeof separator !== 'string') { + throw new TypeError('sql.join() separator must be a string'); + } + /** @type {string[]} */ + const parts = []; + /** @type {unknown[]} */ + const params = []; + for (const item of items) { + // Fragments compose as text; anything else binds. An IN-list + // of plain values is the common case (`join(ids)`), and it + // must not require the caller to reach for raw() — which on + // user data would be an injection. + if (isFragment(item)) { + const frag = /** @type {SqlFragment} */ (item); + parts.push(frag.text); + params.push(...frag.params); + } else { + parts.push('?'); + params.push(item); + } + } + return fragment(parts.join(separator), params); + }; + store.identifier = sqlIdentifier; + store.identifierPath = sqlIdentifierPath; + store.empty = () => fragment('', []); + return store; +}; + +// --- JavaScript virtual tables (Phase 4) ------------------------------------- + +/** + * Normalizes one column spec: `'name'`, `'name TYPE'` or + * `{ name, type }` into the bare column name. + * + * @param {unknown} spec the column spec. + * @param {string} who the call site, for error messages. + * @returns {string} the column name. + * @throws {TypeError} when the spec is malformed. + * @private + */ +function vtabColumnName(spec, who) { + if (typeof spec === 'string') { + const name = spec.split(/\s+/)[0]; + if (name.length > 0) return name; + } else if ( + spec !== null && + typeof spec === 'object' && + typeof (/** @type {{ name?: unknown }} */ (spec).name) === 'string' && + /** @type {{ name?: unknown }} */ (spec).name !== undefined && + /** @type {string} */ (/** @type {{ name?: unknown }} */ (spec).name) + .length > 0 + ) { + return /** @type {{ name: string }} */ (spec).name; + } + throw new TypeError(`${who} column specs must be strings or { name }`); +} + +/** + * @typedef {object} VtabDefinition + * @property {(this: undefined, ...args: unknown[]) => Iterable>} rows + * the row generator: invoked once per query with the table-function + * parameter values; yields arrays (in column order) or objects keyed by + * column name. + * @property {(string | { name: string, type?: string })[]} columns the + * result columns (a `'name TYPE'` string or `{ name, type }`; the type + * is documentation — SQLite virtual tables are typeless). + * @property {string[]} [parameters] a subset of `columns` to declare + * HIDDEN — the table-valued function's arguments + * (`SELECT * FROM name(arg)` passes `arg` to `rows`). + * @since 9.1.0 + */ + +/** + * Registers a read-only virtual table computed by a JavaScript generator + * — better-sqlite3's marquee feature, none of the other JS drivers have + * it, and here it works from both the async paths (one worker round trip + * per query — the whole generator output is materialised at filter time) + * and the sync methods (a direct re-entrant call). + * + * The object form registers an **eponymous-only** module: the table + * exists immediately under `name` (no `CREATE VIRTUAL TABLE`). The + * factory-function form (`db.table(name, (arg, ...) => definition)`) + * registers a named module instantiated per + * `CREATE VIRTUAL TABLE ... USING name(args)`; the arguments arrive as + * the SQL literal strings from the DDL. + * + * Registration is asynchronous but ordered: a query issued right after + * `db.table()` queues behind the registration and sees the table. The + * statement cache is flushed (a cached statement cannot gain the table). + * + * @this {import('./sqlite3-binding.js').Database} + * @param {string} name the module/table name (1..255 bytes). + * @param {VtabDefinition | ((...args: string[]) => unknown)} definition + * the definition object, or a factory function for a named module. + * @returns {import('./sqlite3-binding.js').Database} this database, for chaining. + * @throws {TypeError} when the name or definition is malformed. + * @since 9.1.0 + * @example + * db.table('sequence', { + * columns: ['value'], + * rows: function* (count) { + * for (let i = 0; i < count; i++) yield [i]; + * }, + * }); + * const rows = await db.all('SELECT value FROM sequence(5)'); + */ +Database.prototype.table = function table(name, definition) { + assertNotInSyncCallback(this, 'a virtual table cannot be registered'); + if (typeof name !== 'string' || name.length === 0) { + throw new TypeError('table() requires a non-empty name string'); + } + if (Buffer.byteLength(name, 'utf8') > 255) { + throw new TypeError( + "table() name exceeds SQLite's 255-byte module-name limit", + ); + } + const isFactory = typeof definition === 'function'; + if ( + definition === null || + (typeof definition !== 'object' && typeof definition !== 'function') + ) { + throw new TypeError( + 'table() requires a definition object or a factory function', + ); + } + + /** + * Extracts the shape from the definition (the factory's own columns + * are declared up front by the module). + * + * @param {VtabDefinition | ((...args: string[]) => VtabDefinition)} def + * @param {boolean} allowMissingRows + * @returns {{ columns: string[], params: string[], rows: unknown }} + */ + const parse = (def, allowMissingRows) => { + if (def === null || typeof def !== 'object' || Array.isArray(def)) { + throw new TypeError( + 'table() definition must be an object with columns and rows', + ); + } + const known = new Set(['rows', 'columns', 'parameters']); + for (const key of Object.keys(def)) { + if (!known.has(key)) { + throw new TypeError( + `table() definition received unknown option '${key}'`, + ); + } + } + if (!Array.isArray(def.columns)) { + throw new TypeError( + "table() definition requires a 'columns' array", + ); + } + const columns = def.columns.map((spec) => + vtabColumnName(spec, 'table()'), + ); + const parameters = def.parameters ?? []; + if (!Array.isArray(parameters)) { + throw new TypeError( + "table() option 'parameters' must be an array of column names", + ); + } + for (const param of parameters) { + if (typeof param !== 'string' || param.length === 0) { + throw new TypeError( + "table() 'parameters' entries must be non-empty strings", + ); + } + if (!columns.includes(param)) { + throw new TypeError( + `table() parameter '${param}' is not one of the columns; ` + + 'parameters are the subset of columns declared HIDDEN', + ); + } + } + const rows = /** @type {unknown} */ (def.rows); + if (typeof rows !== 'function' && !allowMissingRows) { + throw new TypeError( + "table() definition requires a 'rows' generator function", + ); + } + return { columns, params: parameters, rows }; + }; + + let columns; + let params; + let factory = null; + let rows = null; + if (isFactory) { + // A factory's instances all share the module's declared shape, so + // the factory itself carries the columns (same spec shapes as a + // definition's); the definitions it returns supply only rows. + factory = definition; + const fn = /** @type {{ columns?: unknown, parameters?: unknown }} */ ( + /** @type {unknown} */ (definition) + ); + if (!Array.isArray(fn.columns)) { + throw new TypeError( + 'a table() factory must declare its columns as factory ' + + ".columns (same shape as a definition's), so the module " + + 'can declare them for every instance', + ); + } + columns = fn.columns.map((spec) => vtabColumnName(spec, 'table()')); + const fparams = fn.parameters ?? []; + if (!Array.isArray(fparams)) { + throw new TypeError( + 'factory.parameters must be an array of column names', + ); + } + for (const param of fparams) { + if (typeof param !== 'string' || !columns.includes(param)) { + throw new TypeError( + `factory parameter '${String(param)}' is not one of the columns`, + ); + } + } + params = fparams; + } else { + const parsed = parse(definition, false); + columns = parsed.columns; + params = parsed.params; + rows = parsed.rows; + } + + // See function(): a cached statement keeps the schema it was compiled + // against, so the cache must not hand one back across a registration. + this._drainStatementCache(); + /** @type {(...args: unknown[]) => unknown} */ ( + /** @type {unknown} */ (this._registerVtab) + )(name, columns, params, factory, rows); + return this; +}; + +/** + * Removes a virtual table module registered with {@link Database#table}. + * In-flight queries complete; a later query against the name fails + * loudly ("this virtual table module was removed"). + * + * @this {import('./sqlite3-binding.js').Database} + * @param {string} name the module name. + * @returns {import('./sqlite3-binding.js').Database} this database, for chaining. + * @throws {TypeError} when the name is malformed. + * @since 9.1.0 + */ +Database.prototype.removeTable = function removeTable(name) { + assertNotInSyncCallback(this, 'a virtual table cannot be removed'); + if (typeof name !== 'string' || name.length === 0) { + throw new TypeError('removeTable() requires a non-empty name string'); + } + this._drainStatementCache(); + /** @type {(...args: unknown[]) => unknown} */ ( + /** @type {unknown} */ (this._removeVtab) + )(name); + return this; +}; + +// The auto-incrementing suffix for db.values() table names. +let valuesTableCounter = 0; +/** + * The per-connection registry of db.values() tables: registration order + * (for the cap) and the drop handles. + * + * @typedef {object} ValuesRegistry + * @property {string[]} order the table names, oldest registration first. + * @property {Map void }>} byName the handles. + */ +/** @type {WeakMap} */ +const valuesTables = new WeakMap(); +const VALUES_TABLES_MAX = 32; + +/** + * Registers one JS array (or any iterable) as a queryable table — the + * rusqlite `rarray()` ergonomics no JS driver has: `WHERE id IN (SELECT + * value FROM v)` and `JOIN` against in-memory data, no string-building. + * + * Returns `{ name, drop() }`: `name` is the (unquoted) table name to use + * in SQL, keyed by position (`key`, 0-based) and element (`value`). + * `drop()` removes one explicitly, and the whole connection is cleaned up + * at close. + * + * Anonymous registrations are capped at 32 per connection: the 33rd drops + * the oldest, whose handle then refers to a table that no longer exists + * (a query against it fails with "this virtual table module was removed"). + * The cap exists so a forgotten `drop()` cannot grow without bound — + * `drop()` each handle when you are done with it, or pass an explicit + * `{ name }`, which opts out of the cap entirely. + * + * @this {import('./sqlite3-binding.js').Database} + * @param {Iterable} iterable the array/iterable to expose. + * @param {{ name?: string }} [options] an explicit table name (then no + * LRU applies — drop() it yourself). + * @returns {{ name: string, drop: () => void }} the table handle. + * @throws {TypeError} when the values or options are malformed. + * @since 9.1.0 + * @example + * const ids = db.values([4, 8, 15]); + * const rows = await db.all( + * `SELECT * FROM users JOIN ${ids.name} v ON users.id = v.value`, + * ); + */ +Database.prototype.values = function values(iterable, options) { + assertNotInSyncCallback(this, 'a values table cannot be registered'); + if (iterable === null || typeof iterable[Symbol.iterator] !== 'function') { + throw new TypeError('values() requires an iterable'); + } + let explicitName; + if (options !== undefined && options !== null) { + if (typeof options !== 'object' || Array.isArray(options)) { + throw new TypeError('values() options must be an object'); + } + for (const key of Object.keys(options)) { + if (key !== 'name') { + throw new TypeError( + `values() received unknown option '${key}'`, + ); + } + } + if (options.name !== undefined) { + if (typeof options.name !== 'string' || options.name.length === 0) { + throw new TypeError( + "values() option 'name' must be a non-empty string", + ); + } + explicitName = options.name; + } + } + let registry = valuesTables.get(this); + if (registry === undefined) { + registry = { order: [], byName: new Map() }; + valuesTables.set(this, registry); + } + const name = explicitName ?? `sqlite_values_${++valuesTableCounter}`; + this.table(name, { + columns: ['key', 'value'], + rows: function* () { + let key = 0; + for (const value of iterable) { + yield [key++, value]; + } + }, + }); + /** @type {() => void} */ + let drop = () => { + this.removeTable(name); + registry?.byName.delete(name); + const at = registry?.order.indexOf(name); + if (at !== undefined && at >= 0) registry?.order.splice(at, 1); + // Dropped once; a second call is a no-op rather than a second + // removeTable (which would refuse or drop a re-registered name). + drop = () => undefined; + }; + const existing = registry.byName.get(name); + if (existing !== undefined) existing.drop(); + registry.byName.set(name, { name, drop: () => drop() }); + if (!explicitName) { + registry.order.push(name); + while (registry.order.length > VALUES_TABLES_MAX) { + const oldest = registry.order.shift(); + const entry = + oldest !== undefined ? registry.byName.get(oldest) : undefined; + if (entry !== undefined) entry.drop(); + } + } + return { name, drop: () => drop() }; +}; + +// --- Hooks, authorizer, progress, WAL and introspection (Deliverable 07) -- +// +// The commit/rollback/wal hooks are installed by the on()/removeListener() +// overrides above: the native sqlite hook exists only while a listener is +// registered, so an installed-but-unused hook costs nothing. + +/** + * Installs (or removes) a declarative authorizer on the connection. + * + * The policy is evaluated inside SQLite itself, in C++ — there is no + * JavaScript callback on the prepare path, so it is fast and safe from any + * thread that prepares a statement. This is the supported way to sandbox + * user-supplied SQL: with `{ default: 'deny' }` everything is refused + * unless a rule explicitly allows it. + * + * `deny` rules are evaluated before `allow` rules, and a denied action + * fails the statement with `SQLITE_AUTH` ("not authorized"). The statement + * cache is flushed on every change: a cached statement was compiled while + * the old policy was in force and would bypass the new one entirely. + * + * @this {import('./sqlite3-binding.js').Database} + * @param {import('./native.js').AuthorizerPolicy | null} [policy] the + * policy to install, or null/undefined to remove the authorizer. + * @returns {import('./sqlite3-binding.js').Database} this database, for chaining. + * @throws {TypeError} when the policy or one of its rules is malformed. + * @since 9.0.0 + * @example + * db.authorizer({ + * default: 'deny', + * allow: [ + * { action: sqlite3.SELECT }, + * { action: sqlite3.READ, table: 'users' }, + * ], + * }); + */ +Database.prototype.authorizer = function (policy) { + assertNotInSyncCallback(this, 'the authorizer policy cannot be changed'); + if (policy === null || policy === undefined) { + this._drainStatementCache(); + /** @type {(...args: unknown[]) => unknown} */ ( + /** @type {unknown} */ (this._setAuthorizer) + )(); + return this; + } + if (typeof policy !== 'object' || Array.isArray(policy)) { + throw new TypeError('authorizer() policy must be an object or null'); + } + const known = new Set(['default', 'allow', 'deny', 'ignore']); + for (const key of Object.keys(policy)) { + if (!known.has(key)) { + throw new TypeError( + `authorizer() received unknown option '${key}'`, + ); + } + } + const decisions = new Set(['allow', 'deny', 'ignore']); + const decisionOf = /** @type {Record} */ ({ + allow: sqlite3.OK, + deny: sqlite3.DENY, + ignore: sqlite3.IGNORE, + }); + const fallback = policy.default === undefined ? 'allow' : policy.default; + if (!decisions.has(fallback)) { + throw new TypeError( + "authorizer() default must be 'allow', 'deny' or 'ignore'", + ); + } + + /** + * Normalizes one rule list into native rows + * [action, verdict, arg1, arg2, database, trigger]. + * + * @param {unknown} rules the raw rule list. + * @param {string} verdict the list's decision name ('allow' etc). + * @param {string} who the list's name in the policy, for error messages. + * @returns {unknown[][]} the native rule rows. + */ + const normalize = (rules, verdict, who) => { + if (rules === undefined || rules === null) return []; + if (!Array.isArray(rules)) { + throw new TypeError( + `authorizer() '${who}' must be an array of rules`, + ); + } + return rules.map((rule, i) => { + if ( + rule === null || + typeof rule !== 'object' || + Array.isArray(rule) + ) { + throw new TypeError( + `authorizer() ${who}[${i}] must be a rule object`, + ); + } + const where = `authorizer() ${who}[${i}]`; + // null = match anything; an explicit '' targets an empty + // argument (previously unexpressible — D08 closes the D07 + // finding). + const row = /** @type {(number | string | null)[]} */ ([ + -1, + decisionOf[verdict], + null, + null, + null, + null, + ]); + if (rule.action !== undefined) { + if ( + typeof rule.action !== 'number' || + !Number.isInteger(rule.action) + ) { + throw new TypeError( + `${where} action must be an integer constant`, + ); + } + row[0] = rule.action; + } + const arg1 = rule.arg1 !== undefined ? rule.arg1 : rule.table; + const arg2 = rule.arg2 !== undefined ? rule.arg2 : rule.column; + const parts = [ + [arg1, 'arg1'], + [arg2, 'arg2'], + [rule.database, 'database'], + [rule.trigger, 'trigger'], + ]; + parts.forEach((part, j) => { + const value = part[0]; + const name = /** @type {string} */ (part[1]); + if (value === undefined || value === null) return; + if (typeof value !== 'string') { + throw new TypeError(`${where} ${name} must be a string`); + } + row[2 + j] = value; + }); + return row; + }); + }; + + // Deny first: the sandbox reading — a deny must not be rescuable by a + // later allow, whatever the array order. + const rows = [ + ...normalize(policy.deny, 'deny', 'deny'), + ...normalize(policy.ignore, 'ignore', 'ignore'), + ...normalize(policy.allow, 'allow', 'allow'), + ]; + + this._drainStatementCache(); + /** @type {(...args: unknown[]) => unknown} */ ( + /** @type {unknown} */ (this._setAuthorizer) + )(decisionOf[fallback], rows); + return this; +}; + +/** + * Installs a progress handler. Two forms: + * + * - `db.progress(period, callback)` — a JavaScript callback invoked every + * `period` VM instructions; returning truthy aborts the running + * statement with `SQLITE_INTERRUPT`. Each invocation is a blocking + * round trip to the JS thread from whatever thread is executing SQL, + * so it is the expensive form: fine for progress bars over a handful + * of long queries, wrong for anything per-row. While it is installed, + * the synchronous methods (`getSync`/`runSync`/`allSync` and + * `prepareSync`) refuse to run — the callback could fire on the thread + * that would have to service it. + * - `db.cancellationToken()` — the recommended form; see there. + * + * Calling `db.progress()` with no callback removes the handler. + * + * @this {import('./sqlite3-binding.js').Database} + * @param {number | (() => unknown)} [period] VM instructions between + * invocations (default 1000), or the callback directly. + * @param {() => unknown} [callback] called with no arguments; a truthy + * return aborts the statement. + * @returns {import('./sqlite3-binding.js').Database} this database, for chaining. + * @throws {TypeError} when the period or callback has the wrong type. + * @since 9.0.0 + * @example + * db.progress(10000, () => shouldStop); + */ +Database.prototype.progress = function (period, callback) { + if (typeof period === 'function' && callback === undefined) { + callback = period; + period = 1000; + } + if (callback === undefined || callback === null) { + // Also the documented removal form: db.progress(). + progressOwner.delete(this); + /** @type {(...args: unknown[]) => unknown} */ ( + /** @type {unknown} */ (this._progressCallback) + )(); + return this; } if (typeof period !== 'number' || !Number.isInteger(period) || period < 1) { throw new TypeError('progress() period must be a positive integer'); @@ -3159,6 +4558,206 @@ const supportedEvents = new Set([ 'preupdate', ]); +// --- diagnostics_channel (Phase 6) ------------------------------------------- +// +// Node's own node:sqlite publishes finished-statement spans on the +// 'sqlite.db.query' channel when it has subscribers; APM tooling can +// subscribe to that name. This package publishes the same payload shape +// there AND on '@appthreat/sqlite3.query' (the named channel is this +// package's documented surface; the sqlite.db.query mirror exists for +// tool compatibility). The SQLITE_TRACE_PROFILE machinery is armed only +// while a subscriber exists, so the cost when nobody listens is nothing +// at all: no hook, no per-query work. + +const DIAGNOSTIC_OWN_CHANNEL = '@appthreat/sqlite3.query'; +const DIAGNOSTIC_NODE_CHANNEL = 'sqlite.db.query'; + +// Every open connection, so a subscriber arriving late can arm tracing on +// connections that already exist. Maintained by the Database wrapper +// (constructor adds, close removes) — and held *weakly*: a connection +// dropped without close() must still be collectable, or this registry +// would pin every Database (and its sqlite handle and file descriptor) +// for the life of the process. Dead references are pruned on iteration. +/** @type {Set>} */ +const liveConnections = new Set(); + +/** + * Registers a connection with the diagnostics registry. + * + * @param {import('./sqlite3-binding.js').Database} db the connection. + * @returns {void} + * @private + */ +function trackConnection(db) { + liveConnections.add(new WeakRef(db)); +} + +/** + * Drops a connection from the diagnostics registry (close()), pruning + * collected entries while it walks. + * + * @param {import('./sqlite3-binding.js').Database} db the connection. + * @returns {void} + * @private + */ +function untrackConnection(db) { + for (const ref of liveConnections) { + const live = ref.deref(); + if (live === undefined || live === db) liveConnections.delete(ref); + } +} + +/** + * The live connections, pruning collected entries as it goes. + * + * @returns {import('./sqlite3-binding.js').Database[]} the connections. + * @private + */ +function trackedConnections() { + /** @type {import('./sqlite3-binding.js').Database[]} */ + const live = []; + for (const ref of liveConnections) { + const db = ref.deref(); + if (db === undefined) liveConnections.delete(ref); + else live.push(db); + } + return live; +} +/** + * One finished-statement span, published on the diagnostics channels. + * + * @typedef {object} QuerySpan + * @property {string} sql the expanded SQL text. + * @property {import('./sqlite3-binding.js').Database} database the connection. + * @property {bigint} duration the measured duration in nanoseconds. + * @property {number} durationMs the measured duration in milliseconds. + * @since 9.1.0 + */ +// The sqlite3.subscribeQueries() consumers (Phase 6): non-empty means the +// profile tracing is armed. +/** @type {Set<(message: QuerySpan) => void>} */ +const diagnosticsChannelSubscribers = new Set(); + +/** + * The internal profile listener forwarding one connection's finished + * statements into the channels. Kept as one shared function so it can be + * removed again on unsubscribe. + * + * @param {import('./sqlite3-binding.js').Database} db the connection. + * @param {string} sql the expanded SQL text. + * @param {number} ms the measured duration in milliseconds. + */ +function publishQuerySpan(db, sql, ms) { + const message = { + sql, + database: db, + duration: BigInt(Math.round(ms * 1e6)), + durationMs: ms, + }; + for (const onMessage of diagnosticsChannelSubscribers) { + try { + onMessage(/** @type {QuerySpan} */ (message)); + } catch { + // A throwing consumer must not break the query pipeline. + } + } + diagnostics_channel.channel(DIAGNOSTIC_OWN_CHANNEL).publish(message); + diagnostics_channel.channel(DIAGNOSTIC_NODE_CHANNEL).publish(message); +} + +// The listener this module installed on each armed connection, so arming +// and disarming touch exactly that one: a user's own 'profile' listener +// must neither block publication (the old `listenerCount === 0` test made +// subscribeQueries silently inert next to one) nor be removed by an +// unsubscribe (removeAllListeners took them with it). +/** @type {WeakMap void>} */ +const armedProfileListeners = new WeakMap(); + +/** + * Arms span publication on one connection (the per-connection half of + * {@link armDiagnostics}). + * + * @param {import('./sqlite3-binding.js').Database} db the connection. + * @returns {void} + * @private + */ +function armDiagnosticsFor(db) { + if (armedProfileListeners.has(db)) return; + /** @type {(sql: string, ms: number) => void} */ + const listener = (sql, ms) => publishQuerySpan(db, sql, ms); + armedProfileListeners.set(db, listener); + // The user may already have a 'profile' listener (and therefore + // tracing already on); adding ours is additive, and configure() is + // idempotent. + db.on('profile', listener); + db.configure('profile', true); +} + +/** + * Disarms span publication on one connection, leaving any listener the + * user registered — and the tracing it needs — in place. + * + * @param {import('./sqlite3-binding.js').Database} db the connection. + * @returns {void} + * @private + */ +function disarmDiagnosticsFor(db) { + const listener = armedProfileListeners.get(db); + if (listener === undefined) return; + armedProfileListeners.delete(db); + db.removeListener('profile', listener); + // Only stop tracing when nobody else is listening for it. + if (db.listenerCount('profile') === 0) db.configure('profile', false); +} + +/** + * Arms or disarms span publication on every live connection. + * + * @param {boolean} arm true to arm, false to disarm. + * @returns {void} + * @private + */ +function armDiagnostics(arm) { + for (const db of trackedConnections()) { + if (arm) armDiagnosticsFor(db); + else disarmDiagnosticsFor(db); + } +} + +/** + * Subscribes to query spans: every finished statement is published as + * `{ sql, database, duration (bigint ns), durationMs }` on the + * `@appthreat/sqlite3.query` diagnostics channel (and mirrored onto + * node:sqlite's `sqlite.db.query` channel name, for APM-tool + * compatibility). The underlying SQLITE_TRACE_PROFILE tracing is armed by + * the first subscriber and disarmed when the last one goes — nothing runs + * while nobody listens. + * + * Returns the unsubscribe function. Connections opened while a + * subscription is active publish their spans too. + * + * @param {(message: { sql: string, database: import('./sqlite3-binding.js').Database, duration: bigint, durationMs: number }) => void} onMessage + * the span consumer. + * @returns {() => void} the unsubscribe function. + * @since 9.1.0 + * @example + * const unsubscribe = sqlite3.subscribeQueries(({ sql, durationMs }) => + * console.log(sql, durationMs.toFixed(3))); + */ +sqlite3.subscribeQueries = function subscribeQueries(onMessage) { + if (typeof onMessage !== 'function') { + throw new TypeError('subscribeQueries() requires a listener function'); + } + diagnosticsChannelSubscribers.add(onMessage); + armDiagnostics(true); + return () => { + diagnosticsChannelSubscribers.delete(onMessage); + if (diagnosticsChannelSubscribers.size === 0) { + armDiagnostics(false); + } + }; +}; + /** * @this {import('./sqlite3-binding.js').Database} * @param {string} type diff --git a/package.json b/package.json index 272c574..cbdb7b8 100644 --- a/package.json +++ b/package.json @@ -79,5 +79,17 @@ ], "type": "module", "main": "./lib/sqlite3.js", - "types": "./lib/sqlite3.d.ts" + "types": "./lib/sqlite3.d.ts", + "exports": { + ".": { + "types": "./lib/sqlite3.d.ts", + "default": "./lib/sqlite3.js" + }, + "./compat": { + "types": "./lib/compat.d.ts", + "default": "./lib/compat.js" + }, + "./package.json": "./package.json", + "./*": "./*" + } } diff --git a/plans/01-project-assessment.md b/plans/01-project-assessment.md new file mode 100644 index 0000000..4601975 --- /dev/null +++ b/plans/01-project-assessment.md @@ -0,0 +1,139 @@ +# 01 — Project assessment: `@appthreat/sqlite3` today + +`@appthreat/sqlite3` v9.0.2 is a Node-API 10, ESM-only, async-first +SQLite binding for Node ≥ 24 (and Electron ≥ 35), forked from +TryGhost/node-sqlite3 and substantially rewritten. It bundles SQLite +3.53.4 and ships prebuilds for six platforms inside the npm tarball +(resolved at runtime by `lib/sqlite3-binding.js`, so pnpm's +install-script block is a no-op). + +## Architecture + +- **Async by default.** Every data method (`run`/`get`/`all`/`each`/ + `map`/`exec`/`prepare`…) runs SQLite on a background worker via a + strictly FIFO queue (`docs/concurrency.md`). The JS event loop stays + free. Dual-mode: trailing callback = classic chainable API, no + callback = promise. +- **Sync fast path.** `getSync`/`runSync`/`allSync`/`prepareSync` execute + on the calling thread and refuse unless the connection is fully idle. + Measured 8–12× faster than the cached async equivalents on macOS + (22–31× on Linux), and at parity with `node:sqlite` on prepared + statements (within 1.03–1.17×; blobs are the one weak column type at + 1.29×). +- **Per-shape generated row builders.** The addon calls back into JS once + per result shape to compile a monomorphic row-construction function + (`makeRowFactory` in `lib/sqlite3.js`), with a C++ store-loop fallback + where codegen is unavailable. +- **JS callbacks (UDFs, collations, progress) cross threads.** Async + statements run on a worker; a JS callback is a blocking round trip to + the main JS thread (~18 µs/call measured). This single fact explains + most of the deliberate API restrictions (below). +- **Worker threads + pool.** The addon is context-aware + (per-environment constructors); `sqlite3.pool(filename, {readers})` + builds one writer + N read-only readers on separate workers, writes + queue, `{signal}` cancellation crosses threads through a + SharedArrayBuffer flag. + +### Deliberate restrictions that follow from the threading model + +1. UDFs/aggregates/window functions **refuse to run** from + `getSync`/`runSync`/`allSync`/`prepareSync` — the JS thread is blocked + inside SQLite and cannot service its own callback (deadlock). It + refuses with an explicit error (`src/function.cc`, + `SyncRefusalMessage`). `docs/performance.md` explicitly records that + this is *policy, not a structural limit* — a re-entrant direct call is + possible future work. +2. While a JS **collation** or JS **progress handler** is registered, the + sync methods refuse entirely (a collation callback has no error + channel). `db.withCollation(name, cmp, fn)` scopes registration to a + block. +3. `commit`/`rollback` hooks are observational only (no veto) — a veto + would need a blocking round trip at commit time. +4. A session and a `'preupdate'` listener cannot coexist (SQLite has one + preupdate hook per connection); both directions fail loudly. + +## Feature inventory (verified in lib/ + src/ + docs/) + +**Connections**: `new Database(file, mode|options, cb)`, `sqlite3.open()` +promise-native, `untrusted: true` hostile-file recipe, Node `--permission` +enforcement on open/ATTACH/`VACUUM INTO`/backup/extension, `configure()` +for `busyTimeout` / `limit` (run-time limits) / hook toggles / +`integerMode` (`number`|`bigint`|`mixed`) / `extensionPolicy` / +`attachPaths`, `serialize()`/`parallelize()`, `interrupt()`, `wait()`, +`state` snapshot, `changes`/`totalChanges` (64-bit), `dbConfig()`, +`checkpoint({mode})`, WAL hook event, `tableInfo()`, SQLCipher + custom +magic source builds, `cached` registry, `verbose()` long-stack traces. + +**Statements**: `prepare`/`prepareSync` (+ promise form gated on +introspection snapshot), `bind`/`run`/`get`/`all`/`map`/`reset`/ +`finalize`, native `fetch(count)` paged reads, `iterate()` async iterator +with backpressure, `stream()` object-mode Readable, statement cache +(`cacheStatements()`, LRU), sync fast paths with `{rowMode:'array'}`, +introspection (`readonly`, `parameterCount`, `parameterNames`, `columns` +with declaredType/database/table/origin), `status()` STMTSTATUS counters, +`lastID`/`lastIDBigInt`/`changes` (mode-aware, lazy RangeError), +`Symbol.dispose`. + +**Advanced**: UDFs / aggregates / window functions (`deterministic`, +`directOnly` default true, `innocuous`, `varargs`); custom collations +(`collation`/`removeCollation`/`withCollation`); rule-list C++ authorizer +(no JS on the prepare path); `'change'`/`'commit'`/`'rollback'`/`'wal'`/ +`'preupdate'` hooks; `progress()` + SharedArrayBuffer +`cancellationToken()` + AbortSignal; sessions (`session()`, `changeset()`, +`patchset()`, `applyChangeset` with per-conflict callbacks and filters, +`invertChangeset`/`concatChangeset`/`iterateChangeset`); stepping +`backup()` with retry policy; `serializeToBytes()`/`deserializeFromBytes()` +(WAL images normalized to rollback format); incremental blob I/O +(`openBlob`, `read`/`write`/`reopen`/`size`, streams); worker pool; +extended result codes (`code`/`errno`/`primaryCode`); strict marshalling +(no `[object Object]`, no silent truncation, arity errors). + +**Compiled-in SQLite extensions** (deps/sqlite3.gyp): FTS3/FTS4/FTS5, +RTREE, JSON, math functions, STAT4, DBSTAT virtual table, sessions + +preupdate hook, column metadata. `SQLITE_DEFAULT_MEMSTATUS=0` (some +global memory statistics are disabled — verify before relying on +`sqlite3_status`-family APIs). + +**Testing/tooling**: 60+ test files (node:test), Electron main+suite+ASAR +CI, glibc/musl Docker matrix, benchmark suite with RME gate and +node:sqlite + better-sqlite3 comparison baselines, generated TypeScript +declarations with CI drift check. + +## Strengths (keep and consolidate) + +1. **The only mature async-first driver** — non-blocking queries with + real backpressured streaming; nothing else in the ecosystem has this + (node:sqlite is sync-only; better-sqlite3 sync-only; bun:sqlite + sync-only). +2. **Plus a competitive sync path** — within ~1.1× of `node:sqlite` on + the shapes measured, faster on prepared inserts and `exec`. +3. **Sessions/changesets beyond anyone in JS** — apply/invert/concat/ + iterate/patchset with per-conflict callbacks (node:sqlite has create + + apply only). +4. **Collations** — better-sqlite3 has *never* had them; bun:sqlite + doesn't either. +5. **Authorizer design** — declarative rules evaluated in C++ inside + SQLite, so no JS runs at prepare time (faster and thread-safe by + construction, vs node:sqlite's JS callback). +6. **Security posture** — permission-model integration, `untrusted` + recipe, extension policy, ATTACH gate: unique among all drivers + surveyed. +7. **Blob streaming with Node streams** — only rusqlite/Python/@db/sqlite + have incremental blob I/O at all; none integrate it with + `stream.pipeline`. + +## Structural constraints to respect in any roadmap + +- Everything that makes SQLite call **back into JS** from the sync path + needs the re-entrancy work (Phase 2 of the roadmap) or must keep + refusing loudly. +- Everything that makes SQLite call back into JS from the **async** path + pays ~18 µs per call (worker→main round trip) — fine for + bounded-row logic, wrong for per-row bulk predicates (documented with + measured crossover). +- The FIFO queue and exclusive-op semantics (`exec`/`close`/`wait`/ + `loadExtension`) are now load-bearing guarantees; new APIs must slot + into the queue discipline rather than bypass it. +- Rows crossing worker boundaries (pool) are structured-clone copies; + bulk-read guidance should continue steering users to a single + connection. diff --git a/plans/02-competitor-research.md b/plans/02-competitor-research.md new file mode 100644 index 0000000..f815bc1 --- /dev/null +++ b/plans/02-competitor-research.md @@ -0,0 +1,307 @@ +# 02 — Competitor research + +Researched 2026-09-09. Facts verified against runtime introspection, +official docs and upstream sources; sources noted per section. + +--- + +## 1. `node:sqlite` (Node.js built-in) + +Sources: runtime introspection on Node v24.16.0; Node v26 docs +(`nodejs.org/api/sqlite.html`, structured `sqlite.json` changelogs); +`src/node_sqlite.cc` on nodejs/node main (4,641 lines). Stability: +experimental at v22.5 → 1.1 at v23.4 → "no longer experimental" v24.2 → +Release-candidate 1.2 at v25.7. + +**Execution model.** Entirely synchronous on the JS main thread; one +connection per thread; the *only* async API is module-level `backup()` +(a libuv-threadpool job with `rate` pages/step and a `progress` +callback). UDFs and aggregates re-enter JS inline on the same thread +(a JS callback can even prepare and run *other* statements mid-query); +the one hard rule is that a statement's own VM cannot re-enter itself +while stepping. + +**DatabaseSync** (v26 surface): `open`/`close`/`isOpen`/`isTransaction`/ +`Symbol.dispose`; `prepare(sql, {readBigInts, returnArrays, +allowBareNamedParameters, allowUnknownNamedParameters, persistent})`; +`exec`; `function(name, {options}, fn)` with +`deterministic`/`directOnly`/`varargs`/`useBigIntArguments`; `aggregate` +with `inverse` (window functions, since v25.5/v24.14); +`loadExtension`/`enableLoadExtension` (requires `allowExtension:true` at +construction); `enableDefensive(active)` (defensive **on by default** +since v25.5/v24.14); `setAuthorizer(callback|null)` returning +`SQLITE_OK/DENY/IGNORE`; `location(dbName)`; `createSession`/ +`applyChangeset` (with `filter` + per-conflict callbacks); +`serialize()`/`deserialize()` (docs say v26.1; present and working on +local 24.16); `createTagStore()`; `limits` getter/setter (11 run-time +limits, v25.8). Constructor options include `enableForeignKeyConstraints` +(default **true**), `timeout` (busy timeout), `defensive`, `limits`. + +**StatementSync**: `get`/`all`/`iterate`/`run` (→ `{changes, +lastInsertRowid}`); `columns()` (name/database/table/column/type); +`sourceSQL`/`expandedSQL` accessors; `setReadBigInts`, +`setReturnArrays`, `setAllowBareNamedParameters`, +`setAllowUnknownNamedParameters`; `close()`/`Symbol.dispose` and +`stat(counter)`/`resetStats()` (v26.8: STMTSTATUS counters — +fullscanStep, sort, autoindex, vmStep, reprepare, run, filterMiss, +filterHit, memused). + +**Session** (named `Session`, not SessionSync): `changeset()`, +`patchset()`, `close()`, `Symbol.dispose`. + +**SQLTagStore** (`createTagStore(maxSize)`, v24.9): an LRU of prepared +statements driven by **tagged template literals** — `store.all\`SELECT … +WHERE id = ${id}\``; interpolation values become positional `?` +parameters, the joined SQL is the cache key, statements prepared with +`SQLITE_PREPARE_PERSISTENT`; arity must match holes exactly. `get/all/ +iterate/run` as tags, `clear()`, `capacity`/`db`/`size` getters. + +**Errors**: `code = 'ERR_SQLITE_ERROR'`, message from `sqlite3_errmsg`, +plus own `errcode` (extended numeric) and `errstr`. Integers outside +±2^53−1 throw `ERR_OUT_OF_RANGE` unless `readBigInts` — same +refuse-to-truncate philosophy as this package. Rows are +null-prototype objects. Bare named parameters allowed by default. +Booleans bindable only since v26.8. + +**Tracing**: `diagnostics_channel` channel **`sqlite.db.query`** +(SQLITE_TRACE_PROFILE) publishing `{sql: expandedSql, database, +duration: ns}` automatically when the channel has subscribers — +zero-instrumentation observability for APM tools. + +**Notable timeline**: v22.5 initial; v23.3 sessions; v23.4 iterate + +unflagged; v23.5 UDF options + loadExtension + constants; v23.8 backup; +v24.0 aggregate + timeout + setReturnArrays + isTransaction; v24.9 +createTagStore; v24.10 setAuthorizer; v24.12–25.1 defensive; v25.5 +window functions; v25.8 limits; v26.1 serialize/deserialize docs; v26.8 +stmt.close/stat/resetStats, booleans, ArrayBuffer binding, re-entrancy +hardening. + +--- + +## 2. better-sqlite3 (v13.0.3) + +Sources: `docs/api.md`, `docs/integer.md`, `docs/threads.md`, +`docs/unsafe.md`, `docs/performance.md`, `lib/` and `src/` on master. + +**Execution model.** Fully synchronous main-thread API (backup is the +single async method, returning a promise with `attached`/`progress` +options and rate control). Marketed on speed; v13 migrated to Node-API +with in-package prebuilds. + +**Database**: ctor options `readonly`, `fileMustExist`, `timeout` +(default 5000 ms), `verbose` (per-SQL logging callback), `nativeBinding`; +`new Database(buffer)` opens a serialized image in memory. Methods: +`prepare`, `exec`, **`pragma(source, {simple})`** (executes `PRAGMA +${source}` in a special pragma mode and returns parsed rows; `{simple: +true}` returns the scalar), **`explain(sql)`** (v13: wraps as +`EXPLAIN ${source}`; parameters may be left unbound), `backup`, +`serialize(options)` → Buffer (deserialization happens via the +constructor's Buffer path), `function`/`aggregate` (with `inverse` for +windows), **`table`** (JS virtual tables), `loadExtension(path, +entryPoint)`, `close`, `defaultSafeIntegers(toggle)`, +`unsafeMode(toggle)` (escape hatch for defensive-mode-blocked ops and +mutate-while-iterating). Properties: `open`, `inTransaction`, `name`, +`memory`, `readonly`. `checkpoint()` was removed in v7 in favour of +`db.pragma('wal_checkpoint(RESTART)')`. + +**`db.table()` — JS virtual tables (v7.4, the marquee feature).** +Read-only virtual tables computed on the fly by a JS **generator +function**: `{ rows: function*(){…}, columns: [...], parameters: +[...] (hidden columns ⇒ table-valued function), safeIntegers, +directOnly }`. An object definition registers an **eponymous-only** +module (the table exists immediately by that name; no +`CREATE VIRTUAL TABLE`); a factory function registers a named module +instantiated per `CREATE VIRTUAL TABLE … USING mod(args)`. Documented +uses: `filesystem_directory`, `regex_matches(pattern, str)` table-valued +regex, `sequence(n)`, CSV files. Write support is deliberately absent. + +**Statement**: `run/get/all/iterate` (+ `return()` on iterators, capped +at 65,535 active iterators); **`pluck()` / `expand()` / `raw()`** as +mutually-exclusive toggles (first-column scalars / table-namespaced rows +with `$` bucket for expressions / array rows); **`bind(...)` permanent** +(one-shot per statement object, then execution-time binds forbidden); +`safeIntegers(toggle)` per statement; `columns()` (best after first +execution); `toString()` (v13: expanded SQL with bound values +substituted); frozen `source`/`reader`/`readonly`/`database`/`busy`. +Named parameters `@x`/`:x`/`$x` all bind from **bare object keys**; +extra keys silently ignored; anonymous values may be spread across +multiple arrays. + +**Transactions.** `db.transaction(fn)` returns a **reusable wrapped +function** with `.deferred()`/`.immediate()`/`.exclusive()` begin-mode +variants; nesting becomes a savepoint automatically; sync-only (returns +a promise ⇒ TypeError). + +**Errors.** `SqliteError.code` = extended code string (e.g. +`SQLITE_CONSTRAINT_UNIQUE`); TypeError = API misuse; RangeError = +count/size violations. UDF arity strictness with **overloads by arity** +under one name; aggregates keep arbitrary JS state between step and +result. + +**Explicit absences** (verified): no custom collations (never existed +across its whole release history), no sessions/changesets, no +authorizer, no serialize-then-deserialize helper beyond the Buffer +constructor path, no async queries, no built-in pool. + +--- + +## 3. `bun:sqlite` (Bun ≤ 1.4.2) + +Sources: `bun-types@1.4.2` `sqlite.d.ts`, main-branch +`src/js/bun/sqlite.ts`, bun.com docs. Synchronous, built into the Bun +runtime (does not run on Node). + +**Database**: ctor options `readonly`, `create`, `readwrite`, +`safeIntegers`, `strict` (strict: missing params throw; named keys given +**bare**, without `$`/`:`/`@`); `db.query(sql)` — LRU-cached +prepared statements (default 20 per Database, `MAX_QUERY_CACHE_SIZE`), +prepared `SQLITE_PREPARE_PERSISTENT`; `prepare` (uncached); `run(sql, +...)` executes multi-statement scripts; `transaction(fn)` with +`.deferred/.immediate/.exclusive` (code copied from better-sqlite3); +`serialize(name?)` → Buffer and **static** `Database.deserialize(bytes, +{readonly, strict, safeIntegers})`; `loadExtension(path, entryPoint)`; +`Database.setCustomSQLite(path)` (swap the SQLite dylib before first +open — needed for extensions on macOS system SQLite); +`fileControl(op, arg)` wrapping `sqlite3_file_control` (e.g. +`SQLITE_FCNTL_PERSIST_WAL`); `close(throwOnError)`; `Symbol.dispose`; +`inTransaction`; `filename`; `handle`. Databases are **not transferable +to Workers** — each worker opens its own connection (WAL recommended); +no shareable/pooled objects. + +**Statement**: `get/all/run/values` (**tuple rows**), `raw()` (all +values as `Uint8Array`), `iterate` (sync), **`as(Class)`** — rows mapped +to class instances *without invoking the constructor* (prototype-only), +`finalize`, `toString()` (expanded SQL of last bindings); +`columnNames`, `columnTypes` (runtime `sqlite3_column_type` of first +row), `declaredTypes`, `paramsCount`. Executing with no parameters +reuses the last bound values. + +**Errors**: `SQLiteError` with `errno` (extended code) and +**`byteOffset`** — the byte offset of the failing token (via +`sqlite3_error_offset`), unique among the drivers surveyed. + +**Important corrections to common claims** (verified): `sql` is **not** +exported from `bun:sqlite` — the tagged-template client is +**`Bun.SQL`** (`import { sql } from "bun"`, covering Postgres/MySQL/ +SQLite since v1.2.21). Helpers named `sql.raw`/`sql.join`/ +`sql.identifier`/`sql.empty`/`sql.blob`/`sql.int`… **do not exist in +Bun** — its equivalents are `sql("users")` (identifier), `sql([...])` +(IN lists), `sql.unsafe()`, `sql.file()`, plus `.values()`/`.raw()`/ +`.simple()` query modifiers and `sql.begin()/reserve()` with pooling. +Also absent from `bun:sqlite`: UDFs, `db.backup`, custom collations, +arrays-as-tables, any async execution. + +--- + +## 4. @libsql/client (Turso) + +Sources: libsql-js and libsql-client-ts READMEs, Turso docs. + +Three modes: local file, **remote HTTP** (`libsql://` + authToken), and +**embedded replicas** (local file + `syncUrl`; `db.sync()` pulls deltas; +`syncInterval` for background sync). Async client with `execute`, +`batch(statements, mode)` — atomic multi-statement with **`"write"`/ +`"read"`/`"deferred"`** modes — interactive `transaction(mode)` objects, +`executeMultiple`, `intMode: number|bigint|string`. The sibling native +package exposes `interrupt()`, a rule-based `authorizer()`, +`loadExtension`, per-statement `timed()`, `raw()`, `reader`. The libsql +**fork** adds native vector search: `F32_BLOB(n)`, `vector32/64/8/1bit/ +sparse` constructors, `vector_distance_cos/l2/dot/jaccard`, and DiskANN +`libsql_vector_idx` + `vector_top_k`. Its docs explicitly list as +unsupported: pragma, backup, serialize, function, aggregate, table, +pluck, expand, bind — a useful negative-space list of what a "serious" +driver is expected to have. + +--- + +## 5. Deno `@db/sqlite` (v0.13) + +Sources: jsr.io/@db/sqlite. + +Pure Deno **FFI** (not WASM): downloads/caches a prebuilt SQLite shared +library; `DENO_SQLITE_PATH` swaps in a custom build. Notable surface: +state properties (`autocommit`, `changes`, `totalChanges`, +`lastInsertRowId`, `inTransaction`, `open`, `path`); `int64` BigInt +toggle; **`parseJson` toggle (auto-parse JSON columns into JS objects)**; +`db.sql` tagged template; `transaction(fn)` with `.deferred/.immediate/ +.exclusive`; `function`/`aggregate` UDFs; **`openBlob`** incremental +blob I/O; `backup(dest, name, pages)` **into another Database**; +`loadExtension`; `isComplete()` (`sqlite3_complete`); `unsafeHandle` +raw-pointer escape hatch. + +--- + +## 6. WASM: wa-sqlite & sql.js + +Sources: upstream READMEs. + +- **wa-sqlite**: SQLite compiled to WASM with **VFS layers written + entirely in JavaScript** — the whole point of the project. Catalogue: + MemoryVFS, IDBBatchAtomicVFS/IDBMirrorVFS (IndexedDB), + OPFSAdaptiveVFS/OPFSAnyContextVFS/OPFSCoopSyncVFS and + **OPFSWriteAheadVFS** (WAL journaling on OPFS). Worker-oriented + deployment; async (Asyncify/JSPI) builds allow async VFS. +- **sql.js**: in-memory-only WASM SQLite; whole-DB import/export + (`new SQL.Database(bytes)` / `db.export()`); JS UDFs; manual + `stmt.free()` lifetimes. No persistence, BigInt binding unsupported. + +Lesson for a native driver: pluggable storage backends and durability +topologies matter in the browser world; on the server, the equivalents +are the pool, `serializeToBytes` handoffs and VFS-level encryption +(SQLCipher) — all already present here. + +--- + +## 7. rusqlite (Rust) — the feature ceiling + +Sources: README + docs.rs. + +Exposes nearly everything SQLite has: **hooks with veto** +(`commit_hook` returning bool forces rollback), `rollback_hook`, +`update_hook`, `wal_hook`, `preupdate_hook`, `progress_handler`, +`busy_handler`, `set_authorizer`; **`unlock_notify`** (wake when a +locked DB frees — unique among everything surveyed); blob I/O with +Read/Write/Seek; serialize/deserialize (incl. from streams); backup +with progress; `changes`/`total_changes`/`is_autocommit`/ +`transaction_state`; **`vtab`** — virtual tables in Rust, plus bundled +`generate_series`, CSV, and **`rarray()`** (bind a Rust array as a +table — unique and extremely useful for `IN (...)`/JOIN patterns); +`limit`/`set_limit` (returns prior value); `column_decltype`; +`get_interrupt_handle`/`is_interrupted`; **session extension incl. +rebasing**; `loadable_extension` (writing SQLite extensions in Rust); +SQLCipher builds; type conversions (uuid, chrono, url, serde_json). + +--- + +## 8. Python stdlib `sqlite3` + +`Connection.autocommit` three-mode control + `in_transaction`; +**adapters/converters** (`register_adapter`/`register_converter` with +`detect_types` for typed column round-trips); `backup(dst, pages, +progress)`; `blobopen` (file-like, indexable/sliceable blob objects); +`serialize`/`deserialize`; `set_trace_callback` (every statement incl. +implicit txn statements); `executescript`; `getlimit`/`setlimit`; +authorizer/progress/window functions/collations; **`iterdump()`** — +streaming SQL dump; `sqlite3.Row` row factory; URI opens incl. +`file:mem1?mode=memory&cache=shared`; `text_factory` for non-UTF-8; +errors carrying `sqlite_errorcode`/`sqlite_errorname`; even a +`python -m sqlite3` CLI. + +--- + +## 9. What ORMs require from a SQLite driver + +- **Drizzle** (better-sqlite3 dialect): constructor, `prepare(sql)` with + `run(args)`→`{changes, lastInsertRowid}`/`all(args)`/`get(args)`/ + `raw()`, `transaction` with `.deferred/.immediate/.exclusive`, + `$client` passthrough. Nothing else. +- **Kysely** (SqliteDialect): `acquireConnection`/`releaseConnection`, + raw `begin`/`commit`/`rollback` SQL, `prepare` with `stmt.reader`, + `stmt.all/run/iterate(args)`, `close()`, plus an `onCreateConnection` + hook (where pragmas get set). Async-native — the natural fit for this + package. + +Takeaway: ORM compatibility is cheap; it is a docs/examples deliverable, +not a core-API one. A first-party Kysely dialect would work against the +existing promise API almost unchanged; a Drizzle driver would map onto +`*Sync` + `transaction`. diff --git a/plans/03-feature-gap-matrix.md b/plans/03-feature-gap-matrix.md new file mode 100644 index 0000000..26818de --- /dev/null +++ b/plans/03-feature-gap-matrix.md @@ -0,0 +1,139 @@ +# 03 — Feature gap matrix + +Legend: ✅ has · ⚠️ partial/different · ❌ lacks. Column order: +**this** = `@appthreat/sqlite3` 9.0.2 · **node** = `node:sqlite` (Node +24–26) · **bs3** = better-sqlite3 13 · **bun** = `bun:sqlite` 1.4. +The last column names any other driver that has the feature +(rusqlite, libsql, Deno `@db/sqlite`, Python stdlib). + +## Core execution model + +| Capability | this | node | bs3 | bun | Elsewhere | +| --- | --- | --- | --- | --- | --- | +| Async, non-blocking queries | ✅ | ❌ (backup only) | ❌ (backup only) | ❌ | — | +| Sync fast path | ✅ | ✅ | ✅ | ✅ | — | +| Async iteration with backpressure | ✅ `iterate`/`stream` | ❌ (sync iterate) | ❌ (sync iterate) | ❌ (sync iterate) | — | +| Statement cache | ✅ opt-in + sync auto | ❌ (tag store is one) | ❌ | ✅ `query()` LRU 20 | — | +| Worker-thread connection pool | ✅ | ❌ | ❌ (docs recipe only) | ❌ | — | +| Multi-connection reader/writer guidance | ✅ pool | ❌ | ⚠️ docs | ⚠️ docs | libsql (server) | +| Cancellation (signal / token / interrupt) | ✅ all three | ❌ | ❌ | ❌ | rusqlite interrupt handle | +| `Symbol.dispose`/`using` | ✅ | ✅ | ❌ | ✅ | — | + +## Query ergonomics + +| Capability | this | node | bs3 | bun | Elsewhere | +| --- | --- | --- | --- | --- | --- | +| Tagged-template queries | ❌ | ✅ `createTagStore` | ❌ | ⚠️ via `Bun.SQL` module | Deno `db.sql` | +| `pragma()` helper w/ parsed results | ❌ | ❌ | ✅ | ❌ | — | +| `explain()` helper | ❌ | ❌ | ✅ (v13) | ❌ | — | +| Array/tuple row mode | ⚠️ sync paths only (`rowMode`) | ✅ `setReturnArrays` | ✅ `raw()` | ✅ `values()` | libsql `raw()` | +| Pluck (first-column) mode | ❌ | ❌ | ✅ | ⚠️ `values`+map | — | +| Expand (table-namespaced rows) | ❌ | ❌ | ✅ | ❌ | — | +| Reusable transaction fn + `.deferred/.immediate/.exclusive` | ⚠️ inline `transaction()` only, auto-savepoints | ❌ | ✅ | ✅ | Deno, Bun.SQL | +| Atomic `batch(statements)` | ❌ | ❌ | ❌ | ⚠️ multi-stmt `run` | libsql ✅ (write/read/deferred) | +| `inTransaction` / txn state introspection | ❌ | ✅ `isTransaction` | ✅ | ✅ | rusqlite `transaction_state`, Python | +| `db.location(name)` (attached-file path) | ❌ | ✅ | ⚠️ `name` property | ⚠️ `filename` | — | +| SQL text dump (`.dump`/`iterdump`) | ❌ | ❌ | ❌ | ❌ | Python `iterdump` | +| `sqlite3_complete`-style util | ❌ | ❌ | ❌ | ❌ | Deno `isComplete` | + +## Values & errors + +| Capability | this | node | bs3 | bun | Elsewhere | +| --- | --- | --- | --- | --- | --- | +| 64-bit-correct integers (bind+read) | ✅ 3 modes, refuse-to-truncate | ✅ `readBigInts` | ✅ `safeIntegers` | ✅ `safeIntegers` | libsql `intMode` | +| Per-statement integer mode override | ❌ (per-connection) | ✅ | ✅ | ⚠️ | — | +| Extended result codes on errors | ✅ `code`/`errno`/`primaryCode` | ✅ `errcode`/`errstr` | ✅ `code` | ✅ | Python `sqlite_errorcode` | +| Error token byte offset | ❌ | ❌ | ❌ | ✅ `byteOffset` | — | +| Strict bind validation (TypeError, arity) | ✅ strictest | ⚠️ | ✅ | ⚠️ `strict:true` | — | +| Boolean binding | ✅ | ⚠️ v26.8+ | ✅ | ✅ | — | +| JSON column auto-parse | ❌ | ❌ | ❌ | ❌ | Deno `parseJson`, Python converters | +| Row→class mapping | ❌ | ❌ | ❌ | ✅ `as(Class)` | Python `row_factory` | + +## User-defined logic in SQL + +| Capability | this | node | bs3 | bun | Elsewhere | +| --- | --- | --- | --- | --- | --- | +| Scalar UDFs (async path) | ✅ | ✅ | ✅ | ❌ | — | +| Scalar UDFs from **sync** calls | ❌ refuses | ✅ | ✅ | n/a | — | +| Aggregates + window functions | ✅ `inverse` | ✅ | ✅ | ❌ | — | +| Custom collations | ✅ (sync-path gated) | ❌ | ❌ never | ❌ | Python, rusqlite | +| Virtual tables in JS | ❌ | ❌ | ✅ `db.table()` | ❌ | rusqlite `vtab`, wa-sqlite | +| Array-as-table binding (`rarray`) | ❌ | ❌ | ❌ | ❌ | rusqlite only | +| On-demand collation factory | ❌ | ❌ | ❌ | ❌ | rusqlite `collation_needed` | +| FTS5 custom tokenizer in JS | ❌ | ❌ | ❌ | ❌ | nobody | + +## Notifications, security, introspection + +| Capability | this | node | bs3 | bun | Elsewhere | +| --- | --- | --- | --- | --- | --- | +| update/change hook | ✅ | ❌ | ❌ | ❌ | rusqlite, Python | +| commit/rollback hooks | ✅ observational | ❌ | ❌ | ❌ | rusqlite (commit **with veto**) | +| preupdate hook (old+new rows) | ✅ | ❌ | ❌ | ❌ | rusqlite | +| WAL hook + checkpoint control | ✅ `wal` event + `checkpoint()` | ❌ | ⚠️ pragma recipe | ⚠️ pragma | rusqlite `wal_hook` | +| Authorizer | ✅ C++ rule list (no JS on prepare path) | ✅ JS callback | ❌ | ❌ | libsql rule-based, Python | +| Run-time limits get/set | ✅ `configure('limit')` | ✅ `limits` prop (v25.8) | ❌ | ❌ | Python, rusqlite | +| Defensive mode | ✅ `dbConfig` + `untrusted` | ✅ default-on | ⚠️ `unsafeMode` inverse | ❌ | — | +| Progress handler | ✅ JS cb + SAB token | ❌ | ❌ | ❌ | rusqlite, Python | +| Statement counters (STMTSTATUS) | ✅ `stmt.status()` | ✅ `stat()` v26.8 | ❌ | ❌ | — | +| Column metadata (origin/decltype) | ✅ `stmt.columns` + `tableInfo` | ✅ `columns()` | ✅ | ⚠️ names/types | rusqlite | +| `expandedSQL` / `normalizedSQL` | ⚠️ trace events only | ✅ / ❌ | ⚠️ `toString()` / ❌ | ⚠️ `toString()` / ❌ | — | +| diagnostics_channel integration | ❌ | ✅ `sqlite.db.query` | ❌ | ❌ | — | +| DB/global status counters | ❌ | ❌ | ❌ | ❌ | rusqlite | +| Node permission-model integration | ✅ unique | ❌ | ❌ | n/a | — | + +## Sessions, backup, snapshots, blobs + +| Capability | this | node | bs3 | bun | Elsewhere | +| --- | --- | --- | --- | --- | --- | +| Session capture + changeset/patchset | ✅ | ✅ | ❌ | ❌ | rusqlite | +| Apply w/ per-conflict callbacks + filter | ✅ | ✅ | ❌ | ❌ | rusqlite | +| Invert / concat / iterate changesets | ✅ | ❌ | ❌ | ❌ | rusqlite | +| **Rebase** changesets | ❌ | ❌ | ❌ | ❌ | rusqlite only | +| `session.diff(table, otherDb)` | ❌ | ❌ | ❌ | ❌ | rusqlite | +| Online backup | ✅ stepping handle, self-paced | ✅ module fn, rate+progress | ✅ promise, rate+progress | ❌ | Python, Deno (→ live db) | +| serialize → bytes | ✅ WAL-normalized | ✅ | ✅ → Buffer | ✅ | Python, wa-sqlite | +| deserialize from bytes | ✅ copied, validated | ✅ | ⚠️ via ctor Buffer | ✅ static | Python | +| Incremental blob I/O | ✅ + Node streams | ❌ | ❌ | ❌ | rusqlite, Python, Deno | + +## Extension, crypto, runtime support + +| Capability | this | node | bs3 | bun | Elsewhere | +| --- | --- | --- | --- | --- | --- | +| Loadable extensions | ✅ + policy allowlist | ✅ gated by ctor flag | ✅ | ✅ (needs custom SQLite on macOS) | — | +| SQLCipher / at-rest encryption | ✅ source build | ❌ | ⚠️ community forks | ⚠️ via custom dylib | rusqlite bundled | +| Custom file magic | ✅ | ❌ | ❌ | ❌ | — | +| Bundled FTS5 / RTREE / JSON / math | ✅ all | ✅ | ✅ | ⚠️ runtime-dependent | — | +| Swappable SQLite build | ✅ `--sqlite=` source flag | ❌ | ⚠️ `nativeBinding` | ✅ `setCustomSQLite` | Deno `DENO_SQLITE_PATH` | +| Electron verified | ✅ CI incl. ASAR | ⚠️ | ⚠️ | n/a | — | +| Bun runtime | ⚠️ install documented; CI not verified | ❌ (their compat module unimplemented) | ⚠️ | ✅ native | — | +| `file_control` (WAL sidecars etc.) | ❌ | ❌ | ❌ | ✅ | — | +| ORM dialects (first-party) | ❌ | ⚠️ community | ⚠️ Drizzle+Kysely dialects exist | ⚠️ Drizzle | libsql first-party | + +## Where this package already leads (no competitor has it) + +True async execution with a sync fast path; async iteration/streaming +with backpressure; worker pool; strict-marshalling defaults with three +integer modes; collations; C++ rule-list authorizer; cancellation tokens +(shared memory) + AbortSignal; preupdate events with old rows; +invert/concat/iterate changesets; WAL-format-normalized serialize; +blob streams; permission-model enforcement; `untrusted` hardening; +extension policy allowlists; verified Electron/ASAR support. + +## The gaps that matter, ranked by user value + +1. **UDFs on the sync path** — the one thing the README's own comparison + concedes to `node:sqlite`; already flagged internally as feasible + (re-entrant direct call). +2. **Changeset rebasing** — the missing half of the sessions story; + rusqlite has it, no JS driver does. Completes a fork-free + replication/sync toolkit. +3. **JS virtual tables / table-valued functions** — better-sqlite3's + marquee feature; combined with this package's async machinery it + could go further (factory modules, streaming generators). +4. **Ergonomics bundle**: `pragma()`, `explain()`, reusable + transactions with begin modes, async-path array/pluck row modes, + tagged-template store, error offsets, `inTransaction`. +5. **Observability**: diagnostics_channel emission, expanded/normalized + SQL accessors, DB status counters. +6. **Adoption surface**: node:sqlite compat shim, Kysely/Drizzle + dialects, Bun CI verification, migration helper. diff --git a/plans/04-roadmap.md b/plans/04-roadmap.md new file mode 100644 index 0000000..675718c --- /dev/null +++ b/plans/04-roadmap.md @@ -0,0 +1,256 @@ +# 04 — Feature roadmap + +Ordered by value-to-effort, sequenced so each phase de-risks the next. +Every item must respect the project's two invariants: the FIFO queue / +exclusive-op discipline, and "call back into JS from the sync path only +via re-entrancy, never by blocking the JS thread on itself". + +Effort scale: **S** ≤ a few days · **M** one to three weeks · **L** a +quarter-ish. Impact: ★–★★★ (adoption/ differentiation). + +--- + +## Phase 1 — Ergonomics parity (all S/M, ship as minors) + +The sync-first drivers make a set of small things trivial; users +migrating from better-sqlite3/node:sqlite keep reaching for them. + +1. **`db.pragma(source, { simple })`** — S · ★★ + Prepare-and-return-parsed-rows around `PRAGMA ${source}`, `{simple: + true}` for the scalar (better-sqlite3 parity; its docs call this *the* + recommended way to run pragmas). Thin wrapper over the existing + `getSync`/async `get`; special-cases the handful of statements-only + pragmas via `exec`. Include `journal_mode = WAL` / `wal_checkpoint( + RESTART)` / `optimize` recipes in docs. +2. **Reusable `db.transaction(fn)` with begin modes** — M · ★★ + Keep the existing inline form; additionally return a reusable wrapped + function (as better-sqlite3, bun, Deno all do) carrying + `.deferred()`/`.immediate()`/`.exclusive()`. Async-aware (unlike bs3, + whose wrapper is sync-only — ours can guard the same + `AsyncLocalStorage` nesting that exists today). BEGIN IMMEDIATE by + default for write transactions is worth considering + documenting + (avoid deferred-write upgrade failures under concurrency). +3. **Array/pluck row modes on the async paths** — S · ★★ + `{ rowMode: 'array' }` exists on sync calls only; extend it (plus a + `pluck` shape: first column) to `get`/`all`/`iterate`/`stream`/ + `fetch` and the pool. The generated row builder already supports + array shapes — this is plumbing, and it is the fastest row shape + (see performance doc). Avoid bs3's mutable `.raw()` toggles; prefer + per-call options to stay dual-mode-safe. +4. **`db.explain(sql, { plan: true })`** — S · ★ + `EXPLAIN QUERY PLAN` rows, parameters optionally unbound (they don't + execute). Small but beloved bs3 v13 feature; pairs with the existing + `stmt.status(FULLSCAN_STEP)` diagnostics. +5. **Error token offset: `err.offset`** — S · ★★ + The vendored 3.53.4 exposes `sqlite3_error_offset()`; bun:sqlite + ships it as `byteOffset` and nobody else does. Attach to + `SqliteError` after failed prepares (`-1` when N/A). Cheap, uniquely + useful for user-facing SQL editors and migrations. +6. **`db.inTransaction` / `db.txnState`** — S · ★ + `sqlite3_get_autocommit` + `sqlite3_txn_state` (idle/read/write). + Every other driver has some form (node `isTransaction`, bs3/bun + `inTransaction`). Trivial binding, read under the connection mutex. +7. **`batch(statements, { mode })`** — M · ★★ + Atomic multi-statement execution in one savepoint, statements as + strings or `{sql, args}`; map libsql's `write/read/deferred` modes + onto BEGIN modes. Wraps existing transaction machinery; big ergonomic + win for migrations/seeding, and a differentiator vs bs3/node which + lack it entirely. +8. **`db.dump()` / `sqlite3.iterdump(db)`** — S · ★ + Streaming `.dump`-style SQL text export (Python `iterdump` parity; + nothing in JS has it). Straight `SELECT * FROM sqlite_schema` + + row-serialization walker; async iterator form fits `iterate()`. + +**Phase 1 exit**: migrating from better-sqlite3 or node:sqlite needs no +adaptation layer for the top-20 utility calls. + +--- + +## Phase 2 — Re-entrant UDFs on the sync path (M/L · ★★★) + +The flagship gap, and the README's own comparison table concedes it: +`node:sqlite` and better-sqlite3 run JS functions inline in sync calls; +this package refuses. `docs/performance.md` ("Future direction: UDFs on +the synchronous fast path") already establishes this is **policy, not a +structural limit**: on the sync path the JS thread is the one executing +SQL, so the trampoline can call the JS function **directly** — same +thread, no round trip — exactly how node:sqlite does it. + +Design sketch: + +- In `src/function.cc`, detect "current thread == JS thread and the call + originates from a sync step" and take the direct path: invoke the JS + function via the env on the stack, convert the result back, and report + a throwing callback through `sqlite3_result_error` (the error channel + functions already have; collations still refuse, as documented). +- **Re-entrancy guard**: while inside such a callback, sync calls on the + *same connection* must throw a clear error ("statement is executing" — + node:sqlite hardened the same rule in v26.8); *other* connections are + fine. The existing "connection fully idle" precondition of the sync + path needs a carve-out for "idle except statements stepped from this + re-entrant frame". +- Window/aggregate `inverse`/`result` paths need the same treatment. +- Bench: the `vers_compare(?, ?)` shape from the perf doc should land in + the suite; the README's node:sqlite UDF caveat section gets rewritten + from "refuses" to "works, and here's the cost model" (async-path UDFs + still pay the ~18 µs round trip; sync-path UDFs become direct). + +Risks: exception safety across sqlite frames (node:sqlite's +`SetIgnoreNextSQLiteError` pattern is the reference), and preserving the +current refusal tests' intent (they become direct-call tests). + +**Phase 2 exit**: `db.function('regexp', …)` + `getSync('… WHERE x +REGEXP ?')` works; the comparison table's biggest ❌ flips. + +--- + +## Phase 3 — Sessions completion: rebase + diff (M · ★★★) + +This package already leads JS on sessions/changesets (apply/invert/ +concat/iterate/patchset). Two C APIs sit unbound in the vendored header, +and together they complete a **fork-free replication story** that no JS +driver has (rusqlite is the only binding anywhere with them): + +1. **`sqlite3.rebaseChangeset(changeset, rebase)`** — wraps + `sqlite3_rebaser_new/configure/apply`: rebase a local changeset + against a "rebase" (conflict resolution) captured during an + `applyChangeset` that used OMIT/REPLACE. Expose `applyChangeset`'s + optional `{ rebase: true }` to harvest the rebase buffer alongside + the apply, then `rebaseChangeset()` to transform subsequent local + changesets — the textbook client-server sync loop. +2. **`session.diff(table, otherDbOrPath)`** — wraps `sqlite3session_diff` + (already compiled in): a changeset of the differences between two + table contents without recording anything — instant "what changed + between these two databases" for sync/verification tooling. +3. **Docs**: a replication cookbook — the pool + sessions + rebase + pattern as a workable offline-first sync engine on stock SQLite. + +Risk: low (pure additive bindings over stable session APIs). Impact: +high — it's a category no other JS driver can enter without copying this +work. + +--- + +## Phase 4 — JavaScript virtual tables (`db.table()`) (L · ★★★) + +better-sqlite3's marquee feature, absent from every other JS driver. +Natural fit here *after* Phase 2: vtab `xBestIndex`/`xFilter`/`xNext` +callbacks fire on whichever thread steps the statement. + +- **Async path**: callbacks marshal worker→JS like UDFs today (~18 µs + per call; fine for generator-driven row production, which is + inherently coarse-grained). +- **Sync path** (post-Phase-2): direct re-entrant calls. + +Scope v1 read-only (same as bs3), matching their proven shape: +`db.table(name, { rows: function* (…) {}, columns: […], parameters: +[…] (hidden columns ⇒ table-valued functions), directOnly })` for +eponymous tables, factory-function form for named modules. Ship the +documented use cases: `sequence`, `regex_matches`, JSON/CSV file tables. + +**The extension nobody has**: rusqlite's `rarray()` — bind a JS array +(or iterable) as a table-valued parameter. With vtab + hidden-column +parameters this becomes natural here: `SELECT * FROM json_each(?1)`-style +ergonomics for JS data (`WHERE id IN (SELECT value FROM ?)`), a genuine +differentiator for ORMs/query builders. + +Risks: vtab cursor lifetimes across the queue discipline (a vtab scan +holding a cursor while the user issues other statements — the blob +`SQLITE_ABORT` invalidation pattern is the precedent to copy); xBestIndex +constraint handling deserves a deliberate, minimal contract (bs3 ignores +constraint passing entirely — do the same in v1). + +--- + +## Phase 5 — Tagged templates & migration affordances (M · ★★) + +1. **Tag store**: `db.createTagStore(maxSize?)` / `sqlite3.tag(db)` — + node:sqlite-compatible tagged-template LRU + (`store.get`/`all`/`iterate`/`run` as template tags; template values + become positional parameters; joined SQL is the cache key). Adds: + composition helpers `sql.raw`, `sql.join`, `sql.identifier`, + `sql.identifierPath`, `sql.empty` (the Kysely/Sequelize-style names — + Bun notably has none of these; Node has only the bare store). Promise- + native (an immediate improvement on node's sync-only store). +2. **`node:sqlite` compat shim** — M · ★★ + `import { DatabaseSync } from '@appthreat/sqlite3/compat'`: a class + mapping `prepare/exec/function/aggregate/loadExtension/…` onto the + sync paths plus re-entrant UDFs (Phase 2). Zero-dependency drop-in for + code written against node:sqlite that outgrows it (needs pools, + sessions, streaming). This is the single cheapest adoption lever + available: the APIs are nearly isomorphic by design. +3. **Migration helper** — S/M · ★ + `sqlite3.migrate(db, migrationsDirOrList)` — `PRAGMA user_version` + based, sequential, runs inside `transaction`. No driver ships one + (libsql ships a CLI). Keep it dependency-free and opt-in. + +--- + +## Phase 6 — Observability, ops & ecosystem (S/M each · ★★) + +1. **`diagnostics_channel` emission** — S · ★★ + `node:diagnostics_channel` channel `@appthreat/sqlite3.query` + publishing `{ sql (expanded), database, duration: ns }` when + subscribed — the machinery already exists (SQLITE_TRACE_PROFILE in + `src/database.cc`); wire `configure('trace', …)` to a channel + subscriber. Optionally *also* mirror into node's `sqlite.db.query` + channel name for APM-tool compatibility (decide: alias vs own name). +2. **`stmt.expandedSQL` / `stmt.normalizedSQL`** — S · ★ + `sqlite3_expanded_sql` (last bindings) and `sqlite3_normalized_sql` + (literals → `?`; requires SQLITE_ENABLE_NORMALIZE — verify the flag, + compile it in if absent). Normalized SQL is what query-metric + dashboards want; only this and `expandedSQL` on statement objects + (bs3/bun expose expanded via `toString()`). +3. **DB status + memory knobs** — S · ★ + `db.status()` over `sqlite3_db_status` (cache hit/miss/spill, schema + used); `db.releaseMemory()` (`sqlite3_db_release_memory`) for pool + pressure; `configure('walAutocheckpoint', n)`; verify what + `SQLITE_DEFAULT_MEMSTATUS=0` disables and document it. +4. **Bun runtime verification** — S · ★★ + The binding loader already handles runtimes without + `process.versions.napi`; add a Bun CI job loading the prebuild and + running the suite (Bun's N-API coverage is the risk). README already + shows `bun add` — make the claim verified rather than hopeful. +5. **ORM dialects** — S/M · ★★ + First-party Kysely dialect (async-native; needs `prepare`+`iterate`+ + `begin/commit` raw SQL — all present) and a Drizzle example mapping to + `*Sync` + transaction-with-modes. Docs + `examples/`, or a tiny + `@appthreat/sqlite3-kysely` package. +6. **Smaller parity items** (bundle as "introspection minor") — S · ★ + `db.limits` getter alongside `configure('limit', …)` (returning + current values, node v25.8-style); `db.location(dbName)`; + `sqlite3.compileOptions` (`sqlite3_compileoption_get`); + `sqlite3.complete(sql)` (`sqlite3_complete`, for REPL/CLIs); + per-statement integer-mode override option (node `setReadBigInts`, + bs3 `safeIntegers` parity) — an options bag on `prepare` rather than + mutable toggles. + +### Explicit non-goals + +- **Client/server, remote replicas, vector search** — libsql's turf; + requires a SQLite fork. Stock answer: document compatibility with + loadable `sqlite-vec`-style extensions instead (extension policy + already exists). +- **Commit-veto hooks** — possible via a blocking round trip on the + async path, but it couples commit latency to the JS thread; keep hooks + observational (documented rationale exists). +- **JS VFS plugins** — wa-sqlite's territory; a native driver's VFS + surface is C. SQLCipher + custom-magic builds cover the server-side + storage-variant cases. +- **Mutable per-statement mode toggles** (bs3 `.pluck()`-style state) — + per-call options fit the dual-mode (callback/promise) API better. + +--- + +## Suggested sequencing + +| Release | Contents | +| --- | --- | +| 9.1 | Phase 1 (ergonomics) + Phase 6 items 1–3 (observability quick wins) | +| 9.2 | Phase 2 (sync-path UDFs) — the headline; rewrite comparison table | +| 9.3 | Phase 3 (rebase + diff) + Phase 6 rest (Bun CI, dialects, shim) | +| 10.0 | Phase 4 (virtual tables + rarray) — new-major surface; Phase 5 tag store/migrations can slip into 9.x independently | + +Budget feeling: Phase 1 ≈ 4–6 focused weeks total; Phase 2 is the one +design-heavy item; Phases 3–6 are additive and parallelizable across +contributors once 2 lands (vtab depends on it for the sync path only). diff --git a/plans/README.md b/plans/README.md new file mode 100644 index 0000000..c9323f7 --- /dev/null +++ b/plans/README.md @@ -0,0 +1,48 @@ +# Plans: competitive research & feature roadmap + +Produced 2026-09-09 against `@appthreat/sqlite3` **v9.0.2** (commit +`25814f7`, branch `master`). + +## What is in here + +| File | Contents | +| --- | --- | +| [01-project-assessment.md](01-project-assessment.md) | What this project is today: architecture, full API inventory, strengths, structural constraints | +| [02-competitor-research.md](02-competitor-research.md) | Deep research on `node:sqlite`, `better-sqlite3`, `bun:sqlite` (+ `Bun.SQL`), plus a broader survey: libsql, Deno `@db/sqlite`, wa-sqlite/sql.js, rusqlite, Python `sqlite3`, and what ORMs (Drizzle/Kysely) require from a driver | +| [03-feature-gap-matrix.md](03-feature-gap-matrix.md) | Capability-by-capability matrix: this package vs each competitor, with gaps and leads | +| [04-roadmap.md](04-roadmap.md) | The recommended roadmap — six phases, ordered, each item with motivation, design sketch, effort and risk | + +## How the research was done + +- **This repo**: read `README.md`, `MIGRATING-TO-V9.md`, all of `docs/`, + `lib/*.js`/`*.d.ts` (the full JS API surface), `deps/sqlite3.gyp` + (compile flags), the vendored SQLite 3.53.4 amalgamation header, and + key parts of `src/*.cc` (trace hooks, configure). +- **node:sqlite**: runtime introspection on Node v24.16.0, the v26 + official docs (`nodejs.org/api/sqlite.html`), and the Node C++ source + (`src/node_sqlite.cc` on `nodejs/node` main). +- **better-sqlite3**: `docs/api.md`, `docs/integer.md`, `docs/threads.md`, + `docs/unsafe.md`, `docs/performance.md` and `lib/`+`src/` from + WiseLibs/better-sqlite3 master (v13.0.3). +- **bun:sqlite**: Bun 1.4.2 official types (`bun-types/sqlite.d.ts`), the + main-branch implementation, and bun.com docs. +- **Others**: upstream READMEs/docs for libsql, Deno `@db/sqlite`, + wa-sqlite, sql.js, rusqlite; Python stdlib docs; Drizzle and Kysely + driver sources. + +## One-paragraph summary + +`@appthreat/sqlite3` is already the most feature-complete SQLite binding +in the Node ecosystem — nothing else offers async-first execution with a +sync fast path, a worker pool, streaming, sessions/changesets, blob I/O, +collations, a C++-side authorizer, and cancellation on one connection +object. The gaps that remain against `node:sqlite` and `better-sqlite3` +are: UDFs on the synchronous path (an architectural item the perf doc +already earmarks), small ergonomics the sync-first drivers make easy +(`pragma()`/`explain()` helpers, reusable transactions with begin modes, +pluck/raw row modes on the async path, tagged templates), changeset +**rebasing** (the missing half of the sessions story, which no JS driver +has), and JS-defined **virtual tables** (better-sqlite3's marquee +feature). The roadmap phases these in order of value-to-effort, keeping +the project's async-first identity and refuse-loudly threading discipline +intact. diff --git a/src/database.cc b/src/database.cc index d67fa75..8c398c7 100644 --- a/src/database.cc +++ b/src/database.cc @@ -56,6 +56,28 @@ Napi::Object Database::Init(Napi::Env env, Napi::Object exports) { InstanceMethod("_applyChangeset", &Database::ApplyChangeset, napi_default_method), InstanceMethod("_serializeToBytes", &Database::SerializeToBytes, napi_default_method), InstanceMethod("_deserialize", &Database::Deserialize, napi_default_method), + // JavaScript virtual tables (Phase 4): internal entry points + // wrapped by lib/sqlite3.js, which validates the definition and + // flushes the statement cache. + InstanceMethod("_registerVtab", &Database::RegisterVtab, napi_default_method), + InstanceMethod("_removeVtab", &Database::RemoveVtab, napi_default_method), + // Phase 1/6 introspection and ops: transaction state, db status + // counters, memory release, run-time limits, attached-file paths. + InstanceAccessor("inTransaction", &Database::InTransactionGetter, nullptr), + InstanceAccessor("txnState", &Database::TxnStateGetter, nullptr), + InstanceMethod("_dbStatus", &Database::DbStatus, napi_default_method), + InstanceMethod("_releaseMemory", &Database::ReleaseMemory, napi_default_method), + InstanceMethod("_getLimit", &Database::GetLimit, napi_default_method), + InstanceMethod("_dbLocation", &Database::DbLocation, napi_default_method), + // True while the JS thread is inside a sqlite call on this + // connection — i.e. while a user callback (function, aggregate, + // virtual-table generator) invoked from a *Sync method is on the + // stack. lib/sqlite3.js consults it to refuse the operations that + // would finalize the statement sqlite is stepping (every + // registration flushes the statement cache) BEFORE they touch any + // state; the native entry points refuse too, but by then the + // flush has already happened. + InstanceAccessor("_inSyncCall", &Database::InSyncCallGetter, nullptr), InstanceAccessor("open", &Database::Open, nullptr), InstanceAccessor("integerMode", &Database::IntegerModeGetter, nullptr), InstanceAccessor("state", &Database::StateGetter, nullptr), @@ -89,6 +111,14 @@ void Database::Process() { auto env = this->Env(); Napi::HandleScope scope(env); + // Virtual-table instance references queued by xDisconnect (a DROP or + // a module replacement can disconnect on a worker) are napi work, so + // this — the JS-thread point every completion and exclusive handler + // funnels through — is where they are deleted. Until now they waited + // for ~Database, leaking one strong generator reference per dropped + // factory instance for the connection's lifetime. + DrainVtabRefs(); + if (db_state == DbState::Closed && !queue.empty()) { // Work queued behind a *failed open* fails with the open's own // error (CANTOPEN etc.), not the generic closed message — it never @@ -184,6 +214,7 @@ void Database::Schedule(Work_Callback callback, Baton* baton, bool exclusive) { Database::Database(const Napi::CallbackInfo& info) : Napi::ObjectWrap(info) { auto env = info.Env(); + uv_mutex_init(&vtab_refs_mutex); if (info.Length() <= 0 || !info[0].IsString()) { Napi::TypeError::New(env, "String expected").ThrowAsJavaScriptException(); @@ -211,6 +242,17 @@ Database::Database(const Napi::CallbackInfo& info) : Napi::ObjectWrap( // Start opening the database. auto* baton = new OpenBaton(this, callback, filename.c_str(), mode); + if (info.Length() > pos && info[pos].IsBoolean() + && info[pos].As().Value()) { + // Synchronous open (the node:sqlite compat shim's DatabaseSync): + // a fresh connection has nothing in flight or queued, so opening + // inline on this thread is safe, and the connection is usable the + // moment the constructor returns — node:sqlite's semantics. The + // same Work_Open/Work_AfterOpen pair the async path runs. + Work_Open(env, baton); + Work_AfterOpen(env, napi_ok, baton); + return; + } Work_BeginOpen(baton); } @@ -329,6 +371,14 @@ Napi::Value Database::IntegerModeGetter(const Napi::CallbackInfo& info) { } } +// True while this thread is inside sqlite on this connection: the *Sync +// methods' guard, observable from JavaScript because a callback they +// invoke re-entrantly must not perform the operations that finalize the +// stepping statement. See lib/sqlite3.js's assertNotInSyncCallback. +Napi::Value Database::InSyncCallGetter(const Napi::CallbackInfo& info) { + return Napi::Boolean::New(info.Env(), sync_sqlite_depth > 0); +} + Napi::Value Database::StateGetter(const Napi::CallbackInfo& info) { auto env = info.Env(); Napi::Object snapshot = Napi::Object::New(env); @@ -557,6 +607,18 @@ Napi::Value Database::Configure(const Napi::CallbackInfo& info) { Baton* baton = new LimitBaton(db, handle, id, value); db->Schedule(SetLimit, baton); } + else if (info[0].StrictEquals( + Napi::String::New(env, "walAutocheckpoint"))) { + if (!info[1].IsNumber()) { + Napi::TypeError::New(env, + "walAutocheckpoint value must be an integer" + ).ThrowAsJavaScriptException(); + return env.Null(); + } + auto* baton = new Baton(db, handle); + baton->timeout = info[1].As().Int32Value(); + db->Schedule(SetWalAutocheckpoint, baton); + } else if (info[0].StrictEquals(Napi::String::New(env, "integerMode"))) { // Pure JS-side marshalling state: applied immediately, no sqlite // handle access, so unlike the other options it needs no baton. @@ -673,6 +735,27 @@ void Database::SetLimit(Baton* b) { baton->db->Process(); } +// configure('walAutocheckpoint', n): sqlite3_wal_autocheckpoint. The value +// rides the same baton slot busyTimeout uses (an int with no sqlite-side +// interpretation until the worker applies it). +void Database::SetWalAutocheckpoint(Baton* b) { + auto baton = std::unique_ptr(b); + + if (baton->db->MayBlockOnWorkerRoundTrip()) { + baton->db->Schedule(SetWalAutocheckpoint, baton.release(), true); + return; + } + + assert(baton->db->IsOpen()); + assert(baton->db->_handle); + assert(!baton->db->MayBlockOnWorkerRoundTrip()); + + sqlite3_wal_autocheckpoint(baton->db->_handle, baton->timeout); + + baton->db->exclusiveHeld = false; + baton->db->Process(); +} + // Recompute the sqlite3_trace_v2 mask from the registered JS hooks and // (un)install the single native callback accordingly. void Database::UpdateTraceMask(Database* db, sqlite3* handle) { @@ -2113,6 +2196,111 @@ Napi::Value Database::TotalChangesGetter(const Napi::CallbackInfo& info) { integer_mode, "db.totalChanges"); } +// --- Transaction state, db status, memory and limit introspection ------ +// +// All of these read live sqlite state, so they follow the db.changes +// pattern: refuse while a worker could be blocked mid-round-trip holding +// the connection mutex; everything else merely serializes on the +// (recursive) mutex inside sqlite, the ordinary main-thread sqlite cost. + +// Shared open/refusal prelude for the live sqlite readers. Returns true +// when the caller may touch _handle. +bool Database::LiveReadGate(Napi::Env env, const char* who) { + if (!IsOpen() || _handle == NULL) { + Napi::Error::New(env, "Database is not open") + .ThrowAsJavaScriptException(); + return false; + } + if (MayBlockOnWorkerRoundTrip()) { + std::string what = std::string(who) + + " cannot be read while a JavaScript function, collation, " + "progress callback or virtual-table generator is mid-call on " + "this connection; read it from a callback or after the query"; + Napi::Error::New(env, what).ThrowAsJavaScriptException(); + return false; + } + return true; +} + +Napi::Value Database::InTransactionGetter(const Napi::CallbackInfo& info) { + auto env = info.Env(); + if (!LiveReadGate(env, "db.inTransaction")) return env.Null(); + // sqlite3_get_autocommit: 0 inside an explicit transaction. + return Napi::Boolean::New(env, + sqlite3_get_autocommit(_handle) == 0); +} + +Napi::Value Database::TxnStateGetter(const Napi::CallbackInfo& info) { + auto env = info.Env(); + if (!LiveReadGate(env, "db.txnState")) return env.Null(); + switch (sqlite3_txn_state(_handle, "main")) { + case SQLITE_TXN_NONE: return Napi::String::New(env, "none"); + case SQLITE_TXN_READ: return Napi::String::New(env, "read"); + case SQLITE_TXN_WRITE: return Napi::String::New(env, "write"); + default: return env.Undefined(); + } +} + +// _dbStatus(op, reset?) -> { current, highwater }: sqlite3_db_status +// counters (cache hits/misses, schema usage, statement memory...). Note +// that SQLITE_DEFAULT_MEMSTATUS=0 in this build zeroes the process-wide +// memory counters; the per-database cache and schema counters work. +Napi::Value Database::DbStatus(const Napi::CallbackInfo& info) { + auto env = info.Env(); + REQUIRE_ARGUMENT_INTEGER(0, op); + bool reset = false; + if (info.Length() > 1 && !info[1].IsUndefined()) { + if (!info[1].IsBoolean()) { + Napi::TypeError::New(env, "reset flag must be a boolean") + .ThrowAsJavaScriptException(); + return env.Null(); + } + reset = info[1].As().Value(); + } + if (!LiveReadGate(env, "db.status()")) return env.Null(); + int current = 0; + int highwater = 0; + int rc = sqlite3_db_status(_handle, op, ¤t, &highwater, + reset ? 1 : 0); + if (rc != SQLITE_OK) { + EXCEPTION(sqlite3_errmsg(_handle), rc, exception); + exception.As().ThrowAsJavaScriptException(); + return env.Null(); + } + Napi::Object result = Napi::Object::New(env); + result.Set("current", Napi::Number::New(env, current)); + result.Set("highwater", Napi::Number::New(env, highwater)); + return result; +} + +// _releaseMemory() -> bytes freed: sqlite3_db_release_memory, the pool +// pressure lever (it releases non-essential page cache). +Napi::Value Database::ReleaseMemory(const Napi::CallbackInfo& info) { + auto env = info.Env(); + if (!LiveReadGate(env, "db.releaseMemory()")) return env.Null(); + int freed = sqlite3_db_release_memory(_handle); + return Napi::Number::New(env, freed); +} + +// _getLimit(id) -> current value: sqlite3_limit(id, -1) reads without +// setting. The JS layer maps names onto ids (db.limits). +Napi::Value Database::GetLimit(const Napi::CallbackInfo& info) { + auto env = info.Env(); + REQUIRE_ARGUMENT_INTEGER(0, id); + if (!LiveReadGate(env, "db.limits")) return env.Null(); + return Napi::Number::New(env, sqlite3_limit(_handle, id, -1)); +} + +// _dbLocation(dbName) -> filesystem path of an attached database ("" for +// in-memory or temp schemas): sqlite3_db_filename. +Napi::Value Database::DbLocation(const Napi::CallbackInfo& info) { + auto env = info.Env(); + REQUIRE_ARGUMENT_STRING(0, name); + if (!LiveReadGate(env, "db.location()")) return env.Null(); + const char* file = sqlite3_db_filename(_handle, name.c_str()); + return Napi::String::New(env, file != NULL ? file : ""); +} + Napi::Value Database::Exec(const Napi::CallbackInfo& info) { auto env = this->Env(); auto* db = this; diff --git a/src/database.h b/src/database.h index de83136..5a2dbec 100644 --- a/src/database.h +++ b/src/database.h @@ -21,6 +21,7 @@ namespace node_sqlite3 { class Database; struct JsFunc; +struct VtabModule; struct FunctionBaton; struct RemoveFunctionBaton; struct UserFunctionOps; @@ -404,6 +405,9 @@ class Database : public Napi::ObjectWrap { bool IsOpen() { return db_state == DbState::Open || db_state == DbState::Closing; } // Terminal: a close completed. The old `locked` tombstone. bool IsClosed() { return db_state == DbState::Closed; } + // The raw mode value for cross-file users (src/vtab.cc's JS-thread + // half converts generator arguments with it). + int IntegerMode() const { return integer_mode; } typedef Async AsyncTrace; typedef Async AsyncProfile; @@ -417,6 +421,7 @@ class Database : public Napi::ObjectWrap { friend class Blob; friend struct UserFunctionOps; friend struct SessionOps; + friend struct VtabOps; // Marks that the JavaScript thread is itself inside a sqlite call on // this connection and therefore cannot service the ThreadSafeFunction @@ -493,6 +498,12 @@ class Database : public Napi::ObjectWrap { sqlite3_close(_handle); _handle = NULL; db_state = DbState::Closed; + // After sqlite3_close every virtual table instance is + // disconnected, so the module holders (and their sqlite3_module + // structs) can be freed. See src/vtab.cc — last, because + // RemoveVtabs drains through the queue this mutex guards. + RemoveVtabs(); + uv_mutex_destroy(&vtab_refs_mutex); } protected: @@ -514,7 +525,8 @@ class Database : public Napi::ObjectWrap { /** Current integerMode as a string: 'number' | 'bigint' | 'mixed'. */ Napi::Value IntegerModeGetter(const Napi::CallbackInfo& info); - + // sync_sqlite_depth > 0, for the JS-side re-entrancy refusals. + Napi::Value InSyncCallGetter(const Napi::CallbackInfo& info); // Read-only snapshot of the connection's scheduling state, computed // on read from the authoritative fields; diagnostics and tests consume // it. The statement cache's hot guard reads the individual accessors @@ -537,6 +549,20 @@ class Database : public Napi::ObjectWrap { static void SetBusyTimeout(Baton* baton); static void SetLimit(Baton* baton); + static void SetWalAutocheckpoint(Baton* baton); + + // --- Live sqlite introspection (Phase 1/6): transaction state, db + // status counters, memory release, run-time limit reads and + // attached-file paths. All follow the db.changes reader pattern (see + // LiveReadGate): refuse while a worker round trip could hold the + // connection mutex, serialize on it otherwise. + bool LiveReadGate(Napi::Env env, const char* who); + Napi::Value InTransactionGetter(const Napi::CallbackInfo& info); + Napi::Value TxnStateGetter(const Napi::CallbackInfo& info); + Napi::Value DbStatus(const Napi::CallbackInfo& info); + Napi::Value ReleaseMemory(const Napi::CallbackInfo& info); + Napi::Value GetLimit(const Napi::CallbackInfo& info); + Napi::Value DbLocation(const Napi::CallbackInfo& info); // Deferred main-thread sqlite work (see MayBlockOnWorkerRoundTrip): // exclusive, so each dispatches only once pending == 0 and the @@ -714,18 +740,28 @@ class Database : public Napi::ObjectWrap { // True when a main-thread sqlite call on this connection could block // on the connection mutex: a JS function, collation or progress - // callback is registered and statement work is in flight, or a - // changeset apply carrying JS conflict/filter handlers is queued or - // in flight (the apply holds the connection mutex for its whole - // run, and its handlers block on this thread) — a worker may be - // sitting inside a round trip holding that mutex while it waits for - // this very thread. Callers on the JS thread must defer their sqlite - // call (the exclusive queue runs it once nothing is in flight) - // instead of touching the handle. Without registered callbacks in-flight work never waits on the JS thread, so the mutex - // is only ever held briefly and blocking on it is fine — which is why - // every pre-existing path is unchanged. + // callback is registered, or a JavaScript virtual table is registered, + // and statement work is in flight; or a changeset apply carrying JS + // conflict/filter handlers is queued or in flight (the apply holds the + // connection mutex for its whole run, and its handlers block on this + // thread) — a worker may be sitting inside a round trip holding that + // mutex while it waits for this very thread. Callers on the JS thread + // must defer their sqlite call (the exclusive queue runs it once + // nothing is in flight) instead of touching the handle. Without + // registered callbacks in-flight work never waits on the JS thread, so + // the mutex is only ever held briefly and blocking on it is fine — + // which is why every pre-existing path is unchanged. + // + // js_vtabs belongs in this set for exactly the same reason as + // js_functions: a worker inside xFilter/xNext waits on the JS thread + // for the generator's next batch while holding the connection mutex. + // Leaving it out deadlocked two concurrent queries against a + // db.table() (or db.values()) table: the first query's completion + // handler called sqlite3_finalize inline, which wants the mutex the + // second query's worker is holding while it waits for this thread. bool MayBlockOnWorkerRoundTrip() { return (!(js_functions.empty() && js_collations.empty() + && js_vtabs.empty() && js_progress == NULL && js_apply_depth == 0)) && pending > 0; } @@ -781,6 +817,21 @@ class Database : public Napi::ObjectWrap { bool EnsureJsChannel(); void ReportRegistrationFailure(int rc); + // --- JavaScript virtual tables (Phase 4). Implementation in + // src/vtab.cc; same lifecycle as the user functions above — exclusive + // registration, refuse-from-sync-callbacks, teardown at close. + Napi::Value RegisterVtab(const Napi::CallbackInfo& info); + Napi::Value RemoveVtab(const Napi::CallbackInfo& info); + static void Work_RegisterVtab(Baton* baton); + static void Work_RemoveVtab(Baton* baton); + // Drops every module registration and frees the holders once no + // instance can remain (Work_BeginClose orders before the actual + // sqlite3_close, which disconnects instances). + void RemoveVtabs(); + // Creates the vtab round-trip channel on demand; reports on 'error'. + bool EnsureVtabChannel(); + void ReleaseVtabChannelIfIdle(); + // Releases the callback channel when no registration is left. Only // called from live-loop contexts (the removal handlers, Work_BeginClose) // — never from a destructor, where it could re-enter the channel's own @@ -929,6 +980,24 @@ class Database : public Napi::ObjectWrap { // registration is gone and nothing can be in flight. napi_threadsafe_function js_channel = NULL; + // --- JavaScript virtual tables (Phase 4; see src/vtab.cc) ---------- + // The live registrations, owned here. A holder leaves this list when + // sqlite drops its last reference to the module (VtabOps::ModuleDestroy) + // and is freed on the JS thread through pending_vtab_modules; what is + // still here at ~Database was never handed to sqlite. + std::vector js_vtabs; + napi_threadsafe_function vtab_channel = NULL; + // Instance rows-references queued by xDisconnect, cursor iterators + // queued by xClose and dead module holders queued by xDestroy (any + // thread) for JS-thread deletion — napi reference work is main-thread + // work. + uv_mutex_t vtab_refs_mutex; + std::vector pending_vtab_refs; + std::vector pending_vtab_modules; + void QueueVtabRef(napi_ref ref); + void QueueVtabModule(VtabModule* module); + void DrainVtabRefs(); + // The JS error thrown inside a user function that caused the current // step failure: attached as `cause` on the SQLite error the statement // then reports. JS-thread-only (set in the channel callback, consumed diff --git a/src/function.cc b/src/function.cc index 8dee071..a41f850 100644 --- a/src/function.cc +++ b/src/function.cc @@ -10,9 +10,12 @@ // makes a blocking round trip through the per-database // ThreadSafeFunction: the JS thread converts, runs the user's JS, // marshals the result back and signals a condition variable. -// - On the JS thread (Database::sync_sqlite_depth > 0), a round trip -// would deadlock — the JS thread is the one blocked inside sqlite — -// so the call is refused with an explicit error instead. +// - On the JS thread (Database::sync_sqlite_depth > 0), the JS thread is +// the one executing SQL, so the implementation is invoked directly — +// re-entrantly, same-thread, like node:sqlite (Phase 2; see +// RunDirectOnJsThread). Collations and the JS progress callback keep +// refusing: they have no error channel, and the sync gate blocks them +// up front. // // Lifetime answers (the checklist items this file must have settled): // @@ -39,8 +42,8 @@ // reachable only at teardown. // - FuncCall ownership: the worker allocates; for waited calls it also // frees (after the wake); fire-and-forget cleanups are freed by the JS -// thread. Enqueue failure (environment shutting down) leaves ownership -// with the worker. +// thread. Direct re-entrant calls are stack-disciplined: allocated and +// freed by the same sqlite callback frame, never crossing threads. #include #include @@ -59,17 +62,6 @@ using namespace node_sqlite3; namespace { -// The error a user-defined function reached from a sync-path statement -// reports instead of deadlocking. prepareSync is not listed because it is -// not gated (and should not be): preparing never invokes a function. -std::string SyncRefusalMessage(const std::string& name) { - return "user-defined function '" + name + "' cannot be invoked from a " - "synchronous method (getSync/runSync/allSync): the " - "JavaScript thread is blocked inside SQLite and cannot run the " - "callback, which would deadlock. Use the asynchronous get/run/all/" - "each instead, or express the logic in SQL."; -} - std::string RemovedMidFlightMessage(const std::string& name) { return "user-defined function '" + name + "' was removed while a call " "to it was still in flight"; @@ -566,6 +558,45 @@ static void ReportRegistrationError(Database* db, int rc) { } // namespace node_sqlite3 +namespace { + +// The re-entrant path for user functions invoked from the synchronous +// methods (Phase 2). sync_sqlite_depth > 0 can only be observed on the JS +// thread inside a *Sync call — this thread is the one executing SQL — so +// the implementation can be invoked directly, exactly like node:sqlite +// does, with no round trip. A throw inside the callback is reported +// through sqlite3_result_error (the error channel functions have); the +// thrown value is kept on the database as the step failure's `cause` by +// SetCallError, as on the async path. +// +// `call` is owned by the caller (deleted after this returns). +void RunDirectOnJsThread(sqlite3_context* ctx, FuncCall* call, + bool has_result) { + node_sqlite3::UserFunctionOps::ExecuteOnJsThread(call->db->Env(), call); + // No exception may escape into sqlite's C frames (and beyond, into the + // sync caller's napi transition); always clear a stray one — the same + // rule as TsfnCallJs, and the check must be napi_is_exception_pending. + bool stray_exception = false; + napi_is_exception_pending(call->db->Env(), &stray_exception); + if (stray_exception) { + napi_value pending = NULL; + napi_get_and_clear_last_exception(call->db->Env(), &pending); + if (!call->errored) { + call->errored = true; + call->error = "internal error while invoking a user-defined " + "function"; + } + } + if (call->errored) { + sqlite3_result_error(ctx, call->error.c_str(), -1); + } + else if (has_result) { + ApplyCell(ctx, call->result); + } +} + +} // namespace + // --- sqlite callbacks (worker thread, or the JS thread under the sync guard) void Database::JsScalarFunc(sqlite3_context* ctx, int argc, @@ -574,7 +605,16 @@ void Database::JsScalarFunc(sqlite3_context* ctx, int argc, Database* db = fn->db; if (db->sync_sqlite_depth > 0) { - sqlite3_result_error(ctx, SyncRefusalMessage(fn->name).c_str(), -1); + // Re-entrant direct call (Phase 2): this is the JS thread inside + // a *Sync call, so the implementation runs right here. + FuncCall* call = new FuncCall(fn, FuncCall::kScalar); + call->args.reserve(argc); + for (int i = 0; i < argc; i++) { + call->args.emplace_back(); + ValueToCell(&call->args.back(), argv[i]); + } + RunDirectOnJsThread(ctx, call, true); + delete call; return; } @@ -593,10 +633,28 @@ void Database::JsAggregateStep(sqlite3_context* ctx, int argc, JsFunc* fn = static_cast(sqlite3_user_data(ctx)); Database* db = fn->db; - if (db->sync_sqlite_depth > 0 || fn->dead) { - const std::string& refusal = (db->sync_sqlite_depth > 0) - ? SyncRefusalMessage(fn->name) : RemovedMidFlightMessage(fn->name); - sqlite3_result_error(ctx, refusal.c_str(), -1); + if (db->sync_sqlite_depth > 0) { + FuncCall* call = new FuncCall(fn, FuncCall::kStep); + call->agg_slot = static_cast( + sqlite3_aggregate_context(ctx, sizeof(AggState*))); + if (call->agg_slot == NULL) { + delete call; + sqlite3_result_error(ctx, "out of memory", -1); + return; + } + call->args.reserve(argc); + for (int i = 0; i < argc; i++) { + call->args.emplace_back(); + ValueToCell(&call->args.back(), argv[i]); + } + RunDirectOnJsThread(ctx, call, false); + delete call; + return; + } + + if (fn->dead) { + sqlite3_result_error(ctx, + RemovedMidFlightMessage(fn->name).c_str(), -1); return; } @@ -630,12 +688,34 @@ void Database::JsAggregateFinal(sqlite3_context* ctx) { sqlite3_aggregate_context(ctx, 0)); AggState* agg = (slot != NULL && *slot != NULL) ? *slot : NULL; - if (db->sync_sqlite_depth > 0 || fn->dead) { - // Reached from a main-thread sqlite3_finalize (the sync methods, - // Statement::Finalize_, the GC safety net) or after removal. A - // round trip would deadlock (or touch freed state); free the - // accumulator without running JS, off-thread of nothing — the - // cleanup itself is what gets deferred. + if (db->sync_sqlite_depth > 0) { + // Re-entrant direct call (Phase 2): the JS thread can run result() + // right here and free the accumulator — no deferred cleanup + // needed, unlike a worker-thread xFinal. + if (fn->dead) { + // Removed under a suspended cursor, and this is the JS thread + // (a *Sync call, Statement::Finalize_, the GC safety net), so + // the accumulator can be released right here instead of being + // enqueued. Running result() against a removed registration + // is not on the table: its implementations may be gone. + if (agg != NULL) { + delete agg; + *slot = NULL; + } + sqlite3_result_null(ctx); + return; + } + FuncCall* call = new FuncCall(fn, FuncCall::kFinal); + call->agg_slot = slot; + RunDirectOnJsThread(ctx, call, true); + delete call; + return; + } + + if (fn->dead) { + // Reached after removal, possibly from a main-thread + // sqlite3_finalize (the GC safety net): free the accumulator + // without running JS — the cleanup itself is what gets deferred. if (agg != NULL) { UserFunctionOps::EnqueueAggCleanup(db, fn, agg); *slot = NULL; @@ -663,7 +743,22 @@ void Database::JsAggregateValue(sqlite3_context* ctx) { JsFunc* fn = static_cast(sqlite3_user_data(ctx)); Database* db = fn->db; - if (db->sync_sqlite_depth > 0 || fn->dead) { + if (db->sync_sqlite_depth > 0) { + if (fn->dead) { + // xValue leaves the state for the aborting statement's + // xFinal, exactly as the worker path does. + sqlite3_result_null(ctx); + return; + } + FuncCall* call = new FuncCall(fn, FuncCall::kValue); + call->agg_slot = static_cast( + sqlite3_aggregate_context(ctx, 0)); + RunDirectOnJsThread(ctx, call, true); + delete call; + return; + } + + if (fn->dead) { sqlite3_result_null(ctx); return; } @@ -681,10 +776,28 @@ void Database::JsAggregateInverse(sqlite3_context* ctx, int argc, JsFunc* fn = static_cast(sqlite3_user_data(ctx)); Database* db = fn->db; - if (db->sync_sqlite_depth > 0 || fn->dead) { - const std::string& refusal = (db->sync_sqlite_depth > 0) - ? SyncRefusalMessage(fn->name) : RemovedMidFlightMessage(fn->name); - sqlite3_result_error(ctx, refusal.c_str(), -1); + if (db->sync_sqlite_depth > 0) { + FuncCall* call = new FuncCall(fn, FuncCall::kInverse); + call->agg_slot = static_cast( + sqlite3_aggregate_context(ctx, sizeof(AggState*))); + if (call->agg_slot == NULL) { + delete call; + sqlite3_result_error(ctx, "out of memory", -1); + return; + } + call->args.reserve(argc); + for (int i = 0; i < argc; i++) { + call->args.emplace_back(); + ValueToCell(&call->args.back(), argv[i]); + } + RunDirectOnJsThread(ctx, call, false); + delete call; + return; + } + + if (fn->dead) { + sqlite3_result_error(ctx, + RemovedMidFlightMessage(fn->name).c_str(), -1); return; } @@ -777,6 +890,23 @@ void Database::JsFuncDestroy(void* data) { // --- JS-visible entry points ----------------------------------------------- +// Registrations (and removals) dispatch inline through Database::Process +// when nothing else is queued — including from inside a user function a +// *Sync call just invoked, mid-step. Redefining a function while sqlite is +// running it (or swapping the implementations a stepping VM may still +// hold) is not a supported sqlite operation, so refuse loudly there; the +// call site can register after the query completes. +static bool RegistrationBlockedBySyncCall(Database* db, Napi::Env env) { + if (db->sync_sqlite_depth == 0) return false; + Napi::Error::New(env, + "user functions cannot be registered or removed from inside a " + "JavaScript callback invoked by a synchronous method on this " + "connection: sqlite is mid-step on the implementation being " + "swapped. Register or remove it before or after the query" + ).ThrowAsJavaScriptException(); + return true; +} + Napi::Value Database::RegisterUserFunction(const Napi::CallbackInfo& info) { auto env = info.Env(); auto* db = this; @@ -785,6 +915,7 @@ Napi::Value Database::RegisterUserFunction(const Napi::CallbackInfo& info) { REQUIRE_ARGUMENT_INTEGER(1, nArg); REQUIRE_ARGUMENT_INTEGER(2, flags); REQUIRE_ARGUMENT_FUNCTION(3, fn); + if (RegistrationBlockedBySyncCall(db, env)) return env.Null(); auto* baton = new FunctionBaton(db, Napi::Function(), name.c_str(), nArg, flags); @@ -806,6 +937,7 @@ Napi::Value Database::RegisterUserAggregate(const Napi::CallbackInfo& info) { REQUIRE_ARGUMENT_FUNCTION(3, start); REQUIRE_ARGUMENT_FUNCTION(4, step); REQUIRE_ARGUMENT_FUNCTION(5, result); + if (RegistrationBlockedBySyncCall(db, env)) return env.Null(); Napi::Function inverse; if (info.Length() > 6 && !info[6].IsUndefined()) { if (!info[6].IsFunction()) { @@ -834,6 +966,7 @@ Napi::Value Database::RegisterUserCollation(const Napi::CallbackInfo& info) { REQUIRE_ARGUMENT_STRING(0, name); REQUIRE_ARGUMENT_FUNCTION(1, fn); + if (RegistrationBlockedBySyncCall(db, env)) return env.Null(); auto* baton = new FunctionBaton(db, Napi::Function(), name.c_str(), 0, 0); @@ -849,6 +982,7 @@ Napi::Value Database::RemoveUserFunction(const Napi::CallbackInfo& info) { auto* db = this; REQUIRE_ARGUMENT_STRING(0, name); + if (RegistrationBlockedBySyncCall(db, env)) return env.Null(); auto* baton = new RemoveFunctionBaton(db, Napi::Function(), name.c_str()); baton->collation = false; @@ -862,6 +996,7 @@ Napi::Value Database::RemoveUserCollation(const Napi::CallbackInfo& info) { auto* db = this; REQUIRE_ARGUMENT_STRING(0, name); + if (RegistrationBlockedBySyncCall(db, env)) return env.Null(); auto* baton = new RemoveFunctionBaton(db, Napi::Function(), name.c_str()); baton->collation = true; diff --git a/src/node_sqlite3.cc b/src/node_sqlite3.cc index ea64b14..c1a1d47 100644 --- a/src/node_sqlite3.cc +++ b/src/node_sqlite3.cc @@ -39,6 +39,35 @@ Napi::Value SetRowFactoryGenerator(const Napi::CallbackInfo& info) { return env.Undefined(); } +// complete(sql): sqlite3_complete — true when a SQL string is a complete +// statement (balanced quotes/semicolons). The REPL/CLI helper; a pure +// string function with no connection. +Napi::Value Complete(const Napi::CallbackInfo& info) { + auto env = info.Env(); + if (info.Length() < 1 || !info[0].IsString()) { + Napi::TypeError::New(env, "complete() requires a SQL string") + .ThrowAsJavaScriptException(); + return env.Null(); + } + std::string sql = info[0].As().Utf8Value(); + return Napi::Boolean::New(env, + sqlite3_complete(sql.c_str()) != 0); +} + +// compileOptions(): the SQLITE_COMPILE_OPTIONS this build was configured +// with (sqlite3_compileoption_get) — what "is FTS5 compiled in" can be +// answered from, at runtime. +Napi::Value CompileOptions(const Napi::CallbackInfo& info) { + auto env = info.Env(); + Napi::Array result = Napi::Array::New(env); + uint32_t i = 0; + for (const char* opt = sqlite3_compileoption_get(0); opt != NULL; + opt = sqlite3_compileoption_get(++i)) { + result.Set(i, Napi::String::New(env, opt)); + } + return result; +} + Napi::Object RegisterModule(Napi::Env env, Napi::Object exports) { Napi::HandleScope scope(env); @@ -57,6 +86,12 @@ Napi::Object RegisterModule(Napi::Env env, Napi::Object exports) { Napi::Function::New(env, ConcatChangeset)); exports.Set("iterateChangeset", Napi::Function::New(env, IterateChangeset)); + exports.Set("rebaseChangeset", + Napi::Function::New(env, RebaseChangeset)); + exports.Set("complete", + Napi::Function::New(env, Complete)); + exports.Set("compileOptions", + Napi::Function::New(env, CompileOptions)); exports.DefineProperties({ DEFINE_CONSTANT_INTEGER(exports, SQLITE_OPEN_READONLY, OPEN_READONLY) @@ -245,6 +280,22 @@ Napi::Object RegisterModule(Napi::Env env, Napi::Object exports) { DEFINE_CONSTANT_INTEGER(exports, SQLITE_STMTSTATUS_FILTER_MISS, STMTSTATUS_FILTER_MISS) DEFINE_CONSTANT_INTEGER(exports, SQLITE_STMTSTATUS_FILTER_HIT, STMTSTATUS_FILTER_HIT) + // sqlite3_db_status counters (db.status). The process-wide memory + // counters read zero in this build (SQLITE_DEFAULT_MEMSTATUS=0); + // the cache and schema counters are the useful ones. + DEFINE_CONSTANT_INTEGER(exports, SQLITE_DBSTATUS_LOOKASIDE_USED, DBSTATUS_LOOKASIDE_USED) + DEFINE_CONSTANT_INTEGER(exports, SQLITE_DBSTATUS_CACHE_USED, DBSTATUS_CACHE_USED) + DEFINE_CONSTANT_INTEGER(exports, SQLITE_DBSTATUS_SCHEMA_USED, DBSTATUS_SCHEMA_USED) + DEFINE_CONSTANT_INTEGER(exports, SQLITE_DBSTATUS_STMT_USED, DBSTATUS_STMT_USED) + DEFINE_CONSTANT_INTEGER(exports, SQLITE_DBSTATUS_LOOKASIDE_HIT, DBSTATUS_LOOKASIDE_HIT) + DEFINE_CONSTANT_INTEGER(exports, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, DBSTATUS_LOOKASIDE_MISS_SIZE) + DEFINE_CONSTANT_INTEGER(exports, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, DBSTATUS_LOOKASIDE_MISS_FULL) + DEFINE_CONSTANT_INTEGER(exports, SQLITE_DBSTATUS_CACHE_HIT, DBSTATUS_CACHE_HIT) + DEFINE_CONSTANT_INTEGER(exports, SQLITE_DBSTATUS_CACHE_MISS, DBSTATUS_CACHE_MISS) + DEFINE_CONSTANT_INTEGER(exports, SQLITE_DBSTATUS_CACHE_WRITE, DBSTATUS_CACHE_WRITE) + DEFINE_CONSTANT_INTEGER(exports, SQLITE_DBSTATUS_CACHE_SPILL, DBSTATUS_CACHE_SPILL) + DEFINE_CONSTANT_INTEGER(exports, SQLITE_DBSTATUS_DEFERRED_FKS, DBSTATUS_DEFERRED_FKS) + // The sqlite3_db_config subset Database#dbConfig exposes. DEFINE_CONSTANT_INTEGER(exports, SQLITE_DBCONFIG_ENABLE_FKEY, DBCONFIG_ENABLE_FKEY) DEFINE_CONSTANT_INTEGER(exports, SQLITE_DBCONFIG_ENABLE_TRIGGER, DBCONFIG_ENABLE_TRIGGER) diff --git a/src/session.cc b/src/session.cc index 31594bf..42a9b5c 100644 --- a/src/session.cc +++ b/src/session.cc @@ -220,6 +220,13 @@ struct ApplyBaton : Database::Baton { Napi::FunctionReference on_filter; napi_threadsafe_function tsfn = NULL; ApplyRoundTrip rt; + // { rebase: true }: harvest the conflict-resolution rebase buffer + // alongside the apply (sqlite3changeset_apply_v2). The buffer records + // each OMIT/REPLACE decision so the same conflicts can be replayed + // onto other changesets via sqlite3.rebaseChangeset(). + bool want_rebase = false; + int rebase_n = 0; + void* rebase = NULL; // sqlite3_malloc'd; freed here or moved to JS int status = SQLITE_OK; std::string message; @@ -229,6 +236,7 @@ struct ApplyBaton : Database::Baton { } virtual ~ApplyBaton() override { if (data != NULL) sqlite3_free(data); + if (rebase != NULL) sqlite3_free(rebase); // The tsfn is released in Work_AfterApplyChangeset. If that never // ran (environment teardown) there is no sound place to release // it from; teardown reclaims it. @@ -525,6 +533,7 @@ Napi::Object Session::Init(Napi::Env env, Napi::Object exports) { auto t = DefineClass(env, "Session", { InstanceMethod("changeset", &Session::Changeset, napi_default_method), InstanceMethod("patchset", &Session::Patchset, napi_default_method), + InstanceMethod("diff", &Session::Diff, napi_default_method), InstanceMethod("close", &Session::Close, napi_default_method), InstanceAccessor("closed", &Session::ClosedGetter, nullptr), }); @@ -742,6 +751,8 @@ void Session::Work_Create(napi_env e, void* data) { sqlite3_mutex* mtx = sqlite3_db_mutex(baton->db->_handle); sqlite3_mutex_enter(mtx); + // Remembered for session.diff(), which needs the schema back. + session->db_name = baton->dbName; int rc = sqlite3session_create(baton->db->_handle, baton->dbName.c_str(), &session->_handle); if (rc == SQLITE_OK) { @@ -961,6 +972,96 @@ void Session::Work_AfterChangeset(napi_env e, napi_status status, void* data) { // --- close ---------------------------------------------------------------------- +// diff(table, fromDb): records the differences between this session's +// table and the same table in the attached database `fromDb` into the +// session (sqlite3session_diff). Neither database is written; harvest the +// recorded changes with changeset(). +Napi::Value Session::Diff(const Napi::CallbackInfo& info) { + auto env = info.Env(); + Session* session = this; + + REQUIRE_ARGUMENT_STRING(0, table); + REQUIRE_ARGUMENT_STRING(1, fromDb); + OPTIONAL_ARGUMENT_FUNCTION(2, callback); + + // sqlite3session_diff compares the named schema's table against the + // session's own; the same schema on both sides is a no-op that reads + // like a bug at the call site. + if (fromDb == session->db_name) { + Napi::Error::New(env, + "session.diff() needs a different attached database than the " + "one the session records ('" + session->db_name + "'): it " + "records the changes that would bring that database's table up " + "to this one's" + ).ThrowAsJavaScriptException(); + return env.Null(); + } + + auto* baton = new DiffBaton(session, callback); + baton->table = table; + baton->fromDb = fromDb; + session->Schedule(Work_BeginDiff, baton); + + return info.This(); +} + +void Session::Work_BeginDiff(Baton* baton) { + SESSION_BEGIN(Diff); +} + +void Session::Work_Diff(napi_env e, void* data) { + auto* baton = static_cast(data); + Session* session = baton->session; + + // Same serialization as Work_Changeset: the session's hash tables are + // the state being appended to. + sqlite3_mutex* mtx = sqlite3_db_mutex(session->db->_handle); + sqlite3_mutex_enter(mtx); + + // The session's own schema (recorded at create time) is the + // "current" side; the table is compared against the same table in + // the attached database `fromDb`. pzErrMsg is sqlite3_malloc'd. + char* err = NULL; + int rc = sqlite3session_diff(session->_handle, + baton->fromDb.c_str(), baton->table.c_str(), &err); + + sqlite3_mutex_leave(mtx); + + session->status = rc; + if (rc != SQLITE_OK) { + if (err != NULL) { + session->message = std::string(err); + sqlite3_free(err); + } + else { + session->message = + std::string(sqlite3_errmsg(session->db->_handle)); + } + } +} + +void Session::Work_AfterDiff(napi_env e, napi_status status, void* data) { + std::unique_ptr baton(static_cast(data)); + AFTER_WORK_TEARDOWN_GUARD(baton); + auto* session = baton->session; + + auto env = session->Env(); + Napi::HandleScope scope(env); + + Session::CallGuard session_call_guard__(session); + + if (session->status != SQLITE_OK) { + Error(baton.get()); + return; + } + + Napi::Function cb = baton->callback.Value(); + if (IS_FUNCTION(cb)) { + Napi::Value argv[] = { env.Null() }; + TRY_CATCH_CALL(session->Value(), cb, 1, argv); + } +} + Napi::Value Session::Close(const Napi::CallbackInfo& info) { auto env = info.Env(); Session* session = this; @@ -1035,10 +1136,12 @@ void Database::CloseLiveSessions(bool owner_dying) { } // _applyChangeset(data, decision, onConflict|null, onFilter|null, -// [callback]). decision is one of the CHANGESET_* return -// constants used when no JS conflict handler is given; onConflict / -// onFilter, when non-null, make the blocking round trip from inside -// sqlite3changeset_apply. +// [callback], [wantRebase]). decision is one of the +// CHANGESET_* return constants used when no JS conflict handler is given; +// onConflict / onFilter, when non-null, make the blocking round trip from +// inside sqlite3changeset_apply. wantRebase harvests the rebase buffer +// (the apply's conflict decisions) and hands it to the callback as its +// second argument. Napi::Value Database::ApplyChangeset(const Napi::CallbackInfo& info) { auto env = info.Env(); auto* db = this; @@ -1094,7 +1197,7 @@ Napi::Value Database::ApplyChangeset(const Napi::CallbackInfo& info) { } Napi::Function callback; - if (info.Length() > 4 && !info[4].IsUndefined()) { + if (info.Length() > 4 && !info[4].IsUndefined() && !info[4].IsNull()) { if (!info[4].IsFunction()) { delete baton; Napi::TypeError::New(env, "Argument 4 must be a function").ThrowAsJavaScriptException(); @@ -1102,6 +1205,9 @@ Napi::Value Database::ApplyChangeset(const Napi::CallbackInfo& info) { } callback = info[4].As(); } + if (info.Length() > 5 && info[5].IsBoolean()) { + baton->want_rebase = info[5].As().Value(); + } // The changeset is applied from a private copy: the apply runs later // on a worker, and a JS-side mutation of the source buffer mid-apply @@ -1163,13 +1269,37 @@ void Database::Work_BeginApplyChangeset(Baton* baton) { void Database::Work_ApplyChangeset(napi_env e, void* data) { auto* baton = static_cast(data); - int rc = sqlite3changeset_apply( - baton->db->_handle, - baton->n, - baton->data, - ApplyFilterTrampoline, - ApplyConflictTrampoline, - baton); + int rc; + if (baton->want_rebase) { + // apply_v2 with an output rebase buffer: every OMIT/REPLACE + // decision the conflict handling made is recorded so the same + // resolutions can be replayed onto later changesets (see + // sqlite3.rebaseChangeset). With no conflicts sqlite sets + // *ppRebase to NULL. + int rebase_n = 0; + void* rebase = NULL; + rc = sqlite3changeset_apply_v2( + baton->db->_handle, + baton->n, + baton->data, + ApplyFilterTrampoline, + ApplyConflictTrampoline, + baton, + &rebase, + &rebase_n, + 0); + baton->rebase_n = rebase_n; + baton->rebase = rebase; + } + else { + rc = sqlite3changeset_apply( + baton->db->_handle, + baton->n, + baton->data, + ApplyFilterTrampoline, + ApplyConflictTrampoline, + baton); + } // A handler error overrides the raw sqlite code as the reported // cause of the (rolled-back) apply. @@ -1216,6 +1346,29 @@ void Database::Work_AfterApplyChangeset(napi_env e, napi_status status, void* da TRY_CATCH_CALL(db->Value(), cb, 1, argv); return; } + if (baton->want_rebase) { + // No conflicts were omitted or replaced: sqlite produces an empty + // buffer. Resolve null rather than a zero-length Uint8Array — + // "nothing to rebase against" reads better at the call site. + if (baton->rebase_n == 0) { + if (baton->rebase != NULL) { + sqlite3_free(baton->rebase); + baton->rebase = NULL; + } + Napi::Value argv[] = { env.Null(), env.Null() }; + TRY_CATCH_CALL(db->Value(), cb, 2, argv); + return; + } + // WrapOwnedBytes takes ownership of the sqlite3_malloc'd buffer + // (zero-copy external ArrayBuffer, freed by its finalizer); + // NULLing the field keeps the baton destructor from double-freeing. + Napi::Value rebase = WrapOwnedBytes(env, baton->rebase, + static_cast(baton->rebase_n)); + baton->rebase = NULL; + Napi::Value argv[] = { env.Null(), rebase }; + TRY_CATCH_CALL(db->Value(), cb, 2, argv); + return; + } Napi::Value argv[] = { env.Null() }; TRY_CATCH_CALL(db->Value(), cb, 1, argv); } @@ -1549,6 +1702,62 @@ Napi::Value ConcatChangeset(const Napi::CallbackInfo& info) { return WrapOwnedBytes(env, p_out, static_cast(n_out)); } +// rebaseChangeset(changeset, rebase): rewrites a changeset against the +// conflict resolutions a previous applyChangeset(..., { rebase: true }) +// harvested — sqlite3_rebaser_configure + sqlite3rebaser_apply. Pure +// memory function: no connection, no mutex, no worker. This is the +// client-server sync primitive no other JS driver exposes (rusqlite is +// the only binding anywhere with it). +Napi::Value RebaseChangeset(const Napi::CallbackInfo& info) { + auto env = info.Env(); + if (info.Length() < 2) { + Napi::TypeError::New(env, "Expected 2 arguments").ThrowAsJavaScriptException(); + return env.Null(); + } + BytesView cs, rb; + if (!GetBytesView(env, info[0], "rebase a changeset", &cs)) { + return env.Null(); + } + if (!GetBytesView(env, info[1], "rebase a changeset", &rb)) { + return env.Null(); + } + int rc = ValidateChangeset(static_cast(cs.length), cs.data); + if (rc != SQLITE_OK) { + EXCEPTION("the changeset is not parseable", rc, exception); + exception.As().ThrowAsJavaScriptException(); + return env.Null(); + } + + sqlite3_rebaser* rebaser = NULL; + rc = sqlite3rebaser_create(&rebaser); + if (rc != SQLITE_OK) { + EXCEPTION("cannot create a changeset rebaser", rc, exception); + exception.As().ThrowAsJavaScriptException(); + return env.Null(); + } + rc = sqlite3rebaser_configure(rebaser, + static_cast(rb.length), const_cast(rb.data)); + if (rc != SQLITE_OK) { + sqlite3rebaser_delete(rebaser); + EXCEPTION("the rebase buffer is not parseable", rc, exception); + exception.As().ThrowAsJavaScriptException(); + return env.Null(); + } + int n_out = 0; + void* p_out = NULL; + rc = sqlite3rebaser_rebase(rebaser, + static_cast(cs.length), const_cast(cs.data), + &n_out, &p_out); + sqlite3rebaser_delete(rebaser); + if (rc != SQLITE_OK) { + if (p_out != NULL) sqlite3_free(p_out); + EXCEPTION("cannot rebase the changeset", rc, exception); + exception.As().ThrowAsJavaScriptException(); + return env.Null(); + } + return WrapOwnedBytes(env, p_out, static_cast(n_out)); +} + // --- ChangesetIter ---------------------------------------------------------------- namespace { diff --git a/src/session.h b/src/session.h index 6e44dbe..9c39805 100644 --- a/src/session.h +++ b/src/session.h @@ -163,6 +163,22 @@ class Session : public Napi::ObjectWrap { static void Work_Changeset(napi_env env, void* data); static void Work_AfterChangeset(napi_env env, napi_status status, void* data); + // diff(table, fromDb): sqlite3session_diff — records the differences + // between the session's table and the same table in another *attached* + // database into the session, without either database being written. + // The changeset is harvested with changeset() afterwards, as usual. + struct DiffBaton : Baton { + std::string table; + std::string fromDb; + DiffBaton(Session* session_, Napi::Function cb_) : + Baton(session_, cb_) {} + virtual ~DiffBaton() override = default; + }; + Napi::Value Diff(const Napi::CallbackInfo& info); + static void Work_BeginDiff(Baton* baton); + static void Work_Diff(napi_env env, void* data); + static void Work_AfterDiff(napi_env env, napi_status status, void* data); + Napi::Value Close(const Napi::CallbackInfo& info); static void Work_BeginClose(Baton* baton); static void Work_Close(napi_env env, void* data); @@ -172,6 +188,11 @@ class Session : public Napi::ObjectWrap { Database* db = NULL; sqlite3_session* _handle = NULL; + // The attached schema this session records ('main' unless created + // with { db: ... }). Kept for diagnostics and for the diff() contract + // (the session's schema is diff's "to" side; the argument names the + // "from" side, and the two must differ). + std::string db_name = "main"; bool inited = false; bool locked = false; @@ -218,6 +239,7 @@ class ChangesetIter : public Napi::ObjectWrap { Napi::Value InvertChangeset(const Napi::CallbackInfo& info); Napi::Value ConcatChangeset(const Napi::CallbackInfo& info); Napi::Value IterateChangeset(const Napi::CallbackInfo& info); +Napi::Value RebaseChangeset(const Napi::CallbackInfo& info); } // namespace node_sqlite3 diff --git a/src/statement.cc b/src/statement.cc index 752a8cb..55199d3 100644 --- a/src/statement.cc +++ b/src/statement.cc @@ -62,7 +62,7 @@ bool ParseSyncReadOptions(const Napi::Value& value, int* row_mode) { if (mode.IsUndefined()) return false; if (!mode.IsString()) { Napi::TypeError::New(env, - "rowMode must be 'object' or 'array'") + "rowMode must be 'object', 'array' or 'pluck'") .ThrowAsJavaScriptException(); return true; } @@ -71,14 +71,39 @@ bool ParseSyncReadOptions(const Napi::Value& value, int* row_mode) { *row_mode = Statement::SYNC_ROW_ARRAY; } else if (requested == "object") { *row_mode = Statement::SYNC_ROW_OBJECT; + } else if (requested == "pluck") { + *row_mode = Statement::SYNC_ROW_PLUCK; } else { Napi::TypeError::New(env, - "rowMode must be 'object' or 'array'") + "rowMode must be 'object', 'array' or 'pluck'") .ThrowAsJavaScriptException(); } return true; } +namespace { + +// Pops the trailing callback (if any) and then a trailing +// `{ rowMode: 'object' | 'array' }` options bag off an asynchronous read +// call (Get/All/Each/Fetch). The bag sits between the bind parameters and +// the callback; a plain object owning `rowMode` could never have been a +// legal bind argument (named bind keys carry a sigil) — the same +// discriminator ParseSyncReadOptions uses. Returns the reduced argument +// count and hands the popped callback back for Bind's preset slot. +// Throws (pending exception) for an invalid rowMode value. +int PopAsyncReadOptions(const Napi::CallbackInfo& info, + Napi::Function* callback, int* row_mode) { + int end = info.Length(); + if (end > 0 && info[end - 1].IsFunction()) { + *callback = info[end - 1].As(); + end--; + } + if (end > 0 && ParseSyncReadOptions(info[end - 1], row_mode)) end--; + return end; +} + +} // namespace + } // namespace Napi::Object Statement::Init(Napi::Env env, Napi::Object exports) { @@ -118,6 +143,15 @@ Napi::Object Statement::Init(Napi::Env env, Napi::Object exports) { nullptr), InstanceAccessor("columns", &Statement::ColumnsGetter, nullptr), InstanceMethod("status", &Statement::Status, napi_default_method), + // SQL text accessors (Phase 6): expanded SQL with the last bound + // values substituted, and normalized SQL with literals folded to `?` + // for query-metric dashboards. + InstanceAccessor("expandedSQL", &Statement::ExpandedSQLGetter, nullptr, + static_cast(napi_configurable)), + InstanceAccessor("normalizedSQL", &Statement::NormalizedSQLGetter, + nullptr, static_cast(napi_configurable)), + InstanceMethod("_setIntegerMode", &Statement::SetIntegerMode, + napi_default_method), }); // Per-env (see Database::AddonData): a worker thread is its own napi @@ -254,6 +288,7 @@ Statement::Statement(const Napi::CallbackInfo& info) : Napi::ObjectWrap_handle)); + error_offset = sqlite3_error_offset(db->_handle); _handle = NULL; } else { @@ -305,6 +340,10 @@ void Statement::Work_Prepare(napi_env e, void* data) { if (stmt->status != SQLITE_OK) { stmt->message = std::string(sqlite3_errmsg(baton->db->_handle)); + // Byte offset of the failing token, read while the mutex is still + // held and the error is current; -1 when this error has no + // position (sqlite only produces offsets for prepare failures). + stmt->error_offset = sqlite3_error_offset(baton->db->_handle); stmt->_handle = NULL; } else { @@ -351,6 +390,14 @@ void Statement::Work_AfterPrepare(napi_env e, napi_status status, void* data) { // on the statement's 'error' event, the documented surface for a // prepare given no callback of its own. EXCEPTION(stmt->message, stmt->status, exception); + // Failed-prepare token position (sqlite3_error_offset), captured + // on the worker while the error was current. Consumed here so no + // later error inherits it. + if (stmt->error_offset >= 0) { + exception_obj.Set("offset", + Napi::Number::New(env, stmt->error_offset)); + stmt->error_offset = -1; + } // A user-defined function that threw during the step kept its JS // error on the database as the pending cause of exactly this // failure (Error() did this for the callback-only path). @@ -404,13 +451,14 @@ template std::unique_ptr } } -template T* Statement::Bind(const Napi::CallbackInfo& info, int start, int last) { +template T* Statement::Bind(const Napi::CallbackInfo& info, int start, + int last, Napi::Function preset_callback) { auto env = info.Env(); Napi::HandleScope scope(env); if (last < 0) last = info.Length(); - Napi::Function callback; - if (last > start && info[last - 1].IsFunction()) { + Napi::Function callback = preset_callback; + if (callback.IsEmpty() && last > start && info[last - 1].IsFunction()) { callback = info[last - 1].As(); last--; } @@ -809,7 +857,12 @@ Napi::Value Statement::Get(const Napi::CallbackInfo& info) { auto env = info.Env(); Statement* stmt = this; - Baton* baton = stmt->Bind(info); + int row_mode = SYNC_ROW_OBJECT; + Napi::Function callback; + int end = PopAsyncReadOptions(info, &callback, &row_mode); + if (env.IsExceptionPending()) return env.Null(); + + RowBaton* baton = stmt->Bind(info, 0, end, callback); if (baton == NULL) { if (!env.IsExceptionPending()) { Napi::TypeError::New(env, "Data type is not supported") @@ -818,6 +871,7 @@ Napi::Value Statement::Get(const Napi::CallbackInfo& info) { return env.Null(); } else { + baton->row_mode = row_mode; stmt->Schedule(Work_BeginGet, baton); return info.This(); } @@ -874,7 +928,8 @@ void Statement::Work_AfterGet(napi_env e, napi_status status, void* data) { if (stmt->status == SQLITE_ROW) { // Create the result array from the data we acquired. stmt->SyncColumnKeys(env, baton->columns); - Napi::Value row = stmt->RowToJS(env, &baton->row); + Napi::Value row = stmt->RowToJS(env, &baton->row, + baton->row_mode); if (env.IsExceptionPending()) { // 'number' integer mode and an unsafe int64: deliver // the RangeError to the callback instead of leaving a @@ -976,7 +1031,12 @@ Napi::Value Statement::All(const Napi::CallbackInfo& info) { auto env = info.Env(); Statement* stmt = this; - Baton* baton = stmt->Bind(info); + int row_mode = SYNC_ROW_OBJECT; + Napi::Function callback; + int end = PopAsyncReadOptions(info, &callback, &row_mode); + if (env.IsExceptionPending()) return env.Null(); + + RowsBaton* baton = stmt->Bind(info, 0, end, callback); if (baton == NULL) { if (!env.IsExceptionPending()) { Napi::TypeError::New(env, "Data type is not supported") @@ -985,6 +1045,7 @@ Napi::Value Statement::All(const Napi::CallbackInfo& info) { return env.Null(); } else { + baton->row_mode = row_mode; stmt->Schedule(Work_BeginAll, baton); return info.This(); } @@ -1048,7 +1109,7 @@ void Statement::Work_AfterAll(napi_env e, napi_status status, void* data) { // Create the result array from the data we acquired. Napi::Array result; const bool failed = !stmt->CellRowsToJS(env, baton->rows, - baton->columns, &result); + baton->columns, baton->row_mode, &result); if (failed) { Napi::Value argv[] = { TakePendingError(env) }; @@ -1081,7 +1142,18 @@ Napi::Value Statement::Each(const Napi::CallbackInfo& info) { completed = info[--last].As(); } - auto baton = stmt->Bind(info, 0, last); + // Item callback, then an optional `{ rowMode }` bag before it (the + // same shape Get/All accept; PopAsyncReadOptions cannot be reused + // here because `last` no longer sits at the end of the call). + int row_mode = SYNC_ROW_OBJECT; + Napi::Function callback; + if (last > 0 && info[last - 1].IsFunction()) { + callback = info[--last].As(); + } + if (last > 0 && ParseSyncReadOptions(info[last - 1], &row_mode)) last--; + if (env.IsExceptionPending()) return env.Null(); + + auto baton = stmt->Bind(info, 0, last, callback); if (baton == NULL) { if (!env.IsExceptionPending()) { Napi::TypeError::New(env, "Data type is not supported") @@ -1090,6 +1162,7 @@ Napi::Value Statement::Each(const Napi::CallbackInfo& info) { return env.Null(); } else { + baton->row_mode = row_mode; baton->completed.Reset(completed, 1); stmt->Schedule(Work_BeginEach, baton); return info.This(); @@ -1103,6 +1176,7 @@ void Statement::Work_BeginEach(Baton* baton) { each_baton->async = new Async(each_baton->stmt, reinterpret_cast(AsyncEach)); each_baton->async->item_cb.Reset(each_baton->callback.Value(), 1); each_baton->async->completed_cb.Reset(each_baton->completed.Value(), 1); + each_baton->async->row_mode = each_baton->row_mode; STATEMENT_BEGIN(Each); } @@ -1203,7 +1277,7 @@ void Statement::AsyncEach(uv_async_t* handle) { napi_value converted = NULL; if (!async->stmt->ConvertCellRow(env, &row, keys, - &converted)) { + async->row_mode, &converted)) { // 'number' integer mode and an unsafe int64: hand the // RangeError to the item callback in place of the row. argv[0] = TakePendingError(env); @@ -1292,10 +1366,10 @@ void Statement::Work_AfterReset(napi_env e, napi_status status, void* data) { } } -// fetch(count, [params], [callback]): steps up to `count` rows and hands -// them back as one batch. Unlike all() the statement is deliberately NOT -// reset between calls, so successive fetches continue one cursor — this is -// the native half of the pull-based async iterator. +// fetch(count, [params], [{rowMode}], [callback]): steps up to `count` +// rows and hands them back as one batch. Unlike all() the statement is +// deliberately NOT reset between calls, so successive fetches continue one +// cursor — this is the native half of the pull-based async iterator. Napi::Value Statement::Fetch(const Napi::CallbackInfo& info) { auto env = info.Env(); Statement* stmt = this; @@ -1307,7 +1381,18 @@ Napi::Value Statement::Fetch(const Napi::CallbackInfo& info) { return env.Null(); } - FetchBaton* baton = stmt->Bind(info, 1); + // The bag/callback scan mirrors PopAsyncReadOptions, offset by the + // leading count argument. + int end = info.Length(); + Napi::Function callback; + if (end > 1 && info[end - 1].IsFunction()) { + callback = info[--end].As(); + } + int row_mode = SYNC_ROW_OBJECT; + if (end > 1 && ParseSyncReadOptions(info[end - 1], &row_mode)) end--; + if (env.IsExceptionPending()) return env.Null(); + + FetchBaton* baton = stmt->Bind(info, 1, end, callback); if (baton == NULL) { if (!env.IsExceptionPending()) { Napi::TypeError::New(env, "Data type is not supported") @@ -1317,6 +1402,7 @@ Napi::Value Statement::Fetch(const Napi::CallbackInfo& info) { } else { baton->count = count; + baton->row_mode = row_mode; stmt->Schedule(Work_BeginFetch, baton); return info.This(); } @@ -1381,7 +1467,7 @@ void Statement::Work_AfterFetch(napi_env e, napi_status status, void* data) { if (baton->rows.size()) { Napi::Array result; const bool failed = !stmt->CellRowsToJS(env, baton->rows, - baton->columns, &result); + baton->columns, baton->row_mode, &result); if (failed) { Napi::Value argv[] = { TakePendingError(env) }; @@ -1427,6 +1513,13 @@ bool Statement::IdleForInline() { void Statement::ThrowStatementError(Napi::Env env) { EXCEPTION(message, status, exception); + // Byte offset of the failing token when this error came from a failed + // prepare (sqlite3_error_offset; -1 otherwise). Consumed here: a later + // error must not inherit a stale prepare position. + if (error_offset >= 0) { + exception_obj.Set("offset", Napi::Number::New(env, error_offset)); + error_offset = -1; + } db->AttachPendingJsError(exception_obj); exception.As().ThrowAsJavaScriptException(); } @@ -1437,6 +1530,18 @@ bool Statement::SyncGate(Napi::Env env) { .ThrowAsJavaScriptException(); return false; } + if (sync_in_flight) { + // A user-defined function invoked from this statement's sync call + // is on the JS stack right now; re-entering the same VM is the one + // re-entrancy rule SQLite has (node:sqlite enforces the same since + // v26.8). Other statements on the connection remain usable. + Napi::Error::New(env, + "this statement is currently executing: a user-defined " + "function it invoked cannot drive the same statement " + "re-entrantly; use another statement or the asynchronous API" + ).ThrowAsJavaScriptException(); + return false; + } if (!IdleForInline()) { Napi::Error::New(env, "database is busy: sync methods require a fully idle database" @@ -1513,9 +1618,10 @@ Napi::Value Statement::GetSync(const Napi::CallbackInfo& info) { const bool bind_supplied = (end > 0); // While this thread is inside sqlite, a user-defined function invoked - // by the statement must refuse to make its round trip (it would wait - // for this very thread) — the guard is what its refusal tests. + // by the statement runs directly on this thread (see src/function.cc); + // the guards are what its re-entrancy checks test. Database::SyncSqliteGuard sync_guard(stmt->db); + Statement::SyncStepGuard step_guard(stmt); // Mirrors Work_Get: step unless the cursor is already exhausted and // no new parameters were supplied. @@ -1568,6 +1674,7 @@ Napi::Value Statement::RunSync(const Napi::CallbackInfo& info) { const bool bind_supplied = (end > 0); Database::SyncSqliteGuard sync_guard(stmt->db); + Statement::SyncStepGuard step_guard(stmt); // Mirrors Work_Run, including the explicit reset for parameterless // re-execution. @@ -1614,6 +1721,7 @@ Napi::Value Statement::AllSync(const Napi::CallbackInfo& info) { const bool bind_supplied = (end > 0); Database::SyncSqliteGuard sync_guard(stmt->db); + Statement::SyncStepGuard step_guard(stmt); if (!bind_supplied) { sqlite3_reset(stmt->_handle); @@ -1718,7 +1826,7 @@ void Statement::SyncColumnKeys(Napi::Env env, const Columns& columns) { Napi::Value Statement::Int64ToJS(Napi::Env env, sqlite3_int64 value, const std::string& what) { - return ConvertInt64ToJS(env, value, db->integer_mode, what); + return ConvertInt64ToJS(env, value, EffectiveIntegerMode(), what); } void Statement::RecordRunResult(sqlite3_int64 id, int changes) { @@ -1918,14 +2026,116 @@ Napi::Value Statement::Status(const Napi::CallbackInfo& info) { return Napi::Number::New(env, value); } +// Shared prelude for the SQL accessors: refuse (with a thrown error) when +// the live statement cannot be safely read right now. Returns true when +// the caller may touch _handle. +bool Statement::SQLAccessorGate(Napi::Env env) { + if (finalized) { + Napi::Error::New(env, "Statement is already finalized") + .ThrowAsJavaScriptException(); + return false; + } + if (!prepared) { + Napi::Error::New(env, + "Statement is not prepared yet").ThrowAsJavaScriptException(); + return false; + } + if (db->MayBlockOnWorkerRoundTrip()) { + Napi::Error::New(env, + "the SQL accessors cannot be read while a JavaScript function, " + "collation or progress callback is mid-call on this " + "connection; read them from a callback or after the query" + ).ThrowAsJavaScriptException(); + return false; + } + if (db->_handle == NULL) { + Napi::Error::New(env, "Database handle is closed") + .ThrowAsJavaScriptException(); + return false; + } + return true; +} + +// The statement's SQL with the most recent bound values substituted +// (sqlite3_expanded_sql). The result is sqlite3_malloc'd and freed here. +Napi::Value Statement::ExpandedSQLGetter(const Napi::CallbackInfo& info) { + auto env = info.Env(); + if (!SQLAccessorGate(env)) return env.Null(); + sqlite3_mutex* mtx = sqlite3_db_mutex(db->_handle); + sqlite3_mutex_enter(mtx); + char* text = sqlite3_expanded_sql(_handle); + std::string copy = text != NULL ? text : ""; + sqlite3_free(text); + sqlite3_mutex_leave(mtx); + return Napi::String::New(env, copy); +} + +// The statement's SQL with literals folded to `?` +// (sqlite3_normalized_sql; requires SQLITE_ENABLE_NORMALIZE). The string +// points into the statement itself and dies with the mutex leave. +Napi::Value Statement::NormalizedSQLGetter(const Napi::CallbackInfo& info) { + auto env = info.Env(); + if (!SQLAccessorGate(env)) return env.Null(); +#ifdef SQLITE_ENABLE_NORMALIZE + sqlite3_mutex* mtx = sqlite3_db_mutex(db->_handle); + sqlite3_mutex_enter(mtx); + const char* text = sqlite3_normalized_sql(_handle); + std::string copy = text != NULL ? text : ""; + sqlite3_mutex_leave(mtx); + return Napi::String::New(env, copy); +#else + Napi::Error::New(env, + "normalizedSQL requires a build with SQLITE_ENABLE_NORMALIZE") + .ThrowAsJavaScriptException(); + return env.Null(); +#endif +} + +// _setIntegerMode(mode | null): per-statement integer-mode override. +// JS-thread only (all row conversion happens there), applied by the JS +// layer immediately after construction. +Napi::Value Statement::SetIntegerMode(const Napi::CallbackInfo& info) { + auto env = info.Env(); + REQUIRE_ARGUMENT_STRING(0, mode); + int value; + if (mode == "number") value = Database::INTEGER_NUMBER; + else if (mode == "bigint") value = Database::INTEGER_BIGINT; + else if (mode == "mixed") value = Database::INTEGER_MIXED; + else { + Napi::TypeError::New(env, + "integer mode must be 'number', 'bigint' or 'mixed'") + .ThrowAsJavaScriptException(); + return env.Null(); + } + integer_mode_override = value; + has_integer_override = true; + return info.This(); +} + bool Statement::ConvertCellRow(Napi::Env env, Row* row, - const std::vector& keys, napi_value* out) { - const int mode = db->integer_mode; + const std::vector& keys, int row_mode, napi_value* out) { + const int mode = EffectiveIntegerMode(); const size_t key_count = keys.size(); + // Pluck serves the first column alone; no row container is built. + if (row_mode == SYNC_ROW_PLUCK) { + if (row->empty()) { + napi_value undef = NULL; + napi_get_undefined(env, &undef); + *out = undef; + return true; + } + bool raised = false; + Napi::Value value = CellToJS(env, (*row)[0], mode, + ValueOrigin(&column_keys_source, 0), true, &raised); + if (raised) return false; + *out = value; + return true; + } + // Same one-call-per-row build as the synchronous path; see // ConvertCurrentRow for why the store loop below is the slow shape. - napi_value factory = RowFactoryForShape(env, SYNC_ROW_OBJECT); + napi_value factory = RowFactoryForShape(env, row_mode); if (factory != NULL && row->size() == column_keys_source.size()) { const int cols = static_cast(row->size()); std::vector cells(row->size()); @@ -1939,6 +2149,24 @@ bool Statement::ConvertCellRow(Napi::Env env, Row* row, return CallRowFactory(env, factory, cells, cols, out); } + // No factory (codegen unavailable, too many columns, or a shape + // mismatch): the store loops, in the requested shape. + if (row_mode == SYNC_ROW_ARRAY) { + napi_value arr; + napi_create_array_with_length(env, row->size(), &arr); + size_t i = 0; + for (auto& cell : *row) { + bool raised = false; + Napi::Value value = CellToJS(env, cell, mode, + ValueOrigin(&column_keys_source, i), true, &raised); + if (raised) return false; + napi_set_element(env, arr, static_cast(i), value); + i++; + } + *out = arr; + return true; + } + napi_value result; napi_create_object(env, &result); @@ -1969,21 +2197,21 @@ bool Statement::ConvertCellRow(Napi::Env env, Row* row, return true; } -Napi::Value Statement::RowToJS(Napi::Env env, Row* row) { +Napi::Value Statement::RowToJS(Napi::Env env, Row* row, int row_mode) { Napi::EscapableHandleScope scope(env); std::vector keys; ResolveColumnKeys(&keys); napi_value result = NULL; - if (!ConvertCellRow(env, row, keys, &result)) { + if (!ConvertCellRow(env, row, keys, row_mode, &result)) { return scope.Escape(env.Null()); } return scope.Escape(Napi::Value(env, result)); } bool Statement::CellRowsToJS(Napi::Env env, Rows& rows, - const Columns& columns, Napi::Array* out) { + const Columns& columns, int row_mode, Napi::Array* out) { SyncColumnKeys(env, columns); Napi::Array result(Napi::Array::New(env, rows.size())); @@ -1999,12 +2227,12 @@ bool Statement::CellRowsToJS(Napi::Env env, Rows& rows, for (size_t start = 0; start < rows.size(); start += kBatch) { Napi::HandleScope batch(env); - ResolveColumnKeys(&keys); + if (row_mode == SYNC_ROW_OBJECT) ResolveColumnKeys(&keys); const size_t end = std::min(start + kBatch, rows.size()); for (size_t i = start; i < end; i++) { napi_value row = NULL; - if (!ConvertCellRow(env, &rows[i], keys, &row)) { + if (!ConvertCellRow(env, &rows[i], keys, row_mode, &row)) { // 'number' integer mode and an unsafe int64: the RangeError // is pending for the caller to deliver. return false; @@ -2122,9 +2350,25 @@ void Statement::ResolveColumnKeys(std::vector* out) { bool Statement::ConvertCurrentRow(Napi::Env env, const std::vector& keys, int row_mode, int cols, napi_value* out) { - const int mode = db->integer_mode; + const int mode = EffectiveIntegerMode(); const size_t key_count = keys.size(); + // Pluck serves the first column alone; no row container is built. + if (row_mode == SYNC_ROW_PLUCK) { + if (cols < 1) { + napi_value undef = NULL; + napi_get_undefined(env, &undef); + *out = undef; + return true; + } + bool raised = false; + Napi::Value value = ColumnToJS(env, _handle, 0, mode, + ValueOrigin(&column_keys_source, 0), &raised); + if (raised) return false; + *out = value; + return true; + } + // The fast shape: convert the cells into a plain argument vector and // let a generated monomorphic function build the row in one call. // Profiling showed the per-column store loops below spend two thirds of @@ -2256,6 +2500,19 @@ Napi::Value Statement::Finalize_(const Napi::CallbackInfo& info) { Statement* stmt = this; OPTIONAL_ARGUMENT_FUNCTION(0, callback); + // Refuse to finalize the statement sqlite is stepping right now: a + // user callback invoked re-entrantly from this statement's own + // getSync/runSync/allSync is on the stack, and sqlite3_finalize on a + // live VM is a use-after-free. Other statements are unaffected. + if (stmt->sync_in_flight) { + Napi::Error::New(env, + "this statement is currently executing: it cannot be " + "finalized from a user-defined function it invoked; finalize " + "it after the query completes" + ).ThrowAsJavaScriptException(); + return env.Null(); + } + auto *baton = new Baton(stmt, callback); stmt->Schedule(Finalize_, baton); diff --git a/src/statement.h b/src/statement.h index 131c1a9..dc7b8a4 100644 --- a/src/statement.h +++ b/src/statement.h @@ -28,9 +28,12 @@ class Statement : public Napi::ObjectWrap { // (`{ rowMode: 'array' }`), which skips the per-cell property stores // entirely — napi_set_element on a pre-sized array has no shape to // transition, which is what makes it the fastest row we can build. + // SYNC_ROW_PLUCK (`{ rowMode: 'pluck' }`) serves only the first result + // column — better-sqlite3's pluck(), as a per-call option. enum SyncRowMode { SYNC_ROW_OBJECT = 0, SYNC_ROW_ARRAY = 1, + SYNC_ROW_PLUCK = 2, }; static Napi::Object Init(Napi::Env env, Napi::Object exports); @@ -94,6 +97,10 @@ class Statement : public Napi::ObjectWrap { Baton(stmt_, cb_) {} Row row; Columns columns; + // Row shape requested through a trailing `{ rowMode: ... }` options + // bag on the asynchronous read paths (Phase 1 ergonomics parity): + // the same SYNC_ROW_* values the synchronous paths use. + int row_mode = SYNC_ROW_OBJECT; virtual ~RowBaton() override = default; }; @@ -110,6 +117,8 @@ class Statement : public Napi::ObjectWrap { Baton(stmt_, cb_) {} Rows rows; Columns columns; + // See RowBaton::row_mode. + int row_mode = SYNC_ROW_OBJECT; virtual ~RowsBaton() override = default; }; @@ -131,6 +140,8 @@ class Statement : public Napi::ObjectWrap { struct EachBaton : Baton { Napi::FunctionReference completed; Async* async; // Isn't deleted when the baton is deleted. + // See RowBaton::row_mode (published into the Async watcher). + int row_mode = SYNC_ROW_OBJECT; EachBaton(Statement* stmt_, Napi::Function cb_) : Baton(stmt_, cb_) {} @@ -210,6 +221,9 @@ class Statement : public Napi::ObjectWrap { // worker thread and read by the main thread, both under the mutex // below, so a mid-stream re-prepare cannot be missed. Columns columns; + // Row shape for the delivered rows (EachBaton::row_mode), set once + // before the work starts and read on the JS thread only. + int row_mode = SYNC_ROW_OBJECT; NODE_SQLITE3_MUTEX_t; bool completed; int retrieved; @@ -329,6 +343,26 @@ class Statement : public Napi::ObjectWrap { Napi::Value ParameterNamesGetter(const Napi::CallbackInfo& info); Napi::Value ColumnsGetter(const Napi::CallbackInfo& info); Napi::Value Status(const Napi::CallbackInfo& info); + // SQL text accessors (Phase 6): the statement's SQL with bound values + // substituted (sqlite3_expanded_sql) and with literals normalized to + // `?` (sqlite3_normalized_sql, needs SQLITE_ENABLE_NORMALIZE). Both + // read the live statement, so they refuse while a worker round trip + // could hold the connection mutex. + Napi::Value ExpandedSQLGetter(const Napi::CallbackInfo& info); + Napi::Value NormalizedSQLGetter(const Napi::CallbackInfo& info); + // Shared refusal prelude of the two SQL accessors: true when the live + // statement may be read right now. + bool SQLAccessorGate(Napi::Env env); + // Per-statement integer-mode override (Phase 6 parity): 0 = none + // (follow the connection's mode), otherwise one of INTEGER_NUMBER / + // INTEGER_BIGINT / INTEGER_MIXED. Applied by the JS layer right after + // construction, so it is only ever read on the JS thread (all row + // conversion happens there). + Napi::Value SetIntegerMode(const Napi::CallbackInfo& info); + int EffectiveIntegerMode() const { + return has_integer_override ? integer_mode_override + : db->integer_mode; + } protected: static void Work_BeginPrepare(Database::Baton* baton); @@ -342,7 +376,12 @@ class Statement : public Napi::ObjectWrap { void Finalize_(); template inline std::unique_ptr BindParameter(const Napi::Value source, T pos); - template T* Bind(const Napi::CallbackInfo& info, int start = 0, int end = -1); + // Binds [start, end) of the call into a baton, popping a trailing + // callback into the baton unless `preset_callback` carries one + // (Get/All/Each/Fetch pop it themselves when a `{ rowMode }` options + // bag sits between the bind parameters and the callback). + template T* Bind(const Napi::CallbackInfo& info, int start = 0, + int end = -1, Napi::Function preset_callback = Napi::Function()); // The bind-argument shapes (one array / N positional / one named // object), shared by Bind (into a Baton, for the queued async // paths) and called directly by the synchronous fast paths, which @@ -368,7 +407,8 @@ class Statement : public Napi::ObjectWrap { // Rebuilds the rooted JS key strings if `columns` differs from the set // they were built from. Call once per batch, before RowToJS. void SyncColumnKeys(Napi::Env env, const Columns& columns); - Napi::Value RowToJS(Napi::Env env, Row* row); + Napi::Value RowToJS(Napi::Env env, Row* row, + int row_mode = SYNC_ROW_OBJECT); // The scopeless core of RowToJS, and the asynchronous counterpart of // ConvertCurrentRow: converts one already-materialised Row into the @@ -376,7 +416,7 @@ class Statement : public Napi::ObjectWrap { // left a pending exception. Callers must store `*out` into a rooted JS // object before their scope closes. bool ConvertCellRow(Napi::Env env, Row* row, - const std::vector& keys, napi_value* out); + const std::vector& keys, int row_mode, napi_value* out); // Converts a whole materialised result set into a JS array, resolving // the column keys once and opening one HandleScope per batch of rows @@ -391,7 +431,7 @@ class Statement : public Napi::ObjectWrap { // Returns false when a row raised the RangeError, leaving it pending // for the caller to deliver to the callback. bool CellRowsToJS(Napi::Env env, Rows& rows, const Columns& columns, - Napi::Array* out); + int row_mode, Napi::Array* out); // The synchronous counterpart of RowToJS: builds the row object from // the live statement, with no intermediate Row. Requires the column // keys to have been synced for the current result shape. `row_mode` @@ -456,6 +496,20 @@ class Statement : public Napi::ObjectWrap { // run on the thread blocked inside SQLite. Throws; false means the // caller must return env.Null(). bool SyncGate(Napi::Env env); + // RAII marker for "this statement's VM is currently executing on the + // JS thread": a user-defined function invoked from the sync path can + // call back into the synchronous methods, but not into *this* + // statement — re-entering a stepping VM is the one hard rule SQLite + // has (node:sqlite enforces the same since v26.8). Other statements + // on the same connection stay legal: the connection mutex is + // recursive, so a nested step just re-enters it. + struct SyncStepGuard { + Statement* stmt; + explicit SyncStepGuard(Statement* s) : stmt(s) { stmt->sync_in_flight = true; } + ~SyncStepGuard() { stmt->sync_in_flight = false; } + SyncStepGuard(const SyncStepGuard&) = delete; + SyncStepGuard& operator=(const SyncStepGuard&) = delete; + }; // Throws the pending status/message as a JS error with errno/code. void ThrowStatementError(Napi::Env env); @@ -473,6 +527,24 @@ class Statement : public Napi::ObjectWrap { bool locked = true; bool finalized = false; + // True while this statement's VM is executing inside a *Sync call on + // the JS thread (see SyncStepGuard): a re-entrant sync call from a + // user-defined function must refuse rather than corrupt the VDBE. + bool sync_in_flight = false; + + // Byte offset of the failing token of the most recent failed prepare, + // from sqlite3_error_offset() (-1 when the last error was not a + // prepare error or there was no error). Attached to the thrown + // SqliteError as `offset` (bun:sqlite exposes the same as byteOffset; + // no other Node driver has it). Written under the connection mutex + // wherever a prepare fails; read on the JS thread when the error is + // built, which the async-work completion ordering serializes. + int error_offset = -1; + + // Per-statement integer-mode override (see SetIntegerMode). + int integer_mode_override = 0; + bool has_integer_override = false; + // Result of the most recent run(), exposed through the lastID, // lastIDBigInt and changes accessors. sqlite3_int64 last_insert_id = 0; diff --git a/src/vtab.cc b/src/vtab.cc new file mode 100644 index 0000000..6c71dd5 --- /dev/null +++ b/src/vtab.cc @@ -0,0 +1,1260 @@ +// JavaScript virtual tables (Phase 4). See src/vtab.h for the design. +// +// JavaScript is reached from xFilter (invoke the generator, take the first +// batch of rows) and from xNext when a cursor runs out of buffered rows — +// never from the prepare path, which is pure C++. Rows are pulled in +// growing batches (kVtabFirstBatch..kVtabMaxBatch) instead of draining the +// iterator, so `SELECT … LIMIT 3` over an unbounded generator stops after +// one batch instead of running forever, and a table larger than memory +// streams. On the sync path each pull is a direct re-entrant call (Phase 2 +// machinery, same guard: sync_sqlite_depth > 0 implies the JS thread); on +// a worker it blocks on the per-database vtab channel like a user +// function. + +#include +#include +#include +#include + +#include +#include +#include + +#include "macros.h" +#include "convert.h" +#include "database.h" +#include "vtab.h" + +using namespace node_sqlite3; + +namespace { + +// One xFilter (or factory-instantiation) round trip: the worker fills the +// request half, the JS thread runs the generator (or factory) and fills +// `rows`, then whoever waits is signalled. Direct sync calls fill the same +// struct inline without the channel. +// How many rows one pull asks the generator for. The first batch is small +// so a LIMIT 1 query does not run a generator 1024 times; later batches +// grow so a full scan pays one round trip per 1024 rows. +const size_t kVtabFirstBatch = 64; +const size_t kVtabMaxBatch = 1024; + +struct VtabCall { + // kCreate instantiates a factory module; kOpen invokes the generator + // and takes the first batch; kPull takes the next batch from the + // iterator a kOpen left on the cursor. + enum Kind { kCreate, kOpen, kPull }; + + Database* db; + VtabModule* module; + Kind kind = kOpen; + // Factory instantiation: the CREATE VIRTUAL TABLE argument strings, + // and the definition's rows generator is published as a raw napi_ref. + std::vector create_args; + napi_ref factory_rows = NULL; + + // xFilter: the hidden-parameter values, and the generator to invoke — + // the instance's factory-produced one (a raw napi_ref, because + // xDisconnect must be able to hand it back to the JS thread from any + // thread), or the module's (a FunctionReference read on this, the JS, + // thread). + napi_ref instance_rows = NULL; + Napi::FunctionReference* module_rows = NULL; + std::vector params; + // Which entries of `params` the query actually supplied: an + // unconstrained HIDDEN parameter reaches the generator as `undefined` + // (SQL NULL is a value someone passed on purpose). + std::vector params_supplied; + + // The live iterator: produced by kOpen, consumed by every kPull, owned + // by the cursor in between (a raw napi_ref for the same reason as + // instance_rows: xClose can run on a worker). + napi_ref iterator = NULL; + size_t want = kVtabFirstBatch; + bool exhausted = false; + + // Result half. + Rows rows_out; + bool errored = false; + std::string error; + + uv_mutex_t mutex; + uv_cond_t cond; + bool done = false; + + VtabCall(Database* db_, VtabModule* module_) : db(db_), module(module_) { + uv_mutex_init(&mutex); + uv_cond_init(&cond); + } + ~VtabCall() { + uv_mutex_destroy(&mutex); + uv_cond_destroy(&cond); + } +}; + +void DisposeVtabCall(VtabCall* call) { + delete call; +} + +// Applies one materialised Cell to a cursor column result slot (same +// mapping ApplyCell uses for function results in src/function.cc). +void ApplyCellToResult(sqlite3_context* ctx, const Cell& cell) { + switch (cell.type) { + case SQLITE_INTEGER: + sqlite3_result_int64(ctx, cell.integer); + break; + case SQLITE_FLOAT: + sqlite3_result_double(ctx, cell.real); + break; + case SQLITE_TEXT: + sqlite3_result_text(ctx, cell.str.data(), + static_cast(cell.str.size()), SQLITE_TRANSIENT); + break; + case SQLITE_BLOB: + sqlite3_result_blob(ctx, + cell.str.empty() ? "" : cell.str.data(), + static_cast(cell.str.size()), SQLITE_TRANSIENT); + break; + default: + sqlite3_result_null(ctx); + } +} + +// Converts one yielded JS value into a Cell via the shared bind converter +// (strict marshalling: no [object Object], no silent coercion). Returns +// false with the call marked errored. +bool YieldedValueToCell(Napi::Env env, Database* db, VtabCall* call, + Napi::Value value, const std::string& what, Cell* out) { + auto field = ConvertToField(value, what); + if (field == nullptr) { + call->errored = true; + napi_value pending = NULL; + napi_get_and_clear_last_exception(env, &pending); + Napi::Value err(env, pending); + if (err.IsObject()) { + Napi::Value msg = err.As().Get("message"); + if (!env.IsExceptionPending() && msg.IsString()) { + call->error = msg.As().Utf8Value(); + return false; + } + napi_get_and_clear_last_exception(env, &pending); + } + call->error = "a row yielded by virtual table module '" + + call->module->name + "' holds an unsupported value"; + return false; + } + switch (field->type) { + case SQLITE_INTEGER: + out->type = SQLITE_INTEGER; + out->integer = static_cast(field.get())->value; + break; + case SQLITE_FLOAT: + out->type = SQLITE_FLOAT; + out->real = static_cast(field.get())->value; + break; + case SQLITE_TEXT: + out->type = SQLITE_TEXT; + out->str = std::move(static_cast(field.get())->value); + break; + case SQLITE_BLOB: { + auto* f = static_cast(field.get()); + out->type = SQLITE_BLOB; + out->str.assign(f->value, f->length); + break; + } + default: + out->type = SQLITE_NULL; + } + return true; +} + +// Reads one row-shaped yielded value into `row`. Returns false with the +// call marked errored. +bool YieldedRowToCells(Napi::Env env, Database* db, VtabCall* call, + Napi::Value yielded, size_t ncols, Row* row) { + VtabModule* module = call->module; + const std::string where = "row " + + std::to_string(call->rows_out.size() + 1) + + " of the virtual table '" + module->name + "'"; + if (yielded.IsArray()) { + Napi::Array arr = yielded.As(); + uint32_t len = arr.Length(); + if (len > ncols) len = static_cast(ncols); + for (uint32_t i = 0; i < len; i++) { + Napi::Value v = arr.Get(i); + if (env.IsExceptionPending()) { + napi_value pending = NULL; + napi_get_and_clear_last_exception(env, &pending); + } + if (!YieldedValueToCell(env, db, call, v, where, &(*row)[i])) { + return false; + } + } + return true; + } + if (yielded.IsObject() && !yielded.IsFunction()) { + Napi::Object obj = yielded.As(); + for (size_t i = 0; i < ncols; i++) { + Napi::Value v = obj.Get(module->columns[i]); + if (env.IsExceptionPending()) { + call->errored = true; + call->error = "a row yielded by virtual table '" + + module->name + "' has a hostile property getter"; + napi_value pending = NULL; + napi_get_and_clear_last_exception(env, &pending); + return false; + } + if (!YieldedValueToCell(env, db, call, v, where, &(*row)[i])) { + return false; + } + } + return true; + } + // A bare value: the single-column-table convenience. + if (ncols == 1) { + return YieldedValueToCell(env, db, call, yielded, where, &(*row)[0]); + } + call->errored = true; + call->error = "a row yielded by virtual table '" + module->name + + "' must be an array or an object (the table has " + + std::to_string(ncols) + " columns)"; + return false; +} + +// Pulls up to call->want rows out of `iter`, appending them to +// call->rows_out and setting call->exhausted when the iterator is done. +void PullRows(Napi::Env env, Database* db, VtabCall* call, Napi::Object iter) { + VtabModule* module = call->module; + const size_t ncols = module->columns.size(); + + Napi::Value next_v = iter.Get("next"); + if (env.IsExceptionPending() || !next_v.IsFunction()) { + call->errored = true; + call->error = "the rows generator of virtual table '" + + module->name + "' did not return an iterable"; + napi_value pending = NULL; + napi_get_and_clear_last_exception(env, &pending); + return; + } + Napi::Function next_fn = next_v.As(); + + while (call->rows_out.size() < call->want) { + Napi::Value step = next_fn.Call(iter, {}); + if (env.IsExceptionPending()) { + call->errored = true; + call->error = "the rows generator of virtual table '" + + module->name + "' threw"; + napi_value pending = NULL; + napi_get_and_clear_last_exception(env, &pending); + Napi::Value err(env, pending); + if (err.IsObject()) { + Napi::Value msg = err.As().Get("message"); + if (!env.IsExceptionPending() && msg.IsString()) { + call->error += ": " + msg.As().Utf8Value(); + } + napi_get_and_clear_last_exception(env, &pending); + } + return; + } + if (!step.IsObject()) { + call->errored = true; + call->error = "the rows generator of virtual table '" + + module->name + "' produced a malformed iteration result"; + return; + } + Napi::Object result = step.As(); + Napi::Value done_v = result.Get("done"); + if (env.IsExceptionPending() || !done_v.IsBoolean()) { + call->errored = true; + call->error = "the rows generator of virtual table '" + + module->name + "' produced a malformed iteration result"; + napi_value pending = NULL; + napi_get_and_clear_last_exception(env, &pending); + return; + } + if (done_v.As().Value()) { + call->exhausted = true; + return; + } + Napi::Value yielded = result.Get("value"); + if (env.IsExceptionPending()) { + napi_value pending = NULL; + napi_get_and_clear_last_exception(env, &pending); + } + Row row(ncols); + if (!YieldedRowToCells(env, db, call, yielded, ncols, &row)) return; + call->rows_out.emplace_back(std::move(row)); + } +} + +// The JS-thread half of one VtabCall. +void ExecuteVtabCallOnJsThread(napi_env nenv, VtabCall* call) { + Napi::Env env(nenv); + Napi::HandleScope scope(env); + Database* db = call->db; + VtabModule* module = call->module; + + if (module->dead) { + call->errored = true; + call->error = "virtual table module '" + module->name + + "' was removed while a query against it was in flight"; + return; + } + + if (call->kind == VtabCall::kCreate) { + // Factory instantiation: run the factory with the CREATE VIRTUAL + // TABLE argument strings; the definition it returns supplies this + // instance's rows generator. + Napi::Function factory = module->factory.IsEmpty() + ? Napi::Function() : module->factory.Value(); + if (factory.IsEmpty()) { + call->errored = true; + call->error = "virtual table module '" + module->name + + "' has no factory"; + return; + } + std::vector argv; + argv.reserve(call->create_args.size()); + for (const auto& arg : call->create_args) { + argv.push_back(Napi::String::New(env, arg)); + } + Napi::Value definition = factory.Call(env.Undefined(), argv); + if (env.IsExceptionPending()) { + call->errored = true; + napi_value pending = NULL; + napi_get_and_clear_last_exception(env, &pending); + Napi::Value err(env, pending); + call->error = "the factory of virtual table module '" + + module->name + "' threw"; + if (err.IsObject()) { + Napi::Value msg = err.As().Get("message"); + if (!env.IsExceptionPending() && msg.IsString()) { + call->error += ": " + msg.As().Utf8Value(); + } + napi_get_and_clear_last_exception(env, &pending); + } + return; + } + Napi::Value rows = definition.IsObject() + ? definition.As().Get("rows") + : env.Undefined(); + if (env.IsExceptionPending()) { + napi_value pending = NULL; + napi_get_and_clear_last_exception(env, &pending); + } + if (!rows.IsFunction()) { + call->errored = true; + call->error = "the factory of virtual table module '" + + module->name + "' must return a definition with a rows " + + "generator function"; + return; + } + napi_create_reference(env, rows, 1, &call->factory_rows); + return; + } + + if (call->kind == VtabCall::kPull) { + // The cursor's iterator, mid-scan: take the next batch. + napi_value iter_v = NULL; + if (call->iterator != NULL) { + napi_get_reference_value(env, call->iterator, &iter_v); + } + if (iter_v == NULL) { + call->errored = true; + call->error = "the rows iterator of virtual table '" + + module->name + "' disappeared mid-scan"; + return; + } + PullRows(env, db, call, Napi::Object(env, iter_v)); + return; + } + + // kOpen (xFilter): convert the parameters, invoke the generator, take + // its iterator and the first batch of rows. + Napi::Function rows_fn; + if (call->instance_rows != NULL) { + napi_value fn = NULL; + napi_get_reference_value(env, call->instance_rows, &fn); + if (fn != NULL) rows_fn = Napi::Function(env, fn); + } + else if (call->module_rows != NULL && !call->module_rows->IsEmpty()) { + rows_fn = call->module_rows->Value(); + } + if (rows_fn.IsEmpty()) { + call->errored = true; + call->error = "virtual table module '" + module->name + + "' has no rows generator"; + return; + } + + const int integer_mode = db->IntegerMode(); + std::vector argv; + argv.reserve(call->params.size()); + for (size_t i = 0; i < call->params.size(); i++) { + const bool supplied = i < call->params_supplied.size() + && call->params_supplied[i]; + argv.push_back(supplied + ? CellToJS(env, call->params[i], integer_mode, + "argument " + std::to_string(i + 1) + + " of the virtual table '" + module->name + "'") + : static_cast(env.Undefined())); + if (env.IsExceptionPending()) { + call->errored = true; + call->error = "cannot pass an argument of the virtual table '" + + module->name + "' to JavaScript"; + napi_value pending = NULL; + napi_get_and_clear_last_exception(env, &pending); + return; + } + } + + Napi::Value iterable_v = rows_fn.Call(env.Undefined(), argv); + if (env.IsExceptionPending() || !iterable_v.IsObject()) { + call->errored = true; + call->error = "the rows generator of virtual table '" + + module->name + "' did not return an iterable"; + napi_value pending = NULL; + napi_get_and_clear_last_exception(env, &pending); + Napi::Value err(env, pending); + if (err.IsObject()) { + Napi::Value msg = err.As().Get("message"); + if (!env.IsExceptionPending() && msg.IsString()) { + call->error = "the rows generator of virtual table '" + + module->name + "' threw: " + + msg.As().Utf8Value(); + } + napi_get_and_clear_last_exception(env, &pending); + } + return; + } + + Napi::Object iterable = iterable_v.As(); + // for..of drives [Symbol.iterator]() first. + Napi::Value iter_fn = iterable.Get( + Napi::Symbol::WellKnown(env, "iterator")); + if (env.IsExceptionPending() || !iter_fn.IsFunction()) { + call->errored = true; + call->error = "the rows generator of virtual table '" + + module->name + "' did not return an iterable"; + napi_value pending = NULL; + napi_get_and_clear_last_exception(env, &pending); + return; + } + Napi::Value iter_v = iter_fn.As().Call(iterable, {}); + if (env.IsExceptionPending() || !iter_v.IsObject()) { + call->errored = true; + call->error = "the rows generator of virtual table '" + + module->name + "' threw"; + napi_value pending = NULL; + napi_get_and_clear_last_exception(env, &pending); + return; + } + Napi::Object iter = iter_v.As(); + // The cursor keeps the iterator alive between pulls; xClose hands the + // reference back to the JS thread (Database::QueueVtabRef). + if (napi_create_reference(env, iter, 1, &call->iterator) != napi_ok) { + call->errored = true; + call->error = "cannot retain the rows iterator of virtual table '" + + module->name + "'"; + return; + } + PullRows(env, db, call, iter); +} + +// The tsfn dispatch: runs the call, then wakes the worker (a direct sync +// call never goes through here). +void VtabCallJs(napi_env nenv, napi_value /*jsCallback*/, void* /*context*/, + void* data) { + if (data == NULL) return; + VtabCall* call = static_cast(data); + ExecuteVtabCallOnJsThread(nenv, call); + // No exception may escape into the tsfn dispatch machinery; always + // clear a stray one (the check must be napi_is_exception_pending — + // napi_get_and_clear_last_exception alone reports a non-NULL "last + // exception" even after every exception has been handled). + bool stray = false; + napi_is_exception_pending(nenv, &stray); + if (stray) { + napi_value pending = NULL; + napi_get_and_clear_last_exception(nenv, &pending); + if (!call->errored) { + call->errored = true; + call->error = "internal error while invoking a virtual table's " + + std::string("rows generator"); + } + } + uv_mutex_lock(&call->mutex); + call->done = true; + uv_cond_signal(&call->cond); + uv_mutex_unlock(&call->mutex); +} + +} // namespace + +namespace node_sqlite3 { + +// The sqlite3_module callbacks. Pure C++ except xConnect's factory half +// and xFilter's generator half, which round-trip (or run directly on the +// sync path). +struct VtabOps { + +// Builds the CREATE TABLE declaration handed to sqlite3_declare_vtab. +// The parameters are a subset of the columns (the better-sqlite3 +// contract): `parameters: ['n']` marks the column `n` HIDDEN, which is +// what turns the module into a table-valued function. +static std::string BuildDeclareSql(const VtabModule* module) { + std::string sql = "CREATE TABLE x("; + for (size_t i = 0; i < module->columns.size(); i++) { + if (i > 0) sql += ","; + sql += "\"" + module->columns[i] + "\""; + for (const auto& param : module->params) { + if (param == module->columns[i]) { + sql += " HIDDEN"; + break; + } + } + } + sql += ")"; + return sql; +} + +// Column index -> parameter position, or -1: the map xBestIndex/xFilter +// agree on. A parameter is whichever column the name names. +static int ParamIndex(const VtabModule* module, int col) { + if (col < 0 || col >= static_cast(module->columns.size())) { + return -1; + } + const std::string& name = module->columns[static_cast(col)]; + for (size_t p = 0; p < module->params.size(); p++) { + if (module->params[p] == name) { + return static_cast(p); + } + } + return -1; +} + +static int Connect(sqlite3* handle, void* aux, int argc, + const char* const* argv, sqlite3_vtab** vtab_out, + char** pzErr) { + auto* module = static_cast(aux); + Database* db = module->db; + + // Factory modules: the instance gets its own rows generator from the + // factory (one round trip / direct call). Eponymous modules use the + // registered generator directly. + auto* instance = new VtabInstance(); + instance->module = module; + + if (module->has_factory) { + VtabCall* call = new VtabCall(db, module); + call->kind = VtabCall::kCreate; + // argv[3..] are the CREATE VIRTUAL TABLE arguments, as SQL text. + for (int i = 3; i < argc; i++) { + call->create_args.emplace_back( + argv[i] != NULL ? argv[i] : ""); + } + if (db->sync_sqlite_depth > 0) { + ExecuteVtabCallOnJsThread(db->Env(), call); + // Same always-clear rule as VtabCallJs: no exception may + // escape into sqlite's C frames from the direct path. + bool stray = false; + napi_is_exception_pending(db->Env(), &stray); + if (stray) { + napi_value pending = NULL; + napi_get_and_clear_last_exception(db->Env(), &pending); + if (!call->errored) { + call->errored = true; + call->error = "internal error while invoking a virtual " + "table factory"; + } + } + } + else { + napi_status st = napi_call_threadsafe_function( + db->vtab_channel, call, napi_tsfn_blocking); + if (st != napi_ok) { + call->errored = true; + call->error = "the JavaScript environment is shutting down"; + } + else { + uv_mutex_lock(&call->mutex); + while (!call->done) uv_cond_wait(&call->cond, &call->mutex); + uv_mutex_unlock(&call->mutex); + } + } + if (call->errored) { + std::string err = call->error; + DisposeVtabCall(call); + delete instance; + *vtab_out = NULL; + if (pzErr != NULL) *pzErr = sqlite3_mprintf("%s", err.c_str()); + return SQLITE_ERROR; + } + instance->rows_ref = call->factory_rows; + call->factory_rows = NULL; + DisposeVtabCall(call); + } + + int rc = sqlite3_declare_vtab(handle, BuildDeclareSql(module).c_str()); + if (rc != SQLITE_OK) { + // Still on the creating thread's responsibility: hand the ref to + // the queue rather than deleting it here (this can run on a + // worker during CREATE VIRTUAL TABLE). + db->QueueVtabRef(instance->rows_ref); + delete instance; + *vtab_out = NULL; + return rc; + } + *vtab_out = &instance->base; + return SQLITE_OK; +} + +static int Disconnect(sqlite3_vtab* vtab) { + auto* instance = reinterpret_cast(vtab); + // Runs on whatever thread dropped the table or closed the database + // (a worker, for close). napi_delete_reference is not callable there, + // so the reference is queued and deleted on the JS thread — see + // Database::DrainVtabRefs. + instance->module->db->QueueVtabRef(instance->rows_ref); + delete instance; + return SQLITE_OK; +} + +// Factory modules are writable-shaped too in sqlite's eyes, but v1 is +// read-only: no xUpdate slot is registered at all. + +static int BestIndex(sqlite3_vtab* vtab, sqlite3_index_info* info) { + auto* instance = reinterpret_cast(vtab); + VtabModule* module = instance->module; + + // Equality constraints on the hidden parameters are the table-valued + // function's arguments. sqlite requires the argvIndex values to be + // 1..N with no gaps and no duplicates (it fails the statement with + // "xBestIndex malfunction" otherwise), so they are handed out in the + // order the constraints are consumed and the argv position -> declared + // parameter mapping travels to xFilter in idxStr. Assigning + // `parameter position + 1` instead — which is what this did — broke + // `WHERE b = ?` on a later parameter and any duplicate constraint on + // one parameter. + // + // A second constraint on an already-supplied parameter is left for + // sqlite to check against the column value (not omitted): omitting it + // would claim `count = 3 AND count = 4` holds. xFilter fills HIDDEN + // columns the generator left NULL with the argument value, so that + // check sees what the caller passed. + std::string mapping; + int argv_n = 0; + std::vector taken(module->params.size(), false); + for (int i = 0; i < info->nConstraint; i++) { + const auto& c = info->aConstraint[i]; + if (!c.usable || c.op != SQLITE_INDEX_CONSTRAINT_EQ) continue; + int p = ParamIndex(module, c.iColumn); + if (p < 0 || taken[static_cast(p)]) continue; + taken[static_cast(p)] = true; + info->aConstraintUsage[i].argvIndex = ++argv_n; + info->aConstraintUsage[i].omit = 1; + if (!mapping.empty()) mapping += ","; + mapping += std::to_string(p); + } + if (argv_n > 0) { + info->idxStr = sqlite3_mprintf("%s", mapping.c_str()); + if (info->idxStr == NULL) return SQLITE_NOMEM; + info->needToFreeIdxStr = 1; + } + info->estimatedCost = argv_n > 0 ? 10.0 : 1000000.0; + info->estimatedRows = argv_n > 0 ? 10 : 1000000; + info->idxNum = argv_n; + return SQLITE_OK; +} + +static int Open(sqlite3_vtab* vtab, sqlite3_vtab_cursor** cursor_out) { + auto* instance = reinterpret_cast(vtab); + auto* cursor = new VtabCursor(); + cursor->base.pVtab = vtab; + cursor->instance = instance; + *cursor_out = &cursor->base; + return SQLITE_OK; +} + +static int Close(sqlite3_vtab_cursor* cursor_base) { + auto* cursor = reinterpret_cast(cursor_base); + // Runs on whichever thread stepped the statement, so the iterator + // reference is handed to the JS thread (napi_delete_reference is not + // callable from a worker). An abandoned scan leaves the generator + // suspended and never resumes it — see src/vtab.h. + if (cursor->iterator != NULL) { + cursor->instance->module->db->QueueVtabRef(cursor->iterator); + cursor->iterator = NULL; + } + delete cursor; + return SQLITE_OK; +} + +static bool RunVtabCall(VtabCall* call, VtabInstance* instance) { + Database* db = instance->module->db; + if (db->sync_sqlite_depth > 0) { + // Direct re-entrant call: this is the JS thread inside a *Sync + // call (Phase 2 machinery). Same always-clear rule as VtabCallJs. + ExecuteVtabCallOnJsThread(db->Env(), call); + bool stray = false; + napi_is_exception_pending(db->Env(), &stray); + if (stray) { + napi_value pending = NULL; + napi_get_and_clear_last_exception(db->Env(), &pending); + if (!call->errored) { + call->errored = true; + call->error = + "internal error while invoking a virtual table's rows " + + std::string("generator"); + } + } + } + else { + napi_status st = napi_call_threadsafe_function( + db->vtab_channel, call, napi_tsfn_blocking); + if (st != napi_ok) { + call->errored = true; + call->error = + "the JavaScript environment is shutting down; the rows " + + std::string("generator of virtual table '") + + instance->module->name + "' cannot run"; + return false; + } + uv_mutex_lock(&call->mutex); + while (!call->done) uv_cond_wait(&call->cond, &call->mutex); + uv_mutex_unlock(&call->mutex); + } + return !call->errored; +} + +// Reports a failed pull on the vtab instance, replacing any previous +// message (sqlite frees zErrMsg when it imports it, but never assume). +static void SetVtabError(VtabInstance* instance, const std::string& message) { + if (instance->base.zErrMsg != NULL) { + sqlite3_free(instance->base.zErrMsg); + } + instance->base.zErrMsg = sqlite3_mprintf("%s", message.c_str()); +} + +// Fills the HIDDEN parameter columns a row left NULL with the argument +// xFilter received for them (see BestIndex). +static void ApplyCursorArgs(VtabCursor* cursor, Row* row) { + VtabModule* module = cursor->instance->module; + for (size_t p = 0; p < cursor->args.size(); p++) { + if (!cursor->has_arg[p]) continue; + // The column this parameter names. + for (size_t col = 0; col < module->columns.size(); col++) { + if (module->columns[col] != module->params[p]) continue; + if (col < row->size() && (*row)[col].type == SQLITE_NULL) { + (*row)[col] = cursor->args[p]; + } + break; + } + } +} + +// One batch: kOpen for the first (xFilter), kPull afterwards (xNext). +static int PullBatch(VtabCursor* cursor, bool first) { + auto* instance = cursor->instance; + VtabModule* module = instance->module; + + VtabCall* call = new VtabCall(module->db, module); + call->kind = first ? VtabCall::kOpen : VtabCall::kPull; + call->want = cursor->batch; + if (first) { + // Factory instances carry their own generator; eponymous instances + // use the module's. + if (instance->rows_ref != NULL) { + call->instance_rows = instance->rows_ref; + } + else { + call->module_rows = &module->rows; + } + call->params = cursor->args; + call->params_supplied = cursor->has_arg; + } + else { + call->iterator = cursor->iterator; + // The rows about to be replaced have all been delivered. + cursor->produced += cursor->rows.size(); + } + + bool ok = RunVtabCall(call, instance); + if (first && call->iterator != NULL) { + // Taken even on failure: the reference exists and must be freed. + cursor->iterator = call->iterator; + call->iterator = NULL; + } + if (!ok) { + std::string err = call->error; + DisposeVtabCall(call); + cursor->rows.clear(); + cursor->pos = 0; + cursor->exhausted = true; + SetVtabError(instance, err); + return SQLITE_ERROR; + } + cursor->rows = std::move(call->rows_out); + cursor->pos = 0; + cursor->exhausted = call->exhausted; + DisposeVtabCall(call); + if (!cursor->args.empty()) { + for (auto& row : cursor->rows) ApplyCursorArgs(cursor, &row); + } + // Grow the next batch: a scan pays one round trip per kVtabMaxBatch + // rows, a LIMIT query only the small first one. + if (cursor->batch < kVtabMaxBatch) { + cursor->batch = cursor->batch * 2 < kVtabMaxBatch + ? cursor->batch * 2 : kVtabMaxBatch; + } + return SQLITE_OK; +} + +static int Filter(sqlite3_vtab_cursor* cursor_base, int /*idxNum*/, + const char* idxStr, int argc, sqlite3_value** argv) { + auto* cursor = reinterpret_cast(cursor_base); + auto* instance = cursor->instance; + VtabModule* module = instance->module; + + // A cursor can be re-filtered (a correlated subquery re-runs it): drop + // the previous scan's iterator first. + if (cursor->iterator != NULL) { + module->db->QueueVtabRef(cursor->iterator); + cursor->iterator = NULL; + } + cursor->rows.clear(); + cursor->pos = 0; + cursor->exhausted = false; + cursor->batch = kVtabFirstBatch; + cursor->produced = 0; + + // idxStr maps argv positions onto declared parameters (BestIndex). + cursor->args.assign(module->params.size(), Cell()); + cursor->has_arg.assign(module->params.size(), false); + int consumed = 0; + if (idxStr != NULL) { + const char* p = idxStr; + while (*p != '\0' && consumed < argc) { + char* end = NULL; + long which = strtol(p, &end, 10); + if (end == p) break; + if (which >= 0 && which < static_cast(cursor->args.size())) { + ValueToCell(&cursor->args[static_cast(which)], + argv[consumed]); + cursor->has_arg[static_cast(which)] = true; + } + consumed++; + p = (*end == ',') ? end + 1 : end; + } + } + + return PullBatch(cursor, true); +} + +static int Next(sqlite3_vtab_cursor* cursor_base) { + auto* cursor = reinterpret_cast(cursor_base); + cursor->pos++; + if (cursor->pos < cursor->rows.size() || cursor->exhausted) { + return SQLITE_OK; + } + // The batch is spent and the generator has more: pull the next one. + return PullBatch(cursor, false); +} + +static int Eof(sqlite3_vtab_cursor* cursor_base) { + auto* cursor = reinterpret_cast(cursor_base); + // xFilter and xNext always leave a row buffered unless the generator + // is done, so an empty remainder means end of table. + return cursor->pos >= cursor->rows.size() ? 1 : 0; +} + +static int Column(sqlite3_vtab_cursor* cursor_base, + sqlite3_context* ctx, int col) { + auto* cursor = reinterpret_cast(cursor_base); + const size_t ncols = cursor->instance->module->columns.size(); + if (col < 0 || static_cast(col) >= ncols + || cursor->pos >= cursor->rows.size()) { + sqlite3_result_null(ctx); + return SQLITE_OK; + } + ApplyCellToResult(ctx, + cursor->rows[cursor->pos][static_cast(col)]); + return SQLITE_OK; +} + +static int Rowid(sqlite3_vtab_cursor* cursor_base, sqlite3_int64* rowid) { + auto* cursor = reinterpret_cast(cursor_base); + // Position in the whole scan, not in the current batch: a per-batch + // counter would hand out the same rowid to every batch. + *rowid = static_cast(cursor->produced + cursor->pos) + 1; + return SQLITE_OK; +} + +// xDestroy for the sqlite3_create_module_v2 registration. sqlite calls it +// once the registration has been replaced or dropped *and* every instance +// created against it has been disconnected (sqlite3VtabModuleUnref), which +// is exactly when the holder may die — including at sqlite3_close, which +// unrefs every module. Waiting for ~Database instead (what this used to +// do) meant a dropped module's generator closure — and everything it +// captured, an entire db.values() array — stayed alive for the life of the +// connection. +// +// It can run on a worker (close, or a DROP dispatched there), so the +// holder is handed to the JS thread, which frees it (and its +// FunctionReferences) from Database::DrainVtabRefs. +static void ModuleDestroy(void* aux) { + auto* module = static_cast(aux); + module->dead = true; + Database* db = module->db; + auto& live = db->js_vtabs; + for (auto it = live.begin(); it != live.end(); ++it) { + if (*it == module) { + live.erase(it); + break; + } + } + db->QueueVtabModule(module); +} + +static void ReportVtabError(Database* db, const std::string& message, + int rc = SQLITE_ERROR) { + Napi::Env env = db->Env(); + if (env.IsExceptionPending()) return; + Napi::HandleScope scope(env); + Napi::Error err = Napi::Error::New(env, message); + err.Value().As().Set("errno", Napi::Number::New(env, rc)); + Napi::Value info[] = { Napi::String::New(env, "error"), err.Value() }; + EMIT_EVENT(db->Value(), 2, info); +} + +// Builds the sqlite3_module for one registration. Eponymous-only modules +// leave xCreate NULL (the table exists by name immediately and cannot be +// CREATE VIRTUAL TABLE'd); factory modules set both xCreate and xConnect. +static sqlite3_module* MakeModule(bool has_factory) { + sqlite3_module* m = new sqlite3_module(); + // iVersion 0 keeps the modern fields unread (no xShadowName etc.). + m->iVersion = 0; + m->xCreate = has_factory ? Connect : NULL; + m->xConnect = Connect; + m->xBestIndex = BestIndex; + m->xDisconnect = Disconnect; + m->xDestroy = has_factory ? Disconnect : NULL; + m->xOpen = Open; + m->xClose = Close; + m->xFilter = Filter; + m->xNext = Next; + m->xEof = Eof; + m->xColumn = Column; + m->xRowid = Rowid; + m->xUpdate = NULL; + m->xBegin = NULL; + m->xSync = NULL; + m->xCommit = NULL; + m->xRollback = NULL; + m->xFindFunction = NULL; + m->xRename = NULL; + m->xSavepoint = NULL; + m->xRelease = NULL; + m->xRollbackTo = NULL; + m->xShadowName = NULL; + return m; +} + +static bool EnsureChannel(Database* db) { + if (db->vtab_channel != NULL) return true; + Napi::Env env = db->Env(); + Napi::Function noop = Napi::Function::New(env, + [](const Napi::CallbackInfo& info) { + return info.Env().Undefined(); + }); + napi_value resource_name = Napi::String::New(env, + "sqlite3.Database.Vtab"); + napi_threadsafe_function tsfn = NULL; + napi_status st = napi_create_threadsafe_function(env, noop, NULL, + resource_name, 0, 1, NULL, NULL, NULL, VtabCallJs, &tsfn); + if (st != napi_ok || tsfn == NULL) return false; + napi_unref_threadsafe_function(env, tsfn); + db->vtab_channel = tsfn; + return true; +} + +static void ReleaseChannelIfIdle(Database* db) { + if (db->vtab_channel != NULL && db->js_vtabs.empty()) { + napi_release_threadsafe_function(db->vtab_channel, + napi_tsfn_release); + db->vtab_channel = NULL; + } +} + +}; // struct VtabOps + +} // namespace node_sqlite3 + +namespace node_sqlite3 { + +// --- JS-visible entry points ----------------------------------------------- + +// The registration baton: the module holder travels to the exclusive +// handler through it. +struct VtabBaton : Database::Baton { + VtabModule* module = NULL; + explicit VtabBaton(Database* db_, VtabModule* module_) : + Baton(db_, Napi::Function()), module(module_) {} + virtual ~VtabBaton() override = default; +}; + +// The stub a removed module is replaced with: every prepare against the +// name fails loudly instead of silently using stale JavaScript. +static int RefusedConnect(sqlite3*, void*, int, const char* const*, + sqlite3_vtab** vtab_out, char** pzErr) { + *vtab_out = NULL; + if (pzErr != NULL) { + *pzErr = sqlite3_mprintf( + "this virtual table module was removed with db.removeTable()"); + } + return SQLITE_ERROR; +} + +// _registerVtab(name, columns[], params[], factory|null, rows|null). +// The JS layer validates the definition, flushes the statement cache and +// refuses while a sync-invoked callback is on the stack. +Napi::Value Database::RegisterVtab(const Napi::CallbackInfo& info) { + auto env = info.Env(); + auto* db = this; + + REQUIRE_ARGUMENT_STRING(0, name); + if (info.Length() < 2 || !info[1].IsArray()) { + Napi::TypeError::New(env, + "Argument 1 must be the column name array").ThrowAsJavaScriptException(); + return env.Null(); + } + if (info.Length() < 3 || !info[2].IsArray()) { + Napi::TypeError::New(env, + "Argument 2 must be the parameter name array").ThrowAsJavaScriptException(); + return env.Null(); + } + if (db->sync_sqlite_depth > 0) { + Napi::Error::New(env, + "virtual tables cannot be registered from inside a JavaScript " + "callback invoked by a synchronous method on this connection" + ).ThrowAsJavaScriptException(); + return env.Null(); + } + + auto* module = new VtabModule(); + module->db = db; + module->name = name; + + auto cols = info[1].As(); + module->columns.reserve(cols.Length()); + for (uint32_t i = 0; i < cols.Length(); i++) { + Napi::Value c = cols.Get(i); + if (!c.IsString()) { + delete module; + Napi::TypeError::New(env, "column names must be strings") + .ThrowAsJavaScriptException(); + return env.Null(); + } + module->columns.push_back(c.As().Utf8Value()); + } + auto params = info[2].As(); + module->params.reserve(params.Length()); + for (uint32_t i = 0; i < params.Length(); i++) { + Napi::Value p = params.Get(i); + if (!p.IsString()) { + delete module; + Napi::TypeError::New(env, "parameter names must be strings") + .ThrowAsJavaScriptException(); + return env.Null(); + } + module->params.push_back(p.As().Utf8Value()); + } + + if (info.Length() > 3 && info[3].IsFunction()) { + module->has_factory = true; + module->factory.Reset(info[3].As(), 1); + } + else if (info.Length() > 4 && info[4].IsFunction()) { + module->rows.Reset(info[4].As(), 1); + } + else { + delete module; + Napi::TypeError::New(env, + "a virtual table definition requires a rows generator (or a " + "factory function)").ThrowAsJavaScriptException(); + return env.Null(); + } + db->Schedule(Work_RegisterVtab, new VtabBaton(db, module), true); + return info.This(); +} + +// _removeVtab(name). +Napi::Value Database::RemoveVtab(const Napi::CallbackInfo& info) { + auto env = info.Env(); + auto* db = this; + + REQUIRE_ARGUMENT_STRING(0, name); + if (db->sync_sqlite_depth > 0) { + Napi::Error::New(env, + "virtual tables cannot be removed from inside a JavaScript " + "callback invoked by a synchronous method on this connection" + ).ThrowAsJavaScriptException(); + return env.Null(); + } + + auto* baton = new Baton(db, Napi::Function()); + baton->message = name; + db->Schedule(Work_RemoveVtab, baton, true); + return info.This(); +} + +void Database::Work_RegisterVtab(Baton* b) { + auto baton = std::unique_ptr(static_cast(b)); + auto* db = baton->db; + auto* module = baton->module; + + assert(db->IsOpen()); + assert(db->_handle); + assert(db->pending == 0); + + // A same-name re-registration replaces the module: sqlite disconnects + // the old registration's instances and unrefs the old Module, which + // calls VtabOps::ModuleDestroy — the holder is freed then, not here, + // because a stepping VM may still hold an instance created against it + // and its sqlite3_module must outlive that. + for (auto* existing : db->js_vtabs) { + if (existing->name == module->name) existing->dead = true; + } + + if (!VtabOps::EnsureChannel(db)) { + VtabOps::ReportVtabError(db, "cannot create the virtual table " + "round-trip channel", SQLITE_NOMEM); + delete module; + db->exclusiveHeld = false; + db->Process(); + return; + } + + module->module = VtabOps::MakeModule(module->has_factory); + int rc = sqlite3_create_module_v2(db->_handle, module->name.c_str(), + module->module, module, VtabOps::ModuleDestroy); + if (rc != SQLITE_OK) { + VtabOps::ReportVtabError(db, + "cannot register virtual table module '" + module->name + + "': " + std::string(sqlite3_errmsg(db->_handle)), rc); + delete module->module; + module->module = NULL; + delete module; + } + else { + db->js_vtabs.push_back(module); + } + + db->exclusiveHeld = false; + db->Process(); +} + +void Database::Work_RemoveVtab(Baton* b) { + auto baton = std::unique_ptr(b); + auto* db = baton->db; + + assert(db->IsOpen()); + assert(db->_handle); + assert(db->pending == 0); + + // sqlite3_drop_modules works from a keep-list (it would also drop the + // built-in fts5/rtree/... modules), so removal replaces the + // registration with a stub whose xConnect refuses. sqlite unrefs (and + // disconnects the instances of) the module being replaced. + static sqlite3_module refused_module = []() { + sqlite3_module m = {}; + m.iVersion = 0; + m.xConnect = RefusedConnect; + return m; + }(); + bool found = false; + for (auto* module : db->js_vtabs) { + if (module->name == baton->message && !module->dead) { + found = true; + module->dead = true; + } + } + int rc = SQLITE_OK; + if (found) { + rc = sqlite3_create_module(db->_handle, baton->message.c_str(), + &refused_module, NULL); + if (rc != SQLITE_OK) { + VtabOps::ReportVtabError(db, + "cannot remove virtual table module '" + baton->message + + "': " + std::string(sqlite3_errmsg(db->_handle)), rc); + } + } + // The holder is freed by VtabOps::ModuleDestroy once sqlite has + // disconnected the last instance created against it; until then the + // dead flag fails any round trip that could still reach one. + VtabOps::ReleaseChannelIfIdle(db); + + db->exclusiveHeld = false; + db->Process(); +} + +void Database::RemoveVtabs() { + // ~Database, after sqlite3_close: every module was unrefed by the + // close, so VtabOps::ModuleDestroy has already queued each holder — + // draining frees them. Whatever is left in js_vtabs was never handed + // to sqlite (a failed sqlite3_create_module, or a connection that + // never opened), so it is ours to free. + DrainVtabRefs(); + for (auto* module : js_vtabs) { + delete module->module; + module->module = NULL; + delete module; + } + js_vtabs.clear(); +} + +void Database::QueueVtabRef(napi_ref ref) { + if (ref == NULL) return; + uv_mutex_lock(&vtab_refs_mutex); + pending_vtab_refs.push_back(ref); + uv_mutex_unlock(&vtab_refs_mutex); +} + +void Database::QueueVtabModule(VtabModule* module) { + if (module == NULL) return; + uv_mutex_lock(&vtab_refs_mutex); + pending_vtab_modules.push_back(module); + uv_mutex_unlock(&vtab_refs_mutex); +} + +void Database::DrainVtabRefs() { + // JS thread only: deleting the queued instance references and module + // holders is napi work (a holder owns FunctionReferences), which is why + // xDisconnect and xDestroy queued them instead. + uv_mutex_lock(&vtab_refs_mutex); + std::vector refs; + refs.swap(pending_vtab_refs); + std::vector modules; + modules.swap(pending_vtab_modules); + uv_mutex_unlock(&vtab_refs_mutex); + for (napi_ref ref : refs) { + napi_delete_reference(Env(), ref); + } + for (VtabModule* module : modules) { + // The sqlite3_module struct dies with the holder: sqlite has + // dropped its last reference to both by now. + delete module->module; + module->module = NULL; + delete module; + } +} + +bool Database::EnsureVtabChannel() { + return VtabOps::EnsureChannel(this); +} + +void Database::ReleaseVtabChannelIfIdle() { + VtabOps::ReleaseChannelIfIdle(this); +} + +} // namespace node_sqlite3 diff --git a/src/vtab.h b/src/vtab.h new file mode 100644 index 0000000..ad17470 --- /dev/null +++ b/src/vtab.h @@ -0,0 +1,113 @@ +#ifndef NODE_SQLITE3_SRC_VTAB_H +#define NODE_SQLITE3_SRC_VTAB_H + +// JavaScript virtual tables (Phase 4): `db.table(name, definition)`. +// +// The better-sqlite3-proven shape, read-only in v1: a JS generator +// function produces rows, described by a column list; entries of +// `parameters` are declared HIDDEN, which turns the module into a +// table-valued function (`SELECT * FROM name(arg)` passes `arg` to the +// generator through an equality constraint — see xBestIndex). A factory +// function instead of a definition registers a named module instantiated +// per `CREATE VIRTUAL TABLE ... USING name(args)`. +// +// Threading: +// +// - xBestIndex/xEof/xColumn/xRowid are pure C++ over the rows buffered on +// the cursor, so the query planner (prepare path) never calls +// JavaScript — the same design principle as the C++ authorizer. +// - xFilter invokes the generator and buffers its first batch of rows; +// xNext pulls another batch only when the cursor has consumed the last +// one (64 rows, doubling to 1024). Batching is what makes `LIMIT 3` +// over an unbounded generator terminate, and what keeps a table larger +// than memory streamable, while a full scan still pays only one round +// trip per 1024 rows. On a worker each pull is one blocking round trip +// to the JS thread; on the synchronous path (sync_sqlite_depth > 0) it +// is a direct re-entrant call, like Phase 2's user functions. +// - A scan abandoned early (LIMIT, or an error) drops the iterator +// without resuming it: the generator is left suspended, so a `finally` +// inside it does not run. Generator cleanup is not a place to release +// resources. +// - Registration/removal are scheduled exclusively and refuse from +// inside a sync-invoked callback (a stepping VM holds the module). +// +// Instances keep a raw VtabModule*. The registry on Database owns the +// holders, and sqlite decides when one may die: it calls the +// sqlite3_create_module_v2 destructor (VtabOps::ModuleDestroy) once the +// registration has been replaced or dropped *and* every instance created +// against it has been disconnected. That destructor can fire on a worker +// (sqlite3_close), where napi calls are illegal, so it hands the holder to +// Database::QueueVtabModule and the JS thread frees it (Process / +// ~Database). RemoveVtabs only mops up holders sqlite never took. + +#include +#include + +#include +#include +#include + +#include "convert.h" +#include "database.h" + +namespace node_sqlite3 { + +// One registered module. Columns/params are captured here so xConnect can +// build the declare_vtab DDL without JavaScript. +struct VtabModule { + Database* db; + std::string name; + std::vector columns; // visible result columns + std::vector params; // HIDDEN table-function parameters + bool has_factory = false; + Napi::FunctionReference factory; // has_factory: (args...) => definition + Napi::FunctionReference rows; // otherwise: the generator itself + sqlite3_module* module = NULL; // the registration handed to sqlite + bool dead = false; // dropped; instances may still hold it +}; + +// Per-instance state (eponymous: one per connection; factory: one per +// CREATE VIRTUAL TABLE). Carries the factory-produced definition when the +// module was registered through a factory. +struct VtabInstance { + sqlite3_vtab base; + VtabModule* module; + // A factory instantiation's own rows generator (NULL for eponymous + // instances, which use the module's). A raw napi_ref because + // xDisconnect — which must hand it back — can run on a worker thread + // (sqlite3_close), where napi calls are not allowed; it queues onto + // Database::pending_vtab_refs instead and the JS thread deletes it. + napi_ref rows_ref = NULL; +}; + +struct VtabCursor { + sqlite3_vtab_cursor base; + VtabInstance* instance; + Rows rows; // the current batch + size_t pos = 0; // position within the batch + // The live iterator (a raw napi_ref: xClose can run on a worker, so + // the reference is queued for the JS thread like rows_ref), the next + // batch size, and whether the generator is done. + napi_ref iterator = NULL; + size_t batch = 0; + bool exhausted = false; + // Rows delivered by earlier batches, so xRowid stays monotonic across + // batch boundaries (it is `produced + pos + 1`). + size_t produced = 0; + // The hidden-parameter values xFilter received, indexed by parameter + // position, and which of them were supplied: a row that leaves a + // HIDDEN column NULL reports the argument the table-valued function + // was called with, so `SELECT count FROM seq(3)` and a second + // equality constraint on the same parameter both behave. + std::vector args; + std::vector has_arg; +}; + +// JS-visible entry points on Database (wrapped by lib/sqlite3.js, which +// validates the definition and flushes the statement cache). Declarations +// live in database.h alongside the other registration surface; this file +// declares the module/instance/cursor types above. + +} // namespace node_sqlite3 + +#endif diff --git a/test/aggregate.test.js b/test/aggregate.test.js index 4a5c5b5..d4e871b 100644 --- a/test/aggregate.test.js +++ b/test/aggregate.test.js @@ -212,14 +212,33 @@ describe('user-defined aggregates', function () { ); }); - it('refuses invocation from the sync methods instead of deadlocking', { + it('runs aggregates directly from the sync methods (re-entrant)', { timeout: 5000, }, async function () { db.aggregate('total', totalAggregate()); - await db.exec('CREATE TABLE t (x INT); INSERT INTO t VALUES (1)'); - assert.throws( - () => db.getSync('SELECT total(x) AS v FROM t'), - /deadlock/, + await db.exec( + 'CREATE TABLE t (x INT); INSERT INTO t VALUES (1), (2), (3)', + ); + assert.strictEqual(db.getSync('SELECT total(x) AS v FROM t').v, 6); + // Window functions (inverse) take the same direct path. + db.aggregate('winsum', { + start: () => 0, + step: (acc, v) => acc + v, + result: (acc) => acc, + inverse: (acc, v) => acc - v, + }); + const rows = db.allSync( + 'SELECT winsum(x) OVER (ORDER BY x ROWS BETWEEN 1 PRECEDING ' + + 'AND CURRENT ROW) AS s FROM t', + ); + assert.deepStrictEqual( + rows.map((r) => r.s), + [1, 3, 5], + ); + // An empty group still evaluates start()+result(). + assert.strictEqual( + db.getSync('SELECT total(x) AS v FROM t WHERE 0').v, + 0, ); }); diff --git a/test/compat.test.js b/test/compat.test.js new file mode 100644 index 0000000..1b15478 --- /dev/null +++ b/test/compat.test.js @@ -0,0 +1,178 @@ +import assert from 'node:assert'; +import { afterEach, beforeEach, describe, it } from 'node:test'; + +import { DatabaseSync } from '@appthreat/sqlite3/compat'; + +import { DatabaseSync as DatabaseSyncRelative } from '../lib/compat.js'; +import sqlite3 from '../lib/sqlite3.js'; + +// Phase 5: the node:sqlite compatibility shim. + +describe('node:sqlite compat shim', function () { + /** @type {DatabaseSync} */ + let db; + + beforeEach(function () { + db = new DatabaseSync(':memory:'); + }); + + afterEach(function () { + if (db.isOpen) db.close(); + }); + + it('is reachable through the package /compat subpath export', function () { + // Regression: the package had no exports map, so the + // '@appthreat/sqlite3/compat' specifier the README documents could + // not resolve at all. + assert.strictEqual(DatabaseSync, DatabaseSyncRelative); + }); + + it('opens synchronously and reports state', function () { + assert.strictEqual(db.isOpen, true); + assert.strictEqual(db.open, true); + assert.strictEqual(db.isTransaction, false); + }); + + it('exec runs multi-statement scripts', function () { + db.exec( + 'CREATE TABLE t (a);\n' + + 'INSERT INTO t VALUES (1);\n' + + "INSERT INTO t VALUES (';not-a-boundary;');\n", + ); + assert.deepStrictEqual( + db.prepare('SELECT COUNT(*) AS n FROM t').get(), + { + n: 2, + }, + ); + }); + + it('prepare/get/all/run/iterate behave like StatementSync', function () { + db.exec('CREATE TABLE t (a); INSERT INTO t VALUES (1), (2)'); + const stmt = db.prepare('SELECT a FROM t WHERE a = ?'); + assert.deepStrictEqual(stmt.get(2), { a: 2 }); + // Re-running reuses the last bindings (node:sqlite's semantics). + assert.deepStrictEqual(stmt.all(), [{ a: 2 }]); + const insert = db.prepare('INSERT INTO t VALUES (?)'); + const result = insert.run(3); + assert.strictEqual(result.changes, 1); + assert.strictEqual(result.lastInsertRowid, 3); + assert.deepStrictEqual(db.prepare('SELECT a FROM t ORDER BY a').all(), [ + { a: 1 }, + { a: 2 }, + { a: 3 }, + ]); + assert.deepStrictEqual([...stmt.iterate(1)], [{ a: 1 }]); + insert.close(); + assert.deepStrictEqual(stmt.columns(), [ + { name: 'a', table: 't', database: 'main', column: 'a' }, + ]); + assert.strictEqual(stmt.sourceSQL, 'SELECT a FROM t WHERE a = ?'); + assert.strictEqual(stmt.expandedSQL, 'SELECT a FROM t WHERE a = 1'); + stmt.close(); + }); + + it('supports the readBigInts and returnArrays toggles', function () { + db.exec('CREATE TABLE t (a)'); + db.prepare('INSERT INTO t VALUES (9223372036854775807)').run(); + const bigint = db.prepare('SELECT a FROM t', { readBigInts: true }); + assert.strictEqual(typeof bigint.get().a, 'bigint'); + const arrays = db.prepare('SELECT 1 AS a, 2 AS b'); + arrays.setReturnArrays(true); + assert.deepStrictEqual(arrays.get(), [1, 2]); + }); + + it('registers functions and aggregates that run synchronously', function () { + db.function('double', (x) => x * 2); + const call = db.prepare('SELECT double(21) AS v'); + assert.strictEqual(call.get().v, 42); + call.close(); + db.aggregate('total', { + start: () => 0, + step: (acc, v) => acc + v, + result: (acc) => acc, + }); + db.exec('CREATE TABLE n (x); INSERT INTO n VALUES (1), (2), (3)'); + assert.strictEqual( + db.prepare('SELECT total(x) AS v FROM n').get().v, + 6, + ); + }); + + it('tracks transactions', function () { + db.exec('CREATE TABLE t (a)'); + db.exec('BEGIN'); + assert.strictEqual(db.isTransaction, true); + db.exec('COMMIT'); + assert.strictEqual(db.isTransaction, false); + }); + + it('enableDefensive toggles defensive mode', function () { + db.exec('CREATE TABLE t (a)'); + db.enableDefensive(true); + assert.throws(() => db.exec('CREATE TABLE sqlite_schemahack (x)')); + db.enableDefensive(false); + }); + + it('location() and the native passthrough work', function () { + // node:sqlite reports null for an in-memory database; the + // package's own location() reports the empty string there. + assert.strictEqual(db.location(), null); + assert.strictEqual(db.native.location(), ''); + assert.ok(db.native instanceof sqlite3.Database); + }); + + it('setReadBigInts changes how columns read, not just lastInsertRowid', function () { + db.exec('CREATE TABLE big (v INTEGER)'); + db.prepare('INSERT INTO big VALUES (?)').run(42); + const stmt = db.prepare('SELECT v FROM big'); + assert.strictEqual(stmt.get().v, 42); + // The integer mode belongs to the prepared statement here, so the + // setter re-prepares; it used to silently affect nothing. + stmt.setReadBigInts(true); + assert.strictEqual(stmt.get().v, 42n); + stmt.setReadBigInts(false); + assert.strictEqual(stmt.get().v, 42); + stmt.close(); + }); + + it('setAuthorizer fails loudly with the documented alternative', function () { + assert.throws(() => db.setAuthorizer(null), /declarative/i); + }); + + it('validates constructor and prepare options', function () { + assert.throws( + () => new DatabaseSync(':memory:', { bogus: 1 }), + /unknown option 'bogus'/, + ); + assert.throws( + () => new DatabaseSync(':memory:', { open: false }), + /no equivalent/, + ); + assert.throws( + () => db.prepare('SELECT 1', { bogus: 1 }), + /unknown option 'bogus'/, + ); + }); + + it('loadExtension is gated by allowExtension', function () { + assert.throws(() => db.loadExtension('whatever.so'), /allowExtension/); + const gated = new DatabaseSync(':memory:', { + allowExtension: true, + }); + gated.enableLoadExtension(true); + assert.throws(() => gated.loadExtension('nope', 'entry'), /entry/); + gated.close(); + }); + + it('dispose support works', function () { + { + using stmt = db.prepare('SELECT 1'); + assert.ok(!stmt.finalized); + } + // Close via the async dispose path. + const closing = db; + db = new DatabaseSync(':memory:'); + return closing[Symbol.asyncDispose](); + }); +}); diff --git a/test/diagnostics.test.js b/test/diagnostics.test.js new file mode 100644 index 0000000..3672fb5 --- /dev/null +++ b/test/diagnostics.test.js @@ -0,0 +1,176 @@ +import assert from 'node:assert'; +import diagnostics_channel from 'node:diagnostics_channel'; +import { afterEach, beforeEach, describe, it } from 'node:test'; + +import sqlite3 from '../lib/sqlite3.js'; + +// Phase 6: the diagnostics_channel query spans. + +// Spans are delivered through a uv_async handle, so they arrive on a +// later loop turn — not necessarily the next one. A single setImmediate +// happened to be enough on macOS and was not on Windows; poll instead. +/** + * @param {() => boolean} predicate the condition to wait for. + * @param {string} what described in the timeout message. + * @returns {Promise} resolves once the predicate holds. + */ +async function waitFor(predicate, what) { + const deadline = Date.now() + 5000; + while (!predicate()) { + if (Date.now() > deadline) { + throw new Error(`timed out waiting for ${what}`); + } + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + +describe('diagnostics_channel', function () { + /** @type {sqlite3.Database} */ + let db; + + beforeEach(async function () { + db = new sqlite3.Database(':memory:'); + await db.exec('CREATE TABLE t (a)'); + }); + + afterEach(async function () { + await db.close(); + }); + + it('publishes query spans while subscribed and stops after', async function () { + /** @type {any[]} */ + const spans = []; + const unsubscribe = sqlite3.subscribeQueries((span) => + spans.push(span), + ); + try { + await db.get('SELECT * FROM t'); + await db.run('INSERT INTO t VALUES (1)'); + await waitFor(() => spans.length >= 2, 'two query spans'); + assert.strictEqual(spans.length, 2); + assert.strictEqual(spans[0].sql, 'SELECT * FROM t'); + assert.strictEqual(spans[0].database, db); + assert.ok(typeof spans[0].durationMs === 'number'); + assert.ok(typeof spans[0].duration === 'bigint'); + } finally { + unsubscribe(); + } + const before = spans.length; + await db.get('SELECT 1'); + // Nothing should arrive; give it the same grace a span would get. + await new Promise((resolve) => setTimeout(resolve, 50)); + assert.strictEqual(spans.length, before); + }); + + it('coexists with a profile listener the application registered', async function () { + /** @type {string[]} */ + const mine = []; + /** @type {any[]} */ + const spans = []; + const listener = (/** @type {string} */ sql) => mine.push(sql); + db.on('profile', listener); + db.configure('profile', true); + + const unsubscribe = sqlite3.subscribeQueries((span) => + spans.push(span), + ); + await db.get('SELECT 1 AS v'); + await waitFor( + () => spans.length >= 1 && mine.length >= 1, + 'the span and the application listener', + ); + // Arming used to bail out whenever any 'profile' listener existed, + // which made subscribeQueries silently publish nothing. + assert.strictEqual(spans.length, 1); + assert.strictEqual(mine.length, 1); + + unsubscribe(); + // ... and unsubscribing used to removeAllListeners('profile'), + // taking the application's listener with it. + assert.strictEqual(db.listenerCount('profile'), 1); + await db.get('SELECT 2 AS v'); + await waitFor(() => mine.length >= 2, "the application's second span"); + assert.strictEqual(mine.length, 2); + assert.strictEqual(spans.length, 1); + db.removeListener('profile', listener); + }); + + it('does not pin connections that are never closed', async function () { + // The registry that lets a late subscriber arm existing + // connections holds them weakly; a strong Set leaked the + // connection, its sqlite handle and its file descriptor. + const { execFileSync } = await import('node:child_process'); + const script = ` + import sqlite3 from './lib/sqlite3.js'; + const tick = () => new Promise((r) => setTimeout(r, 20)); + let collected = 0; + const registry = new FinalizationRegistry(() => collected++); + for (let i = 0; i < 20; i++) { + const db = await sqlite3.open(':memory:'); + registry.register(db, i); + await db.get('SELECT 1'); + } + global.gc(); await tick(); global.gc(); await tick(); global.gc(); + await tick(); + console.log(collected); + `; + const out = execFileSync( + process.execPath, + ['--expose-gc', '--input-type=module', '-e', script], + { encoding: 'utf8', cwd: new URL('..', import.meta.url) }, + ); + assert.ok( + Number(out.trim()) >= 19, + `only ${out.trim()} of 20 unclosed connections were collected`, + ); + }); + + it('mirrors onto the node:sqlite channel name', async function () { + /** @type {any[]} */ + const mirrored = []; + const onMessage = (message) => mirrored.push(message); + diagnostics_channel.subscribe('sqlite.db.query', onMessage); + const unsubscribe = sqlite3.subscribeQueries(() => undefined); + try { + await db.get('SELECT 42 AS v'); + await waitFor(() => mirrored.length >= 1, 'the mirrored span'); + assert.strictEqual(mirrored.length, 1); + assert.strictEqual(mirrored[0].sql, 'SELECT 42 AS v'); + assert.ok(typeof mirrored[0].duration === 'bigint'); + } finally { + unsubscribe(); + diagnostics_channel.unsubscribe('sqlite.db.query', onMessage); + } + }); + + it('a late subscriber arms an existing connection', async function () { + /** @type {any[]} */ + const spans = []; + const unsubscribe = sqlite3.subscribeQueries((span) => + spans.push(span), + ); + try { + await db.all('SELECT 1'); + await waitFor(() => spans.length >= 1, 'the late subscriber span'); + assert.strictEqual(spans.length, 1); + } finally { + unsubscribe(); + } + }); + + it('validates the listener', function () { + assert.throws(() => sqlite3.subscribeQueries(7), TypeError); + }); + + it('a throwing consumer does not break the query', async function () { + const unsubscribe = sqlite3.subscribeQueries(() => { + throw new Error('consumer bug'); + }); + try { + const row = await db.get('SELECT 7 AS v'); + assert.strictEqual(row.v, 7); + } finally { + unsubscribe(); + } + }); +}); diff --git a/test/ergonomics.test.js b/test/ergonomics.test.js new file mode 100644 index 0000000..8daafe2 --- /dev/null +++ b/test/ergonomics.test.js @@ -0,0 +1,583 @@ +import assert from 'node:assert'; +import { afterEach, beforeEach, describe, it } from 'node:test'; + +import sqlite3 from '../lib/sqlite3.js'; + +// Phase 1 (ergonomics parity) and the Phase 6 quick wins: pragma(), +// explain(), batch(), dump()/iterdump, the reusable transaction form, +// async-path row modes (array/pluck), err.offset, inTransaction/txnState, +// db.status()/limits/location/releaseMemory, complete()/compileOptions() +// and the expandedSQL/normalizedSQL accessors. + +describe('ergonomics parity', function () { + /** @type {sqlite3.Database} */ + let db; + + beforeEach(async function () { + db = new sqlite3.Database(':memory:'); + await db.exec( + 'CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT);\n' + + "INSERT INTO t (v) VALUES ('a'), ('b'), ('c')", + ); + }); + + afterEach(async function () { + await db.close(); + }); + + describe('pragma()', function () { + it('resolves the pragma rows', async function () { + const rows = await db.pragma('journal_mode'); + assert.strictEqual(rows.length, 1); + assert.strictEqual(rows[0].journal_mode, 'memory'); + }); + + it('resolves the scalar with { simple: true }', async function () { + await db.pragma('user_version = 7'); + assert.strictEqual( + await db.pragma('user_version', { simple: true }), + 7, + ); + }); + + it('accepts argument forms', async function () { + const rows = await db.pragma('table_info(t)'); + assert.strictEqual(rows.length, 2); + assert.strictEqual(rows[0].name, 'id'); + }); + + it('simple resolves undefined for statements-only pragmas', async function () { + assert.strictEqual( + await db.pragma('user_version = 9', { simple: true }), + undefined, + ); + assert.strictEqual( + await db.pragma('user_version', { simple: true }), + 9, + ); + }); + + it('rejects malformed input', async function () { + await assert.rejects(db.pragma(''), TypeError); + await assert.rejects(db.pragma(7), TypeError); + await assert.rejects( + db.pragma('user_version', { bogus: true }), + /unknown option 'bogus'/, + ); + }); + }); + + describe('explain()', function () { + it('resolves the EXPLAIN QUERY PLAN rows without executing', async function () { + const plan = await db.explain('SELECT * FROM t WHERE id = 1'); + assert.ok(plan.length > 0); + assert.ok('detail' in plan[0]); + }); + + it('supports parameters left unbound', async function () { + const plan = await db.explain('SELECT * FROM t WHERE id = ?'); + assert.ok(plan.length > 0); + }); + + it('{ full: true } yields the VDBE program', async function () { + const program = await db.explain('SELECT 1', { full: true }); + assert.ok(program.length > 0); + assert.ok('opcode' in program[0]); + }); + + it('rejects malformed input', async function () { + await assert.rejects(db.explain(''), TypeError); + await assert.rejects( + db.explain('SELECT 1', { bogus: 1 }), + /unknown option 'bogus'/, + ); + }); + }); + + describe('batch()', function () { + it('runs statements atomically', async function () { + const results = await db.batch([ + { sql: 'INSERT INTO t (v) VALUES (?)', args: 'x' }, + { sql: 'INSERT INTO t (v) VALUES (?)', args: 'y' }, + ]); + assert.strictEqual(results.length, 2); + assert.strictEqual(results[0].changes, 1); + assert.strictEqual( + (await db.get('SELECT COUNT(*) AS n FROM t')).n, + 5, + ); + }); + + it('rolls the whole batch back on failure', async function () { + await assert.rejects( + db.batch([ + { sql: 'INSERT INTO t (v) VALUES (?)', args: 'x' }, + 'INSERT INTO nonexistent VALUES (1)', + ]), + ); + assert.strictEqual( + (await db.get('SELECT COUNT(*) AS n FROM t')).n, + 3, + ); + }); + + it('accepts strings, [sql, params] pairs and objects', async function () { + const results = await db.batch([ + "INSERT INTO t (v) VALUES ('s')", + ['INSERT INTO t (v) VALUES (?)', 'p'], + { sql: 'INSERT INTO t (v) VALUES (?)', args: 'o' }, + ]); + assert.strictEqual(results.length, 3); + assert.strictEqual( + (await db.get('SELECT COUNT(*) AS n FROM t')).n, + 6, + ); + }); + + it('accepts a scalar or named-object args, not just arrays', async function () { + // Documented shapes that used to throw "Spread syntax requires + // ...iterable[Symbol.iterator] to be a function". + const results = await db.batch([ + { sql: 'INSERT INTO t (v) VALUES (?)', args: 'scalar' }, + { sql: 'INSERT INTO t (v) VALUES ($v)', args: { $v: 'named' } }, + ]); + assert.strictEqual(results.length, 2); + assert.deepStrictEqual( + await db.all( + "SELECT v FROM t WHERE v IN ('scalar','named') ORDER BY v", + ), + [{ v: 'named' }, { v: 'scalar' }], + ); + }); + + it('collects the rows of a RETURNING statement', async function () { + const results = await db.batch([ + { sql: "INSERT INTO t (v) VALUES ('r') RETURNING id, v" }, + { sql: '/* a comment */ SELECT COUNT(*) AS n FROM t' }, + ]); + assert.strictEqual( + /** @type {any[]} */ (results[0])[0].v, + 'r', + 'RETURNING rows must not be thrown away', + ); + assert.strictEqual(/** @type {any[]} */ (results[1])[0].n, 4); + }); + + it('resolves read-shaped statements as rows', async function () { + const results = await db.batch([ + 'SELECT COUNT(*) AS n FROM t', + 'PRAGMA user_version', + ]); + assert.strictEqual(results[0][0].n, 3); + assert.ok('user_version' in results[1][0]); + }); + + it('maps libsql modes onto BEGIN forms', async function () { + await db.batch(['INSERT INTO t (v) VALUES (1)'], { + mode: 'read', + }); + await assert.rejects( + db.batch([], { mode: 'bogus' }), + /mode must be/, + ); + }); + }); + + describe('dump() / iterdump', function () { + it('produces restorable SQL text', async function () { + const dump = await db.dump(); + assert.ok(dump.startsWith('PRAGMA foreign_keys=OFF;')); + assert.ok(dump.includes('CREATE TABLE t')); + assert.ok(dump.includes('INSERT INTO "t"("id","v") VALUES')); + + const restored = new sqlite3.Database(':memory:'); + await restored.exec(dump); + assert.strictEqual( + (await restored.get('SELECT COUNT(*) AS n FROM t')).n, + 3, + ); + await restored.close(); + }); + + it('iterdump streams statement by statement', async function () { + /** @type {string[]} */ + const statements = []; + for await (const statement of sqlite3.iterdump(db)) { + statements.push(statement); + } + assert.ok(statements.length >= 4); + assert.ok(statements.at(-1)?.includes('COMMIT')); + }); + + it('keeps AUTOINCREMENT counters, user_version and ±Infinity', async function () { + const source = new sqlite3.Database(':memory:'); + await source.exec( + 'CREATE TABLE a (id INTEGER PRIMARY KEY AUTOINCREMENT, x REAL,' + + ' d GENERATED ALWAYS AS (x * 2));\n' + + 'INSERT INTO a (x) VALUES (1), (2);\n' + + 'DELETE FROM a;\n' + + 'PRAGMA user_version = 12', + ); + await source.run( + 'INSERT INTO a (x) VALUES (?)', + Number.POSITIVE_INFINITY, + ); + const dump = await source.dump(); + // A JavaScript `Infinity` literal is not SQL; the overflowing + // decimal is what SQLite reads back as +inf. + assert.ok(!/Infinity/.test(dump), dump); + assert.ok(dump.includes('sqlite_sequence')); + assert.ok(dump.includes('PRAGMA user_version = 12')); + // A generated column cannot be an INSERT target. + assert.ok(!/INSERT INTO "a"\("id","x","d"\)/.test(dump), dump); + + const restored = new sqlite3.Database(':memory:'); + await restored.exec(dump); + assert.strictEqual( + (await restored.get('SELECT x, d FROM a')).x, + Number.POSITIVE_INFINITY, + ); + assert.strictEqual( + await restored.pragma('user_version', { simple: true }), + 12, + ); + // The AUTOINCREMENT high-water mark survived, so no rowid is + // handed out twice. + const inserted = await restored.get( + 'INSERT INTO a (x) VALUES (5) RETURNING id', + ); + assert.strictEqual(inserted.id, 4); + await restored.close(); + await source.close(); + }); + + it('round-trips a virtual table with its content', async function () { + const source = new sqlite3.Database(':memory:'); + await source.exec( + 'CREATE VIRTUAL TABLE docs USING fts5(body);\n' + + "INSERT INTO docs (body) VALUES ('hello world');\n" + + // A table that only looks like a shadow table. + 'CREATE TABLE docs_notes (note);\n' + + "INSERT INTO docs_notes VALUES ('keep me')", + ); + const dump = await source.dump(); + const restored = new sqlite3.Database(':memory:'); + await restored.exec(dump); + assert.deepStrictEqual( + await restored.all( + "SELECT body FROM docs WHERE docs MATCH 'hello'", + ), + [{ body: 'hello world' }], + ); + assert.deepStrictEqual( + await restored.all('SELECT note FROM docs_notes'), + [{ note: 'keep me' }], + ); + assert.strictEqual( + await restored.pragma('integrity_check', { simple: true }), + 'ok', + ); + await restored.close(); + await source.close(); + }); + + it('reads inside a transaction and releases it when abandoned', async function () { + const iterator = sqlite3.iterdump(db); + await iterator.next(); + // A deferred transaction is open (it takes its read lock at + // the first read), so the walk is a point-in-time snapshot. + assert.strictEqual(db.inTransaction, true); + await iterator.return(); + assert.strictEqual(db.inTransaction, false); + // ... and a completed walk commits it. + await db.dump(); + assert.strictEqual(db.inTransaction, false); + }); + + it('serializes blobs and quotes safely', async function () { + await db.exec('CREATE TABLE blobs (b BLOB)'); + await db.run( + 'INSERT INTO blobs VALUES (?)', + Buffer.from([0x00, 0xde, 0xad]), + ); + await db.run("INSERT INTO t (v) VALUES ('it''s quoted')"); + const dump = await db.dump(); + const restored = new sqlite3.Database(':memory:'); + await restored.exec(dump); + const row = await restored.get('SELECT b FROM blobs'); + assert.deepStrictEqual( + Buffer.from(/** @type {Buffer} */ (row.b)), + Buffer.from([0x00, 0xde, 0xad]), + ); + await restored.close(); + }); + }); + + describe('createTransaction()', function () { + it('returns a reusable wrapper', async function () { + const insert = db.createTransaction((tx, v) => + tx.run('INSERT INTO t (v) VALUES (?)', v), + ); + await insert('x'); + await insert('y'); + assert.strictEqual( + (await db.get('SELECT COUNT(*) AS n FROM t')).n, + 5, + ); + }); + + it('carries .deferred/.immediate/.exclusive variants', async function () { + let seenMode = ''; + const probe = db.createTransaction(async (tx, tag) => { + seenMode = tag; + await tx.run('INSERT INTO t (v) VALUES (?)', tag); + }); + await probe.deferred('d'); + await probe.immediate('i'); + await probe.exclusive('e'); + assert.strictEqual(seenMode, 'e'); + assert.strictEqual( + (await db.get('SELECT COUNT(*) AS n FROM t')).n, + 6, + ); + }); + + it('rolls back on throw and stays reusable', async function () { + const insertUnlessFlagged = db.createTransaction((tx, v, fail) => { + const step = tx.run('INSERT INTO t (v) VALUES (?)', v); + return fail + ? step.then(() => { + throw new Error('body failure'); + }) + : step; + }); + await assert.rejects( + insertUnlessFlagged('boom', true), + /body failure/, + ); + await insertUnlessFlagged.immediate('boom', false); + assert.strictEqual( + (await db.get("SELECT COUNT(*) AS n FROM t WHERE v = 'boom'")) + .n, + 1, + ); + }); + + it('validates its arguments', function () { + assert.throws( + () => db.createTransaction(), + /requires a function body/, + ); + assert.throws( + () => db.createTransaction(() => undefined, { mode: 'x' }), + /mode must be/, + ); + }); + }); + + describe('async-path row modes', function () { + it('array rows on get/all/iterate/each/fetch', async function () { + assert.deepStrictEqual( + await db.all('SELECT id, v FROM t', { rowMode: 'array' }), + [ + [1, 'a'], + [2, 'b'], + [3, 'c'], + ], + ); + assert.deepStrictEqual( + await db.get('SELECT id, v FROM t WHERE id = 2', { + rowMode: 'array', + }), + [2, 'b'], + ); + const rows = []; + for await (const row of db.iterate('SELECT id FROM t', { + rowMode: 'array', + })) { + rows.push(row); + } + assert.deepStrictEqual(rows, [[1], [2], [3]]); + const each = await new Promise((resolve, reject) => { + /** @type {unknown[]} */ + const out = []; + db.each( + 'SELECT id FROM t', + { rowMode: 'array' }, + (err, row) => { + if (err) reject(err); + else out.push(row); + }, + () => resolve(out), + ); + }); + assert.deepStrictEqual(each, [[1], [2], [3]]); + const stmt = await db.prepare('SELECT id FROM t'); + assert.deepStrictEqual( + await new Promise((resolve, reject) => { + stmt.fetch(2, { rowMode: 'array' }, (err, rows) => { + if (err) reject(err); + else resolve(rows); + }); + }), + [[1], [2]], + ); + await stmt.finalize(); + }); + + it('pluck rows serve the first column', async function () { + assert.strictEqual( + await db.get('SELECT id, v FROM t WHERE id = 1', { + rowMode: 'pluck', + }), + 1, + ); + assert.deepStrictEqual( + await db.all('SELECT v FROM t ORDER BY id', { + rowMode: 'pluck', + }), + ['a', 'b', 'c'], + ); + assert.strictEqual( + db.getSync('SELECT v FROM t WHERE id = 2', { + rowMode: 'pluck', + }), + 'b', + ); + }); + + it('works alongside parameters', async function () { + assert.deepStrictEqual( + await db.all('SELECT id FROM t WHERE id > ?', 1, { + rowMode: 'array', + }), + [[2], [3]], + ); + }); + }); + + describe('err.offset', function () { + it('carries the failing token byte offset of a failed prepare', async function () { + await assert.rejects( + db.prepare('SELECT * FRUM t'), + (err) => err.offset === 9, + ); + assert.throws( + () => db.prepareSync('SELECT * FRUM t'), + (err) => err.offset === 9, + ); + }); + + it('is absent when the error carries no position', async function () { + // A step-time failure (constraint violation), not a prepare one. + await assert.rejects( + db.run("INSERT INTO t (id, v) VALUES (1, 'dup')"), + (err) => err.offset === undefined, + ); + }); + }); + + describe('inTransaction / txnState', function () { + it('track explicit transactions', async function () { + assert.strictEqual(db.inTransaction, false); + assert.strictEqual(db.txnState, 'none'); + await db.exec('BEGIN'); + assert.strictEqual(db.inTransaction, true); + await db.get('SELECT * FROM t'); + assert.strictEqual(db.txnState, 'read'); + await db.exec("INSERT INTO t (v) VALUES ('d')"); + assert.strictEqual(db.txnState, 'write'); + await db.exec('COMMIT'); + assert.strictEqual(db.inTransaction, false); + }); + + it('works inside the transaction helper', async function () { + await db.transaction(async () => { + assert.strictEqual(db.inTransaction, true); + }); + assert.strictEqual(db.inTransaction, false); + }); + }); + + describe('status, limits, location, memory', function () { + it('db.status() reads counters by name and constant', async function () { + await db.all('SELECT * FROM t'); + const hit = db.status('cacheHit'); + assert.ok(typeof hit.current === 'number'); + assert.ok(typeof hit.highwater === 'number'); + const byConstant = db.status(sqlite3.DBSTATUS_CACHE_HIT); + assert.ok(typeof byConstant.current === 'number'); + assert.throws(() => db.status('bogus'), /unknown counter 'bogus'/); + }); + + it('db.limits reads every run-time limit', function () { + const limits = db.limits; + assert.strictEqual(limits.attached, 10); + assert.strictEqual(limits.column, 2000); + assert.ok(Number.isInteger(limits.length)); + }); + + it('db.location() resolves attached paths', function () { + assert.strictEqual(db.location(), ''); + assert.strictEqual(db.location('main'), ''); + }); + + it('db.releaseMemory() returns a number', function () { + assert.ok(typeof db.releaseMemory() === 'number'); + }); + }); + + describe('complete() and compileOptions()', function () { + it('complete() detects finished statements', function () { + assert.strictEqual(sqlite3.complete('SELECT 1;'), true); + assert.strictEqual(sqlite3.complete('SELECT'), false); + assert.throws(() => sqlite3.complete(7), TypeError); + }); + + it('compileOptions() lists the build configuration', function () { + const options = sqlite3.compileOptions(); + assert.ok(Array.isArray(options)); + assert.ok(options.includes('ENABLE_FTS5')); + assert.ok(options.includes('ENABLE_SESSION')); + assert.ok(options.includes('ENABLE_NORMALIZE')); + }); + }); + + describe('SQL accessors and per-statement integer mode', function () { + it('expandedSQL substitutes the last bound values', async function () { + const stmt = await db.prepare('SELECT * FROM t WHERE id = ?'); + await stmt.get(2); + assert.strictEqual( + stmt.expandedSQL, + 'SELECT * FROM t WHERE id = 2', + ); + await stmt.finalize(); + }); + + it('normalizedSQL folds literals to ?', async function () { + const stmt = await db.prepare( + "SELECT * FROM t WHERE v = 'x' AND id = 5", + ); + assert.ok(stmt.normalizedSQL.includes('?')); + assert.ok(!stmt.normalizedSQL.includes("'x'")); + await stmt.finalize(); + }); + + it('prepareSync accepts { integerMode }', function () { + const stmt = db.prepareSync('SELECT 9223372036854775807 AS v', { + integerMode: 'bigint', + }); + assert.strictEqual(typeof stmt.getSync().v, 'bigint'); + stmt.finalize(); + }); + + it('async prepare accepts { integerMode }', async function () { + const stmt = await db.prepare('SELECT 9223372036854775807 AS v', { + integerMode: 'bigint', + }); + const row = await stmt.get(); + assert.strictEqual(typeof row.v, 'bigint'); + await stmt.finalize(); + }); + }); +}); diff --git a/test/function.test.js b/test/function.test.js index cbfdfba..f19c33d 100644 --- a/test/function.test.js +++ b/test/function.test.js @@ -189,31 +189,144 @@ describe('user-defined functions', function () { db.configure('integerMode', 'number'); }); - it('refuses invocation from the sync methods instead of deadlocking', { + it('invokes functions directly from the sync methods (re-entrant)', { timeout: 5000, }, async function () { - db.function('nope', () => 1); - for (const sql of ['SELECT nope()', 'SELECT nope() FROM t']) { - assert.throws( - () => db.getSync(sql), - (err) => - /cannot be invoked from a\s+synchronous method/.test( - err.message, - ) && - /deadlock/.test(err.message) && - /getSync\/runSync\/allSync/.test(err.message), - ); - assert.throws(() => db.runSync(sql)); - assert.throws(() => db.allSync(sql)); - } - // prepareSync statements refuse at step time too. - const stmt = db.prepareSync('SELECT nope()'); - assert.throws(() => stmt.getSync(), /deadlock/); + db.function('yep', { varargs: true }, (...xs) => (xs[0] ?? 0) + 1); + assert.strictEqual(db.getSync('SELECT yep() AS v').v, 1); + assert.strictEqual(db.getSync('SELECT yep(41) AS v').v, 42); + assert.strictEqual(db.allSync('SELECT yep(1) AS v')[0].v, 2); + db.runSync('SELECT yep(2)'); + // prepareSync statements call through at step time too. + const stmt = db.prepareSync('SELECT yep(5)'); + assert.strictEqual(stmt.getSync()['yep(5)'], 6); stmt.finalize(); // And the connection is fine afterwards. assert.strictEqual(db.getSync('SELECT 7 AS v').v, 7); }); + it('reports a throwing function to the sync caller and keeps the connection usable', { + timeout: 5000, + }, function () { + db.function('boom', () => { + throw new Error('sync UDF failure'); + }); + assert.throws( + () => db.getSync('SELECT boom()'), + (err) => + /user-defined function 'boom' threw/.test(err.message) && + err.cause instanceof Error && + err.cause.message === 'sync UDF failure', + ); + assert.throws(() => db.allSync('SELECT boom()')); + assert.strictEqual(db.getSync('SELECT 1 AS v').v, 1); + }); + + it('lets a sync-invoked function drive other statements and refuse its own', { + timeout: 5000, + }, async function () { + await db.exec('CREATE TABLE probe (v)'); + /** @type {sqlite3.Statement} */ + let stmt; + db.function('reenter', function reenter() { + // Other statements on the same connection work... + db.runSync('INSERT INTO probe VALUES (1)'); + // ...but driving this statement's own VM must refuse (the + // one re-entrancy rule SQLite has). + assert.throws(() => stmt.getSync(), /currently executing/); + return 1; + }); + stmt = db.prepareSync('SELECT reenter() AS v'); + assert.strictEqual(stmt.getSync().v, 1); + stmt.finalize(); + assert.strictEqual(db.getSync('SELECT COUNT(*) AS n FROM probe').n, 1); + db.removeFunction('reenter'); + }); + + it('refuses every cache-flushing call from inside a sync-path callback', { + timeout: 5000, + }, async function () { + // Regression: these all flushed the statement cache before the + // native refusal ran, finalizing the statement SQLite was stepping + // — a use-after-free that segfaulted the process. + /** @type {string[]} */ + const outcomes = []; + /** @param {string} label @param {() => unknown} body */ + const attempt = (label, body) => { + try { + body(); + outcomes.push(`${label}: allowed`); + } catch (err) { + outcomes.push( + `${label}: ${/** @type {Error} */ (err).message}`, + ); + } + }; + const store = db.createTagStore(); + db.function('meddle', function meddle() { + attempt('function', () => db.function('added', () => 1)); + attempt('aggregate', () => + db.aggregate('agg', { + start: () => 0, + step: (acc) => acc, + result: (acc) => acc, + }), + ); + attempt('collation', () => db.collation('coll', () => 0)); + attempt('removeFunction', () => db.removeFunction('meddle')); + attempt('removeCollation', () => db.removeCollation('nope')); + attempt('table', () => + db.table('inner', { + columns: ['a'], + rows: function* () { + yield [1]; + }, + }), + ); + attempt('removeTable', () => db.removeTable('inner')); + attempt('values', () => db.values([1, 2, 3])); + attempt('authorizer', () => db.authorizer(null)); + attempt('close', () => db.close(() => undefined)); + attempt('tagStore.clear', () => store.clear()); + return 1; + }); + await db.wait(); + assert.strictEqual(db.getSync('SELECT meddle() AS v').v, 1); + for (const outcome of outcomes) { + assert.match( + outcome, + /from inside a JavaScript callback invoked by a synchronous method/, + outcome, + ); + } + assert.strictEqual(outcomes.length, 11); + // The connection is intact, and the refused registrations did not + // happen. + assert.strictEqual(db.getSync('SELECT 1 AS v').v, 1); + await assert.rejects(db.get('SELECT added()'), /no such function/); + // ... and they work again once the query is over. + db.function('added', () => 7); + await db.wait(); + assert.strictEqual(db.getSync('SELECT added() AS v').v, 7); + }); + + it('refuses to finalize the statement it is executing', function () { + /** @type {sqlite3.Statement} */ + let stmt; + db.function('selffinalize', function selfFinalize() { + // Finalizing the live VM from its own callback is the same + // use-after-free; the native guard refuses it. + assert.throws( + () => stmt.finalize(() => undefined), + /currently executing/, + ); + return 3; + }); + stmt = db.prepareSync('SELECT selffinalize() AS v'); + assert.strictEqual(stmt.getSync().v, 3); + stmt.finalize(() => undefined); + }); + it('keeps the sync methods working for functions they never call', function () { db.function('unused', () => 1); assert.strictEqual(db.getSync('SELECT 5 AS v').v, 5); diff --git a/test/migrate.test.js b/test/migrate.test.js new file mode 100644 index 0000000..251e8e8 --- /dev/null +++ b/test/migrate.test.js @@ -0,0 +1,134 @@ +import assert from 'node:assert'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, it } from 'node:test'; + +import sqlite3 from '../lib/sqlite3.js'; + +// Phase 5: the user_version-based migration runner. + +describe('migrate()', function () { + /** @type {sqlite3.Database} */ + let db; + + beforeEach(async function () { + db = new sqlite3.Database(':memory:'); + }); + + afterEach(async function () { + await db.close(); + }); + + it('applies pending migrations in order and bumps user_version', async function () { + const { applied, from, to } = await sqlite3.migrate(db, [ + { name: '0001-create', sql: 'CREATE TABLE a (x)' }, + { + name: '0002-seed', + up: (tx) => tx.run('INSERT INTO a VALUES (1)'), + }, + ]); + assert.deepStrictEqual(applied, ['0001-create', '0002-seed']); + assert.strictEqual(from, 0); + assert.strictEqual(to, 2); + assert.strictEqual( + await db.pragma('user_version', { simple: true }), + 2, + ); + assert.strictEqual((await db.get('SELECT COUNT(*) AS n FROM a')).n, 1); + }); + + it('is idempotent', async function () { + await sqlite3.migrate(db, [{ name: 'a', sql: 'CREATE TABLE a (x)' }]); + const second = await sqlite3.migrate(db, [ + { name: 'a', sql: 'CREATE TABLE b (x)' }, + ]); + assert.deepStrictEqual(second.applied, []); + }); + + it('rolls a failing migration back and names it', async function () { + await sqlite3.migrate(db, [{ name: 'a', sql: 'CREATE TABLE a (x)' }]); + await assert.rejects( + sqlite3.migrate(db, [ + { name: 'a', sql: 'CREATE TABLE a (x)' }, + { + name: 'b', + sql: 'CREATE TABLE b (x);\nINSERT INTO nonexistent VALUES (1)', + }, + ]), + /migration b \(version 2\) failed/, + ); + // user_version is still 1: the failed migration did not land. + assert.strictEqual( + await db.pragma('user_version', { simple: true }), + 1, + ); + }); + + it('loads migrations from a directory, ordered numerically', async function () { + const dir = mkdtempSync(path.join(os.tmpdir(), 'migrate-')); + try { + writeFileSync( + path.join(dir, '002-second.sql'), + 'CREATE TABLE b (x)', + ); + writeFileSync(path.join(dir, '10-tenth.sql'), 'CREATE TABLE j (x)'); + writeFileSync( + path.join(dir, '001-first.sql'), + 'CREATE TABLE a (x)', + ); + const { applied } = await sqlite3.migrate(db, dir); + assert.deepStrictEqual(applied, [ + '001-first', + '002-second', + '10-tenth', + ]); + } finally { + rmSync(dir, { recursive: true }); + } + }); + + it('validates the migration list', async function () { + await assert.rejects(sqlite3.migrate(db, 7), /array of migrations/); + await assert.rejects( + sqlite3.migrate(db, [/** @type {any} */ ({ name: 'x' })]), + /must be \{ name, sql \} or \{ name, up \}/, + ); + await assert.rejects( + sqlite3.migrate(db, [ + /** @type {any} */ + ({ name: 'x', sql: 'SELECT 1', up: () => undefined }), + ]), + /both sql and up/, + ); + }); + + it('wraps a non-Error rejection instead of throwing over it', async function () { + await assert.rejects( + sqlite3.migrate(db, [ + { + name: '0001-throws-a-string', + up: () => Promise.reject('nope'), + }, + ]), + (err) => + err instanceof Error && + /0001-throws-a-string \(version 1\) failed: nope/.test( + err.message, + ) && + err.cause === 'nope', + ); + assert.strictEqual( + await db.pragma('user_version', { simple: true }), + 0, + ); + }); + + it('reports the database version when it is ahead of the list', async function () { + await db.pragma('user_version = 5'); + const result = await sqlite3.migrate(db, [ + { name: '0001-a', sql: 'CREATE TABLE a (x)' }, + ]); + assert.deepStrictEqual(result, { applied: [], from: 5, to: 5 }); + }); +}); diff --git a/test/session_rebase.test.js b/test/session_rebase.test.js new file mode 100644 index 0000000..3d725a1 --- /dev/null +++ b/test/session_rebase.test.js @@ -0,0 +1,202 @@ +import assert from 'node:assert'; +import { mkdtempSync, rmSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, it } from 'node:test'; + +import sqlite3 from '../lib/sqlite3.js'; + +// Phase 3: changeset rebasing (the client-server sync primitive no other +// JS driver exposes) and session.diff. + +describe('changeset rebasing', function () { + /** @type {sqlite3.Database} */ + let local; + + beforeEach(async function () { + local = new sqlite3.Database(':memory:'); + await local.exec('CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)'); + }); + + afterEach(async function () { + await local.close(); + }); + + /** + * Builds a database with t(1, 'server') — a "remote" that already + * applied a conflicting change. + * + * @param {string} value the row's v. + * @returns {Promise} the remote connection. + */ + async function remoteWith(value) { + const remote = new sqlite3.Database(':memory:'); + await remote.exec('CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)'); + await remote.run('INSERT INTO t VALUES (1, ?)', value); + return remote; + } + + it('harvests a rebase buffer from a conflicting apply', async function () { + await local.run("INSERT INTO t VALUES (1, 'local')"); + const session = local.session({ table: 't' }); + await local.run("UPDATE t SET v = 'local2' WHERE id = 1"); + const changeset = await session.changeset(); + await session.close(); + + const remote = await remoteWith('server'); + // OMIT: the remote's row wins; the rebase buffer records that. + const rebase = await remote.applyChangeset(changeset, { + conflict: 'omit', + rebase: true, + }); + assert.ok(rebase instanceof Uint8Array); + assert.ok(rebase.length > 0); + assert.strictEqual((await remote.get('SELECT v FROM t')).v, 'server'); + await remote.close(); + }); + + it('rebases a later local changeset against the harvested resolutions', async function () { + // Round one: the client learns its update lost to the server's. + await local.run("INSERT INTO t VALUES (1, 'v1')"); + const session1 = local.session({ table: 't' }); + await local.run("UPDATE t SET v = 'client-edit' WHERE id = 1"); + const round1 = await session1.changeset(); + await session1.close(); + + const remote = await remoteWith('server-edit'); + const rebase = await remote.applyChangeset(round1, { + conflict: 'omit', + rebase: true, + }); + await remote.close(); + + // Round two: the client makes a *new* local change (starting from + // its own state) and rebases it onto the resolved history — the + // textbook sync loop. The rebased changeset applies cleanly to + // the server state. + await local.run("UPDATE t SET v = 'after-rebase' WHERE id = 1"); + const session2 = local.session({ table: 't' }); + await local.run("INSERT INTO t VALUES (2, 'new-row')"); + const round2 = await session2.changeset(); + await session2.close(); + + const rebased = sqlite3.rebaseChangeset(round2, rebase); + assert.ok(rebased instanceof Uint8Array); + assert.ok(rebased.length > 0); + + const converged = await remoteWith('server-edit'); + await converged.applyChangeset(rebased); + // Rebase semantics: the OMIT resolution recorded in round one + // means the client's further edit to that same row is rebased + // away — the server's value stands. The new row (no conflict) + // lands normally. + assert.strictEqual( + (await converged.get('SELECT v FROM t WHERE id = 1')).v, + 'server-edit', + ); + assert.strictEqual( + (await converged.get('SELECT v FROM t WHERE id = 2')).v, + 'new-row', + ); + await converged.close(); + }); + + it('resolves undefined (no rebase) when no conflicts occurred', async function () { + await local.run("INSERT INTO t VALUES (1, 'only-client')"); + const session = local.session({ table: 't' }); + const changeset = await session.changeset(); + await session.close(); + + const remote = await remoteWith('anything'); + const rebase = await remote.applyChangeset(changeset, { + rebase: true, + }); + // No conflicts: nothing to rebase against. + assert.strictEqual(rebase, null); + await remote.close(); + }); + + it('rebaseChangeset validates its inputs', function () { + assert.throws(() => sqlite3.rebaseChangeset(7), TypeError); + assert.throws( + () => sqlite3.rebaseChangeset(new Uint8Array(4), 7), + TypeError, + ); + }); + + it('validates the { rebase } option', async function () { + await assert.rejects( + local.applyChangeset(new Uint8Array(0), { + rebase: 'yes', + }), + /'rebase' must be a boolean/, + ); + }); +}); + +describe('session.diff()', function () { + it('records the differences between two attached databases', async function () { + const dir = mkdtempSync(path.join(os.tmpdir(), 'diff-')); + const mainPath = path.join(dir, 'main.db'); + const otherPath = path.join(dir, 'other.db'); + try { + const main = new sqlite3.Database(mainPath); + await main.exec('CREATE TABLE t (id INTEGER PRIMARY KEY, v)'); + await main.exec( + "INSERT INTO t VALUES (1, 'same'), (2, 'old'), (3, 'gone')", + ); + await main.close(); + + const other = new sqlite3.Database(otherPath); + await other.exec('CREATE TABLE t (id INTEGER PRIMARY KEY, v)'); + await other.exec( + "INSERT INTO t VALUES (1, 'same'), (2, 'new'), (4, 'added')", + ); + await other.close(); + + const db = new sqlite3.Database(mainPath); + await db.exec(`ATTACH '${otherPath}' AS other`); + const session = db.session({ table: 't' }); + await session.diff('t', 'other'); + const changeset = await session.changeset(); + await session.close(); + assert.ok(changeset.length > 0); + + /** @type {any[]} */ + const changes = []; + for (const change of sqlite3.iterateChangeset(changeset)) { + changes.push(change); + } + const byId = new Map( + changes.map((c) => [c.newRow?.[0] ?? c.oldRow?.[0], c.op]), + ); + // The recorded changeset transforms `other`'s table into + // `main`'s: row 2 must be UPDATEd back to 'old', row 3 + // (main-only) INSERTed, row 4 (other-only) DELETEd; row 1 + // matches and is absent. + assert.strictEqual(byId.get(2), 'update'); + assert.strictEqual(byId.get(3), 'insert'); + assert.strictEqual(byId.get(4), 'delete'); + assert.strictEqual(byId.has(1), false); + await db.close(); + } finally { + rmSync(dir, { recursive: true }); + } + }); + + it('reports schema mismatches loudly', async function () { + const db = new sqlite3.Database(':memory:'); + await db.exec( + "CREATE TABLE t (id INTEGER PRIMARY KEY, v);\nATTACH ':memory:' AS other", + ); + const session = db.session({ table: 't' }); + await assert.rejects( + new Promise((_, reject) => { + session.diff('t', 'other', (err) => reject(err)); + }), + /no such table/i, + ); + await session.close(); + await db.close(); + }); +}); diff --git a/test/sync.test.js b/test/sync.test.js index dc591f2..64cda4e 100644 --- a/test/sync.test.js +++ b/test/sync.test.js @@ -685,12 +685,12 @@ describe('sync read paths: rowMode array', function () { stmt.finalize(); }); - it('rejects a rowMode that is not object or array', function () { + it('rejects a rowMode that is not object, array or pluck', function () { assert.throws( () => db.getSync('SELECT i FROM m', { rowMode: 'bogus' }), (err) => err instanceof TypeError && - err.message === "rowMode must be 'object' or 'array'", + err.message === "rowMode must be 'object', 'array' or 'pluck'", ); assert.throws( () => db.allSync('SELECT i FROM m', { rowMode: 7 }), diff --git a/test/tagstore.test.js b/test/tagstore.test.js new file mode 100644 index 0000000..85f6ac5 --- /dev/null +++ b/test/tagstore.test.js @@ -0,0 +1,149 @@ +import assert from 'node:assert'; +import { afterEach, beforeEach, describe, it } from 'node:test'; + +import sqlite3 from '../lib/sqlite3.js'; + +// Phase 5: the tagged-template store (db.createTagStore) and its +// composition helpers. + +describe('tag store', function () { + /** @type {sqlite3.Database} */ + let db; + + beforeEach(async function () { + db = new sqlite3.Database(':memory:'); + await db.exec( + 'CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT);\n' + + "INSERT INTO t (v) VALUES ('a'), ('b')", + ); + }); + + afterEach(async function () { + await db.close(); + }); + + it('runs template queries with positional parameters', async function () { + const store = db.createTagStore(); + const id = 1; + assert.deepStrictEqual( + await store.get`SELECT * FROM t WHERE id = ${id}`, + { + id: 1, + v: 'a', + }, + ); + assert.deepStrictEqual(await store.all`SELECT id FROM t ORDER BY id`, [ + { id: 1 }, + { id: 2 }, + ]); + const result = await store.run`INSERT INTO t (v) VALUES (${'c'})`; + assert.strictEqual(result.changes, 1); + let seen = 0; + for await (const _row of store.iterate`SELECT id FROM t`) { + seen++; + } + assert.strictEqual(seen, 3); + }); + + it('caches statements by the joined SQL and LRU-evicts', async function () { + const store = db.createTagStore(2); + await store.get`SELECT 1 AS v`; + await store.get`SELECT 1 AS v`; + assert.strictEqual(store.size, 1); + await store.get`SELECT 2 AS v`; + await store.get`SELECT 3 AS v`; + assert.strictEqual(store.size, 2); + store.clear(); + assert.strictEqual(store.size, 0); + assert.strictEqual(store.capacity, 2); + assert.strictEqual(store.db, db); + }); + + it('composes identifiers and raw SQL', async function () { + const store = db.createTagStore(); + const table = store.identifier('t'); + const column = store.identifier('v'); + const rows = + await store.all`SELECT ${column} FROM ${table} ORDER BY id`; + assert.deepStrictEqual(rows, [{ v: 'a' }, { v: 'b' }]); + const literal = store.raw("'a' AS literal"); + assert.deepStrictEqual(await store.get`SELECT ${literal}`, { + literal: 'a', + }); + }); + + it('identifierPath quotes each part', async function () { + const store = db.createTagStore(); + assert.strictEqual( + store.identifierPath('main.t.id').text, + '"main"."t"."id"', + ); + assert.strictEqual(store.identifier('users').text, '"users"'); + assert.throws( + () => store.identifier('users; DROP TABLE t'), + /plain identifier/, + ); + assert.throws(() => store.identifierPath('a..b'), /empty part/); + // A dot inside a quoted part belongs to the name. + assert.strictEqual(store.identifierPath('"a.b".c').text, '"a.b"."c"'); + assert.throws( + () => store.identifierPath('"unterminated'), + /unterminated quoted part/, + ); + // An already-quoted name passes through (the guard used to reject + // every name containing a quote, making that branch unreachable). + assert.strictEqual(store.identifier('"odd name"').text, '"odd name"'); + assert.throws(() => store.identifier('od"d'), /not a plain identifier/); + assert.throws(() => store.identifier(''), /non-empty string/); + }); + + it('join() builds IN-lists and similar', async function () { + const store = db.createTagStore(); + const ids = store.join( + [1, 2].map((v) => store.raw(String(v))), + ', ', + ); + const rows = await store.all`SELECT id FROM t WHERE id IN (${ids})`; + assert.deepStrictEqual(rows, [{ id: 1 }, { id: 2 }]); + // Plain values bind as parameters, which is what an IN-list of + // user data needs — reaching for raw() there would be an + // injection. + const bound = store.join([1, 2]); + assert.strictEqual(bound.text, '?, ?'); + assert.deepStrictEqual( + await store.all`SELECT id FROM t WHERE id IN (${bound})`, + [{ id: 1 }, { id: 2 }], + ); + assert.strictEqual(store.empty().text, ''); + assert.throws(() => store.join([]), /non-empty array/); + assert.throws(() => store.join([1], 5), /separator must be a string/); + }); + + it('only splices fragments, never look-alike values', async function () { + const store = db.createTagStore(); + // An object that merely looks like a fragment (from JSON, say) + // must not become SQL: this was an injection. It binds instead, + // and the strict bind marshalling refuses a plain object loudly. + const hostile = JSON.parse('{"text":"1 OR 1=1","params":[]}'); + await assert.rejects( + () => store.all`SELECT id FROM t WHERE id = ${hostile}`, + /unsupported type/, + ); + // The genuine article still composes. + const real = store.raw('1 OR 1=1'); + assert.ok( + (await store.all`SELECT id FROM t WHERE id = ${real}`).length > 1, + ); + }); + + it('validates usage and capacity', async function () { + const store = db.createTagStore(); + // Calling the tag as a plain function is the misuse case. + assert.throws( + // @ts-expect-error deliberate misuse + () => store.get('SELECT 1'), + /used as a template tag/, + ); + assert.throws(() => db.createTagStore(0), /positive integer/); + }); +}); diff --git a/test/vtab.test.js b/test/vtab.test.js new file mode 100644 index 0000000..d022d29 --- /dev/null +++ b/test/vtab.test.js @@ -0,0 +1,508 @@ +import assert from 'node:assert'; +import { afterEach, beforeEach, describe, it } from 'node:test'; + +import sqlite3 from '../lib/sqlite3.js'; + +// Phase 4: JavaScript virtual tables — db.table() in both the eponymous +// and factory forms, db.values(), the sync path's direct calls, error +// propagation and teardown. + +describe('virtual tables', function () { + /** @type {sqlite3.Database} */ + let db; + + beforeEach(async function () { + db = new sqlite3.Database(':memory:'); + await db.exec( + 'CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT);\n' + + "INSERT INTO users (name) VALUES ('a'), ('b'), ('c')", + ); + }); + + afterEach(async function () { + await db.close(); + }); + + it('serves an eponymous table from a generator', async function () { + db.table('letters', { + columns: ['c'], + rows: function* () { + yield ['x']; + yield ['y']; + }, + }); + assert.deepStrictEqual(await db.all('SELECT c FROM letters'), [ + { c: 'x' }, + { c: 'y' }, + ]); + }); + + it('passes HIDDEN parameters to the generator (table-valued function)', async function () { + db.table('sequence', { + columns: ['value', 'count'], + parameters: ['count'], + rows: function* seq(count) { + for (let i = 0; i < count; i++) yield [i, count]; + }, + }); + assert.deepStrictEqual(await db.all('SELECT value FROM sequence(3)'), [ + { value: 0 }, + { value: 1 }, + { value: 2 }, + ]); + // Hidden columns do not appear in SELECT *. + assert.deepStrictEqual(await db.all('SELECT * FROM sequence(1)'), [ + { value: 0 }, + ]); + }); + + it('delivers several HIDDEN parameters in declaration order', async function () { + // Regression: xBestIndex used to hand xFilter the constraints in + // reverse (argvIndex = size - p), so a two-parameter table + // function received its arguments swapped. + db.table('pair', { + columns: ['first', 'second', 'n', 'm'], + parameters: ['n', 'm'], + rows: function* pair(n, m) { + yield [n, m, 'seen']; + }, + }); + assert.deepStrictEqual(await db.all('SELECT * FROM pair(10, 20)'), [ + { first: 10, second: 20 }, + ]); + // The WHERE-constraint spelling reaches the same xBestIndex path. + assert.deepStrictEqual( + await db.all('SELECT * FROM pair WHERE n = 1 AND m = 2'), + [{ first: 1, second: 2 }], + ); + assert.deepStrictEqual(db.allSync('SELECT * FROM pair(7, 9)'), [ + { first: 7, second: 9 }, + ]); + }); + + it('runs from the synchronous methods too', function () { + db.table('sequence', { + columns: ['value'], + parameters: ['value'], + rows: function* seq(count) { + for (let i = 0; i < count; i++) yield [i]; + }, + }); + assert.deepStrictEqual(db.allSync('SELECT value FROM sequence(2)'), [ + { value: 0 }, + { value: 1 }, + ]); + }); + + it('joins against real tables', async function () { + db.table('ids', { + columns: ['n'], + rows: function* () { + yield [1]; + yield [2]; + }, + }); + const rows = await db.all( + 'SELECT users.name FROM users JOIN ids ON users.id = ids.n', + ); + assert.deepStrictEqual(rows, [{ name: 'a' }, { name: 'b' }]); + }); + + it('accepts object rows keyed by column name', async function () { + db.table('objrows', { + columns: ['a', 'b'], + rows: function* () { + yield { a: 1, b: 'two' }; + }, + }); + assert.deepStrictEqual(await db.all('SELECT * FROM objrows'), [ + { a: 1, b: 'two' }, + ]); + }); + + it('accepts bare values for single-column tables', async function () { + db.table('singles', { + columns: ['v'], + rows: function* () { + yield 7; + yield 'eight'; + }, + }); + assert.deepStrictEqual(await db.all('SELECT v FROM singles'), [ + { v: 7 }, + { v: 'eight' }, + ]); + }); + + it('applies the strict marshalling to yielded values', async function () { + db.table('badcell', { + columns: ['v'], + rows: function* () { + yield [{ not: 'bindable' }]; + }, + }); + await assert.rejects( + db.all('SELECT v FROM badcell'), + /unsupported type/i, + ); + }); + + it('stops pulling an unbounded generator at LIMIT', async function () { + let produced = 0; + db.table('naturals', { + columns: ['n'], + rows: function* () { + for (let i = 0; ; i++) { + // A generator with no end is the canonical sequence + // table; materialising it would never return. + if (++produced > 100_000) throw new Error('ran away'); + yield [i]; + } + }, + }); + assert.deepStrictEqual(await db.all('SELECT n FROM naturals LIMIT 3'), [ + { n: 0 }, + { n: 1 }, + { n: 2 }, + ]); + // One batch, not one row and not everything. + assert.ok(produced < 1000, `produced ${produced}`); + }); + + it('survives concurrent queries against the same table', async function () { + // Regression (deadlock): a worker inside the generator round trip + // holds the connection mutex while it waits for the JS thread, so + // the first query's completion handler must not call + // sqlite3_finalize inline — it would want that mutex. Virtual + // tables were missing from MayBlockOnWorkerRoundTrip(), so two + // concurrent queries hung the process, event loop and all. + db.table('nums', { + columns: ['v'], + rows: function* () { + for (let i = 0; i < 300; i++) yield [i]; + }, + }); + const results = await Promise.all([ + db.all('SELECT count(*) AS c FROM nums'), + db.all('SELECT count(*) AS c FROM nums'), + db.all('SELECT v FROM nums LIMIT 2'), + db.get('SELECT max(v) AS m FROM nums'), + ]); + assert.deepStrictEqual(results[0], [{ c: 300 }]); + assert.deepStrictEqual(results[1], [{ c: 300 }]); + assert.deepStrictEqual(results[2], [{ v: 0 }, { v: 1 }]); + assert.deepStrictEqual(results[3], { m: 299 }); + }); + + it('survives concurrent queries against a values() table', async function () { + const rows = Array.from({ length: 2500 }, (_, i) => i); + const table = db.values(rows); + const [a, b] = await Promise.all([ + db.get(`SELECT count(*) AS c FROM ${table.name}`), + db.get(`SELECT sum(value) AS s FROM ${table.name}`), + ]); + assert.strictEqual(a.c, 2500); + assert.strictEqual(b.s, (2499 * 2500) / 2); + table.drop(); + }); + + it('reports a generator that throws mid-scan and stays usable', async function () { + db.table('flaky', { + columns: ['v'], + rows: function* () { + for (let i = 0; i < 5000; i++) { + // Past the first batch, so the failure lands in xNext. + if (i === 100) throw new Error('mid-scan failure'); + yield [i]; + } + }, + }); + await assert.rejects(db.all('SELECT v FROM flaky'), /mid-scan failure/); + assert.strictEqual((await db.get('SELECT 1 AS v')).v, 1); + }); + + it('re-filters a cursor for each row of a correlated subquery', async function () { + db.table('upto', { + columns: ['v', 'n'], + parameters: ['n'], + rows: function* (n) { + for (let i = 0; i < Number(n ?? 0); i++) yield [i]; + }, + }); + assert.deepStrictEqual( + await db.all( + 'SELECT id, (SELECT count(*) FROM upto(id)) AS c FROM users', + ), + [ + { id: 1, c: 1 }, + { id: 2, c: 2 }, + { id: 3, c: 3 }, + ], + ); + }); + + it('keeps rowids monotonic across batch boundaries', async function () { + db.table('many', { + columns: ['n'], + rows: function* () { + for (let i = 0; i < 5000; i++) yield [i]; + }, + }); + const rows = /** @type {any[]} */ ( + await db.all('SELECT rowid AS r, n FROM many') + ); + assert.strictEqual(rows.length, 5000); + assert.strictEqual(rows[0].r, 1); + assert.strictEqual(rows.at(-1)?.r, 5000); + assert.strictEqual( + new Set(rows.map((row) => row.r)).size, + 5000, + 'a per-batch counter would repeat rowids', + ); + assert.strictEqual( + (await db.get('SELECT COUNT(DISTINCT n) AS n FROM many')).n, + 5000, + ); + }); + + it('streams the same rows through the sync path', function () { + db.table('lots', { + columns: ['n'], + rows: function* () { + for (let i = 0; i < 3000; i++) yield [i]; + }, + }); + const rows = db.allSync('SELECT n FROM lots'); + assert.strictEqual(rows.length, 3000); + assert.strictEqual(db.getSync('SELECT n FROM lots LIMIT 1').n, 0); + }); + + it('accepts constraints on any hidden parameter, in any number', async function () { + db.table('pair', { + columns: ['v', 'a', 'b'], + parameters: ['a', 'b'], + rows: function* (a, b) { + yield [`${a}/${b}`]; + }, + }); + // Only the second parameter constrained: the argvIndex values must + // still be 1..N without gaps, or sqlite fails the statement with + // "xBestIndex malfunction". + assert.deepStrictEqual(await db.all('SELECT v FROM pair WHERE b = 2'), [ + { v: 'undefined/2' }, + ]); + assert.deepStrictEqual(await db.all('SELECT v FROM pair(1, 2)'), [ + { v: '1/2' }, + ]); + // A duplicate equality on one parameter must not double-assign it. + db.table('seq', { + columns: ['value', 'count'], + parameters: ['count'], + rows: function* (count) { + for (let i = 0; i < Number(count ?? 0); i++) yield [i]; + }, + }); + assert.deepStrictEqual( + await db.all('SELECT value FROM seq WHERE count = 2 AND count = 2'), + [{ value: 0 }, { value: 1 }], + ); + // Contradictory constraints select nothing rather than erroring. + assert.deepStrictEqual( + await db.all('SELECT value FROM seq WHERE count = 2 AND count = 3'), + [], + ); + }); + + it('reports a hidden parameter the generator did not yield', async function () { + db.table('echo', { + columns: ['value', 'n'], + parameters: ['n'], + rows: function* (n) { + for (let i = 0; i < Number(n); i++) yield [i]; + }, + }); + // The argument fills the HIDDEN column the generator left NULL. + assert.deepStrictEqual(await db.all('SELECT value, n FROM echo(2)'), [ + { value: 0, n: 2 }, + { value: 1, n: 2 }, + ]); + }); + + it('releases a dropped table generator (and what it captured)', async function () { + // Regression: dead module holders used to live until the connection + // was destroyed, pinning the closure — and any array it captured. + const { execFileSync } = await import('node:child_process'); + const script = ` + import sqlite3 from './lib/sqlite3.js'; + const tick = () => new Promise((r) => setTimeout(r, 20)); + let collected = 0; + const registry = new FinalizationRegistry(() => collected++); + const db = await sqlite3.open(':memory:'); + for (let i = 0; i < 20; i++) { + const rows = new Array(1000).fill(i); + registry.register(rows, i); + const table = db.values(rows); + await db.all(\`SELECT count(*) FROM \${table.name}\`); + table.drop(); + await db.wait(); + } + global.gc(); await tick(); global.gc(); await tick(); global.gc(); + await tick(); + console.log(collected); + await db.close(); + `; + const out = execFileSync( + process.execPath, + ['--expose-gc', '--input-type=module', '-e', script], + { encoding: 'utf8', cwd: new URL('..', import.meta.url) }, + ); + // The last iteration's array is still referenced by the loop body. + assert.ok( + Number(out.trim()) >= 19, + `only ${out.trim()} of 20 dropped values() arrays were collected`, + ); + }); + + it('propagates generator throws as query errors', async function () { + db.table('broken', { + columns: ['v'], + // Throwing before the first yield is the case under test: a + // generator function that throws on its first next(). + rows: function* () { + if (Date.now() > 0) throw new Error('generator exploded'); + yield 0; + }, + }); + await assert.rejects( + db.all('SELECT v FROM broken'), + (err) => + /generator exploded/.test(err.message) || + /broken/.test(err.message), + ); + // The connection survives. + assert.strictEqual((await db.get('SELECT 1 AS v')).v, 1); + }); + + it('supports factory modules with CREATE VIRTUAL TABLE', async function () { + // Factory arguments arrive as the SQL literal strings from the + // DDL ("10", "12"); coerce as needed. + /** @type {any} */ + const range = (lo, hi) => ({ + rows: function* () { + for (let i = Number(lo); i <= Number(hi); i++) yield [i]; + }, + }); + range.columns = ['value']; + db.table('mod_range', range); + await db.exec('CREATE VIRTUAL TABLE r3 USING mod_range(10, 12)'); + assert.deepStrictEqual(await db.all('SELECT * FROM r3'), [ + { value: 10 }, + { value: 11 }, + { value: 12 }, + ]); + }); + + it('validates definitions loudly', function () { + assert.throws(() => db.table(''), /non-empty name/); + assert.throws(() => db.table('x', 7), /definition object/); + assert.throws( + () => db.table('x', { columns: ['a'] }), + /requires a 'rows' generator/, + ); + assert.throws( + () => db.table('x', { columns: 'no' }), + /requires a 'columns' array/, + ); + assert.throws( + () => + db.table('x', { + columns: ['a'], + parameters: ['b'], + rows: function* () { + yield ['a']; + }, + }), + /parameter 'b' is not one of the columns/, + ); + assert.throws( + () => db.table('x', { columns: ['a'], rows: null }), + /requires a 'rows' generator/, + ); + // v1 is read-only with no pattern push-down: the option must not + // be accepted silently. + assert.throws( + () => + db.table('x', { + columns: ['a'], + pattern: 'x%', + rows: function* () { + yield ['a']; + }, + }), + /unknown option 'pattern'/, + ); + }); + + it('removeTable makes the name fail loudly', async function () { + db.table('temp1', { + columns: ['v'], + rows: function* () { + yield [1]; + }, + }); + assert.ok((await db.all('SELECT v FROM temp1')).length === 1); + db.removeTable('temp1'); + await assert.rejects( + db.all('SELECT v FROM temp1'), + /was removed|no such module/i, + ); + }); + + it('values() exposes a JS array as a table', async function () { + const ids = db.values([1, 3]); + const rows = await db.all( + `SELECT users.name FROM users JOIN ${ids.name} ON users.id = ${ids.name}.value`, + ); + assert.deepStrictEqual(rows, [{ name: 'a' }, { name: 'c' }]); + ids.drop(); + await assert.rejects( + db.all(`SELECT * FROM ${ids.name}`), + /was removed|no such module/i, + ); + }); + + it('values() handles strings, nulls and blobs', async function () { + const values = db.values(['x', null, Buffer.from([1, 2])]); + const rows = await db.all( + `SELECT key, value FROM ${values.name} ORDER BY key`, + ); + assert.strictEqual(rows.length, 3); + assert.strictEqual(rows[0].value, 'x'); + assert.strictEqual(rows[1].value, null); + assert.deepStrictEqual( + Buffer.from(/** @type {Buffer} */ (rows[2].value)), + Buffer.from([1, 2]), + ); + }); + + it('values() validates its input', function () { + assert.throws(() => db.values(7), /requires an iterable/); + assert.throws( + () => db.values([1], { bogus: 1 }), + /unknown option 'bogus'/, + ); + }); + + it('closes cleanly with tables and values registered', async function () { + db.table('closeme', { + columns: ['v'], + rows: function* () { + yield [1]; + }, + }); + db.values([1, 2, 3]); + await db.close(); + // afterEach would double-close; reopen so the hook stays valid. + db = new sqlite3.Database(':memory:'); + }); +}); diff --git a/tools/gen-types.js b/tools/gen-types.js index f2760c9..df06f28 100644 --- a/tools/gen-types.js +++ b/tools/gen-types.js @@ -34,22 +34,53 @@ // declaration can neither drift from the JSDoc nor be silently dropped. import { execFileSync } from 'node:child_process'; import { readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; const root = path.dirname(path.dirname(fileURLToPath(import.meta.url))); -const tsc = path.join(root, 'node_modules', '.bin', 'tsc'); +// The compiler's own entry point, run through this node — not the +// node_modules/.bin shim, which on Windows is tsc.CMD and made +// execFileSync fail with ENOENT *after* the outputs below were deleted. +// Resolved through the package manifest because TypeScript 7 does not +// list its bin in `exports`, so require.resolve cannot reach it directly. +const typescriptManifest = createRequire(import.meta.url).resolve( + 'typescript/package.json', +); +const tsc = path.join( + path.dirname(typescriptManifest), + JSON.parse(readFileSync(typescriptManifest, 'utf8')).bin.tsc, +); const generated = ['sqlite3.d.ts', 'promises.d.ts', 'trace.d.ts', 'pool.d.ts']; // Stale outputs first, so resolution during the run sees the sources. +// Their contents are kept until the emit succeeds: a failed run must not +// leave the checkout without the declarations it came with. +/** @type {Map} */ +const previous = new Map(); for (const file of generated) { - rmSync(path.join(root, 'lib', file), { force: true }); + const at = path.join(root, 'lib', file); + try { + previous.set(at, readFileSync(at, 'utf8')); + } catch { + // Not generated yet; nothing to restore. + } + rmSync(at, { force: true }); } -execFileSync(tsc, ['-p', path.join(root, 'tsconfig.types.json')], { - stdio: 'inherit', - cwd: root, -}); +try { + execFileSync( + process.execPath, + [tsc, '-p', path.join(root, 'tsconfig.types.json')], + { + stdio: 'inherit', + cwd: root, + }, + ); +} catch (err) { + for (const [at, text] of previous) writeFileSync(at, text); + throw err; +} const emitDir = path.join(root, 'types-gen'); const entry = path.join(root, 'lib', 'sqlite3.d.ts'); From da241a24e6d5c3025510ed82079eb9b46dd21550 Mon Sep 17 00:00:00 2001 From: Team AppThreat Date: Sun, 13 Sep 2026 23:24:38 +0100 Subject: [PATCH 2/3] fix(tests): parse leak-child output without styling escapes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two GC leak tests (unclosed-connection collection, dropped values()-generator collection) failed on every CI runner while passing locally: setup-node's '24' resolves to Node 24.21+, which styles console.log() output with ANSI escapes whenever color output is forced — and it is forced under pnpm/CI. execFileSync captured '\x1b[33m19\x1b[39m' from the child, Number() of that is NaN, and NaN >= 19 failed while the log rendered a deceptively clean '19'. The children now print String(collected) (strings print verbatim) and the parents parse the digit run from the output, so styling can never reach the comparison. The leak checks themselves were passing all along — 19 of 20, with the 20th held by the loop body by design. Verified on Node 24.21.0 in an ubuntu-22.04 container with FORCE_COLOR=1 and CI=true: full suite 1012 tests, 0 failures. --- test/diagnostics.test.js | 13 ++++++++++--- test/vtab.test.js | 15 +++++++++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/test/diagnostics.test.js b/test/diagnostics.test.js index 3672fb5..6a4a940 100644 --- a/test/diagnostics.test.js +++ b/test/diagnostics.test.js @@ -112,16 +112,23 @@ describe('diagnostics_channel', function () { } global.gc(); await tick(); global.gc(); await tick(); global.gc(); await tick(); - console.log(collected); + // String() on purpose: newer Node (24.21+) styles a NUMBER + // passed to console.log with ANSI when color is forced (as it + // is under pnpm/CI), and the parent Number()-parses this + // output. A string prints verbatim. + console.log(String(collected)); `; const out = execFileSync( process.execPath, ['--expose-gc', '--input-type=module', '-e', script], { encoding: 'utf8', cwd: new URL('..', import.meta.url) }, ); + // Belt and braces: parse the digit run, ignoring any styling + // escapes around it. + const collectedCount = Number(out.match(/\d+/)?.[0] ?? '0'); assert.ok( - Number(out.trim()) >= 19, - `only ${out.trim()} of 20 unclosed connections were collected`, + collectedCount >= 19, + `only ${collectedCount} of 20 unclosed connections were collected`, ); }); diff --git a/test/vtab.test.js b/test/vtab.test.js index d022d29..21f284f 100644 --- a/test/vtab.test.js +++ b/test/vtab.test.js @@ -348,7 +348,11 @@ describe('virtual tables', function () { } global.gc(); await tick(); global.gc(); await tick(); global.gc(); await tick(); - console.log(collected); + // String() on purpose: newer Node (24.21+) styles a NUMBER + // passed to console.log with ANSI when color is forced (as it + // is under pnpm/CI), and the parent Number()-parses this + // output. A string prints verbatim. + console.log(String(collected)); await db.close(); `; const out = execFileSync( @@ -356,10 +360,13 @@ describe('virtual tables', function () { ['--expose-gc', '--input-type=module', '-e', script], { encoding: 'utf8', cwd: new URL('..', import.meta.url) }, ); - // The last iteration's array is still referenced by the loop body. + // Belt and braces: parse the digit run, ignoring any styling + // escapes around it. The last iteration's array is still + // referenced by the loop body. + const collectedCount = Number(out.match(/\d+/)?.[0] ?? '0'); assert.ok( - Number(out.trim()) >= 19, - `only ${out.trim()} of 20 dropped values() arrays were collected`, + collectedCount >= 19, + `only ${collectedCount} of 20 dropped values() arrays were collected`, ); }); From eca41673022463457a41d505771aafc8468701f0 Mon Sep 17 00:00:00 2001 From: Team AppThreat Date: Sun, 13 Sep 2026 23:57:24 +0100 Subject: [PATCH 3/3] chore: bump version to 9.1.0 The PR dry-run publish gate fails while package.json still says 9.0.2: the content no longer matches the published 9.0.2 tarball, so npm refuses ('You cannot publish over the previously published versions'). Everything in this PR is already documented @since 9.1.0. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index cbdb7b8..835a6e4 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@appthreat/sqlite3", "description": "Asynchronous, non-blocking SQLite3 bindings. Modern rewrite of TryGhost/node-sqlite3", - "version": "9.0.2", + "version": "9.1.0", "homepage": "https://github.com/AppThreat/node-sqlite3", "author": "Team AppThreat ", "binary": {