diff --git a/Cargo.lock b/Cargo.lock index c9df533..f6e7d2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1277,7 +1277,7 @@ dependencies = [ [[package]] name = "zu" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=8aa27d9c9df4087522f4314ea5b1d5850df27f8d#8aa27d9c9df4087522f4314ea5b1d5850df27f8d" +source = "git+https://github.com/tamnd/zu?rev=95c7c9909f3a2624515d27eb436da52936016960#95c7c9909f3a2624515d27eb436da52936016960" dependencies = [ "zu-common", "zu-encoding", @@ -1293,7 +1293,7 @@ dependencies = [ [[package]] name = "zu-common" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=8aa27d9c9df4087522f4314ea5b1d5850df27f8d#8aa27d9c9df4087522f4314ea5b1d5850df27f8d" +source = "git+https://github.com/tamnd/zu?rev=95c7c9909f3a2624515d27eb436da52936016960#95c7c9909f3a2624515d27eb436da52936016960" dependencies = [ "thiserror", ] @@ -1301,7 +1301,7 @@ dependencies = [ [[package]] name = "zu-encoding" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=8aa27d9c9df4087522f4314ea5b1d5850df27f8d#8aa27d9c9df4087522f4314ea5b1d5850df27f8d" +source = "git+https://github.com/tamnd/zu?rev=95c7c9909f3a2624515d27eb436da52936016960#95c7c9909f3a2624515d27eb436da52936016960" dependencies = [ "ruzstd", "zu-common", @@ -1310,7 +1310,7 @@ dependencies = [ [[package]] name = "zu-exec" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=8aa27d9c9df4087522f4314ea5b1d5850df27f8d#8aa27d9c9df4087522f4314ea5b1d5850df27f8d" +source = "git+https://github.com/tamnd/zu?rev=95c7c9909f3a2624515d27eb436da52936016960#95c7c9909f3a2624515d27eb436da52936016960" dependencies = [ "zu-common", "zu-query", @@ -1320,7 +1320,7 @@ dependencies = [ [[package]] name = "zu-query" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=8aa27d9c9df4087522f4314ea5b1d5850df27f8d#8aa27d9c9df4087522f4314ea5b1d5850df27f8d" +source = "git+https://github.com/tamnd/zu?rev=95c7c9909f3a2624515d27eb436da52936016960#95c7c9909f3a2624515d27eb436da52936016960" dependencies = [ "crossbeam-deque", "zu-common", @@ -1331,7 +1331,7 @@ dependencies = [ [[package]] name = "zu-s3" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=8aa27d9c9df4087522f4314ea5b1d5850df27f8d#8aa27d9c9df4087522f4314ea5b1d5850df27f8d" +source = "git+https://github.com/tamnd/zu?rev=95c7c9909f3a2624515d27eb436da52936016960#95c7c9909f3a2624515d27eb436da52936016960" dependencies = [ "crc32c", "object_store", @@ -1342,7 +1342,7 @@ dependencies = [ [[package]] name = "zu-sqlite" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=8aa27d9c9df4087522f4314ea5b1d5850df27f8d#8aa27d9c9df4087522f4314ea5b1d5850df27f8d" +source = "git+https://github.com/tamnd/zu?rev=95c7c9909f3a2624515d27eb436da52936016960#95c7c9909f3a2624515d27eb436da52936016960" dependencies = [ "rusqlite", "zu-common", @@ -1352,7 +1352,7 @@ dependencies = [ [[package]] name = "zu-storage" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=8aa27d9c9df4087522f4314ea5b1d5850df27f8d#8aa27d9c9df4087522f4314ea5b1d5850df27f8d" +source = "git+https://github.com/tamnd/zu?rev=95c7c9909f3a2624515d27eb436da52936016960#95c7c9909f3a2624515d27eb436da52936016960" dependencies = [ "zu-common", "zu-encoding", @@ -1361,7 +1361,7 @@ dependencies = [ [[package]] name = "zu-vector" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=8aa27d9c9df4087522f4314ea5b1d5850df27f8d#8aa27d9c9df4087522f4314ea5b1d5850df27f8d" +source = "git+https://github.com/tamnd/zu?rev=95c7c9909f3a2624515d27eb436da52936016960#95c7c9909f3a2624515d27eb436da52936016960" dependencies = [ "zu-common", ] @@ -1369,7 +1369,7 @@ dependencies = [ [[package]] name = "zu-zu1" version = "0.0.1" -source = "git+https://github.com/tamnd/zu?rev=8aa27d9c9df4087522f4314ea5b1d5850df27f8d#8aa27d9c9df4087522f4314ea5b1d5850df27f8d" +source = "git+https://github.com/tamnd/zu?rev=95c7c9909f3a2624515d27eb436da52936016960#95c7c9909f3a2624515d27eb436da52936016960" dependencies = [ "crc32c", "loom", diff --git a/Cargo.toml b/Cargo.toml index 19e42a5..c1d3ec5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,8 @@ crate-type = ["cdylib"] # with (ADR 0002), so a revision is the honest way to say which one. # A local checkout is used instead with a `paths` override in # `.cargo/config.toml`, which is untracked on purpose. -zudb = { package = "zu", git = "https://github.com/tamnd/zu", rev = "8aa27d9c9df4087522f4314ea5b1d5850df27f8d" } -zu-common = { git = "https://github.com/tamnd/zu", rev = "8aa27d9c9df4087522f4314ea5b1d5850df27f8d" } +zudb = { package = "zu", git = "https://github.com/tamnd/zu", rev = "95c7c9909f3a2624515d27eb436da52936016960" } +zu-common = { git = "https://github.com/tamnd/zu", rev = "95c7c9909f3a2624515d27eb436da52936016960" } # N-API by way of napi-rs (ADR 0002). `napi9` is the version of N-API # this addon declares it needs, which is what makes one binary work # across Node 24, Node 26, Electron and Bun without a rebuild: the diff --git a/README.md b/README.md index 343ea61..c90840d 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,28 @@ The rows are an array, so iterating them is `for (const row of rows)` and nothin - **A refusal is a rejection.** A closed connection, a statement that is not a string and a parameter of a type nothing can bind are all refused inside the promise rather than thrown out of the call, so one `await` catches everything one statement can do and no caller has to wrap the same call twice. That holds for the arguments too: passing a number where a statement goes is a `ZuUsageError` the promise rejects with, not a `TypeError` off the stack. - **Parameters are named, and nothing about them is guessed.** An object keyed by the names the statement uses, without the `$`. An array is refused rather than bound by position, because zu has no positional parameters and binding one by index would run the statement with none of the values the caller passed and say nothing about it. A value that contains itself is refused too, at a nesting depth no real value reaches. +## A database with no file + +`connect()` with nothing after it is a database in memory, and it makes no file anywhere. + +```ts +import { connect } from "zudb"; + +await using conn = await connect(); + +await conn.exec(`INSERT (p:Person {id: 1, name: 'ada'})`); +for (const { name } of await conn.query(`MATCH (p:Person) RETURN p.name AS name`)) { + console.log(name); +} +``` + +`connect(":memory:")` is the same thing spelled the way every embedded database spells it, and it makes no file called `:memory:` either, which is what it used to do. Options may stand where the path would, so `connect({ threads: 2 })` is a call and not a mistake. + +It is the whole engine and not a reduced one: writes, transactions, the appender, registered frames and streams, all of it, on bytes that are not a file. `conn.memory` says which kind you have, since `path` cannot quite answer it on a filesystem that allows a colon in a name. Nothing survives the last connection, which is the point: a test, a script, or five minutes with the language costs no cleanup and leaves no `social.zu1` in a directory somebody has to notice later. + ## What works today -`connect`, `query`, `exec`, `stream`, `close`, `dispose` and `await using`. Named parameters both ways, including lists, records and nesting. Every scalar the engine has, plus nodes, edges and paths with their tables named rather than numbered, and `ZuDate`, `ZuTime`, `ZuTimestamp` and `ZuDuration`, with `{ temporal: true }` and `toTemporal()` for the runtimes that have `Temporal`. Read-only connections, memory and thread limits. `bigIntMode`, per statement or per connection. An `AbortSignal` on any statement. The full error surface above, and `isZuError` to recognize it. Streaming, as an async iterable, as batches and as a Web Stream. Transactions, with `inTransaction` on the connection. An appender, for loading rows a batch at a time, and `load` for building a whole database out of columns and an edge list. Registered frames, so an Arrow table or an object of typed arrays is something a statement can match on without the rows being copied. `columnar`, for a result read down its columns as the buffers themselves rather than across its rows as objects. Prepared statements, compiled at the line that asked and run as often as wanted, and `explain` and `profile`, as a tree a program walks and as the listing a person reads. Both module formats, typed separately. +`connect`, `query`, `exec`, `stream`, `close`, `dispose` and `await using`. Named parameters both ways, including lists, records and nesting. Every scalar the engine has, plus nodes, edges and paths with their tables named rather than numbered, and `ZuDate`, `ZuTime`, `ZuTimestamp` and `ZuDuration`, with `{ temporal: true }` and `toTemporal()` for the runtimes that have `Temporal`. Read-only connections, databases in memory, memory and thread limits. `bigIntMode`, per statement or per connection. An `AbortSignal` on any statement. The full error surface above, and `isZuError` to recognize it. Streaming, as an async iterable, as batches and as a Web Stream. Transactions, with `inTransaction` on the connection. An appender, for loading rows a batch at a time, and `load` for building a whole database out of columns and an edge list. Registered frames, so an Arrow table or an object of typed arrays is something a statement can match on without the rows being copied. `columnar`, for a result read down its columns as the buffers themselves rather than across its rows as objects. Prepared statements, compiled at the line that asked and run as often as wanted, and `explain` and `profile`, as a tree a program walks and as the listing a person reads. Both module formats, typed separately. Build it with `npm run build`, and run the suite with `npm test`. Nothing is published yet, so `npm i zudb` is not a thing you can type at anybody's terminal, but everything it will do is built and installed on every run of the release workflow. diff --git a/binding.d.cts b/binding.d.cts index 4058d86..0ebaae3 100644 --- a/binding.d.cts +++ b/binding.d.cts @@ -809,6 +809,11 @@ export declare class Connection { get path(): string /** Whether this connection refuses every statement that writes. */ get readOnly(): boolean + /** + * Whether the database behind it is in memory rather than on + * disk, in which case nothing survives the last connection to it. + */ + get memory(): boolean /** Whether the connection is still open. */ get open(): boolean /** @@ -1377,8 +1382,15 @@ export declare function abiVersion(): string * Creates one when the path holds nothing, which is what a first * program expects and what every embedded database does. A read-only * connection never creates anything. + * + * With no path, with `null`, or with `':memory:'`, the database is in + * memory and no file is made anywhere. It is the whole engine and not + * a reduced one, so it takes writes and transactions and the appender + * exactly as a database on disk does, and it is gone when the last + * connection to it is. Options may stand where the path would in that + * case, so `connect({ threads: 2 })` is a call and not a mistake. */ -export declare function connect(path: string, options?: ConnectOptions | undefined | null): Promise +export declare function connect(path?: string | ConnectOptions | undefined | null, options?: ConnectOptions | undefined | null): Promise /** * What a connection can be opened with. diff --git a/etc/zudb.api.md b/etc/zudb.api.md index 851fcbb..33d71a0 100644 --- a/etc/zudb.api.md +++ b/etc/zudb.api.md @@ -22,7 +22,7 @@ export class Appender { } // @public -export function connect(path: string, options?: ConnectOptions | undefined | null): Promise +export function connect(path?: string | ConnectOptions | undefined | null, options?: ConnectOptions | undefined | null): Promise // @public export class Connection { @@ -34,6 +34,7 @@ export class Connection { exec(statement: string, params?: Record | null, options?: ZuStatementOptions | null): Promise explain(statement: string): Promise get inTransaction(): boolean + get memory(): boolean get open(): boolean get path(): string prepare(statement: string): Promise diff --git a/src/conn.rs b/src/conn.rs index 607a7cf..09d1f79 100644 --- a/src/conn.rs +++ b/src/conn.rs @@ -131,20 +131,38 @@ pub struct Connection { spelling: Spelling, path: String, read_only: bool, + /// Whether the database behind it is in memory, which is the one + /// thing [`Self::path`] cannot quite say: a file could be called + /// `:memory:` on any filesystem that allows a colon. + memory: bool, } +/// The name a database in memory is asked for by, and answers to. +/// +/// The spelling every embedded database has used for thirty years, +/// which is the reason it is this and not something better: a caller +/// who types it has already been taught what it means somewhere else. +pub(crate) const MEMORY: &str = ":memory:"; + /// Opens the database at `path` and connects to it. /// /// Creates one when the path holds nothing, which is what a first /// program expects and what every embedded database does. A read-only /// connection never creates anything. +/// +/// With no path, with `null`, or with `':memory:'`, the database is in +/// memory and no file is made anywhere. It is the whole engine and not +/// a reduced one, so it takes writes and transactions and the appender +/// exactly as a database on disk does, and it is gone when the last +/// connection to it is. Options may stand where the path would in that +/// case, so `connect({ threads: 2 })` is a call and not a mistake. #[napi( - ts_args_type = "path: string, options?: ConnectOptions | undefined | null", + ts_args_type = "path?: string | ConnectOptions | undefined | null, options?: ConnectOptions | undefined | null", ts_return_type = "Promise" )] pub fn connect( env: &Env, - path: Unknown<'_>, + path: Option>, options: Option, ) -> AsyncTask { // Whether this runtime has `Temporal` is a question only the thread @@ -152,17 +170,55 @@ pub fn connect( // the thread that opens the database, where the answer decides // whether there is anything to open. let has_temporal = temporal::present(env).unwrap_or(false); - let path = text(&path, "path"); + let (path, options, refused) = arguments(path, options); AsyncTask::new(ConnectTask { - path: path.as_deref().unwrap_or_default().to_string(), - refused: path.err(), + memory: path.is_none(), + path: path.unwrap_or_else(|| MEMORY.to_string()), + refused, options, has_temporal, }) } +/// Which of the three shapes the call was written in. +/// +/// A path and options, options alone, or neither. The first argument +/// is read here rather than declared, because a value that is a string +/// in one call and an object in the next is a value napi would refuse +/// before this client got to say anything about it. +/// +/// The path comes back as `None` when the database is in memory, which +/// is the one thing the three shapes have to agree on. +fn arguments( + first: Option>, + second: Option, +) -> (Option, Option, Option) { + let Some(first) = first else { + return (None, second, None); + }; + let kind = match first.get_type() { + Ok(kind) => kind, + Err(err) => return (None, second, Some(err.reason)), + }; + match kind { + ValueType::Undefined | ValueType::Null => (None, second, None), + ValueType::Object => match ConnectOptions::from_unknown(first) { + Ok(options) => (None, Some(options), None), + Err(err) => (None, second, Some(err.reason)), + }, + _ => match text(&first, "path") { + Ok(path) if path == MEMORY => (None, second, None), + Ok(path) => (Some(path), second, None), + Err(message) => (None, second, Some(message)), + }, + } +} + pub struct ConnectTask { path: String, + /// Whether the database is in memory, in which case [`Self::path`] + /// is the name it is asked for by rather than a name to open. + memory: bool, /// What this client refused the call with, before any of it ran. refused: Option, options: Option, @@ -225,8 +281,16 @@ impl<'task> ScopedTask<'task> for ConnectTask { config = config.threads(threads as usize); } } - Ok(open(PathBuf::from(&self.path), read_only, config) - .map(|opened| Opened { spelling, ..opened }) + let opened = match self.memory { + true => memory(config), + false => open(PathBuf::from(&self.path), read_only, config), + }; + Ok(opened + .map(|opened| Opened { + spelling, + read_only, + ..opened + }) .map_err(Failure::Engine)) } @@ -240,6 +304,7 @@ impl<'task> ScopedTask<'task> for ConnectTask { spelling: opened.spelling, path: opened.path, read_only: opened.read_only, + memory: opened.memory, } .into_instance(env)?; wire_disposal(env, &mut instance, "dispose")?; @@ -279,6 +344,25 @@ pub struct Opened { spelling: Spelling, path: String, read_only: bool, + memory: bool, +} + +/// Opens a database in memory, then connects. +/// +/// The path is the name it was asked for by rather than the one the +/// engine spells it with: the engine mints a unique name per database +/// so two of them never share a writer, and that counter is its +/// business and not a caller's. +fn memory(config: Config) -> std::result::Result { + let database = Database::memory_with(config)?; + let conn = database.connect()?; + Ok(Opened { + conn, + spelling: Spelling::default(), + path: MEMORY.to_string(), + read_only: false, + memory: true, + }) } /// Opens or creates, then connects. @@ -300,6 +384,7 @@ fn open(path: PathBuf, read_only: bool, config: Config) -> std::result::Result bool { + self.memory + } + /// Whether the connection is still open. #[napi(getter)] pub fn open(&self) -> bool { diff --git a/test/connect.test.mjs b/test/connect.test.mjs index 0491550..122b960 100644 --- a/test/connect.test.mjs +++ b/test/connect.test.mjs @@ -1,5 +1,6 @@ import assert from 'node:assert/strict' -import { mkdtemp, rm, stat } from 'node:fs/promises' +import { existsSync } from 'node:fs' +import { mkdtemp, readdir, rm, stat } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import test from 'node:test' @@ -112,3 +113,74 @@ test('the executor takes the limits it was opened with', async (t) => { await conn.exec("INSERT (p:person {id: 1, name: 'ada'})") assert.equal((await conn.query('MATCH (p:person) RETURN p.name AS name')).length, 1) }) + +test('connecting with no path is a database in memory that makes no file', async (t) => { + const dir = await mkdtemp(join(tmpdir(), 'zu-node-')) + t.after(() => rm(dir, { recursive: true, force: true })) + + const conn = await connect() + t.after(() => conn.close()) + + assert.equal(conn.memory, true) + assert.equal(conn.path, ':memory:') + assert.equal(conn.readOnly, false) + await conn.exec("INSERT (p:person {id: 1, name: 'ada'})") + assert.deepEqual([...(await conn.query('MATCH (p:person) RETURN p.name AS name'))], [{ name: 'ada' }]) + assert.deepEqual(await readdir(dir), [], 'nothing was written anywhere') +}) + +test('the name every embedded database spells it with means the same thing', async (t) => { + // The bug this replaces: `':memory:'` used to make a file called + // `:memory:` in whatever directory the caller was standing in, which + // is why the check below is on the directory this runs from. + const conn = await connect(':memory:') + t.after(() => conn.close()) + + assert.equal(conn.memory, true) + assert.equal(conn.path, ':memory:') + await conn.exec("INSERT (p:person {id: 1, name: 'ada'})") + assert.equal(existsSync(':memory:'), false, 'no file is called that') +}) + +test('options may stand where the path would', async (t) => { + const conn = await connect({ threads: 2, bigIntMode: 'number' }) + t.after(() => conn.close()) + + assert.equal(conn.memory, true) + await conn.exec("INSERT (p:person {id: 1, name: 'ada'})") + const rows = await conn.query('MATCH (p:person) RETURN p.id AS id') + assert.deepEqual([...rows], [{ id: 1 }], 'the options were read, so INT64 is a number') +}) + +test('two databases in memory share nothing', async (t) => { + const one = await connect() + const two = await connect() + t.after(() => { one.close(); two.close() }) + + await one.exec("INSERT (p:person {id: 1, name: 'ada'})") + assert.equal((await one.query('MATCH (p:person) RETURN p.name AS name')).length, 1) + assert.equal((await two.query('MATCH (p:person) RETURN p.name AS name')).length, 0) +}) + +test('a database in memory takes a transaction and rolls it back', async (t) => { + const conn = await connect() + t.after(() => conn.close()) + + await conn.exec("INSERT (p:person {id: 1, name: 'ada'})") + const work = await conn.transaction() + await conn.exec("INSERT (p:person {id: 2, name: 'zoe'})") + await work.rollback() + assert.equal((await conn.query('MATCH (p:person) RETURN p.name AS name')).length, 1) +}) + +test('a database on disk is not one in memory', async (t) => { + const { conn } = await fresh(t) + assert.equal(conn.memory, false) +}) + +test('a database in memory cannot be opened read-only', async (t) => { + await assert.rejects(() => connect({ readOnly: true }), (err) => { + assert.equal(err.name, 'ZuUsageError') + return true + }) +}) diff --git a/test/plan.test.mjs b/test/plan.test.mjs index ef92b37..111cf48 100644 --- a/test/plan.test.mjs +++ b/test/plan.test.mjs @@ -11,6 +11,8 @@ import assert from 'node:assert/strict' import test from 'node:test' +import { connect } from 'zudb' + import { fresh, isZuError, twoPeople } from './helper.mjs' const BY_NAME = 'MATCH (p:person) WHERE p.name = $name RETURN p.id AS id' @@ -181,11 +183,30 @@ test('a profile is what the operators really did', async (t) => { assert.equal(scan.pulls, 1) assert.equal(scan.rows, 2) assert.equal(scan.flat, 2) + // The estimate comes off the catalog's summary of the table, and rows + // written without folding the file have not reached it yet, so a table + // two statements old estimates low. That is the engine's to answer; + // what this client owes is that the number and the q-error derived + // from it arrive at all. + assert.ok(scan.estimate > 0) + assert.equal(scan.qerror, scan.rows / scan.estimate) + assert.ok(scan.nanos > 0) +}) + +test('the estimate is the catalog summary once the file holds it', async (t) => { + // Written down because the difference between this and the profile + // above is a real one a reader would otherwise take for noise. + const { path, conn } = await twoPeople(t) + conn.close() + + const again = await connect(path) + t.after(() => again.close()) + const run = await again.profile('MATCH (p:person) RETURN p.name AS name') + const scan = run.stages[0].ops.find((op) => op.op === 'Scan') + + assert.equal(scan.rows, 2) assert.equal(scan.estimate, 2) - // The optimizer was right about a table it has the statistics for, - // which is what a q-error of one means. assert.equal(scan.qerror, 1) - assert.ok(scan.nanos > 0) }) test('an operator the optimizer has nothing to say about carries nulls', async (t) => { diff --git a/test/readme.test.mjs b/test/readme.test.mjs index c6e34d3..62d9adc 100644 --- a/test/readme.test.mjs +++ b/test/readme.test.mjs @@ -22,7 +22,7 @@ import assert from 'node:assert/strict' import { execFile } from 'node:child_process' -import { mkdtemp, readFile, rm, stat, symlink, writeFile, mkdir } from 'node:fs/promises' +import { mkdtemp, readFile, readdir, rm, stat, symlink, writeFile, mkdir } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' import test from 'node:test' @@ -81,7 +81,7 @@ async function installed(t, program) { } test('the README prints programs and fragments and knows which is which', async () => { - assert.equal((await programs()).length, 2, "the README's whole programs") + assert.equal((await programs()).length, 3, "the README's whole programs") assert.ok((await blocks('ts')).length > (await programs()).length, 'and its fragments') }) @@ -102,6 +102,19 @@ test('the first minute of the README runs as printed', { skip: !NODE }, async (t await stat(join(dir, 'social.zu1')) }) +test('the program with no path leaves the directory empty', { skip: !NODE }, async (t) => { + // The claim the section makes is the one worth testing: a reader who + // runs it finds nothing beside them afterwards. + const program = (await programs()).find((block) => block.includes('connect()')) + assert.ok(program, 'the block that opens a database in memory') + + const { dir, file } = await installed(t, program) + const { stdout } = await run(process.execPath, [file], { cwd: dir }) + + assert.equal(stdout.trim(), 'ada') + assert.deepEqual(await readdir(dir), ['main.ts', 'node_modules', 'package.json'].sort()) +}) + test('every whole program in the README runs', { skip: !NODE }, async (t) => { for (const program of await programs()) { const { dir, file } = await installed(t, program)