diff --git a/.eslintrc.cjs b/.eslintrc.cjs deleted file mode 100644 index 600c8ef..0000000 --- a/.eslintrc.cjs +++ /dev/null @@ -1,16 +0,0 @@ -module.exports = { - "extends": "eslint:recommended", - "parserOptions": { - "sourceType": "module" - }, - "env": { - "es2017": true, - "node": true - }, - "rules": { - "indent": ["error", 4], - "linebreak-style": ["error", "unix"], - "semi": ["error", "always"], - "no-cond-assign": ["error", "always"] - } -}; diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..ab473e1 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,63 @@ +# Benchmarks (Deliverable 13). A dedicated workflow rather than a job in +# ci.yml, because the nightly `schedule` trigger would wake every job in +# that file; here it wakes exactly this one. +# +# Posture: INFORMATIONAL. `continue-on-error: true` means this job cannot +# fail a build, and that is deliberate — the D11 precedent (sqlcipher is +# post-merge-only so an apt hiccup cannot gate every push) applies more +# strongly here: timing on shared GitHub runners is noisier than apt, and +# a flaky gate is worse than an ungated check. The gate itself still +# exists and still reports: bench:compare exits 2 on a >10% median +# regression, the per-case verdicts are in the log, and the JSON is +# uploaded as an artifact so a regression can be promoted into +# bench/baseline.json deliberately (`--baseline-from`). Local runs on a +# quiet machine are the enforceable gate. +# +# Runner: ubuntu-22.04 is a pinned label (ubuntu-latest floats across a +# mixed pool — results from a mixed pool are noise). Node is pinned to 24 +# to match the environment the committed baselines were captured on. +name: bench +on: + workflow_dispatch: + schedule: + # Nightly at 03:07 UTC — off the top of the hour, when runner + # contention from everyone else's cron spikes. + - cron: '7 3 * * *' + push: + branches: + - 'release/*' + +jobs: + bench: + runs-on: ubuntu-22.04 + timeout-minutes: 40 + # Informational: see the workflow header. A red bench job annotates + # and uploads; it cannot block a merge or a publish. + continue-on-error: true + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + scope: '@appthreat' + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: Install dependencies + run: pnpm install --frozen-lockfile --ignore-scripts + - name: Rebuild native module (required after --ignore-scripts) + run: pnpm run rebuild + # RME gate + baseline comparison. On a shared runner expect some + # REJECTED cases; that is the harness refusing to report noise, not + # a failure. The verdict lines and the exit code land in the log. + - name: Run benchmarks and compare against baseline + run: pnpm run bench:compare --json bench-results.json + - name: Upload bench results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bench-results-${{ github.sha }} + path: bench-results.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3594a34..ee924ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,7 @@ on: push: branches: - master + - 'release/*' tags: - '*' env: @@ -13,6 +14,103 @@ concurrency: group: ${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: + # Lint needs no native build: biome ships as optionalDependencies and + # its bin resolves without lifecycle scripts, so --ignore-scripts is + # safe here. lint:jsdoc hard-fails on any undocumented shipped + # declaration (Deliverable 04). + lint: + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + scope: '@appthreat' + - name: Install dependencies + run: pnpm install --frozen-lockfile --ignore-scripts + - name: Lint (check only; CI never runs --write) + run: pnpm run lint:check + - name: JSDoc completeness (hard fail) + run: pnpm run lint:jsdoc + + # Type declarations: tsd assertions plus a strict/node16 consumer + # compile, then a regeneration diff proving the committed .d.ts files + # match the JSDoc and the hand-written declarations. No native build + # is involved — gen-types only typechecks lib/*.js. + types: + runs-on: ubuntu-22.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + scope: '@appthreat' + - name: Install dependencies + run: pnpm install --frozen-lockfile --ignore-scripts + - name: Type tests (tsd + strict/node16 consumer) + run: pnpm run test:types + - name: Regenerated declarations must match the committed ones + run: | + pnpm run gen-types + git diff --exit-code -- lib/sqlite3.d.ts lib/promises.d.ts lib/trace.d.ts + + + # Fast test feedback via a dev rebuild instead of a full prebuild. + # Leak check: the suite must exit on its own — nothing here passes + # --force-exit, so a leaked handle or worker hangs the run and the + # job times out (a failure, not a silent pass). The stray-.only guard + # runs inside `pnpm run test` and fails rather than silently skipping. + test: + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-22.04 + # Arm64 Linux was previously covered only by the slower `build` + # matrix, which is about producing the artifact; the suite now + # runs here on every push too. + - ubuntu-22.04-arm + - macos-latest + # Added after a Windows-only path bug (a /\/test$/ strip that + # never matches a backslash path) survived five deliverables + # because the suite ran on Linux and macOS only — the Electron + # job was the first thing ever to run it on Windows. + - windows-latest + # 24 is the engines floor and the LTS line; 26 is Current (and the + # local dev Node). napi means one binary serves both majors, but + # the C++ still compiles against each major's headers and the JS + # surface still meets each major's runtime changes. + node: + - 24 + - 26 + name: test (${{ matrix.os }}, node=${{ matrix.node }}) + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ matrix.node }} + scope: '@appthreat' + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: Install dependencies + run: pnpm install --frozen-lockfile --ignore-scripts + - name: Rebuild native module (required after --ignore-scripts) + run: pnpm run rebuild + - name: Run tests + run: pnpm run test + build: runs-on: ${{ matrix.os }} strategy: @@ -44,16 +142,28 @@ jobs: name: ${{ matrix.os }} (host=${{ matrix.host }}, target=${{ matrix.target }}) steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + # Reads the version from the packageManager field in package.json. + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: ${{ matrix.node }} architecture: ${{ matrix.host }} scope: '@appthreat' - - name: Add yarn - run: npm install -g yarn + - name: Get pnpm store directory + shell: bash + run: | + echo "PNPM_STORE_PATH=$(pnpm store path)" >> $GITHUB_ENV + - name: Cache pnpm store + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.PNPM_STORE_PATH }} + key: pnpm-store-${{ runner.os }}-${{ matrix.host }}-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + pnpm-store-${{ runner.os }}-${{ matrix.host }}- - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: 3.12 - name: Add msbuild to PATH @@ -62,7 +172,7 @@ jobs: with: msbuild-architecture: ${{ matrix.target }} - name: Install dependencies - run: yarn install --ignore-scripts + run: pnpm install --frozen-lockfile --ignore-scripts - name: Check Node compatibility run: node tools/semver-check.js @@ -85,7 +195,7 @@ jobs: echo "CXXFLAGS=${CXXFLAGS:-} -include ../src/gcc-preinclude.h" >> $GITHUB_ENV - name: Build binaries - run: yarn prebuild --arch ${{ env.TARGET }} --tag-libc + run: pnpm run prebuild --arch ${{ env.TARGET }} --tag-libc - name: Print binary info if: contains(matrix.os, 'ubuntu') @@ -97,14 +207,13 @@ jobs: file prebuilds/*/*.node - name: Run tests - run: yarn test + run: pnpm run test - name: Upload prebuilds to artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: prebuilds-${{ matrix.os }}-${{ matrix.host }}-${{ matrix.target }}-${{ matrix.node }} path: prebuilds/ - retention-days: 7 build-qemu: runs-on: ubuntu-24.04-arm @@ -124,13 +233,13 @@ jobs: node: 24 name: ${{ matrix.variant }} (node=${{ matrix.node }}, target=${{ matrix.target }}) steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up QEMU - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 - name: Setup Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - name: Build binaries and test run: | @@ -150,23 +259,307 @@ jobs: SAFE_TARGET=$(echo "${{ matrix.target }}" | tr '/' '-') echo "SAFE_TARGET=$SAFE_TARGET" >> $GITHUB_ENV - name: Upload QEMU prebuilds to artifacts - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: prebuilds-qemu-${{ matrix.variant }}-${{ env.SAFE_TARGET }}-${{ matrix.node }} path: prebuilds/ - retention-days: 7 + + # Electron (Deliverable 10). Depends on `build` and consumes its + # prebuilds artifact — it must test the shipped binary, not a local + # rebuild; PREBUILDS_ONLY (set inside test/electron/run.mjs) enforces + # that even if a build/ directory were present. One pinned Electron + # version (the devDependency), three platforms. Linux needs a display + # for the app-environment harness and the sandbox disabled in the + # runner container. + electron: + needs: build + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + artifact: prebuilds-ubuntu-22.04-x64-x64-24 + - os: macos-latest + artifact: prebuilds-macos-latest-arm64-arm64-24 + - os: windows-latest + artifact: prebuilds-windows-latest-x64-x64-24 + name: electron (${{ matrix.os }}) + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + scope: '@appthreat' + - name: Install dependencies (electron's binary downloads on first launch) + run: pnpm install --frozen-lockfile --ignore-scripts + - name: Download the shipped prebuilds + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ matrix.artifact }} + path: prebuilds/ + - name: Install xvfb (Linux) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y xvfb + - name: Suite inside Electron's Node build + app-environment harness + if: runner.os != 'Linux' + run: pnpm run test:electron + - name: Suite inside Electron's Node build + app-environment harness (xvfb) + if: runner.os == 'Linux' + env: + ELECTRON_DISABLE_SANDBOX: '1' + run: xvfb-run -a pnpm run test:electron + + # The packaged-ASAR cases: slow (packages Electron twice from a real + # packed-tarball install), so ubuntu-only and not on every push — + # post-merge on release/v9 and on demand. + electron-asar: + needs: build + # Tags included: publish depends on this job, so it has to actually + # run on the event that publishes rather than being skipped past. + if: >- + github.event_name == 'workflow_dispatch' + || (github.event_name == 'push' + && (startsWith(github.ref, 'refs/heads/release/') + || startsWith(github.ref, 'refs/tags/'))) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + scope: '@appthreat' + - name: Install dependencies + run: pnpm install --frozen-lockfile --ignore-scripts + - name: Download the shipped prebuilds + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: prebuilds-ubuntu-22.04-x64-x64-24 + path: prebuilds/ + - name: Install xvfb + run: sudo apt-get update && sudo apt-get install -y xvfb + - name: Packaged-ASAR tests (unpacked + sealed-in-archive) + env: + ELECTRON_DISABLE_SANDBOX: '1' + run: xvfb-run -a pnpm run test:electron:asar + + # SQLCipher source build (Deliverable 11 §2.4): the external-SQLite + # build path must keep working, and the smoke test proves the link + # really is SQLCipher (a wrong key must fail), not the vendored plain + # SQLite. Like electron-asar it runs post-merge on release/ branches, + # on tags and on demand, so a build hiccup cannot gate every push. + # + # SQLCipher is built from source rather than installed from apt, and + # that is not incidental: this package uses the session extension and + # the preupdate hook (D08), and Ubuntu's libsqlcipher-dev exports + # neither (`nm -D libsqlcipher.so | grep sqlite3session_create` → 0), + # so the addon cannot link against it at all. The SQLite base version + # matters too — src/node_sqlite3.cc exports extended result codes that + # only exist from 3.53; SQLCipher 4.18 is built on 3.53.4, the same + # amalgamation this repo vendors, and older SQLCipher releases fail to + # compile. All of this was reproduced in a container before it was + # written down. + sqlcipher: + if: >- + github.event_name == 'workflow_dispatch' + || (github.event_name == 'push' + && (startsWith(github.ref, 'refs/heads/release/') + || startsWith(github.ref, 'refs/tags/'))) + runs-on: ubuntu-22.04 + timeout-minutes: 30 + env: + SQLCIPHER_TAG: v4.18.0 + SQLITE_PREFIX: /opt/sqlcipher + # SQLCipher's current configure installs the library as + # libsqlite3.so with headers at /include, NOT as + # libsqlcipher.*/include/sqlcipher — that layout belongs to the + # distro packaging. The libname follows the artifact, not the name. + SQLITE_LIBNAME: sqlite3 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + scope: '@appthreat' + - name: Install dependencies + run: pnpm install --frozen-lockfile --ignore-scripts + - name: Build SQLCipher from source + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends libssl-dev tcl + git clone --depth 1 --branch "$SQLCIPHER_TAG" \ + https://github.com/sqlcipher/sqlcipher.git /tmp/sqlcipher + cd /tmp/sqlcipher + # --session is required (see the job header). SQLCipher's own + # mandatory defines have to be repeated here because setting + # CFLAGS replaces its defaults rather than adding to them: drop + # them and the build stops with "SQLCipher must be compiled + # with -DSQLITE_EXTRA_INIT=…". + CFLAGS="-DSQLITE_HAS_CODEC -DSQLITE_ENABLE_COLUMN_METADATA \ + -DSQLITE_ENABLE_PREUPDATE_HOOK \ + -DSQLITE_EXTRA_INIT=sqlcipher_extra_init \ + -DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown \ + -DSQLITE_TEMP_STORE=2" \ + LDFLAGS="-lcrypto" \ + ./configure --prefix="$SQLITE_PREFIX" --session --fts5 --rtree --dbstat + make -j"$(nproc)" + sudo make install + # Fail here, loudly, rather than in a confusing C++ error later. + nm -D "$SQLITE_PREFIX/lib/lib$SQLITE_LIBNAME.so" \ + | grep -q sqlite3session_create + - name: Build against SQLCipher + env: + # node-gyp 13 forwards everything after `--` to gyp as build-file + # names, so `rebuild -- --sqlite=…` dies with "not found while + # trying to load". The binding.gyp `sqlite` and `sqlite_libname` + # variables are gyp defines: set them through the GYP_DEFINES + # environment variable, which gyp reads on every run. + GYP_DEFINES: 'sqlite=${{ env.SQLITE_PREFIX }} sqlite_libname=${{ env.SQLITE_LIBNAME }}' + # binding.gyp adds <(sqlite)/include; Debian keeps the SQLCipher + # headers under $SQLITE_PREFIX/include/sqlcipher, so point the + # compiler at them too (the same CPPFLAGS dance as docs/security.md). + CPPFLAGS: '-I${{ env.SQLITE_PREFIX }}/include' + LDFLAGS: '-l${{ env.SQLITE_LIBNAME }} -L${{ env.SQLITE_PREFIX }}/lib' + run: pnpm run rebuild + - name: 'Smoke: encrypted database round trip' + run: | + node --input-type=module -e " + import sqlite3 from './lib/sqlite3.js'; + const file = '/tmp/sqlcipher-smoke.db'; + const good = 'correct horse battery staple'; + const db = await sqlite3.open(file); + await db.exec(\`PRAGMA key = '\${good}'; CREATE TABLE t (x); INSERT INTO t VALUES (42)\`); + await db.close(); + const reopen = await sqlite3.open(file); + await reopen.exec(\`PRAGMA key = '\${good}'\`); + const row = await reopen.get('SELECT x FROM t'); + if (row?.x !== 42) throw new Error('encrypted round trip lost the row'); + await reopen.close(); + const wrong = await sqlite3.open(file); + await wrong.exec(\`PRAGMA key = 'wrong key'\`); + try { + await wrong.get('SELECT x FROM t'); + throw new Error('wrong key read the table: this is NOT a SQLCipher build'); + } catch (err) { + if (!/not a database|file is not a database/.test(err.message)) throw err; + } + await wrong.close(); + console.log('SQLCIPHER_OK'); + " + + # The Node-API promise, end to end: the binary built once under Node 24 + # by the `build` matrix must run unmodified under every supported Node. + # This job never compiles — no rebuild step, and PREBUILDS_ONLY makes + # node-gyp-build refuse build/Release — so green here means the shipped + # artifact itself passed the suite, not a fresh build of it. (The `test` + # matrix compiles from source per Node major; this one consumes.) + prebuild-consumer: + needs: build + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-22.04 + artifact: prebuilds-ubuntu-22.04-x64-x64-24 + - os: macos-latest + artifact: prebuilds-macos-latest-arm64-arm64-24 + - os: windows-latest + artifact: prebuilds-windows-latest-x64-x64-24 + name: prebuild-consumer (${{ matrix.os }}, node=26) + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 26 + scope: '@appthreat' + - name: Install dependencies + run: pnpm install --frozen-lockfile --ignore-scripts + - name: Download the shipped prebuilds + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ matrix.artifact }} + path: prebuilds/ + - name: Run tests against the prebuilt binary + env: + PREBUILDS_ONLY: '1' + run: pnpm run test + + # The musl half of the same promise. alpine3.20 — the variant the musl + # prebuild is built on — has no Node 26 image (node:26-alpine tags start + # at 3.22), so rather than moving the build floor this job *consumes* + # the alpine3.20-built artifact inside a node:26-alpine3.22 container. + # Same pnpm trick as tools/BinaryBuilder.Dockerfile: pnpm 11 ships + # static musl-safe binaries. x64 only — the arm64 musl artifact is built + # and suite-tested by build-qemu; this adds the Node-major dimension, + # not the arch one. + prebuild-consumer-musl: + needs: build-qemu + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + # Only to resolve the exact packageManager version; the container + # installs its own copy. + - name: Download the shipped musl prebuilds + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: prebuilds-qemu-alpine3.20-linux-amd64-24 + path: prebuilds/ + - name: Suite against the musl prebuild under Node 26 + run: | + docker run --rm -v "$PWD:/ws" -w /ws -e PREBUILDS_ONLY=1 node:26-alpine3.22 sh -c " + npm install --global pnpm@$(pnpm --version) && + pnpm install --frozen-lockfile --ignore-scripts && + pnpm run test + " publish: - needs: [build, build-qemu] + # Everything that can prove the package works gates publishing — + # previously only [build, build-qemu] did, so a tag could publish to + # npm with the suite, the types, the lint and every Electron job red. + # + # always() with an explicit failure/cancellation check, rather than a + # bare needs list, because electron-asar does not run on every event: + # a skipped dependency would otherwise skip publish too, including + # the pull-request dry run. Skipped is tolerated here; failed and + # cancelled are not. + needs: + - lint + - types + - test + - build + - build-qemu + - electron + - electron-asar + - prebuild-consumer + - prebuild-consumer-musl + if: >- + always() + && !contains(needs.*.result, 'failure') + && !contains(needs.*.result, 'cancelled') runs-on: ubuntu-22.04 permissions: contents: write packages: write id-token: write steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24 scope: '@appthreat' @@ -178,12 +571,13 @@ jobs: path: prebuilds/ merge-multiple: true - - name: Add yarn - run: npm install -g yarn + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + # Reads the version from the packageManager field in package.json. - name: Install dependencies run: | - yarn install --ignore-scripts + pnpm install --frozen-lockfile --ignore-scripts npm publish --dry-run - name: Publish to npm diff --git a/.gitignore b/.gitignore index 85ae861..cf7beb3 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ package-lock.json yarn.lock prebuilds somefile + +# types-gen: scratch declaration emit (gen-types copies into lib/) +/types-gen/ diff --git a/MIGRATING-TO-V9.md b/MIGRATING-TO-V9.md new file mode 100644 index 0000000..488e818 --- /dev/null +++ b/MIGRATING-TO-V9.md @@ -0,0 +1,423 @@ +# Migrating to v9 + +v9 fixes a class of silent data-corruption bugs in the value marshalling +between JavaScript and SQLite, and adds the promise API, async iteration, +disposal and cancellation. Code that was already receiving **wrong +values** will now see errors instead; code that was correct keeps working. +Everything below follows from those two principles. + +## Calls without a callback now return promises + +`run`, `get`, `all`, `map`, `exec`, `close`, `wait`, `loadExtension` on a +`Database`, and `bind`, `run`, `get`, `all`, `map`, `reset`, `finalize` on +a `Statement` (plus `step`/`finish` on `Backup`) are dual-mode: a trailing +function keeps the callback contract byte-for-byte (still returns `this`, +still chainable); without one you get a promise. + +- `db.run(sql, params)` resolves `{ lastID, changes, lastIDBigInt }`. + `lastID` applies the integer mode and keeps the 'number'-mode + `RangeError` for unsafe rowids **lazy** — it throws only when read, so + awaiting an insert into a big-rowid table never throws by itself. + `lastIDBigInt` is exact in every mode. +- Strict-binding errors that used to throw synchronously from a + callback-less call (bad bind values, arity mismatches) are now + rejections; the orphaned statement is still finalized. +- A callback-less call that failed used to emit `'error'` on the database; + that failure is now a rejection. Calls that pass a callback (including + the whole pre-v9 test suite's usage) are unchanged. +- `each()` is callback-only; calling it without callbacks throws a + `TypeError` pointing at `iterate()`. Streaming without callbacks was a + silent no-op before. +- `db.prepare()` and `db.backup()` keep their synchronous return in every + form — ~60 places in the callback regression suite rely on it. A + prepare error still surfaces on the statement's `'error'` event. + +New in this release: top-level `sqlite3.open()` (promise-native open), +`db.iterate()`/`stmt.iterate()` (pull-based async iteration with +backpressure, backed by the new native `Statement#fetch(count)`), +`db.stream()` (object-mode `Readable`), `db.transaction()` (BEGIN/COMMIT +with rollback on throw, automatic savepoints when nested), and +`Symbol.asyncDispose`/`Symbol.dispose` for `await using`/`using`. + +## Cancellation is connection-wide + +Promise-mode calls, `iterate()` and `transaction()` accept a trailing +`{ signal }` options object (an `AbortSignal`). An already-aborted signal +rejects before scheduling anything. Aborting afterwards calls +`db.interrupt()` and rejects with the signal's reason — **interrupting +every in-flight statement on that connection**, not just the awaited one. +That is a SQLite constraint: `sqlite3_interrupt` has no per-statement +form. Work that was queued but not started when the abort lands may still +run to completion; its result is dropped. The signal listener is removed +when the call settles, so one long-lived signal does not accumulate +listeners. + +## User-defined functions, aggregates, window functions and collations (new) + +`db.function()`, `db.aggregate()`, `db.collation()`, `db.removeFunction()` +and `db.removeCollation()` are new — there is nothing to migrate. Two +restrictions they bring are behavioural and worth knowing up front: + +- A JS function invoked from a **synchronous method** + (`getSync`/`runSync`/`allSync`/`prepareSync`) fails with an explicit + `SQLITE_ERROR` explaining the deadlock it refused to attempt, instead + of hanging. The synchronous methods keep working for statements that + never reach a JS function. +- While a JS **collation** is registered, the synchronous methods throw + until `removeCollation()` is called: a collation callback cannot report + an error mid-comparison, so refusing up front is the only sound + behaviour. + +Also new: each round trip to JS costs ~18 µs (measured; see the README's +"User-defined functions" section for the full cost table), and every +registration, replacement or removal flushes the statement cache. + +## Hooks, authorizer, progress and introspection (new) + +Everything in the README's "Hooks, authorizer, progress and +introspection" section is new — there is nothing to migrate. Three +behavioural notes worth knowing up front: + +- `'commit'` and `'rollback'` events are **observational**: the commit + or rollback has already happened when the listener runs, and a return + value cannot veto it. (A vetoing hook would have to block the + committing worker thread on the JS thread.) +- Registering the same event twice used to silently *uninstall* the + native hook (the old code toggled on every `addListener`); a second + `db.on('change', …)` now keeps the hook installed, and removing one + listener of several no longer stops the others from firing. The + single-listener usage the old suite exercised behaves identically. +- A JavaScript progress callback (`db.progress(n, cb)`) blocks the + synchronous methods while registered, exactly like a JS collation; + the SharedArrayBuffer cancellation token does not. + +## `map()` single-column results are rows, not `undefined` + +`db.map('SELECT id FROM t')` used to build `{ id: undefined }` (the +two-column code path read a missing second column). A single-column +result now maps the key to the whole row, consistent with the three-plus +column rule. + +## Integer reads now refuse to truncate (default `'number'` mode) + +In v8, any INTEGER column value outside the safe integer range +(±2^53−1) came back as a silently rounded `number`. In v9 the default +throws instead: + +```js +db.get("SELECT some_big_rowid FROM t", (err, row) => { + // v9: err instanceof RangeError, message names the column and value +}); +``` + +Pick the behaviour you want with `configure('integerMode', mode)` +(readable as `db.integerMode`): + +- `'number'` (default) — `number` when safe, `RangeError` otherwise. +- `'bigint'` — every INTEGER column and `lastID` is a `BigInt`. +- `'mixed'` — `number` when safe, `BigInt` otherwise. Recommended for + anything touching `rowid`s. + +`Statement#lastIDBigInt` is exact in every mode if you need a large +rowid without switching modes. + +## Integers bind as true 64-bit values + +Integral numbers within the int64 range now bind via +`sqlite3_bind_int64` instead of `sqlite3_bind_int`, and `BigInt` +parameters are accepted (exact; `RangeError` outside the signed 64-bit +range). In v8 anything above int32 was silently stored as a REAL, which +broke `WHERE` matches on STRICT tables and typeof-based checks: + +```js +db.runSync("INSERT INTO t VALUES (?)", 2 ** 40); +// v8: typeof(a) === 'real'; v9: typeof(a) === 'integer' +``` + +`lastID` follows the integer mode: a `RangeError` in `'number'` mode +when the rowid is unsafe (note `db.runSync` reads `lastID` eagerly for +its result object), a `BigInt` in `'bigint'`/`'mixed'`. + +## Objects no longer bind as the string `"[object Object]"` + +In v8 every plain object, array, `Map` and class instance bound as the +eleven-byte TEXT `"[object Object]"` (arrays passed directly were also +treated as named-parameter maps). In v9 these throw a `TypeError` +naming the parameter index and constructor: + +```js +db.run("INSERT INTO t VALUES (?)", { a: 1 }, (err) => { /* v8: stored "[object Object]" */ }); +db.runSync("INSERT INTO t VALUES (?)", [{ a: 1 }]); +// v9 TypeError: Cannot bind parameter 1: unsupported type Object. +// Serialize it explicitly (e.g. JSON.stringify) before binding. +``` + +Serialize explicitly: `JSON.stringify(value)`, or bind the object's +fields as named parameters (`INSERT INTO t VALUES ($a)` with `{ $a: 1 }`). + +`undefined` binds as NULL, matching `null` — object shorthand like +`{ $x: obj.maybeMissing }` keeps working. The one historic shape kept +on purpose: an argument list consisting only of `undefined` against a +statement with no parameters is ignored +(`db.run(sql, undefined, cb)` still runs the statement). + +## Parameter-count mismatches and unknown named parameters are errors + +- Too few parameters (`SELECT ?, ?` with one value): v8 silently bound + NULL for the missing ones; v9 reports an error. +- Too many parameters, or extra keys on a named-parameter object: v8 + ignored them (or surfaced a bare `SQLITE_RANGE`); v9 reports + `"supplied N parameter(s) but the statement takes M"`. +- A named parameter that does not exist in the SQL + (`db.get('SELECT $a', { $b: 1 })`): v9 reports + `"unknown named parameter \"$b\""`. + +## Blob binding accepts every binary view + +`Uint8Array` (byte range honoured, including non-zero `byteOffset`), +`DataView`, `ArrayBuffer`, typed arrays over `SharedArrayBuffer`, and +Node `Buffer` all bind as BLOBs of their exact byte range. Reads still +return Node `Buffer`s. + +## `err.code` is now the extended result code + +v8 reported only the 26 primary codes. v9 enables SQLite extended +result codes, so a unique violation is `SQLITE_CONSTRAINT_UNIQUE` +instead of `SQLITE_CONSTRAINT`. Every error carries: + +- `err.code` — extended name (`'SQLITE_CONSTRAINT_UNIQUE'`), +- `err.errno` — extended numeric code, +- `err.primaryCode` — primary name (`'SQLITE_CONSTRAINT'`). + +Migrate `err.code === 'SQLITE_CONSTRAINT'` checks to +`err.primaryCode === 'SQLITE_CONSTRAINT'` (or match the specific +extended code). The `SQLITE_CONSTRAINT_*`, `SQLITE_BUSY_*`, +`SQLITE_READONLY_*`, `SQLITE_IOERR_*`, `SQLITE_CANTOPEN_*` and other +extended families are exported as constants, as are the previously +missing open flags `OPEN_NOMUTEX`, `OPEN_MEMORY`, `OPEN_EXRESCODE`. + +## The connection state is native truth: `db.state` + +The internal scheduling state is now exposed read-only from the native +side and the JS-side mirrors are gone: + +- `db.state` is a frozen snapshot `{ open, closing, locked, serialized, + pending, queued }`, computed on read; the same fields are also + individual read-only accessors (`db.serialized`, `db.closing`, ...). +- `db._serialized` and `db._closing` (undocumented JS-side mirrors, + maintained by monkey-patching `serialize`/`parallelize`) **no longer + exist**. `serialize()`/`parallelize()` are now the native methods + directly — code that relied on the mirrors should read `db.state` or + the individual accessors. +- The mirror deletion fixes a real bug: anything that reached the native + `serialize` other than through the patched prototype (a saved + prototype reference, `Reflect.apply`) desynchronised `_serialized`, + and the statement cache then kept taking its fast path while the + connection was serialized — silently breaking the FIFO guarantee + `serialize()` promises. +- `db._queueBusy()` (undocumented, `@internal`) is **deprecated** in + favour of `db.state` and will be removed in a later minor release. +- `stmt.finalized` is new read-only state: true once a statement was + finalized (explicitly, after a failed prepare, or by the GC safety + net). + +## Concurrent `transaction()` calls now fail loudly + +Nesting used to be tracked with a connection-wide counter, so a second +transaction started *concurrently* (not nested inside the first's body) +silently rode inside the first as a savepoint: its "commit" was a +`RELEASE` the first transaction's rollback would have undone, and its +work only persisted if the unrelated first transaction committed. +Nesting is now tracked per async flow (`AsyncLocalStorage`): calls made +from inside a transaction body — including across `await` — still nest +via savepoints, but a concurrent top-level `BEGIN` rejects with +`a transaction is already active on this connection`. Serialize +concurrent flows yourself (or open a second connection). + +## A failed prepare no longer strands the calls queued behind it + +When preparing a statement failed, the calls already queued against that +statement were discarded without their callbacks ever being invoked. In +callback style the call simply never came back; in promise style the +promise never settled. Those calls are now failed with the prepare's own +error, so they reject (or call back) like any other failure. + +This is most visible with `AbortSignal`: `sqlite3_interrupt()` aborts a +prepare just as readily as it aborts a running step, so an abort landing +in that window used to wedge the connection. It also fixes +`stmt.iterate()` on a statement whose own prepare failed, where `next()` +waited forever on a dropped fetch and `return()` never settled. + +The statement still reports the failure on its own `'error'` event when +the prepare was given no callback, so that surface is unchanged. `each()` +delivers the error to its completion handler — or, if it was called +without one, to its row callback; it is never handed to both, and the row +callback is never invoked with a row-shaped call it has no row for. + +## New surface: worker-thread safety and the connection pool + +The addon now loads correctly in `worker_threads` workers (per- +environment constructors, and `worker.terminate()` with a query in +flight no longer crashes the process, with one remaining caveat noted in +[docs/concurrency.md](docs/concurrency.md#terminating-a-worker)), and +`sqlite3.pool(filename, options?)` builds a +worker pool over one file: a single writer connection plus read-only +readers, with `pool.read`/`get`/`write`/`exec`/`transaction`/`close`, +`{ signal }` cancellation, preserved error diagnostics, and +`Symbol.asyncDispose`. It is opt-in and promise-only — a single +`Database` remains the primary object. Two behavioural notes for code +that used workers informally before: + +- `db.get()`/`db.getSync()` on the statement cache with **no bind + parameters** used to return `undefined` from the second identical call + onwards (the cached statement re-stepped its cursor). Both now re-run + from the first row. A user-held `stmt.get()` keeps its documented + cursor-stepping behaviour (`marshalling.test.js` pins it); only the + Database-level convenience changed. +- Blob columns in **pool** results are `Uint8Array`, not `Buffer` + (structured clone does not preserve the subclass). Bind values are + affected in neither direction — a `Buffer` binds as a blob as always. + +See docs/concurrency.md for the concurrency model in full. + +## New surfaces: sessions, snapshots and blob handles + +Everything here is additive; no v8 behaviour changed to make room. + +- `db.session(options?)` records changes and harvests them as + `session.changeset()`/`patchset()` (`Uint8Array`); `db.applyChangeset( + bytes, { conflict, filter })` replays them inside one savepoint, and + `sqlite3.invertChangeset`/`concatChangeset`/`iterateChangeset` work on + any changeset bytes. A session and a `'preupdate'` listener cannot + coexist on one connection (SQLite has one preupdate hook); both + directions fail loudly. +- The `'preupdate'` event carries `{ op, database, table, rowid, + oldRowid, oldRow, newRow }` — the old row values the `'change'` event + has never been able to give you. +- `db.serializeToBytes(dbName?)` is a `Uint8Array` snapshot of the + database. The name is deliberate: `db.serialize()` still means "run + these statements in FIFO order". `sqlite3.deserializeFromBytes(bytes, + { readonly, resizable })` builds a fresh connection from bytes + (copied; corrupt input rejects with `SQLITE_NOTADB`). +- `db.openBlob({ table, column, rowid, readOnly? })` returns a handle + with `read`/`write` at offsets, `size`, `reopen(rowid)`, + `close()`, `createReadStream()` and `createWriteStream()`. Any write + to the row invalidates open handles (`SQLITE_ABORT`, with a message + saying so). +- `Session` and `Blob` are exported alongside `Database`, `Statement` + and `Backup`, and both support `using`/`await using` disposal. A + session or blob left open is closed by `db.close()`; closing twice is + a benign no-op. +- Changeset constants (`CHANGESET_OMIT/REPLACE/ABORT`, the + `CHANGESET_DATA/…/FOREIGN_KEY` conflict codes) live on the namespace + like the other SQLite constants. +- The declarative authorizer now distinguishes an explicitly-passed + empty-string rule field (matches only an empty argument) from an + omitted one (matches anything); previously the empty-string target + was unexpressible. + +## A throwing completion callback no longer wedges the connection + +`exec`, `open`, `close` and `loadExtension` fired their completion +callback as their last JS action and drained the database queue +afterwards. A callback that **threw** used to skip that drain — the +early return left everything queued behind the call undispatched +forever, so every later query on the connection never settled. The +drain now runs on every exit path (`Database::ProcessGuard`), including +the throwing one, and work queued behind a close that completes is +failed with `SQLITE_MISUSE` instead of hanging. The thrown error still +surfaces as an uncaught exception exactly as before. + +## Electron is now supported and verified + +The Node-API 10 prebuild loads in Electron **without any rebuild** +(Node-API is ABI-stable across runtimes); the verified minimum is +**Electron 35** (`engines.electron >= 35`, the first major whose +bundled Node exposes Node-API 10). On older Electron (32–34, Node-API +9) the load does not throw — it segfaults inside module registration — +so `lib/sqlite3-binding.js` now checks `process.versions.napi` *before* +loading and throws an error naming the floors. Binding load failures +everywhere now name the resolved package root and, when the package +sits inside an `app.asar` archive, the `asarUnpack` configuration that +fixes it (the original `node-gyp-build` error is preserved as +`err.cause`). node-webkit support and its build instructions were +removed. See [docs/electron.md](docs/electron.md). + +## Minor notes + +- `Date` still binds as epoch milliseconds (REAL) and reads back as a + `number` — unchanged, now documented. An opt-in TEXT form may arrive + in a later minor release. +- `RegExp` still binds as its source string (`"/re/g"`). +- `NaN` binds as NULL (SQLite `bind_double` semantics) — unchanged. +- `-0` binds as INTEGER `0` (SQLite has no signed integer zero). +- Strings containing lone surrogates are converted to U+FFFD at the + UTF-8 boundary, as in v8. +- `lastID`/`changes` are now prototype accessors rather than own + enumerable properties assigned after each run; they read `undefined` + before the first run, and `JSON.stringify` of a statement no longer + includes them. + +## Node permission model, extension policy, untrusted files + +Under Node's `--permission` flag (with `--allow-addons`, which this +package requires to load at all), every open path now checks the target +against the process's fs allowances and refuses with an +`ERR_ACCESS_DENIED`-shaped error that names the path and the flag that +permits it. A writable open needs `fs.write` for the file **and its +directory** (SQLite writes `-journal`/`-wal`/`-shm` beside it — grant +`--allow-fs-write="/*"`). `ATTACH` and `VACUUM INTO` are denied +unless their target is allowlisted with +`db.configure('attachPaths', [...])`, `db.backup()` destinations are +checked like opens, and `loadExtension` is refused unless allowlisted +with `db.configure('extensionPolicy', { allow: [...] })`. With the +permission model off — the overwhelmingly common case — behaviour is +unchanged and the checks cost one property read. Full details and the +explicit list of what remains open: +[docs/security.md](docs/security.md). + +New in the same delivery: + +- `sqlite3.open(filename, { mode, untrusted })` (and the same options + object in the `Database` constructor): `untrusted: true` applies the + hostile-file hardening recipe (defensive mode, untrusted schema, + `writable_schema` off, extension loading permanently disabled, + conservative run-time limits, deny-all ATTACH gate). +- `db.configure('extensionPolicy', { allow } | { deny: true })` — + restrict or permanently disable `loadExtension` on one connection. +- `db.configure('attachPaths', [...] | null)` — the ATTACH-gate + allowlist (works without the permission model too, as defence in + depth). +- Behaviour change at the margins: work queued behind a **failed open** + used to sit stranded forever (the connection stayed in the Opening + state); it now settles with the open's own error, and the connection + behaves as closed (a later `close()` reports the usual `SQLITE_MISUSE`). +- `new Database(filename, )` now + throws a `TypeError` instead of silently ignoring the argument. + +## TypeScript consumers + +The type declarations are now generated (`pnpm run gen-types`) from the +JSDoc in `lib/*.js` plus two hand-written declaration files (`lib/native.d.ts` +for the addon's shape, `lib/augment.d.ts` for the JS layer's members), and +CI fails if they drift. Behavioural changes visible in the types: + +- Parameterized promise calls now resolve to `Promise`: `db.all(sql, 1)` + is `Promise` (previously it fell through to a callback overload + and typed as the database). Callback calls still type as the receiver. +- Promise-mode methods that accept an `AbortSignal` type the trailing + `{ signal }` options object (`SignalOptions`); `iterate()`/`stream()` + accept it too. +- Bind parameters are typed (`BindValue`/`BindParams`), so binding e.g. a + `Symbol` is a compile error, matching the runtime strict-binding rule. +- The SQLite constants have literal types (`sqlite3.OPEN_READONLY` is + `1`), so flag combinations are checkable. +- `db.get`'s callback row is `row?: T` and the promise resolves + `T | undefined` in every mode, matching the runtime. +- The namespace carries `cached.objects`, and the classes expose + `db.filename`/`db.mode`/`stmt.sql` and the `Backup` own-properties, + none of which were declared before. +- The ~128 module-level `export const` constants that the old + declarations listed never existed at runtime (importing them by name + returned `undefined`); they are gone from the types — use the default + namespace object, which does carry them. diff --git a/README.md b/README.md index 5671b3e..22610ef 100644 --- a/README.md +++ b/README.md @@ -19,46 +19,65 @@ Asynchronous, non-blocking [SQLite3](https://sqlite.org/) bindings for [Node.js] # Installing -You can use [`npm`](https://github.com/npm/cli) or [`yarn`](https://github.com/yarnpkg/yarn) to install `sqlite3`: - -- (recommended) Latest published package: +Use whichever package manager you like: ```bash npm install @appthreat/sqlite3 # or +pnpm add @appthreat/sqlite3 +# or yarn add @appthreat/sqlite3 +# or +bun add @appthreat/sqlite3 ``` - GitHub's `master` branch: `npm install https://github.com/AppThreat/node-sqlite3/tarball/master` -### Prebuilt binaries +Requires Node.js >= 24. See [docs/install.md](docs/install.md) for the full +installation guide: prebuild coverage, source builds, custom SQLite/SQLCipher, +and troubleshooting. -`@appthreat/sqlite3` v6+ was rewritten to use [Node-API](https://nodejs.org/api/n-api.html) so prebuilt binaries do not need to be built for specific Node versions. `sqlite3` currently builds for both Node-API v3 and v6. Check the [Node-API version matrix](https://nodejs.org/api/n-api.html#node-api-version-matrix) to ensure your Node version supports one of these. The prebuilt binaries should be supported on Node v10+. +### Prebuilt binaries -The module uses [`prebuild-install`](https://github.com/prebuild/prebuild-install) to download the prebuilt binary for your platform, if it exists. These binaries are hosted on GitHub Releases for `sqlite3` versions above 5.0.2, and they are hosted on S3 otherwise. The following targets are currently provided: +`@appthreat/sqlite3` v6+ was rewritten to use [Node-API](https://nodejs.org/api/n-api.html), so a single prebuilt binary per platform covers every supported Node version — nothing is compiled or downloaded at install time for the platforms below: - `darwin-arm64` - `darwin-x64` -- `linux-arm64` -- `linux-x64` -- `linuxmusl-arm64` -- `linuxmusl-x64` -- `win32-ia32` +- `linux-arm64` (glibc and musl) +- `linux-x64` (glibc and musl) +- `win32-arm64` - `win32-x64` -Unfortunately, [prebuild](https://github.com/prebuild/prebuild/issues/174) cannot differentiate between `armv6` and `armv7`, and instead uses `arm` as the `{arch}`. Until that is fixed, you will still need to install `sqlite3` from [source](#source-install). - -Support for other platforms and architectures may be added in the future if CI supports building on them. +The prebuilds are bundled **inside the npm tarball** and resolved at +**runtime**, not by an install script. In particular, **pnpm 10+ users need no +`onlyBuiltDependencies` allowlist**: pnpm blocks dependencies' install scripts +by default, and that block is a no-op here because `lib/sqlite3-binding.js` +locates the prebuild itself when the module is first imported. -If your environment isn't supported, it'll use `node-gyp` to build SQLite, but you will need to install a C++ compiler and linker. +Support for other platforms and architectures may be added in the future if CI +supports building on them. Everywhere else, `@appthreat/sqlite3` builds from +source via `node-gyp` — see +[docs/install.md](docs/install.md#source-builds) for the toolchain +requirements and the pnpm specifics for source builds. ### Other ways to install -It is also possible to make your own build of `sqlite3` from its source instead of its npm package ([See below.](#source-install)). +It is also possible to make your own build of `sqlite3` from its source instead of its npm package ([See below.](#source-install)). -The `sqlite3` module also works with [node-webkit](https://github.com/rogerwang/node-webkit) if node-webkit contains a supported version of Node.js engine. [(See below.)](#building-for-node-webkit) +SQLite's [SQLCipher extension](https://github.com/sqlcipher/sqlcipher) is also supported. [(See below.)](#sqlcipher-encrypted-databases) -SQLite's [SQLCipher extension](https://github.com/sqlcipher/sqlcipher) is also supported. [(See below.)](#building-for-sqlcipher) +## Electron + +No rebuild, no `electron-rebuild`: v9 ships Node-API 10 prebuilds, and Node-API +is ABI-stable across runtimes — the same binary Node loads is the one Electron +loads. The minimum is **Electron 35** (the first major whose bundled Node +exposes Node-API 10; verified by loading the prebuild in Electron 35 and 44), +recorded in `engines.electron`. Source builds against Electron headers are only +needed for SQLCipher or a custom `sqlite_magic` — see +[docs/electron.md](docs/electron.md) for process placement (main vs. utility +process vs. preload), ASAR/`asarUnpack` configuration for electron-builder and +electron-forge, bundler externals, `userData` paths, and the SQLCipher rebuild +path. # API @@ -66,7 +85,7 @@ See the [API documentation](https://github.com/AppThreat/node-sqlite3/wiki/API) # Usage -**Note:** the module must be [installed](#installing) before use. +**Note:** the module must be [installed](#installing) before use. This package is now ESM only. @@ -91,6 +110,49 @@ db.serialize(() => { db.close(); ``` +## Promises, async iteration and disposal (v9) + +Every data method is dual-mode: pass a trailing callback for the classic +behaviour (returning `this`, chainable), or omit it to get a promise. `run` +resolves `{ lastID, lastIDBigInt, changes }`; `get`/`all`/`map` resolve the +rows; `exec`/`close`/`wait` resolve `undefined`. Errors carry the v9 +`code`/`errno`/`primaryCode` triple. + +```js +const db = await sqlite3.open(":memory:"); // promise-native open +await db.exec("CREATE TABLE lorem (info TEXT)"); +const { lastID } = await db.run("INSERT INTO lorem VALUES (?)", "Ipsum 1"); +const row = await db.get("SELECT * FROM lorem WHERE rowid = ?", lastID); +``` + +Stream large results with real backpressure — batches are pulled from +SQLite (64..1024 rows) only as fast as the consumer reads them: + +```js +for await (const row of db.iterate("SELECT * FROM big")) { ... } +db.stream("SELECT * FROM big").pipe(someTransform); // object-mode Readable +``` + +Transactions, cancellation and `await using` disposal: + +```js +await db.transaction(async (tx) => { // ROLLBACK on throw, nested savepoints + await tx.run("INSERT INTO lorem VALUES (?)", "Ipsum 2"); +}); + +const rows = await db.all("SELECT * FROM big", { signal }); // AbortSignal: +// an already-aborted signal rejects before scheduling; aborting in flight +// interrupts the whole connection (a SQLite constraint) and rejects with +// the signal's reason. + +await using db2 = await sqlite3.open("app.db"); // closed however the block exits +await using stmt = db2.prepare("SELECT 1"); // finalized the same way +``` + +`each()` stays callback-only — the async iterator is its promise-based +replacement. `db.prepare()` and `db.backup()` keep their synchronous return +in every form. + ## Performance options Two opt-in fast paths avoid the per-call prepare and threadpool round-trip @@ -120,18 +182,40 @@ const row = db.getSync("SELECT * FROM t WHERE rowid = ?", 42); // row | undefi const info = db.runSync("INSERT INTO t (a) VALUES (?)", 42); // { lastID, changes } const rows = db.allSync("SELECT * FROM t"); const stmt = db.prepareSync("SELECT ? AS v"); // statement-level variants +// Bulk-reader row shape: one array per row, values in result-column order. +const flat = db.allSync("SELECT * FROM t", { rowMode: "array" }); ``` -`getSync/runSync/allSync` execute on the calling thread — roughly 6x faster -than the async equivalents for interactive lookups. They throw when the +`getSync`/`allSync` (not the async paths) accept a trailing +`{ rowMode: 'array' }` option: rows come back as arrays instead of +objects — duplicate column names keep every value instead of collapsing, +and the per-cell property stores disappear entirely, making it the +fastest row shape the sync paths can build. CSV export, ETL and bulk +feeds are the intended users; the default object shape is unchanged. +(A named bind parameter could never have the bare key `rowMode` — bind +keys carry a sigil — so the option is unambiguous.) + +`getSync/runSync/allSync` execute on the calling thread. On the benchmark +suite (`pnpm run bench`, [docs/performance.md](docs/performance.md)), +cached single-row lookups are **8–12× faster** than the cached async +`get`/`run` +equivalents on arm64 macOS (10.4–11.8× for `getSync`, flat +from batches of 1 to 10,000; `runSync` 8.2× at one operation rising to +~11.5× at 10,000 as per-round overhead amortises) — and **22–31×** on +Linux, where the async threadpool round trip costs more. For large +result sets sync and async are level (20,000 rows × 4 cols measured +within the noise floor): the marshalling is the same work either way, +and it dominates the threadpool round trip. They throw when the database is not fully idle: async work in flight or queued, or when called from inside an async completion callback (defer with `setImmediate` or use `db.wait`). They accept no callback argument. Like any synchronous database API, a busy database file can block the event loop for up to the configured `busyTimeout`. -Without `cacheStatements()` these methods prepare and finalize a statement -per call; enabling the cache is what makes them fast. +These `Database`-level forms keep their own statement cache, so they do +not prepare and finalize a statement per call; that is automatic and +does not need `cacheStatements()`, which is opt-in and governs the +asynchronous calls. ### Scheduling change @@ -144,123 +228,514 @@ old queue-jumping behaviour may see operations complete in a different order. Parallel throughput is unchanged: the queue is only non-empty once something has had to wait. -## Source install +## Value marshalling (v9) -To skip searching for pre-compiled binaries, and force a build from source, use +### Integer modes -```bash -npm install --build-from-source +```js +db.getSync("SELECT COUNT(*) AS n FROM t").n; // number (default) +db.configure("integerMode", "mixed"); // or 'number' | 'bigint' +db.integerMode; // 'mixed' ``` -The sqlite3 module depends only on libsqlite3. However, by default, an internal/bundled copy of sqlite will be built and statically linked, so an externally installed sqlite3 is not required. +Integers are stored as true 64-bit values on both the bind and the read +path, and `BigInt` parameters bind exactly. Reads follow the configured +mode: + +| Mode | INTEGER columns and `lastID` | +|---|---| +| `'number'` (default) | `number` when safely representable, otherwise a `RangeError` — never a silently truncated double | +| `'bigint'` | always `BigInt` | +| `'mixed'` | `number` when safe, `BigInt` otherwise — recommended for anything touching `rowid`s | + +`Statement#lastIDBigInt` returns the last insert rowid as a `BigInt` in +every mode, so `'number'`-mode code can still read a large rowid without +switching modes. + +### Accepted bind values + +`string`, `number` (integral values within the int64 range bind as +INTEGER; the double `2**63` clamps to `2**63-1`), `bigint` (`RangeError` +outside the signed 64-bit range), `boolean` (0/1), `null` and `undefined` +(both NULL), `Date` (epoch milliseconds as REAL — documented, lossy in +type), `RegExp` (its source string), and any binary view: Node `Buffer`, +`Uint8Array`/`Float64Array`/… (byte range honoured), `DataView` +(byte range honoured), `ArrayBuffer`. + +Everything else — plain objects, arrays, `Map`, class instances, symbols, +functions — throws a `TypeError` naming the parameter index and the +constructor. Bind the number of parameters the statement takes: too few +(previously silently NULL) and too many (previously ignored) are both +errors now, and a named parameter absent from the SQL (`sqlite3_bind_parameter_index` +returning 0) throws as well. + +### Extended result codes + +Errors carry three properties: `err.code` (the extended name, e.g. +`SQLITE_CONSTRAINT_UNIQUE`), `err.errno` (the extended number) and +`err.primaryCode` (the primary name, e.g. `SQLITE_CONSTRAINT`). The +`SQLITE_CONSTRAINT_*`, `SQLITE_BUSY_*`, `SQLITE_READONLY_*`, +`SQLITE_IOERR_*`, `SQLITE_CANTOPEN_*`, `SQLITE_LOCKED_*`, +`SQLITE_CORRUPT_*`, `SQLITE_ERROR_*`, `SQLITE_ABORT_ROLLBACK` and +`SQLITE_AUTH_USER` constants are exported, as are the previously missing +open flags `OPEN_NOMUTEX`, `OPEN_MEMORY` and `OPEN_EXRESCODE`. + +## User-defined functions, aggregates and collations (v9) -If you wish to install against an external sqlite then you need to pass the `--sqlite` argument to `npm` wrapper: +```js +// Scalar functions — this makes WHERE x REGEXP ? work: +db.function('regexp', { deterministic: true }, + (pattern, value) => new RegExp(pattern).test(value) ? 1 : 0); + +// Aggregates: start() builds an accumulator, step() folds a row into it, +// result() produces the value. Providing inverse makes it a window +// function usable with OVER (...). +db.aggregate('median', { + start: () => [], + step: (acc, v) => { acc.push(v); return acc; }, + result: (acc) => { + acc.sort((a, b) => a - b); + return acc.length ? acc[acc.length >> 1] : null; + }, +}); +await db.get('SELECT median(salary) AS m FROM employees'); -```bash -npm install --build-from-source --sqlite=/usr/local +// Collations — ORDER BY, indexes, COLLATE: +db.collation('german', (a, b) => a.localeCompare(b, 'de')); +await db.all('SELECT name FROM t ORDER BY name COLLATE german'); + +db.removeFunction('regexp'); +db.removeCollation('german'); ``` -If building against an external sqlite3 make sure to have the development headers available. Mac OS X ships with these by default. If you don't have them installed, install the `-dev` package with your package manager, e.g. `apt-get install libsqlite3-dev` for Debian/Ubuntu. Make sure that you have at least `libsqlite3` >= 3.6. +Arguments and return values use exactly the bind-marshalling rules above +(int64/BigInt, buffers for blobs, strict types: an unsupported return +value is an error, never a coerced string). Without `varargs: true` the +arity comes from the implementation's `length` (minus the accumulator for +aggregates), and calls with any other argument count are SQL errors. + +Options: `deterministic` (required for index/generated-column use, and a +false claim corrupts results — opt-in), `directOnly` (default **true**: +schema SQL — triggers, views, CHECK constraints, index expressions — +cannot invoke the function; opt out explicitly), `innocuous`, `varargs`. +Window functions (aggregates with `inverse`) are registered through +`sqlite3_create_window_function`, which has no flag slot, so the flag +options do not apply to them. + +### The threading model, and what it costs + +SQLite invokes a function callback on whatever thread is executing the +statement — here a worker thread. Each call therefore makes a blocking +round trip to the JS thread: the worker marshals the arguments and waits +while the JS thread runs your function and posts the result back. + +Measured cost (`pnpm run bench`, Apple Silicon, Node 26): **~18 µs per +call**. Consequences, with one decimal of honesty: + +| Filtering 100,000 rows | Time | +|---|---| +| the predicate in SQL | 5 ms | +| the predicate in JS after `all()` | 25 ms | +| the predicate in a JS function per row | 1,830 ms | + +A JS function called per row is the wrong tool for bulk filtering — +fetch and filter in JS (or write the predicate in SQL). A JS collation is +even sharper: sorting 100k rows costs O(N log N) round trips (~17 s). +Where they shine is pushing *logic* into a query — a regexp, a domain +checksum, a custom aggregate over a bounded group. + +Two deliberate restrictions follow from the threading model: + +- A JS function reached from a **synchronous method** + (`getSync`/`runSync`/`allSync`/`prepareSync`) fails with an explicit + error instead of deadlocking: the JS thread is the one blocked inside + SQLite there and cannot run the callback. Use the async API. +- While a JS **collation** is registered, the synchronous methods refuse + to run entirely (remove it with `removeCollation()` or use the async + API): a comparison would need the blocked JS thread, and unlike + functions, a collation callback has no way to report an error. + +Errors: a throwing callback surfaces as a `SQLITE_ERROR` whose message +names the function, with the original JS error attached as `err.cause`; +the connection stays usable. Registration and replacement are refused +with `SQLITE_BUSY` (reported on the connection's `'error'` event) while +a cursor is suspended mid-query; the statement cache is flushed on every +registration, replacement and removal, so no statement compiled against +the old implementation is handed back. + +## Hooks, authorizer, progress and introspection (v9) -Note, if building against homebrew-installed sqlite on OS X you can do: +```js +// Transaction hooks. commit fires after the transaction commits — every +// change event of that transaction is delivered first, which is what +// makes the pair useful for cache invalidation. The hooks are +// observational: the commit (or rollback) has already happened when the +// listener runs, and no return value can veto it. +db.on('change', (type, database, table, rowid) => { /* ... */ }); +db.on('commit', () => { /* ... */ }); +db.on('rollback', () => { /* ... */ }); + +// WAL hook: fires after a commit appends frames to the WAL. +db.on('wal', (database, pages) => { /* ... */ }); +``` -```bash -npm install --build-from-source --sqlite=/usr/local/opt/sqlite/ +A hook's native sqlite callback exists only while at least one listener +is registered — an installed-but-unused hook costs nothing. In WAL mode, +`db.checkpoint()` is the lever for keeping the WAL bounded: + +```js +const { busy, logFrames, checkpointedFrames } = + await db.checkpoint({ mode: 'truncate' }); ``` -## Custom file header (magic) +### Sandboxing SQL with the authorizer -The default sqlite file header is "SQLite format 3". You can specify a different magic, though this will make standard tools and libraries unable to work with your files. +```js +db.authorizer({ + default: 'deny', + allow: [ + { action: sqlite3.SELECT }, + { action: sqlite3.READ, table: 'users' }, + ], +}); +db.authorizer(null); // remove +``` -```bash -npm install --build-from-source --sqlite_magic="MyCustomMagic15" +The policy is a rule list evaluated inside SQLite itself, in C++ — no +JavaScript runs on the prepare path, so it is fast and thread-safe by +construction. `deny` rules win over `allow` rules; a denied action fails +the statement with `SQLITE_AUTH` ("not authorized"). The ~35 action +constants (`sqlite3.SELECT`, `sqlite3.READ`, `sqlite3.INSERT`, +`sqlite3.ATTACH`, …) and the decisions (`sqlite3.DENY`, +`sqlite3.IGNORE`) are exported. The statement cache is flushed on every +policy change: a cached statement was compiled under the old policy and +would bypass the new one. + +### Cancelling queries + +```js +// The cancellation token: an atomic flag in a SharedArrayBuffer that +// the native progress handler polls. Zero JS cost per check, and +// cancel() works from any thread — post token.buffer to a Worker. +const token = db.cancellationToken(); +db.all(longRunningSql).catch(() => {}); +setTimeout(() => token.cancel(), 100); + +// token.signal is a real AbortSignal, so the promise form rejects with +// your reason: +db.all(longRunningSql, { signal: token.signal }).catch((reason) => {}); ``` -Note that the magic _must_ be exactly 15 characters long (16 bytes including null terminator). +Cancellation is connection-wide, like `db.interrupt()`: the abort +reaches every statement running on the connection. While a token exists, +each query pays one relaxed atomic load per `period` VM instructions +(default 1000) — within measurement noise in the benchmark suite. -## Building for node-webkit +A JavaScript callback form exists for progress reporting — +`db.progress(10000, () => shouldStop)` calls the callback every 10,000 +VM instructions and aborts the statement when it returns truthy — but +each invocation is a blocking round trip to the JS thread (the same +~18 µs class as JS functions), so it is for progress bars over long +queries, not per-row work. While it is registered, the synchronous +methods refuse to run (a callback would fire on the thread that must +service it). The token form has no such restriction. -Because of ABI differences, `sqlite3` must be built in a custom to be used with [node-webkit](https://github.com/rogerwang/node-webkit). +### Statement and connection introspection -To build `sqlite3` for node-webkit: +```js +const stmt = await db.prepare('SELECT name AS who FROM users WHERE id = ?'); +stmt.readonly; // true — sqlite3_stmt_readonly +stmt.parameterCount; // 1 +stmt.parameterNames; // ['?1'] (null entries for positional `?`) +stmt.columns; // [{ name: 'who', declaredType: 'TEXT', + // database: 'main', table: 'users', origin: 'name' }] +stmt.status(sqlite3.STMTSTATUS_FULLSCAN_STEP); // >0: the query scanned + // without an index + +db.changes; // rows changed by the most recent statement (64-bit) +db.totalChanges; // every change since open (64-bit) +await db.tableInfo('users'); // column metadata incl. collation, defaults +await db.dbConfig(sqlite3.DBCONFIG_DEFENSIVE, true); // safe db_config switches +``` -1. Install [`nw-gyp`](https://github.com/rogerwang/nw-gyp) globally: `npm install nw-gyp -g` _(unless already installed)_ +The statement accessors serve a snapshot taken when the statement was +prepared, so reading them never touches the sqlite handle and cannot +race a running query; fields SQLite reports as absent (an expression +column has no origin, a typeless column no declared type) are omitted +rather than nulled. Integer modes apply to `changes`/`totalChanges` as +everywhere else. `tableInfo` runs a `PRAGMA table_info`, so a +deny-by-default authorizer must allow `sqlite3.PRAGMA`. -2. Build the module with the custom flags of `--runtime`, `--target_arch`, and `--target`: +## Sessions, changesets and the preupdate event (v9) -```bash -NODE_WEBKIT_VERSION="0.8.6" # see latest version at https://github.com/rogerwang/node-webkit#downloads -npm install sqlite3 --build-from-source --runtime=node-webkit --target_arch=ia32 --target=$(NODE_WEBKIT_VERSION) +A session records every INSERT, UPDATE and DELETE made through the +connection (tables need a primary key to be recordable), and harvests +them as a changeset — a `Uint8Array` you can store, ship, or apply to +another connection: + +```js +const session = db.session({ table: 'users' }); // or every table +await db.run('UPDATE users SET name = ? WHERE id = ?', 'x', 1); +const changeset = await session.changeset(); // Uint8Array +const patchset = await session.patchset(); // new rows only, smaller +await session.close(); + +await target.applyChangeset(changeset, { conflict: 'replace' }); +await target.applyChangeset(changeset, { + // the fully general form — runs per conflict: + conflict: (info) => (info.conflict === 'notFound' ? 'omit' : 'replace'), + filter: (table) => table !== 'audit', +}); + +for (const op of sqlite3.iterateChangeset(changeset)) { + console.log(op.op, op.table, op.oldRow, op.newRow); +} +const inverse = sqlite3.invertChangeset(changeset); // undoes the apply +const both = sqlite3.concatChangeset(a, b); // a then b ``` -You can also run this command from within a `sqlite3` checkout: +`applyChangeset` wraps the apply in one savepoint: either every change +lands or the whole apply rolls back. `conflict` decides what happens on +a collision — `'abort'` (the default) rolls back, `'omit'` skips the +change, `'replace'` overwrites the row — or a function returning one of +those per conflict. The function form is a blocking round trip from the +applying thread (like a user-defined JS function), so it must not use +the synchronous methods on that connection. -```bash -npm install --build-from-source --runtime=node-webkit --target_arch=ia32 --target=$(NODE_WEBKIT_VERSION) +The `'preupdate'` event fires for every write with the row's before and +after values — the old values `change` events cannot give you: + +```js +db.on('preupdate', ({ op, table, rowid, oldRowid, oldRow, newRow }) => { + audit.log(op, table, oldRow, newRow); +}); +``` + +`oldRowid` and `rowid` differ exactly on a rowid-changing update. +One preupdate hook exists per connection and is shared with the session +machinery, so a session and a `'preupdate'` listener cannot coexist on +one connection — attempting either direction fails loudly instead of +silently stopping the other. + +## In-memory snapshots: serializeToBytes / deserializeFromBytes (v9) + +The whole database as bytes — snapshotting, shipping a prebuilt +database, fast fixtures, moving a database between threads: + +```js +const bytes = await db.serializeToBytes(); // Uint8Array snapshot +const copy = await sqlite3.deserializeFromBytes(bytes, { + readonly: false, + resizable: true, +}); +``` + +`serializeToBytes` returns the exact bytes a file copy would contain +(the FIFO-ordering `db.serialize()` keeps its old meaning). The bytes +are named deliberately: overloading `serialize()` would be the worst +API decision available. `deserializeFromBytes` **copies** into +SQLite-owned memory — handing a JS buffer to SQLite directly is a +use-after-free waiting to happen — and rejects corrupt input with +`SQLITE_NOTADB` rather than crashing later. + +## Incremental blob I/O (v9) + +Reading a 500 MB blob as one value materialises it as a single buffer; +`openBlob` gives you a handle that streams it instead: + +```js +const blob = await db.openBlob({ table: 'files', column: 'data', rowid: 1 }); +const chunk = new Uint8Array(65536); +const n = await blob.read(chunk, 0); // n bytes at blob offset 0 +await blob.write(source, 4096); // write at an offset +blob.size; // sqlite3_blob_bytes + +await pipeline(blob.createReadStream(), fs.createWriteSink(path)); +await pipeline(fs.createReadStream(path), blob.createWriteStream()); +await blob.close(); +``` + +Streams read and write in chunks (default 64 KiB), so memory stays flat +regardless of the blob's size. Any write to the row invalidates open +handles with `SQLITE_ABORT` (and a message saying so); an aborted handle +cannot be reopened — close and open a fresh one — while `blob.reopen( +rowid)` cheaply re-aims a *healthy* handle at another row. The blob +cannot grow through the handle: size the column first (e.g. +`UPDATE ... SET data = zeroblob(n)`) and then stream into it. Writing +through a blob handle surfaces as a `'preupdate'` delete event (the new +values are not yet available inside `sqlite3_blob_write`). + +## Worker threads and the connection pool (v9) + +The addon is context-aware: it loads cleanly in every `worker_threads` +worker, and each environment gets its own constructors. Two supported +ways to use it from workers — plus the pool, which is the batteries- +included version: + +**Path handoff** — the worker opens its own connection to the same file +(WAL mode gives real read concurrency): + +```js +const w = new Worker('./db-worker.js', { + workerData: { filename: 'app.db' }, +}); +``` + +**Bytes handoff** — move an in-memory database across threads with one +copy (`serializeToBytes()` → transfer → `deserializeFromBytes()`): + +```js +const bytes = await db.serializeToBytes(); +const movable = bytes.slice().buffer; // plain ArrayBuffer copy +w.postMessage({ bytes: movable }, [movable]); +// worker: await sqlite3.deserializeFromBytes(new Uint8Array(bytes)) ``` -Remember the following: +**The pool** — one writer plus N read-only reader connections, each on +its own worker; writes queue instead of racing to `SQLITE_BUSY`: + +```js +const pool = await sqlite3.pool('app.db', { readers: 4 }); -- You must provide the right `--target_arch` flag. `ia32` is needed to target 32bit node-webkit builds, while `x64` will target 64bit node-webkit builds (if available for your platform). +const rows = await pool.read('SELECT * FROM t WHERE a = ?', [1]); +const one = await pool.get('SELECT b FROM t WHERE a = ?', [1]); +await pool.write('INSERT INTO t (b) VALUES (?)', ['hi']); -- After the `sqlite3` package is built for node-webkit it cannot run in the vanilla Node.js (and vice versa). - - For example, `npm test` of the node-webkit's package would fail. +await pool.transaction(async (tx) => { + const row = await tx.get('SELECT a FROM t'); // pinned to the writer + await tx.write('UPDATE t SET a = ?', [row.a + 1]); +}); -Visit the “[Using Node modules](https://github.com/rogerwang/node-webkit/wiki/Using-Node-modules)” article in the node-webkit's wiki for more details. +await pool.close(); // drains, closes every connection, no worker survives +``` -## Building for SQLCipher +Queries accept `{ signal }` (cancellation crosses the thread boundary +through a shared-memory flag), errors keep `code`/`errno`/`primaryCode`, +and `await using pool` works. Rows are structured-cloned across the +boundary: blob columns come back as `Uint8Array` (not `Buffer`) and huge +result sets pay a copy — the pool is for many small queries, not bulk +reads. See [docs/concurrency.md](docs/concurrency.md) for the full +picture: `serialize()`/`parallelize()` semantics, WAL, busy timeouts, +and when to use one connection, several, or the pool. -For instructions on building SQLCipher, see [Building SQLCipher for Node.js](https://coolaj86.com/articles/building-sqlcipher-for-node-js-on-raspberry-pi-2/). Alternatively, you can install it with your local package manager. +## Source install -To run against SQLCipher, you need to compile `sqlite3` from source by passing build options like: +To skip searching for pre-compiled binaries, and force a build from source, use ```bash -npm install sqlite3 --build-from-source --sqlite_libname=sqlcipher --sqlite=/usr/ +npm install --build-from-source ``` -If your SQLCipher is installed in a custom location (if you compiled and installed it yourself), you'll need to set some environment variables: +The sqlite3 module depends only on libsqlite3. However, by default, an internal/bundled copy of sqlite will be built and statically linked, so an externally installed sqlite3 is not required. + +If you wish to install against an external sqlite then you need to pass the `--sqlite` argument to `npm` wrapper: + +```bash +npm install --build-from-source --sqlite=/usr/local +``` -### On OS X with Homebrew +If building against an external sqlite3 make sure to have the development headers available. Mac OS X ships with these by default. If you don't have them installed, install the `-dev` package with your package manager, e.g. `apt-get install libsqlite3-dev` for Debian/Ubuntu. Make sure that you have at least `libsqlite3` >= 3.6. -Set the location where `brew` installed it: +Note, if building against homebrew-installed sqlite on OS X you can do: ```bash -export LDFLAGS="-L`brew --prefix`/opt/sqlcipher/lib" -export CPPFLAGS="-I`brew --prefix`/opt/sqlcipher/include/sqlcipher" -npm install sqlite3 --build-from-source --sqlite_libname=sqlcipher --sqlite=`brew --prefix` +npm install --build-from-source --sqlite=/usr/local/opt/sqlite/ ``` -### On most Linuxes (including Raspberry Pi) +## Custom file header (magic) -Set the location where `make` installed it: +The default sqlite file header is “SQLite format 3”. You can specify a different magic, though this will make standard tools and libraries unable to work with your files. ```bash -export LDFLAGS="-L/usr/local/lib" -export CPPFLAGS="-I/usr/local/include -I/usr/local/include/sqlcipher" -export CXXFLAGS="$CPPFLAGS" -npm install sqlite3 --build-from-source --sqlite_libname=sqlcipher --sqlite=/usr/local --verbose +npm install --build-from-source --sqlite_magic=”MyCustomMagic15” ``` +Note that the magic _must_ be exactly 15 characters long (16 bytes including null terminator). + +## SQLCipher (encrypted databases) + +SQLCipher is supported via a **source build** — no prebuild ships with +SQLCipher, because the encryption runtime must come from your system's +SQLCipher. Build flags, Homebrew/Linux paths and the Electron variant are +in [docs/security.md#sqlcipher](docs/security.md#sqlcipher). + ### Custom builds and Electron -Running `sqlite3` through [electron-rebuild](https://github.com/electron/electron-rebuild) does not preserve the SQLCipher extension, so some additional flags are needed to make this build Electron compatible. Your `npm install sqlite3 --build-from-source` command needs these additional flags (be sure to replace the target version with the current Electron version you are working with): +The default build needs **no Electron-specific step at all**: v9 ships Node-API +10 prebuilds and Node-API is ABI-stable across runtimes, so the prebuild loads +in Electron >= 35 unchanged (see [Electron](#electron) above and +[docs/electron.md](docs/electron.md)). + +Running a **source** build (SQLCipher, custom `sqlite_magic`) against Electron +headers needs extra flags for `npm install sqlite3 --build-from-source` +(replace the target with your Electron version): ```bash ---runtime=electron --target=18.2.1 --dist-url=https://electronjs.org/headers +--runtime=electron --target=44.0.0 --dist-url=https://electronjs.org/headers ``` -In the case of MacOS with Homebrew, the command should look like the following: +The SQLite location and library name go through `GYP_DEFINES`, not +command-line flags — node-gyp 13 treats anything after `--` as a +build-file name. For macOS with Homebrew: ```bash -npm install sqlite3 --build-from-source --sqlite_libname=sqlcipher --sqlite=`brew --prefix` --runtime=electron --target=18.2.1 --dist-url=https://electronjs.org/headers +export GYP_DEFINES="sqlite=$(brew --prefix) sqlite_libname=sqlcipher" +npm install @appthreat/sqlite3 --build-from-source \ + --runtime=electron --target=44.0.0 --dist-url=https://electronjs.org/headers ``` +SQLCipher needs the session extension enabled, which packaged builds +usually omit — see [docs/security.md](docs/security.md#sqlcipher). + +# Security + +The security posture — what this package does and does not protect +against, the Node `--permission` interaction (and how the checks refuse +out-of-scope file access), the `untrusted: true` recipe for hostile +database files, extension-loading policy, and the vendored-SQLite CVE +policy — is documented in [docs/security.md](docs/security.md). +Vulnerability reporting is in [SECURITY.md](SECURITY.md). + # Testing ```bash -npm test +pnpm run test ``` +# Developing + +Development of this repo itself requires **pnpm >= 11** (`corepack enable`, or a +standalone install). Clone, then: + +```bash +pnpm install # also builds the native binding via the install script +pnpm run rebuild # recompile after changing C++ (node-gyp rebuild) +pnpm run lint # biome check --write (autofix; CI runs lint:check) +pnpm run test # node:test, 20s per-test timeout, files run in parallel +pnpm run prebuild # produce the shipping prebuilds/ artifacts +pnpm run test:electron # the full suite + app-env harness inside Electron +pnpm run test:matrix # the suite across glibc/musl containers (needs Docker) +``` + +`test:matrix` exists for the failures that do not reproduce on a developer +machine — a musl-only segfault, or a race that needs an older glibc, a +specific Node and a busy CPU before it shows up at all: + +```bash +node tools/test-matrix.mjs --list # the targets and why each exists +node tools/test-matrix.mjs --cpus=1 --load=6 # simulate a slow CI runner +node tools/test-matrix.mjs --repeat=20 --cmd='node --test test/foo.test.js' +``` + +It rebuilds the addon and regenerates fixtures inside each container, +ignoring your local `node_modules/`, `build/`, `prebuilds/` and `test/tmp/`, +so a result does not depend on working-tree leftovers. + +Always use `pnpm run rebuild`, never bare `pnpm rebuild` — the latter is a +pnpm builtin that rebuilds *dependencies*, not this repo's `rebuild` script. +See [docs/install.md](docs/install.md#development) for the full guide, +including the stale-`prebuilds/` trap when iterating on C++. + # Contributors - [Daniel Lockyer](https://github.com/daniellockyer) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..5c9770c --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,51 @@ +# Security policy + +## Reporting a vulnerability + +Report vulnerabilities privately to **cloud@appthreat.com** (the package +maintainers, Team AppThreat). Include: + +- a minimal reproducer (code and, where relevant, a database file), +- the affected versions — of this package **and** of the vendored SQLite + (`sqlite3.VERSION` reports it at runtime), +- the environment (Node/Electron version, OS and architecture, and + whether the build is a shipped prebuild or a source build). + +Please do not open public issues for unreported vulnerabilities. We aim +to respond within a week. + +## Scope + +This package embeds the SQLite amalgamation and exposes it to Node. A +report is in scope if it concerns: + +- the JavaScript layer (`lib/`) or the native addon (`src/`) of this + package, or +- a vulnerability in the vendored SQLite version that this package + ships, including its default build configuration (the compile-time + defines are in `deps/sqlite3.gyp`). + +Out of scope: vulnerabilities requiring the attacker to already control +executed SQL (SQL is trusted input — see +[docs/security.md](docs/security.md)), or already-allowed extension +loading, which is arbitrary code execution **by design**. + +## Vendored-SQLite CVE response + +The amalgamation is pinned (`deps/sqlite-amalgamation-3530400`, SQLite +3.53.4) and the package inherits its CVEs. When a SQLite CVE is +published: + +1. the maintainers assess whether the vulnerable code is reachable in + this package's build (the amalgamation is compiled with a specific + feature set — `deps/sqlite3.gyp` — which can exclude a vulnerable + feature entirely); +2. if reachable, the amalgamation is bumped to the fixed SQLite version + in a dedicated version bump (never folded silently into a feature + change), and a release ships with the CVE identifier in the notes; +3. the policy for reporting SQLite CVEs upstream is the SQLite project's + own process — this package does not adjudicate SQLite bugs, it tracks + releases. + +Check what you are running with `sqlite3.VERSION` and compare against +[SQLite's change log](https://sqlite.org/changes.html). diff --git a/bench/baseline.json b/bench/baseline.json new file mode 100644 index 0000000..8e4cf57 --- /dev/null +++ b/bench/baseline.json @@ -0,0 +1,492 @@ +{ + "schemaVersion": 1, + "note": "Per-environment medians captured deliberately via `pnpm run bench:update`. Compare only within one platform-arch signature; ratios travel across platforms, absolute milliseconds do not. See docs/performance.md.", + "environments": { + "darwin-arm64": { + "capturedAt": "2026-08-28T15:15:46.037Z", + "environment": { + "node": "v26.7.0", + "platform": "darwin", + "arch": "arm64", + "cpuModel": "Apple M4 Pro", + "cpuCount": 14, + "container": "none", + "sqliteVersion": "3.53.4", + "packageVersion": "9.0.0", + "gitSha": "9d1b165+dirty", + "exposeGc": true + }, + "config": { + "warmupMs": 500, + "targetSampleMs": 20, + "minSampleMs": 10, + "samples": 32, + "rmeThresholdPct": 5, + "allocSamples": 16 + }, + "noiseFloorPct": 1.69, + "cases": { + "calibration/cached get (A)": { + "medianPerOpMs": 0.00969796831683166, + "rme": 0.0037269340276194535, + "n": 32 + }, + "calibration/cached get (B)": { + "medianPerOpMs": 0.009863701525590698, + "rme": 0.004187862406014868, + "n": 32 + }, + "read/all: 1,000 rows × 1 cols": { + "medianPerOpMs": 0.00019694062499999744, + "rme": 0.0035390488884709517, + "n": 32 + }, + "read/all: 1,000 rows × 4 cols": { + "medianPerOpMs": 0.0005801994142857178, + "rme": 0.008822914801284028, + "n": 32 + }, + "read/all: 1,000 rows × 16 cols": { + "medianPerOpMs": 0.0007895133400000122, + "rme": 0.003647816767726226, + "n": 32 + }, + "read/all: 20,000 rows × 1 cols": { + "medianPerOpMs": 0.00017489565833333245, + "rme": 0.002304933813144754, + "n": 32 + }, + "read/all: 20,000 rows × 4 cols": { + "medianPerOpMs": 0.0005700145750000046, + "rme": 0.015064483219558755, + "n": 32 + }, + "read/all: 20,000 rows × 16 cols": { + "medianPerOpMs": 0.0008035374999999476, + "rme": 0.01435060591448197, + "n": 32 + }, + "read/all: 200,000 rows × 1 cols": { + "medianPerOpMs": 0.0001727987475000009, + "rme": 0.0026397471428332565, + "n": 32 + }, + "read/all: 200,000 rows × 4 cols": { + "medianPerOpMs": 0.0005963269775000027, + "rme": 0.010142994495006757, + "n": 32 + }, + "read/all: 200,000 rows × 16 cols": { + "medianPerOpMs": 0.0008312881249999918, + "rme": 0.012439279100731802, + "n": 32 + }, + "read/all: 20,000 rows × 8 cols mostly NULL": { + "medianPerOpMs": 0.0003000316000000263, + "rme": 0.003993190495112072, + "n": 32 + }, + "read/get: single row (prepared statement)": { + "medianPerOpMs": 0.009340008226691286, + "rme": 0.02094349669679764, + "n": 32 + }, + "read/each: 20,000 rows × 4 cols": { + "medianPerOpMs": 0.0004374598999999762, + "rme": 0.02220086343926602, + "n": 32 + }, + "read/iterate: 20,000 rows × 4 cols (for await)": { + "medianPerOpMs": 0.0006283104125000136, + "rme": 0.0017938286992478323, + "n": 32 + }, + "read/map: 20,000 rows × 4 cols": { + "medianPerOpMs": 0.00018397729500000423, + "rme": 0.0026531942432093027, + "n": 32 + }, + "marshalling/integer ×20,000 (mode 'number')": { + "medianPerOpMs": 0.00015366369285711698, + "rme": 0.0056768914114139184, + "n": 32 + }, + "marshalling/integer ×20,000 (mode 'mixed')": { + "medianPerOpMs": 0.00015133273928570688, + "rme": 0.003199241633174314, + "n": 32 + }, + "marshalling/integer ×20,000 (mode 'bigint')": { + "medianPerOpMs": 0.0001584303791666571, + "rme": 0.0037690845855636213, + "n": 32 + }, + "marshalling/float ×20,000": { + "medianPerOpMs": 0.00015613680833333394, + "rme": 0.0037428457507714574, + "n": 32 + }, + "marshalling/short text ×20,000": { + "medianPerOpMs": 0.00017635867916669667, + "rme": 0.0044919110516548205, + "n": 32 + }, + "marshalling/long text 4 KiB ×20,000": { + "medianPerOpMs": 0.0010080718750001324, + "rme": 0.04386791833430546, + "n": 32 + }, + "marshalling/unicode text ×20,000": { + "medianPerOpMs": 0.0002711018250000052, + "rme": 0.006827854257269843, + "n": 32 + }, + "marshalling/NULL ×20,000": { + "medianPerOpMs": 0.0001421171107142852, + "rme": 0.004952756392286643, + "n": 32 + }, + "marshalling/blob 64 B ×20,000": { + "medianPerOpMs": 0.00040669739999993907, + "rme": 0.016549743126066815, + "n": 32 + }, + "marshalling/blob 4,095 B ×20,000 (copy boundary)": { + "medianPerOpMs": 0.0009462229250000746, + "rme": 0.010747216360287174, + "n": 32 + }, + "marshalling/blob 4 KiB ×20,000 (external boundary)": { + "medianPerOpMs": 0.0009557770999997957, + "rme": 0.019720000615258, + "n": 32 + }, + "marshalling/blob 64 KiB ×4,096": { + "medianPerOpMs": 0.004892501708984476, + "rme": 0.006678422671972584, + "n": 32 + }, + "marshalling/blob 1 MiB ×256": { + "medianPerOpMs": 0.0505144042968837, + "rme": 0.008677396124863495, + "n": 32 + }, + "marshalling/blob round-trip: 2,000 × 256 KiB": { + "medianPerOpMs": 0.0862761562500018, + "rme": 0.010704843494945885, + "n": 32 + }, + "marshalling/blob stream: 100 MiB round trip": { + "medianPerOpMs": 25.36427100000583, + "rme": 0.02604015309573482, + "n": 12 + }, + "write/run: prepared insert ×1,000": { + "medianPerOpMs": 0.009780635499999335, + "rme": 0.009128369010994298, + "n": 32 + }, + "write/db.run: prepare per call ×1,000": { + "medianPerOpMs": 0.01953454149999743, + "rme": 0.004900338331619955, + "n": 32 + }, + "write/db.run: statement cache ×1,000": { + "medianPerOpMs": 0.010292124999999942, + "rme": 0.00639343187155091, + "n": 32 + }, + "write/insert: ×1,000 in one transaction (file db)": { + "medianPerOpMs": 0.008301768499999727, + "rme": 0.010317831676465506, + "n": 32 + }, + "write/exec: 100-statement script": { + "medianPerOpMs": 0.0009272858986175969, + "rme": 0.0022666699638115573, + "n": 32 + }, + "sync-vs-async/get: batch of 1 (async)": { + "medianPerOpMs": 0.009791235207100764, + "rme": 0.012134892007183169, + "n": 32 + }, + "sync-vs-async/getSync: batch of 1": { + "medianPerOpMs": 0.0008874035924909155, + "rme": 0.012269443791701577, + "n": 32 + }, + "sync-vs-async/run: batch of 1 (async)": { + "medianPerOpMs": 0.011594576721117241, + "rme": 0.007438332043428026, + "n": 32 + }, + "sync-vs-async/runSync: batch of 1": { + "medianPerOpMs": 0.0014106449808862064, + "rme": 0.003312640316983785, + "n": 32 + }, + "sync-vs-async/get: batch of 10 (async)": { + "medianPerOpMs": 0.00951121874999808, + "rme": 0.005328183822300695, + "n": 32 + }, + "sync-vs-async/getSync: batch of 10": { + "medianPerOpMs": 0.0008840561002660271, + "rme": 0.006924353896609049, + "n": 32 + }, + "sync-vs-async/run: batch of 10 (async)": { + "medianPerOpMs": 0.010146995526317225, + "rme": 0.0037626901385526446, + "n": 32 + }, + "sync-vs-async/runSync: batch of 10": { + "medianPerOpMs": 0.0009075822222219198, + "rme": 0.006233701182635286, + "n": 32 + }, + "sync-vs-async/get: batch of 100 (async)": { + "medianPerOpMs": 0.009453452380954071, + "rme": 0.0053683083785561045, + "n": 32 + }, + "sync-vs-async/getSync: batch of 100": { + "medianPerOpMs": 0.0009012518362833039, + "rme": 0.012867745927609597, + "n": 32 + }, + "sync-vs-async/run: batch of 100 (async)": { + "medianPerOpMs": 0.010354067894739655, + "rme": 0.006044763794948211, + "n": 32 + }, + "sync-vs-async/runSync: batch of 100": { + "medianPerOpMs": 0.0008894688738737468, + "rme": 0.0035887648102007055, + "n": 32 + }, + "sync-vs-async/get: batch of 10,000 (async)": { + "medianPerOpMs": 0.009693289599999844, + "rme": 0.004799180868325526, + "n": 32 + }, + "sync-vs-async/getSync: batch of 10,000": { + "medianPerOpMs": 0.0009233104500002811, + "rme": 0.007562800248003012, + "n": 32 + }, + "sync-vs-async/run: batch of 10,000 (async)": { + "medianPerOpMs": 0.01038860835000014, + "rme": 0.003473357430041331, + "n": 32 + }, + "sync-vs-async/runSync: batch of 10,000": { + "medianPerOpMs": 0.000903226050000012, + "rme": 0.007648178161089595, + "n": 32 + }, + "sync-vs-async/allSync: 20,000 rows × 4 cols": { + "medianPerOpMs": 0.000550493225000173, + "rme": 0.02698337023125632, + "n": 32 + }, + "sync-vs-async/allSync (arrays): 20,000 rows × 4 cols": { + "medianPerOpMs": 0.0005516958374999376, + "rme": 0.028843826014843046, + "n": 32 + }, + "sync-vs-async/getSync (native path): single row": { + "medianPerOpMs": 0.0008299687849305071, + "rme": 0.005289395455631841, + "n": 32 + }, + "sync-vs-async/runSync bind shape: positional (4 params)": { + "medianPerOpMs": 0.0006988330249107766, + "rme": 0.006926162823073936, + "n": 32 + }, + "sync-vs-async/runSync bind shape: array (4 params)": { + "medianPerOpMs": 0.0008459519516929329, + "rme": 0.006520301226771621, + "n": 32 + }, + "sync-vs-async/runSync bind shape: named (4 params)": { + "medianPerOpMs": 0.001335067239874103, + "rme": 0.007391983434984686, + "n": 32 + }, + "baseline/node:sqlite/get: single row (prepared)": { + "medianPerOpMs": 0.0007913134543704837, + "rme": 0.010000361121269763, + "n": 32 + }, + "baseline/node:sqlite/all: 20,000 rows × 4 cols": { + "medianPerOpMs": 0.00048599478750002164, + "rme": 0.01263408612192968, + "n": 32 + }, + "baseline/node:sqlite/all (returnArrays): 20,000 rows × 4 cols": { + "medianPerOpMs": 0.0003662840249999135, + "rme": 0.01625372323235868, + "n": 32 + }, + "baseline/node:sqlite/insert: prepared ×1,000": { + "medianPerOpMs": 0.0008017324799997732, + "rme": 0.015185863494007128, + "n": 32 + }, + "baseline/node:sqlite/exec: 100-statement script": { + "medianPerOpMs": 0.0009950230402008212, + "rme": 0.00564629232466017, + "n": 32 + }, + "overhead/stmt.get: 1,000 (callback)": { + "medianPerOpMs": 0.008731927250002627, + "rme": 0.003781038144424262, + "n": 32 + }, + "overhead/stmt.get: 1,000 (promise)": { + "medianPerOpMs": 0.007589041666666162, + "rme": 0.009123604759363992, + "n": 32 + }, + "overhead/db.run cached: 1,000": { + "medianPerOpMs": 0.010310728999997082, + "rme": 0.006809536697360925, + "n": 32 + }, + "overhead/db.run cached + trace listener: 1,000": { + "medianPerOpMs": 0.01108589574999496, + "rme": 0.015426876984339746, + "n": 32 + }, + "overhead/db.run cached + profile listener: 1,000": { + "medianPerOpMs": 0.01046060425000178, + "rme": 0.007965641660189393, + "n": 32 + }, + "overhead/db.run cached + commit listener: 1,000 autocommits": { + "medianPerOpMs": 0.011050021000002744, + "rme": 0.007898729332822617, + "n": 32 + }, + "overhead/db.run cached + change+commit listeners: 1,000": { + "medianPerOpMs": 0.01136067725000612, + "rme": 0.02679657588234892, + "n": 32 + }, + "overhead/db.run cached after listener removal: 1,000": { + "medianPerOpMs": 0.010350572750001447, + "rme": 0.006288903191322981, + "n": 32 + }, + "overhead/stmt.get: 10,000 with cancellation token": { + "medianPerOpMs": 0.0068260083500019395, + "rme": 0.0226700087101886, + "n": 32 + }, + "overhead/get: statement cache hit": { + "medianPerOpMs": 0.00878956250000192, + "rme": 0.018678645837412133, + "n": 32 + }, + "overhead/get: statement cache miss": { + "medianPerOpMs": 0.022672478999986197, + "rme": 0.017508788683339317, + "n": 32 + }, + "overhead/get: statement cache disabled": { + "medianPerOpMs": 0.02033195849999902, + "rme": 0.005787574522060754, + "n": 32 + }, + "overhead/filter 20k: in SQL (a % 7 = 0)": { + "medianPerOpMs": 0.00004254674479164654, + "rme": 0.008042110546036148, + "n": 32 + }, + "overhead/filter 20k: JS function per row": { + "medianPerOpMs": 0.019099090600000635, + "rme": 0.004400623137515085, + "n": 24 + }, + "overhead/filter 20k: JS after all()": { + "medianPerOpMs": 0.00018817833499997505, + "rme": 0.023877748067778287, + "n": 32 + }, + "overhead/JS round trip: 20k minimal calls": { + "medianPerOpMs": 0.019344109399999435, + "rme": 0.006697586708236123, + "n": 24 + }, + "overhead/JS aggregate: 20k steps": { + "medianPerOpMs": 0.01912320519999921, + "rme": 0.006471632171778836, + "n": 24 + }, + "overhead/JS collation: sort 10k as text": { + "medianPerOpMs": 0.14468844370000006, + "rme": 0.005902092649290284, + "n": 12 + }, + "overhead/db.transaction: 200 empty bodies": { + "medianPerOpMs": 0.017129982916655233, + "rme": 0.004562561407113668, + "n": 32 + }, + "overhead/raw BEGIN+COMMIT: 200 pairs": { + "medianPerOpMs": 0.014806949642858983, + "rme": 0.02134543380095957, + "n": 32 + }, + "overhead/open+close: 1,000 :memory: connections": { + "medianPerOpMs": 0.023048764534871945, + "rme": 0.00691837011360869, + "n": 32 + }, + "concurrency/50 concurrent queries: parallelize()": { + "medianPerOpMs": 0.1792708350000612, + "rme": 0.006310284659307708, + "n": 32 + }, + "concurrency/50 concurrent queries: serialize()": { + "medianPerOpMs": 0.19860395999989122, + "rme": 0.002106377939999691, + "n": 32 + }, + "concurrency/pool.read: 1,000 round trips": { + "medianPerOpMs": 0.022911020999992614, + "rme": 0.006504326018594557, + "n": 32 + }, + "concurrency/pool.get: 1,000 round trips": { + "medianPerOpMs": 0.02245662500000617, + "rme": 0.002709231128884739, + "n": 32 + }, + "concurrency/pool.write: 1,000 round trips": { + "medianPerOpMs": 0.044271061999999806, + "rme": 0.021072834213669747, + "n": 32 + }, + "concurrency/pool.all: 20,000 rows (postMessage transfer)": { + "medianPerOpMs": 0.0012413708499996574, + "rme": 0.003072873025580276, + "n": 32 + }, + "concurrency/200 concurrent reads: pool (4 readers)": { + "medianPerOpMs": 0.44879562500005704, + "rme": 0.011054313349695694, + "n": 32 + }, + "concurrency/200 concurrent reads: single connection": { + "medianPerOpMs": 0.4383939575000113, + "rme": 0.005248680919611739, + "n": 32 + } + } + } + } +} diff --git a/bench/bench.js b/bench/bench.js deleted file mode 100644 index 09fba97..0000000 --- a/bench/bench.js +++ /dev/null @@ -1,164 +0,0 @@ -// Micro-benchmarks for the hot paths targeted by the marshalling -// optimisations: row conversion (all/each), bind marshalling (run), -// and blob transfers. Run: node bench/bench.js -import sqlite3 from '../lib/sqlite3.js'; - -function bench(name, fn) { - return new Promise((resolve) => { - // warmup - fn(() => { - const start = process.hrtime.bigint(); - fn(() => { - const ms = Number(process.hrtime.bigint() - start) / 1e6; - resolve({ name, ms }); - }); - }); - }); -} - -const results = []; - -async function main() { - const db = new sqlite3.Database(':memory:'); - const db2 = new sqlite3.Database(':memory:'); - const db3 = new sqlite3.Database(':memory:'); - db.exec('CREATE TABLE t (a INTEGER, b REAL, c TEXT, d BLOB)'); - db.exec('CREATE TABLE t2 (a INTEGER, b REAL, c TEXT, d BLOB)'); - db2.exec('CREATE TABLE t2 (a INTEGER, b REAL, c TEXT, d BLOB)'); - db3.exec('CREATE TABLE t (a INTEGER, b REAL, c TEXT, d BLOB)'); - await new Promise((r) => { - const s = db3.prepare('INSERT INTO t VALUES (?, ?, ?, ?)'); - const buf = Buffer.alloc(64); - for (let i = 0; i < 20000; i++) { buf[0] = i & 0xff; s.run(i, i + 0.5, 'text-value-' + i, buf); } - s.finalize(r); - }); - db.exec('CREATE TABLE t3 (d BLOB)'); - - await new Promise((r) => { - const stmt = db.prepare('INSERT INTO t VALUES (?, ?, ?, ?)'); - const buf = Buffer.alloc(64); - for (let i = 0; i < 20000; i++) { - buf[0] = i & 0xff; - stmt.run(i, i + 0.5, 'text-value-' + i, buf); - } - stmt.finalize(r); - }); - - results.push(await bench('all: 20k rows x 4 cols (read cache)', (done) => { - db.all('SELECT a, b, c, d FROM t', () => done()); - })); - - results.push(await bench('each: 20k rows x 4 cols', (done) => { - let n = 0; - db.each('SELECT a, b, c, d FROM t', () => { n++; }, () => done()); - })); - - results.push(await bench('run: 10k inserts (bind+exec)', (done) => { - db.exec('DELETE FROM t2', () => { - const stmt = db.prepare('INSERT INTO t2 VALUES (?, ?, ?, ?)'); - const buf = Buffer.alloc(64); - for (let i = 0; i < 10000; i++) { - buf[0] = i & 0xff; - stmt.run(i, i + 0.5, 'text-value-' + i, buf); - } - stmt.finalize(() => done()); - }); - })); - - results.push(await bench('db.run: 10k (prepare per call)', (done) => { - db.exec('DELETE FROM t2', () => { - let i = 0; - const next = () => { - if (i === 10000) return done(); - db.run('INSERT INTO t2 VALUES (?, ?, ?, ?)', i, i + 0.5, 'text-value-' + i, Buffer.alloc(64), () => { i++; next(); }); - }; - next(); - }); - })); - - results.push(await bench('db.run + trace: 10k', (done) => { - const onTrace = function() {}; - db.on('trace', onTrace); - db.exec('DELETE FROM t2', () => { - let i = 0; - const next = () => { - if (i === 10000) { - db.removeListener('trace', onTrace); - return done(); - } - db.run('INSERT INTO t2 VALUES (?, ?, ?, ?)', i, i + 0.5, 'text-value-' + i, Buffer.alloc(64), () => { i++; next(); }); - }; - next(); - }); - })); - - results.push(await bench('db.run + profile: 10k', (done) => { - const onProfile = function() {}; - db.on('profile', onProfile); - db.exec('DELETE FROM t2', () => { - let i = 0; - const next = () => { - if (i === 10000) { - db.removeListener('profile', onProfile); - return done(); - } - db.run('INSERT INTO t2 VALUES (?, ?, ?, ?)', i, i + 0.5, 'text-value-' + i, Buffer.alloc(64), () => { i++; next(); }); - }; - next(); - }); - })); - - results.push(await bench('db.run cached: 10k', (done) => { - db2.cacheStatements(); - db2.exec('DELETE FROM t2', () => { - let i = 0; - const next = () => { - if (i === 10000) return done(); - db2.run('INSERT INTO t2 VALUES (?, ?, ?, ?)', i, i + 0.5, 'text-value-' + i, Buffer.alloc(64), () => { i++; next(); }); - }; - next(); - }); - })); - - // Pure synchronous loop: the intended usage pattern for the sync API. - { - db3.cacheStatements(); - let warm = db3.getSync('SELECT a, b, c, d FROM t WHERE rowid = ?', 1); - const t0 = process.hrtime.bigint(); - for (let i = 0; i < 10000; i++) { - warm = db3.getSync('SELECT a, b, c, d FROM t WHERE rowid = ?', (i % 20000) + 1); - } - if (warm === undefined) throw new Error('lookup failed'); - results.push({ name: 'db.getSync cached: 10k lookups', ms: Number(process.hrtime.bigint() - t0) / 1e6 }); - } - - results.push(await bench('get: 10k single-row lookups', (done) => { - const stmt = db.prepare('SELECT a, b, c, d FROM t WHERE rowid = ?'); - let i = 0; - const next = () => { - if (i === 10000) return stmt.finalize(() => done()); - i++; - stmt.get((i % 20000) + 1, next); - }; - next(); - })); - - results.push(await bench('blob: 2k x 256KB round-trip', (done) => { - const buf = Buffer.alloc(256 * 1024); - for (let j = 0; j < buf.length; j++) buf[j] = j & 0xff; - db.exec('DELETE FROM t3', () => { - const stmt = db.prepare('INSERT INTO t3 (d) VALUES (?)'); - for (let i = 0; i < 2000; i++) stmt.run(buf); - stmt.finalize(() => { - db.all('SELECT d FROM t3', () => done()); - }); - }); - })); - - for (const r of results) { - console.log(r.name.padEnd(40), r.ms.toFixed(1).padStart(8) + ' ms'); - } - await new Promise((r) => db.close(() => db2.close(() => db3.close(r)))); -} - -main(); diff --git a/bench/cases/baselines.js b/bench/cases/baselines.js new file mode 100644 index 0000000..60f86f4 --- /dev/null +++ b/bench/cases/baselines.js @@ -0,0 +1,158 @@ +// Comparison baselines (Deliverable 13 §2.3): node:sqlite (built in — +// the default choice for Node users today, so every sync-path number is +// reported next to it) and better-sqlite3 (the incumbent sync binding, +// behind --compare and an optional install; it is NOT a devDependency — +// `npm i --no-save better-sqlite3` before running with --compare). +// +// The mirrors use the same fixtures and statement shapes as the package +// cases they are ratio'd against. +import { intRows } from './shared.js'; + +/** @typedef {import('../harness.js').CaseSpec} CaseSpec */ + +/** + * The four mirror cases against a synchronous baseline driver + * (node:sqlite's DatabaseSync or better-sqlite3's Database). + * + * @param {any} db the baseline connection (prepared-statement API: prepare/get/all/run/exec). + * @param {string} prefix case-name prefix, e.g. 'baseline/node:sqlite'. + * @param {{ getSyncCase: string, allSyncCase: string, allSyncArrayCase: string, insertCase: string, execCase: string }} ratios case names to ratio against. + * @returns {CaseSpec[]} the mirror cases. + */ +export function baselineCases(db, prefix, ratios) { + db.exec('CREATE TABLE t (c0 INTEGER, c1 REAL, c2 TEXT, c3 BLOB)'); + db.exec(intRows(20000, "x, x + 0.5, 'text-value-' || x, zeroblob(64)")); + db.exec('CREATE TABLE w (c0 INTEGER, c1 REAL, c2 TEXT, c3 BLOB)'); + const buf = Buffer.alloc(64); + const K = 1000; + + /** @type {CaseSpec[]} */ + return [ + { + name: `${prefix}/get: single row (prepared)`, + group: 'baseline', + ratioTo: ratios.getSyncCase, + iter: (_env, n) => { + const stmt = db.prepare('SELECT * FROM t WHERE rowid = ?'); + for (let i = 0; i < n; i++) { + stmt.get((i % 20000) + 1); + } + }, + }, + { + name: `${prefix}/all: 20,000 rows × 4 cols`, + group: 'baseline', + ops: 20000, + ratioTo: ratios.allSyncCase, + iter: (_env, n) => { + const stmt = db.prepare('SELECT * FROM t'); + for (let i = 0; i < n; i++) { + const rows = stmt.all(); + if (rows.length !== 20000) throw new Error('bad count'); + } + }, + }, + { + // The baseline's own array row shape: node:sqlite's + // setReturnArrays(true), better-sqlite3's raw(true). This is + // the mirror the package's `{ rowMode: 'array' }` case is + // ratio'd against (D16). + name: `${prefix}/all (returnArrays): 20,000 rows × 4 cols`, + group: 'baseline', + ops: 20000, + ratioTo: ratios.allSyncArrayCase, + iter: (_env, n) => { + const stmt = db.prepare('SELECT * FROM t'); + if (typeof stmt.setReturnArrays === 'function') { + stmt.setReturnArrays(true); + } else if (typeof stmt.raw === 'function') { + stmt.raw(true); + } + for (let i = 0; i < n; i++) { + const rows = stmt.all(); + if (rows.length !== 20000) throw new Error('bad count'); + } + }, + }, + { + name: `${prefix}/insert: prepared ×1,000`, + group: 'baseline', + ops: K, + ratioTo: ratios.insertCase, + note: 'each round clears the table first (timed, not counted)', + iter: (_env, n) => { + const del = db.prepare('DELETE FROM w'); + const stmt = db.prepare('INSERT INTO w VALUES (?, ?, ?, ?)'); + for (let r = 0; r < n; r++) { + del.run(); + for (let i = 0; i < K; i++) { + stmt.run(i, i + 0.5, `text-value-${i}`, buf); + } + } + }, + }, + { + name: `${prefix}/exec: 100-statement script`, + group: 'baseline', + ops: 100, + ratioTo: ratios.execCase, + iter: (_env, n) => { + const script = Array.from( + { length: 100 }, + (_, i) => + `INSERT INTO w VALUES (${i}, ${i}.5, 's${i}', x'00')`, + ).join(';\n'); + for (let i = 0; i < n; i++) { + db.exec(`DELETE FROM w;\n${script}`); + } + }, + }, + ]; +} + +/** + * Builds the node:sqlite mirror cases when the built-in module is + * importable (Node >= 22.5; unflagged since 23.4). + * + * @param {{ getSyncCase: string, allSyncCase: string, allSyncArrayCase: string, insertCase: string, execCase: string }} ratios case names to ratio against. + * @returns {Promise<{ cases: CaseSpec[], dispose: () => void } | { skipped: string }>} the mirror cases, or a skip reason. + */ +export async function nodeSqliteCases(ratios) { + let mod; + try { + mod = await import('node:sqlite'); + } catch (err) { + return { + skipped: `node:sqlite not available: ${/** @type {Error} */ (err).message}`, + }; + } + const db = new mod.DatabaseSync(':memory:'); + return { + cases: baselineCases(db, 'baseline/node:sqlite', ratios), + dispose: () => db.close(), + }; +} + +/** + * Builds the better-sqlite3 mirror cases when the package is installed + * and --compare was passed. Never a devDependency of this repo. + * + * @param {{ getSyncCase: string, allSyncCase: string, allSyncArrayCase: string, insertCase: string, execCase: string }} ratios case names to ratio against. + * @returns {Promise<{ cases: CaseSpec[], dispose: () => void } | { skipped: string }>} the mirror cases, or a skip reason. + */ +export async function betterSqliteCases(ratios) { + let mod; + try { + mod = await import('better-sqlite3'); + } catch { + return { + skipped: + 'better-sqlite3 not installed (optional; npm i --no-save better-sqlite3)', + }; + } + const db = mod.default(':memory:'); + return { + cases: baselineCases(db, 'baseline/better-sqlite3', ratios), + dispose: () => db.close(), + }; +} diff --git a/bench/cases/concurrency.js b/bench/cases/concurrency.js new file mode 100644 index 0000000..5998ac6 --- /dev/null +++ b/bench/cases/concurrency.js @@ -0,0 +1,184 @@ +// Concurrency cases (Deliverable 13 §2.2): parallelize() vs serialize() +// under N concurrent queries, and the worker pool against a single +// connection — including the postMessage row-transfer cost that decides +// whether the pool is worth using (D09 filed "a real number for the pool +// under contention"; this is it). +import { join } from 'node:path'; + +import { intRows } from './shared.js'; + +/** @typedef {import('../harness.js').CaseSpec} CaseSpec */ + +/** + * parallelize() vs serialize(): 50 concurrent 1,000-row queries on one + * connection, wall-clock. Same connection, same queries — only the + * scheduling mode differs. + * + * @param {any} db an open connection with a 1,000-row table `c`. + * @returns {CaseSpec[]} the two scheduling cases. + */ +export function schedulingCases(db) { + db.exec('CREATE TABLE c (v INTEGER)'); + db.exec(intRows(1000, 'x', 'c')); + const CONCURRENT = 50; + + /** @param {boolean} serialized @returns {Promise} one round */ + const round = async (serialized) => { + /** @type {Promise[]} */ + let started = []; + const mode = serialized ? db.serialize : db.parallelize; + mode.call(db, () => { + started = Array.from({ length: CONCURRENT }, () => + db.all('SELECT * FROM c'), + ); + }); + const rows = await Promise.all(started); + for (const r of rows) { + if (/** @type {any[]} */ (r).length !== 1000) + throw new Error('bad count'); + } + }; + + return [ + { + name: 'concurrency/50 concurrent queries: parallelize()', + group: 'concurrency', + ops: CONCURRENT, + iter: async (_env, n) => { + for (let i = 0; i < n; i++) await round(false); + }, + }, + { + name: 'concurrency/50 concurrent queries: serialize()', + group: 'concurrency', + ops: CONCURRENT, + ratioTo: 'concurrency/50 concurrent queries: parallelize()', + iter: async (_env, n) => { + for (let i = 0; i < n; i++) await round(true); + }, + }, + ]; +} + +/** + * The pool cases (Deliverable 09 keepers plus the contention pair): + * round trips, 200 concurrent reads pool-vs-single-connection, and the + * postMessage row-transfer cost (pool.all of 20k rows against the same + * query on a local connection). + * + * @param {typeof import('../../lib/sqlite3.js').default} sqlite3 the driver. + * @param {any} localDb a local connection with a 20,000-row table `c20` (the pool file gets the same data). + * @param {{ dir: string }} scratch scratch directory for the pool file. + * @returns {Promise<{ cases: CaseSpec[], dispose: () => Promise }>} the pool cases and disposer. + */ +export async function poolCases(sqlite3, localDb, scratch) { + localDb.exec('CREATE TABLE c20 (c0 INTEGER, c1 REAL, c2 TEXT, c3 BLOB)'); + localDb.exec( + intRows(20000, "x, x + 0.5, 'text-value-' || x, zeroblob(64)", 'c20'), + ); + + const file = join(scratch.dir, 'bench-pool.db'); + const pool = await sqlite3.pool(file, { readers: 4 }); + await pool.exec('CREATE TABLE t (a INTEGER, b TEXT)'); + await pool.exec('CREATE TABLE c20 (c0 INTEGER, c1 REAL, c2 TEXT, c3 BLOB)'); + await pool.write( + "INSERT INTO c20 SELECT x, x + 0.5, 'text-value-' || x, zeroblob(64) " + + 'FROM (WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt WHERE x < 20000) SELECT x FROM cnt)', + ); + await pool.write('INSERT INTO t VALUES (?, ?)', [1, 'seed']); + + /** @type {CaseSpec[]} */ + const cases = [ + { + name: 'concurrency/pool.read: 1,000 round trips', + group: 'concurrency', + ops: 1000, + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + for (let r = 0; r < 1000; r++) { + await pool.read('SELECT a, b FROM t WHERE a = ?', [1]); + } + } + }, + }, + { + name: 'concurrency/pool.get: 1,000 round trips', + group: 'concurrency', + ops: 1000, + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + for (let r = 0; r < 1000; r++) { + await pool.get('SELECT a, b FROM t WHERE a = ?', [1]); + } + } + }, + }, + { + name: 'concurrency/pool.write: 1,000 round trips', + group: 'concurrency', + ops: 1000, + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + for (let r = 0; r < 1000; r++) { + await pool.write('INSERT INTO t VALUES (?, ?)', [ + r, + 'x', + ]); + } + } + await pool.exec('DELETE FROM t'); + await pool.write('INSERT INTO t VALUES (?, ?)', [1, 'seed']); + }, + }, + { + name: 'concurrency/pool.all: 20,000 rows (postMessage transfer)', + group: 'concurrency', + ops: 20000, + ratioTo: 'read/all: 20,000 rows × 4 cols', + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + const rows = await pool.read('SELECT * FROM c20'); + if (rows.length !== 20000) throw new Error('bad count'); + } + }, + }, + { + name: 'concurrency/200 concurrent reads: pool (4 readers)', + group: 'concurrency', + ops: 200, + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + const rows = await Promise.all( + Array.from({ length: 200 }, () => + pool.read('SELECT * FROM c20 WHERE c0 % 100 = 0'), + ), + ); + if (rows.length !== 200) throw new Error('bad count'); + } + }, + }, + { + name: 'concurrency/200 concurrent reads: single connection', + group: 'concurrency', + ops: 200, + ratioTo: 'concurrency/200 concurrent reads: pool (4 readers)', + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + const rows = await Promise.all( + Array.from({ length: 200 }, () => + localDb.all('SELECT * FROM c20 WHERE c0 % 100 = 0'), + ), + ); + if (rows.length !== 200) throw new Error('bad count'); + } + }, + }, + ]; + + return { + cases, + dispose: async () => { + await pool.close(); + }, + }; +} diff --git a/bench/cases/index.js b/bench/cases/index.js new file mode 100644 index 0000000..5eaa74c --- /dev/null +++ b/bench/cases/index.js @@ -0,0 +1,195 @@ +// Composes the full suite. Case files are imported explicitly — never +// globbed — for the same reason tools/run-tests.mjs exists: a shell glob +// enumerated zero files on Windows while exiting 0. + +import { betterSqliteCases, nodeSqliteCases } from './baselines.js'; +import { poolCases, schedulingCases } from './concurrency.js'; +import { + blobRoundTripCase, + blobStreamCase, + marshallingCases, +} from './marshalling.js'; +import { + cacheTrioCases, + openCloseCase, + overheadCases, + transactionCases, + udfCases, +} from './overhead.js'; +import { readCases } from './read.js'; +import { + colDefsFor, + colsFor, + connectionRegistry, + intRows, + scratchDir, +} from './shared.js'; +import { syncCases } from './sync.js'; +import { writeCases } from './write.js'; + +/** @typedef {import('../harness.js').CaseSpec} CaseSpec */ + +/** + * The calibration case: a cached async single-row get — the README's + * "interactive lookup" shape. Measured twice as two independent cases; + * their same-run difference is the suite's noise floor, and every ratio + * the harness prints is checked against it. + * + * @param {any} db a cache-enabled connection with a 20,000-row table `t`. + * @param {string} name case name (A or B). + * @returns {CaseSpec} the calibration case. + */ +function calibrationCase(db, name) { + return { + name, + group: 'calibration', + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + await db.get( + 'SELECT * FROM t WHERE rowid = ?', + (i % 20000) + 1, + ); + } + }, + }; +} + +/** + * Builds every case and every fixture the suite needs. + * + * @param {typeof import('../lib/sqlite3.js').default} sqlite3 the driver. + * @param {{ compare: boolean }} opts whether --compare was passed (enables the better-sqlite3 mirror). + * @returns {Promise<{ cases: CaseSpec[], dispose: () => Promise, skipped: string[] }>} the composed suite. + */ +export async function buildSuite(sqlite3, opts) { + const registry = connectionRegistry(sqlite3); + const scratch = scratchDir(); + /** @type {(() => Promise | void)[]} */ + const disposers = []; + /** @type {string[]} */ + const skipped = []; + + // Calibration pair: two identical connections, measured back to back. + for (const label of ['calA', 'calB']) { + const db = registry.mem(label); + db.exec( + `CREATE TABLE t (${colDefsFor(4)}); ${intRows(20000, colsFor(4))}`, + ); + db.cacheStatements(); + } + /** @type {CaseSpec[]} */ + const cases = [ + calibrationCase(registry.all[0], 'calibration/cached get (A)'), + calibrationCase(registry.all[1], 'calibration/cached get (B)'), + ]; + + // read group (own connection, no cache) + cases.push(...readCases(registry.mem('read'))); + + // marshalling group: three integer-mode connections + keepers. The + // default mode is 'number'; the other two are set explicitly per + // connection so the modes cannot contaminate each other's numbers. + const marshalNumber = registry.mem('marshal-number'); + const marshalMixed = registry.mem('marshal-mixed'); + const marshalBigint = registry.mem('marshal-bigint'); + cases.push( + ...marshallingCases({ + number: marshalNumber, + mixed: marshalMixed, + bigint: marshalBigint, + }), + ); + await setIntegerMode(marshalMixed, 'mixed'); + await setIntegerMode(marshalBigint, 'bigint'); + cases.push(blobRoundTripCase(registry.mem('blob-rt'))); + cases.push(blobStreamCase(registry.mem('blob-stream'))); + + // write group: plain + cached connections + cases.push( + ...writeCases(registry.mem('write'), registry.mem('write-cached')), + ); + + // sync-vs-async group: two cache-enabled connections + cases.push(...syncCases(registry.mem('sync'), registry.mem('async'))); + + // baseline mirrors + const ratioNames = { + getSyncCase: 'sync-vs-async/getSync: batch of 1', + allSyncCase: 'sync-vs-async/allSync: 20,000 rows × 4 cols', + allSyncArrayCase: + 'sync-vs-async/allSync (arrays): 20,000 rows × 4 cols', + insertCase: 'sync-vs-async/runSync: batch of 1', + execCase: 'write/exec: 100-statement script', + }; + const nodeSqlite = await nodeSqliteCases(ratioNames); + if ('cases' in nodeSqlite) { + cases.push(...nodeSqlite.cases); + disposers.push(nodeSqlite.dispose); + } else { + skipped.push(nodeSqlite.skipped); + } + if (opts.compare) { + const better = await betterSqliteCases(ratioNames); + if ('cases' in better) { + cases.push(...better.cases); + disposers.push(better.dispose); + } else { + skipped.push(better.skipped); + } + } + + // overhead group + cases.push(...overheadCases(registry.mem('overhead'))); + cases.push( + ...cacheTrioCases({ + hit: registry.mem('cache-hit'), + miss: registry.mem('cache-miss'), + disabled: registry.mem('cache-off'), + }), + ); + cases.push(...udfCases(registry.mem('udf'))); + cases.push(...transactionCases(registry.mem('txn'))); + cases.push(openCloseCase(sqlite3)); + + // concurrency group + cases.push(...schedulingCases(registry.mem('scheduling'))); + { + const pool = await poolCases( + sqlite3, + registry.mem('pool-local'), + scratch, + ); + cases.push(...pool.cases); + disposers.push(pool.dispose); + } + + // Deterministic drain barrier: every fixture table was created with + // un-awaited exec() calls (queued FIFO per connection); wait() queues + // at each tail and resolves only once reached, so every connection is + // provably idle before the first sample — sync methods refuse + // otherwise, and a busy queue would fail cases non-deterministically. + await Promise.all(registry.all.map((db) => db.wait())); + + return { + cases, + skipped, + dispose: async () => { + await Promise.allSettled( + disposers.map((fn) => Promise.resolve().then(fn)), + ); + await registry.dispose(); + scratch.cleanup(); + }, + }; +} + +/** + * Sets the integer mode on a connection, awaiting the queued configure. + * + * @param {any} db the connection. + * @param {'number' | 'mixed' | 'bigint'} mode the mode. + * @returns {Promise} resolves once configured. + */ +async function setIntegerMode(db, mode) { + await db.configure('integerMode', mode); +} diff --git a/bench/cases/marshalling.js b/bench/cases/marshalling.js new file mode 100644 index 0000000..c49201e --- /dev/null +++ b/bench/cases/marshalling.js @@ -0,0 +1,229 @@ +// Marshalling cases (Deliverable 13 §2.2): each value type in isolation, +// read as a single column so per-op cost is per-value conversion. This is +// the regression guard for the Deliverable 02/03b marshalling work — the +// hottest code in the addon (GetRow/RowToJS/CellToJS). +// +// Integer modes get three connections because configure('integerMode') +// is per-connection and the modes must not contaminate each other's +// numbers. All connections enable the statement cache: allSync on a +// cached statement is the purest read path this package has. +// +// Allocation is measured for every case here (alloc: true): the +// marshalling work is fundamentally about allocation, and external +// buffers (blobs >= 4096 bytes become zero-copy external buffers in +// CellToJS, src/convert.cc) do not show up in heapUsed at all — watch +// the `external` and `arrayBuffers` counters for those. + +/** @typedef {import('../harness.js').CaseSpec} CaseSpec */ + +/** + * One single-column marshalling case. + * + * @param {string} name case name. + * @param {any} db connection to read on (cache enabled). + * @param {string} table table name (single column `v`). + * @param {number} rows row count. + * @returns {CaseSpec} the case. + */ +function colCase(name, db, table, rows) { + const sql = `SELECT v FROM ${table}`; + return { + name, + group: 'marshalling', + ops: rows, + iter: (_env, n) => { + for (let i = 0; i < n; i++) { + const out = db.allSync(sql); + if (out.length !== rows) throw new Error('bad row count'); + } + }, + alloc: true, + allocIter: (env, n) => { + for (let i = 0; i < n; i++) { + env.keep = db.allSync(sql); + } + }, + }; +} + +/** + * Builds the marshalling cases on the given connections. + * + * @param {{ number: any, mixed: any, bigint: any }} dbs one cache-enabled connection per integer mode. + * @returns {CaseSpec[]} the marshalling cases. + */ +export function marshallingCases(dbs) { + const db = dbs.number; + + /** @param {string} cols @param {string} table */ + const make = (table, cols, rows) => { + db.exec(`CREATE TABLE ${table} (v)`); + db.exec( + `INSERT INTO ${table} SELECT ${cols} FROM (WITH RECURSIVE cnt(x) AS ` + + `(SELECT 1 UNION ALL SELECT x+1 FROM cnt WHERE x < ${rows}) SELECT x FROM cnt)`, + ); + }; + + const floatCols = 'x + 0.5'; + const shortTextCols = "'short-' || x"; + const longTextCols = "printf('%4096s', '') || x"; + const unicodeCols = "'説明コード🌟パフォーマンス' || x"; + const nullCols = 'NULL'; + const blob = (n) => `zeroblob(${n})`; + + // The int-mode tables exist on all three connections. + for (const modeDb of [dbs.number, dbs.mixed, dbs.bigint]) { + modeDb.exec('CREATE TABLE m_int (v)'); + modeDb.exec( + 'INSERT INTO m_int SELECT x FROM (WITH RECURSIVE cnt(x) AS ' + + '(SELECT 1 UNION ALL SELECT x+1 FROM cnt WHERE x < 20000) SELECT x FROM cnt)', + ); + modeDb.cacheStatements(); + } + + make('m_float', floatCols, 20000); + make('m_shorttext', shortTextCols, 20000); + make('m_longtext', longTextCols, 20000); + make('m_unicode', unicodeCols, 20000); + make('m_null', nullCols, 20000); + make('m_blob64', blob(64), 20000); + // 4095/4096 straddle the external-buffer boundary in CellToJS: + // < 4096 copies, >= 4096 moves the payload into a zero-copy external + // Buffer. The pair exists to keep that boundary honest. + make('m_blob4095', blob(4095), 20000); + make('m_blob4k', blob(4096), 20000); + make('m_blob64k', blob(65536), 4096); + make('m_blob1m', blob(1024 * 1024), 256); + db.cacheStatements(); + + return [ + colCase( + "marshalling/integer ×20,000 (mode 'number')", + dbs.number, + 'm_int', + 20000, + ), + colCase( + "marshalling/integer ×20,000 (mode 'mixed')", + dbs.mixed, + 'm_int', + 20000, + ), + colCase( + "marshalling/integer ×20,000 (mode 'bigint')", + dbs.bigint, + 'm_int', + 20000, + ), + colCase('marshalling/float ×20,000', db, 'm_float', 20000), + colCase('marshalling/short text ×20,000', db, 'm_shorttext', 20000), + colCase('marshalling/long text 4 KiB ×20,000', db, 'm_longtext', 20000), + colCase('marshalling/unicode text ×20,000', db, 'm_unicode', 20000), + colCase('marshalling/NULL ×20,000', db, 'm_null', 20000), + colCase('marshalling/blob 64 B ×20,000', db, 'm_blob64', 20000), + colCase( + 'marshalling/blob 4,095 B ×20,000 (copy boundary)', + db, + 'm_blob4095', + 20000, + ), + colCase( + 'marshalling/blob 4 KiB ×20,000 (external boundary)', + db, + 'm_blob4k', + 20000, + ), + colCase('marshalling/blob 64 KiB ×4,096', db, 'm_blob64k', 4096), + colCase('marshalling/blob 1 MiB ×256', db, 'm_blob1m', 256), + ]; +} + +/** + * The blob round-trip keeper from the pre-v9 bench (bind + read back), + * at the old 2k × 256 KiB shape so history stays comparable. + * + * @param {any} db a cache-free connection. + * @returns {CaseSpec} the case. + */ +export function blobRoundTripCase(db) { + db.exec('CREATE TABLE m_rt (d BLOB)'); + const buf = Buffer.alloc(256 * 1024); + for (let j = 0; j < buf.length; j++) buf[j] = j & 0xff; + return { + name: 'marshalling/blob round-trip: 2,000 × 256 KiB', + group: 'marshalling', + ops: 2000, + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + await db.exec('DELETE FROM m_rt'); + await new Promise((resolve, reject) => { + const stmt = db.prepare('INSERT INTO m_rt (d) VALUES (?)'); + for (let r = 0; r < 2000; r++) stmt.run(buf); + stmt.finalize((err) => (err ? reject(err) : resolve())); + }); + const rows = await db.all('SELECT d FROM m_rt'); + if (rows.length !== 2000) throw new Error('bad count'); + } + }, + }; +} + +/** + * The incremental-blob stream round trip (Deliverable 08 keeper): 100 MiB + * through createWriteStream and back through createReadStream. Sample + * count is reduced — one iteration is ~half a second — and the case is + * honest that its per-op number is a coarse whole-operation figure. + * + * @param {any} db a cache-free connection. + * @returns {CaseSpec} the case. + */ +export function blobStreamCase(db) { + return { + name: 'marshalling/blob stream: 100 MiB round trip', + group: 'marshalling', + samples: 12, + note: 'whole-operation figure; few samples by construction', + setup: async () => { + const { pipeline } = await import('node:stream/promises'); + await db.exec( + 'CREATE TABLE big (id INTEGER PRIMARY KEY, data BLOB)', + ); + await db.exec( + 'INSERT INTO big VALUES (1, zeroblob(100 * 1024 * 1024))', + ); + const blob = await new Promise((resolve, reject) => { + const b = db.openBlob( + { table: 'big', column: 'data', rowid: 1 }, + (/** @type {Error | null} */ err) => + err ? reject(err) : resolve(b), + ); + }); + const src = Buffer.alloc(1024 * 1024, 0xab); + return { pipeline, blob, src }; + }, + teardown: async (env) => { + await new Promise((resolve) => env.blob.close(resolve)); + }, + iter: async (env, n) => { + for (let i = 0; i < n; i++) { + await env.pipeline( + (async function* () { + for (let j = 0; j < 100; j++) yield env.src; + })(), + env.blob.createWriteStream(), + ); + let readBytes = 0; + await env.pipeline( + env.blob.createReadStream(), + async (source) => { + for await (const chunk of source) { + readBytes += chunk.length; + } + }, + ); + if (readBytes !== 100 * 1024 * 1024) + throw new Error('short read'); + } + }, + }; +} diff --git a/bench/cases/overhead.js b/bench/cases/overhead.js new file mode 100644 index 0000000..d40fb47 --- /dev/null +++ b/bench/cases/overhead.js @@ -0,0 +1,445 @@ +// Overhead cases (Deliverable 13 §2.2): promise vs callback per call, +// trace/profile listeners, the statement cache hit/miss/disabled trio, +// JS scalar functions per row, and the per-deliverable keepers (hooks, +// cancellation token, transaction wrapper, open+close). +import { intRows, seqCallbacks } from './shared.js'; + +/** @typedef {import('../harness.js').CaseSpec} CaseSpec */ + +/** + * Builds the callback/promise, trace/profile, hook and token cases. + * + * @param {any} db an open connection with an empty write table `ow`. + * @returns {CaseSpec[]} overhead cases sharing that connection. + */ +export function overheadCases(db) { + db.exec('CREATE TABLE ow (c0 INTEGER, c1 REAL, c2 TEXT, c3 BLOB)'); + db.cacheStatements(); + const RUN_SQL = 'INSERT INTO ow VALUES (?, ?, ?, ?)'; + const buf = Buffer.alloc(64); + const K = 1000; + + /** One cached-write round: clear, then K inserts. */ + const writeRound = async () => { + await db.exec('DELETE FROM ow'); + for (let i = 0; i < K; i++) { + await db.run(RUN_SQL, i, i + 0.5, `text-value-${i}`, buf); + } + }; + + /** @type {CaseSpec[]} */ + const cases = [ + { + name: 'overhead/stmt.get: 1,000 (callback)', + group: 'overhead', + ops: 1000, + iter: async (_env, n) => { + for (let r = 0; r < n; r++) { + await seqCallbacks(1000, (i, done) => + db.get('SELECT 42 AS v, ? AS p', i, done), + ); + } + }, + }, + { + name: 'overhead/stmt.get: 1,000 (promise)', + group: 'overhead', + ops: 1000, + ratioTo: 'overhead/stmt.get: 1,000 (callback)', + iter: async (_env, n) => { + for (let r = 0; r < n; r++) { + for (let i = 0; i < 1000; i++) { + await db.get('SELECT 42 AS v, ? AS p', i); + } + } + }, + }, + { + name: 'overhead/db.run cached: 1,000', + group: 'overhead', + ops: K, + note: 'each round clears the table first (timed, not counted)', + iter: async (_env, n) => { + for (let r = 0; r < n; r++) await writeRound(); + }, + }, + ]; + + // trace/profile: attach a no-op listener for the round, remove it + // after, so each case is self-contained on the shared connection. + for (const kind of ['trace', 'profile']) { + cases.push({ + name: `overhead/db.run cached + ${kind} listener: 1,000`, + group: 'overhead', + ops: K, + ratioTo: 'overhead/db.run cached: 1,000', + note: 'each round clears the table first (timed, not counted)', + iter: async (_env, n) => { + const listener = () => { + /* no-op listener: measures dispatch cost only */ + }; + db.on(kind, listener); + try { + for (let r = 0; r < n; r++) await writeRound(); + } finally { + db.removeListener(kind, listener); + } + }, + }); + } + + // The D07 write-path hooks. "after removal" is the structural zero: + // the native hook exists only while a listener is registered, and the + // case proves removal returns to the cached baseline. + cases.push( + { + name: 'overhead/db.run cached + commit listener: 1,000 autocommits', + group: 'overhead', + ops: K, + ratioTo: 'overhead/db.run cached: 1,000', + note: 'each round clears the table first (timed, not counted)', + iter: async (_env, n) => { + const listener = () => { + /* no-op listener: measures dispatch cost only */ + }; + db.on('commit', listener); + try { + for (let r = 0; r < n; r++) await writeRound(); + } finally { + db.removeListener('commit', listener); + } + }, + }, + { + name: 'overhead/db.run cached + change+commit listeners: 1,000', + group: 'overhead', + ops: K, + ratioTo: 'overhead/db.run cached: 1,000', + note: 'each round clears the table first (timed, not counted)', + iter: async (_env, n) => { + const onCommit = () => { + /* no-op listener: measures dispatch cost only */ + }; + const onChange = () => { + /* no-op listener: measures dispatch cost only */ + }; + db.on('commit', onCommit); + db.on('change', onChange); + try { + for (let r = 0; r < n; r++) await writeRound(); + } finally { + db.removeListener('commit', onCommit); + db.removeListener('change', onChange); + } + }, + }, + { + name: 'overhead/db.run cached after listener removal: 1,000', + group: 'overhead', + ops: K, + ratioTo: 'overhead/db.run cached: 1,000', + note: 'each round clears the table first (timed, not counted)', + iter: async (_env, n) => { + const listener = () => { + /* no-op listener: measures dispatch cost only */ + }; + db.on('commit', listener); + db.removeListener('commit', listener); + for (let r = 0; r < n; r++) await writeRound(); + }, + }, + { + name: 'overhead/stmt.get: 10,000 with cancellation token', + group: 'overhead', + ops: 10000, + ratioTo: 'read/get: single row (prepared statement)', + setup: () => { + const token = db.cancellationToken(); + const stmt = db.prepare('SELECT 42 AS v'); + return { token, stmt }; + }, + teardown: async (env) => { + await new Promise((resolve) => env.stmt.finalize(resolve)); + env.token.destroy(); + }, + iter: async (env, n) => { + for (let r = 0; r < n; r++) { + for (let i = 0; i < 10000; i++) { + await env.stmt.get(); + } + } + }, + }, + ); + + return cases; +} + +/** + * The statement-cache trio (§2.2): hit (same SQL every call), miss (never + * the same SQL twice — a 16-entry cache over 1,000 distinct statements), + * and disabled (no cache; a prepare per call through the database queue). + * Three connections, because cacheStatements() cannot be turned off. + * + * @param {{ hit: any, miss: any, disabled: any }} dbs three connections, each with a 1,000-row lookup table `g`. + * @returns {CaseSpec[]} the cache trio cases. + */ +export function cacheTrioCases(dbs) { + for (const db of [dbs.hit, dbs.miss, dbs.disabled]) { + db.exec('CREATE TABLE g (v INTEGER)'); + db.exec(intRows(1000, 'x', 'g')); + } + dbs.hit.cacheStatements(64); + dbs.miss.cacheStatements(16); + + /** @type {CaseSpec[]} */ + const cases = [ + { + name: 'overhead/get: statement cache hit', + group: 'overhead', + ops: 1000, + iter: async (_env, n) => { + for (let r = 0; r < n; r++) { + for (let i = 0; i < 1000; i++) { + await dbs.hit.get( + 'SELECT v FROM g WHERE rowid = ?', + (i % 1000) + 1, + ); + } + } + }, + }, + { + name: 'overhead/get: statement cache miss', + group: 'overhead', + ops: 1000, + ratioTo: 'overhead/get: statement cache hit', + iter: async (_env, n) => { + let call = 0; + for (let r = 0; r < n; r++) { + for (let i = 0; i < 1000; i++) { + // Every call prepares: never the same SQL twice, + // and the 16-entry cache keeps evicting. + await dbs.miss.get( + `SELECT v FROM g WHERE rowid = ? /*${call++}*/`, + (i % 1000) + 1, + ); + } + } + }, + }, + { + name: 'overhead/get: statement cache disabled', + group: 'overhead', + ops: 1000, + ratioTo: 'overhead/get: statement cache hit', + iter: async (_env, n) => { + for (let r = 0; r < n; r++) { + for (let i = 0; i < 1000; i++) { + await dbs.disabled.get( + 'SELECT v FROM g WHERE rowid = ?', + (i % 1000) + 1, + ); + } + } + }, + }, + ]; + return cases; +} + +/** + * The user-defined-function keepers (Deliverable 06): the JS round trip + * is the cost that decides when a JS function is the wrong tool — a JS + * function called from a query running on a worker thread pays a + * cross-thread round trip per invocation, which is the number to beat. + * + * Row counts are 20,000 (10,000 for the collation sort): a JS call per + * row costs tens of microseconds, so 100k rows per sample — the old + * one-shot bench's shape — would put a single sample over two seconds. + * + * @param {any} db an open connection with a 20,000-row table `f`. + * @returns {CaseSpec[]} the UDF cases. + */ +export function udfCases(db) { + db.exec('CREATE TABLE f (a INT, b REAL, c TEXT, d BLOB)'); + db.exec(intRows(20000, "x, x + 0.5, 'text-' || x, zeroblob(64)", 'f')); + + /** @type {CaseSpec[]} */ + const cases = [ + { + name: 'overhead/filter 20k: in SQL (a % 7 = 0)', + group: 'overhead', + ops: 20000, + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + const rows = await db.all( + 'SELECT a FROM f WHERE a % 7 = 0', + ); + if (rows.length !== 2857) throw new Error('bad count'); + } + }, + }, + { + name: 'overhead/filter 20k: JS function per row', + group: 'overhead', + ops: 20000, + samples: 24, + iter: async (_env, n) => { + db.function('seventh', { deterministic: true }, (a) => + a % 7 === 0 ? 1 : 0, + ); + try { + for (let i = 0; i < n; i++) { + const rows = await db.all( + 'SELECT a FROM f WHERE seventh(a) = 1', + ); + if (rows.length !== 2857) throw new Error('bad count'); + } + } finally { + db.removeFunction('seventh'); + } + }, + }, + { + name: 'overhead/filter 20k: JS after all()', + group: 'overhead', + ops: 20000, + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + const rows = await db.all('SELECT a FROM f'); + const kept = rows.filter( + (/** @type {{a: number}} */ r) => r.a % 7 === 0, + ); + if (kept.length !== 2857) throw new Error('bad count'); + } + }, + }, + { + name: 'overhead/JS round trip: 20k minimal calls', + group: 'overhead', + ops: 20000, + samples: 24, + iter: async (_env, n) => { + db.function('noop', { deterministic: true }, (_a) => 1); + try { + for (let i = 0; i < n; i++) { + await db.all('SELECT noop(a) FROM f'); + } + } finally { + db.removeFunction('noop'); + } + }, + }, + { + name: 'overhead/JS aggregate: 20k steps', + group: 'overhead', + ops: 20000, + samples: 24, + iter: async (_env, n) => { + db.aggregate('accumulate', { + start: () => 0, + step: (/** @type {number} */ acc, _v) => acc + 1, + result: (/** @type {number} */ acc) => acc, + }); + try { + for (let i = 0; i < n; i++) { + const row = await db.get( + 'SELECT accumulate(a) AS v FROM f', + ); + if (row.v !== 20000) throw new Error('bad count'); + } + } finally { + db.removeFunction('accumulate'); + } + }, + }, + { + name: 'overhead/JS collation: sort 10k as text', + group: 'overhead', + ops: 10000, + samples: 12, + note: 'O(n log n) JS comparisons — the per-row figure is per sorted row', + iter: async (_env, n) => { + db.collation( + 'natsort', + (/** @type {string} */ x, /** @type {string} */ y) => + x < y ? -1 : x > y ? 1 : 0, + ); + try { + for (let i = 0; i < n; i++) { + await db.all( + 'SELECT a FROM f WHERE a <= 10000 ORDER BY CAST(a AS TEXT) COLLATE natsort', + ); + } + } finally { + db.removeCollation('natsort'); + } + }, + }, + ]; + return cases; +} + +/** + * The db.transaction() wrapper keepers (D05 follow-up): deliberately + * empty bodies measure the wrapper (AsyncLocalStorage, flow-store copy, + * validation) against raw BEGIN/COMMIT. + * + * @param {any} db an open connection. + * @returns {CaseSpec[]} the transaction-wrapper cases. + */ +export function transactionCases(db) { + return [ + { + name: 'overhead/db.transaction: 200 empty bodies', + group: 'overhead', + ops: 200, + iter: async (_env, n) => { + for (let r = 0; r < n; r++) { + for (let i = 0; i < 200; i++) { + await db.transaction(async () => undefined); + } + } + }, + }, + { + name: 'overhead/raw BEGIN+COMMIT: 200 pairs', + group: 'overhead', + ops: 200, + iter: async (_env, n) => { + for (let r = 0; r < n; r++) { + for (let i = 0; i < 200; i++) { + await db.exec('BEGIN'); + await db.exec('COMMIT'); + } + } + }, + }, + ]; +} + +/** + * The open/close keeper (Deliverable 11): every connection goes through + * the Database wrapper whose permission-model gate costs one property + * read with the model off. + * + * @param {typeof import('../../lib/sqlite3.js').default} sqlite3 the driver. + * @returns {CaseSpec} the case. + */ +export function openCloseCase(sqlite3) { + return { + name: 'overhead/open+close: 1,000 :memory: connections', + group: 'overhead', + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + const conn = new sqlite3.Database(':memory:'); + await new Promise((resolve, reject) => { + conn.once('open', resolve); + conn.once('error', reject); + }); + await new Promise((resolve) => conn.close(resolve)); + } + }, + }; +} diff --git a/bench/cases/read.js b/bench/cases/read.js new file mode 100644 index 0000000..a46083b --- /dev/null +++ b/bench/cases/read.js @@ -0,0 +1,187 @@ +// Read-path cases (Deliverable 13 §2.2): `all` across a rows × columns +// matrix, plus `get`/`each`/`iterate`/`map`, a wide-text row set and a +// mostly-NULL row set. Marshalling optimisations that only help narrow +// integer columns should show up here as exactly that. +import { colDefsFor, colsFor, fmt, intRows } from './shared.js'; + +/** + * Builds the read cases on a fresh connection: tables r__ for + * the size matrix, plus the wide and mostly-NULL sets. + * + * @param {any} db an open, cache-free connection. + * @returns {import('../harness.js').CaseSpec[]} the read cases. + */ +export function readCases(db) { + const sizes = [ + [1000, 1], + [1000, 4], + [1000, 16], + [20000, 1], + [20000, 4], + [20000, 16], + [200000, 1], + [200000, 4], + [200000, 16], + ]; + + /** @type {import('../harness.js').CaseSpec[]} */ + const cases = []; + for (const [rows, cols] of sizes) { + db.exec(`CREATE TABLE r_${rows}_${cols} (${colDefsFor(cols)})`); + db.exec(intRows(rows, colsFor(cols), `r_${rows}_${cols}`)); + cases.push({ + name: `read/all: ${fmt(rows)} rows × ${cols} cols`, + group: 'read', + ops: rows, + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + const rowsOut = await db.all( + `SELECT * FROM r_${rows}_${cols}`, + ); + if (rowsOut.length !== rows) + throw new Error('bad row count'); + } + }, + alloc: rows === 20000 && (cols === 1 || cols === 4), + allocIter: async (env, n) => { + for (let i = 0; i < n; i++) { + env.keep = await db.all(`SELECT * FROM r_${rows}_${cols}`); + } + }, + }); + } + + // Wide rows: 8 columns of ~100-char text — the shape that stresses + // string marshalling rather than integer conversion. + db.exec( + 'CREATE TABLE r_wide (c0 TEXT, c1 TEXT, c2 TEXT, c3 TEXT, c4 TEXT, c5 TEXT, c6 TEXT, c7 TEXT)', + ); + db.exec( + intRows( + 20000, + Array.from( + { length: 8 }, + (_, i) => `'w${i}-' || printf('%096d', x)`, + ).join(', '), + 'r_wide', + ), + ); + cases.push({ + name: 'read/all: 20,000 rows × 8 cols wide text', + group: 'read', + ops: 20000, + // ~80 MB of string allocation per sample makes GC pauses a real + // part of this case; more samples keep the median honest about it. + samples: 48, + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + const rowsOut = await db.all('SELECT * FROM r_wide'); + if (rowsOut.length !== 20000) throw new Error('bad row count'); + } + }, + alloc: true, + allocIter: async (env, _n) => { + env.keep = await db.all('SELECT * FROM r_wide'); + }, + }); + + // Mostly-NULL rows: 7 of 8 columns NULL — NULL marshalling and the + // fixed per-row overhead, without payload conversion work. + db.exec( + 'CREATE TABLE r_null (c0 INTEGER, c1 INTEGER, c2 INTEGER, c3 INTEGER, c4 INTEGER, c5 INTEGER, c6 INTEGER, c7 INTEGER)', + ); + db.exec( + intRows(20000, 'x, NULL, NULL, NULL, NULL, NULL, NULL, NULL', 'r_null'), + ); + cases.push({ + name: 'read/all: 20,000 rows × 8 cols mostly NULL', + group: 'read', + ops: 20000, + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + const rowsOut = await db.all('SELECT * FROM r_null'); + if (rowsOut.length !== 20000) throw new Error('bad row count'); + } + }, + alloc: true, + allocIter: async (env, _n) => { + env.keep = await db.all('SELECT * FROM r_null'); + }, + }); + + // Single-row get through a prepared statement: the interactive lookup. + // rowid lookup, not a table scan. + cases.push({ + name: 'read/get: single row (prepared statement)', + group: 'read', + iter: async (_env, n) => { + const stmt = db.prepare('SELECT * FROM r_20000_4 WHERE rowid = ?'); + for (let i = 0; i < n; i++) { + await stmt.get((i % 20000) + 1); + } + await stmt.finalize(); + }, + }); + + cases.push({ + name: 'read/each: 20,000 rows × 4 cols', + group: 'read', + ops: 20000, + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + await new Promise((resolve, reject) => { + let seen = 0; + db.each( + 'SELECT * FROM r_20000_4', + () => { + seen++; + }, + (/** @type {Error | null} */ err) => { + if (err) reject(err); + else if (seen !== 20000) + reject(new Error('bad row count')); + else resolve(); + }, + ); + }); + } + }, + }); + + cases.push({ + name: 'read/iterate: 20,000 rows × 4 cols (for await)', + group: 'read', + ops: 20000, + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + let seen = 0; + for await (const _row of db.iterate( + 'SELECT * FROM r_20000_4', + )) { + seen++; + } + if (seen !== 20000) throw new Error('bad row count'); + } + }, + }); + + cases.push({ + name: 'read/map: 20,000 rows × 4 cols', + group: 'read', + ops: 20000, + // map() returns an object keyed by the first column, not an array. + iter: async (_env, n) => { + for (let i = 0; i < n; i++) { + const mapped = await db.map('SELECT c0 FROM r_20000_4'); + if ( + !Object.hasOwn(mapped, '1') || + !Object.hasOwn(mapped, '20000') + ) { + throw new Error('bad map keys'); + } + } + }, + }); + + return cases; +} diff --git a/bench/cases/shared.js b/bench/cases/shared.js new file mode 100644 index 0000000..059fdd2 --- /dev/null +++ b/bench/cases/shared.js @@ -0,0 +1,151 @@ +// Shared fixture and helper code for the bench cases. Everything here is +// setup, not measurement: the harness times `iter` bodies only. +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +/** + * Formats an integer with thousands separators, locale-independently, so + * case names are identical on every machine (they are the baseline keys). + * + * @param {number} n the number. + * @returns {string} formatted string. + */ +export function fmt(n) { + return String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ','); +} + +/** + * SQL that materialises `n` integer rows 1..n via a recursive CTE, with + * the given per-row column expressions. + * + * @param {number} n row count. + * @param {string} cols comma-separated column expressions over `x`. + * @param {string} table target table name (default `t`). + * @returns {string} the INSERT ... SELECT statement. + */ +export function intRows(n, cols, table = 't') { + return ( + `INSERT INTO ${table} SELECT ` + + cols + + ' FROM (WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt WHERE x < ' + + n + + ') SELECT x FROM cnt)' + ); +} + +/** + * Column expressions for an `x`-driven row with the requested width. + * 1 column: the integer; 4: int, real, short text, 64-byte blob; + * 16: a wider mix of ints, reals and texts. + * + * @param {number} width 1, 4 or 16 columns. + * @returns {string} comma-separated expressions. + */ +export function colsFor(width) { + if (width === 1) return 'x'; + if (width === 4) { + return "x, x + 0.5, 'text-value-' || x, zeroblob(64)"; + } + const parts = ['x', 'x + 0.5']; + for (let i = 2; i < width; i++) { + if (i % 3 === 0) parts.push(`x * ${i}`); + else if (i % 3 === 1) parts.push(`x + ${i}.5`); + else parts.push(`'col-${i}-' || x`); + } + return parts.join(', '); +} + +/** + * The column list matching `colsFor`, for CREATE TABLE. + * + * @param {number} width 1, 4 or 16 columns. + * @returns {string} comma-separated `name type` definitions. + */ +export function colDefsFor(width) { + if (width === 1) return 'c0 INTEGER'; + if (width === 4) return 'c0 INTEGER, c1 REAL, c2 TEXT, c3 BLOB'; + const parts = ['c0 INTEGER', 'c1 REAL']; + for (let i = 2; i < width; i++) { + parts.push( + i % 3 === 0 + ? `c${i} INTEGER` + : `c${i} ${i % 3 === 1 ? 'REAL' : 'TEXT'}`, + ); + } + return parts.join(', '); +} + +/** + * Waits for `n` sequential callback-style operations. Used to time the + * callback API without a promise wrapper on the per-call path. + * + * @param {number} n how many operations to run. + * @param {(i: number, done: () => void) => void} call issues operation `i`; must invoke `done` when it completes. + * @returns {Promise} resolves when all n operations completed, in order. + */ +export function seqCallbacks(n, call) { + return new Promise((resolve) => { + let i = 0; + const next = () => { + if (i === n) { + resolve(); + return; + } + call(i, next); + i++; + }; + next(); + }); +} + +/** + * Opens every connection the suite needs on one shared handle object, so + * `dispose()` can close them all at the end. In-memory databases are used + * wherever the fixture fits comfortably in RAM: they keep the OS page + * cache out of the measurements. + * + * @param {typeof import('../lib/sqlite3.js').default} sqlite3 the driver. + * @returns {{ mem: () => any, all: any[], dispose: () => Promise}} connection registry. + */ +export function connectionRegistry(sqlite3) { + /** @type {any[]} */ + const conns = []; + return { + /** + * Opens (and registers) a new in-memory database. + * + * @param {string} [label] diagnostic label. + * @returns {any} the open database. + */ + mem(label = 'mem') { + const db = new sqlite3.Database(':memory:'); + Reflect.set(db, 'benchLabel', label); + conns.push(db); + return db; + }, + all: conns, + /** Closes every registered connection. */ + async dispose() { + for (const db of conns) { + await new Promise((resolve) => db.close(resolve)); + } + conns.length = 0; + }, + }; +} + +/** + * Creates a scratch directory for file-backed fixtures (the pool bench). + * + * @returns {{ dir: string, cleanup: () => void }} the directory path and a cleanup callback. + */ +export function scratchDir() { + const dir = mkdtempSync(join(tmpdir(), 'node-sqlite3-bench-')); + return { + dir, + cleanup() { + rmSync(dir, { recursive: true, force: true }); + }, + }; +} diff --git a/bench/cases/sync.js b/bench/cases/sync.js new file mode 100644 index 0000000..f8c6547 --- /dev/null +++ b/bench/cases/sync.js @@ -0,0 +1,202 @@ +// Sync vs async (Deliverable 13 §2.2): getSync/runSync/allSync against +// their async equivalents at 1, 10, 100 and 10,000 operations. This is +// where README's "roughly 6x faster" claim lives — the harness publishes +// the curve (per-op cost as the batch grows), not one number, and every +// ratio is checked against the same-run noise floor before it is called +// a result. +// +import { fmt } from './shared.js'; + +// Both sides use the statement cache: that is the documented fast-path +// pairing (README shows getSync after cacheStatements(), and the async +// equivalent of a cached sync call is a cached async call). + +/** Batch sizes the curve is measured at. */ +const SIZES = [1, 10, 100, 10000]; + +/** + * Builds the sync-vs-async cases on two cache-enabled connections with + * identical 20,000-row read tables and empty write tables. + * + * @param {any} dbSync connection for the sync cases (cache enabled). + * @param {any} dbAsync connection for the async cases (cache enabled). + * @returns {import('../harness.js').CaseSpec[]} the sync-vs-async cases. + */ +export function syncCases(dbSync, dbAsync) { + const ddl = 'CREATE TABLE t (c0 INTEGER, c1 REAL, c2 TEXT, c3 BLOB)'; + const seed = + "INSERT INTO t SELECT x, x + 0.5, 'text-value-' || x, zeroblob(64) " + + 'FROM (WITH RECURSIVE cnt(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM cnt WHERE x < 20000) SELECT x FROM cnt)'; + for (const db of [dbSync, dbAsync]) { + db.exec( + `${ddl}; CREATE TABLE wt (c0 INTEGER, c1 REAL, c2 TEXT, c3 BLOB); ${seed}`, + ); + db.cacheStatements(); + } + const GET_SQL = 'SELECT * FROM t WHERE rowid = ?'; + const RUN_SQL = 'INSERT INTO wt VALUES (?, ?, ?, ?)'; + const buf = Buffer.alloc(64); + + /** @type {import('../harness.js').CaseSpec[]} */ + const cases = []; + + for (const size of SIZES) { + const label = fmt(size); + + cases.push({ + name: `sync-vs-async/get: batch of ${label} (async)`, + group: 'sync-vs-async', + ops: size, + iter: async (_env, n) => { + for (let r = 0; r < n; r++) { + for (let i = 0; i < size; i++) { + await dbAsync.get(GET_SQL, (i % 20000) + 1); + } + } + }, + }); + cases.push({ + name: `sync-vs-async/getSync: batch of ${label}`, + group: 'sync-vs-async', + ops: size, + ratioTo: `sync-vs-async/get: batch of ${label} (async)`, + iter: (_env, n) => { + for (let r = 0; r < n; r++) { + for (let i = 0; i < size; i++) { + dbSync.getSync(GET_SQL, (i % 20000) + 1); + } + } + }, + }); + + cases.push({ + name: `sync-vs-async/run: batch of ${label} (async)`, + group: 'sync-vs-async', + ops: size, + note: 'each round clears the table first (timed, not counted)', + iter: async (_env, n) => { + for (let r = 0; r < n; r++) { + dbAsync.runSync('DELETE FROM wt'); + for (let i = 0; i < size; i++) { + await dbAsync.run( + RUN_SQL, + i, + i + 0.5, + `text-value-${i}`, + buf, + ); + } + } + }, + }); + cases.push({ + name: `sync-vs-async/runSync: batch of ${label}`, + group: 'sync-vs-async', + ops: size, + ratioTo: `sync-vs-async/run: batch of ${label} (async)`, + note: 'each round clears the table first (timed, not counted)', + iter: (_env, n) => { + for (let r = 0; r < n; r++) { + dbSync.runSync('DELETE FROM wt'); + for (let i = 0; i < size; i++) { + dbSync.runSync( + RUN_SQL, + i, + i + 0.5, + `text-value-${i}`, + buf, + ); + } + } + }, + }); + } + + // allSync against the async `read/all: 20,000 rows × 4 cols` case: + // the crossover point — one threadpool round trip amortised over + // 20,000 marshalled rows should narrow the gap to near nothing. + cases.push({ + name: 'sync-vs-async/allSync: 20,000 rows × 4 cols', + group: 'sync-vs-async', + ops: 20000, + ratioTo: 'read/all: 20,000 rows × 4 cols', + iter: (_env, n) => { + for (let i = 0; i < n; i++) { + const rows = dbSync.allSync('SELECT * FROM t'); + if (rows.length !== 20000) throw new Error('bad row count'); + } + }, + }); + + // The `{ rowMode: 'array' }` bulk-reader shape: no per-row property + // stores. Ratio'd against node:sqlite's returnArrays mirror (D16): + // with the property stores gone, what remains is the per-value + // Node-API transfer tax, which this case isolates from the row shape. + cases.push({ + name: 'sync-vs-async/allSync (arrays): 20,000 rows × 4 cols', + group: 'sync-vs-async', + ops: 20000, + ratioTo: + 'baseline/node:sqlite/all (returnArrays): 20,000 rows × 4 cols', + iter: (_env, n) => { + for (let i = 0; i < n; i++) { + const rows = dbSync.allSync('SELECT * FROM t', { + rowMode: 'array', + }); + if (rows.length !== 20000) throw new Error('bad row count'); + } + }, + }); + + // getSync on a prepared statement, no JS wrapper and no statement + // cache lookup: the statement-to-statement comparison with + // node:sqlite that localises the sync call's fixed cost (the JS + // wrapper adds the rest; see D16's decomposition). + cases.push({ + name: 'sync-vs-async/getSync (native path): single row', + group: 'sync-vs-async', + ratioTo: 'baseline/node:sqlite/get: single row (prepared)', + iter: (_env, n) => { + const stmt = dbSync.prepareSync(GET_SQL); + for (let i = 0; i < n; i++) { + stmt.getSync((i % 20000) + 1); + } + stmt.finalize(); + }, + }); + + // The three bind shapes against one statement, so the cost of the + // ergonomic call form is visible next to the terse one. + // + // Named binding is the shape most code actually reaches for, and it + // is the only one whose cost is not obvious from the call site: it + // enumerates the object's keys, classifies each as a name or a + // position, and resolves each name to a bind index. Left unmeasured, + // that work is free to grow. The array and positional cases are here + // as its reference points, not for their own sake. + for (const [label, bind] of [ + ['positional', (stmt, i) => stmt.runSync(i, 'x', 1.5, 'yz')], + ['array', (stmt, i) => stmt.runSync([i, 'x', 1.5, 'yz'])], + [ + 'named', + (stmt, i) => stmt.runSync({ $a: i, $b: 'x', $c: 1.5, $d: 'yz' }), + ], + ]) { + cases.push({ + name: `sync-vs-async/runSync bind shape: ${label} (4 params)`, + group: 'sync-vs-async', + iter: (_env, n) => { + const sql = + label === 'named' + ? 'INSERT INTO wt VALUES ($a, $b, $c, $d)' + : 'INSERT INTO wt VALUES (?, ?, ?, ?)'; + dbSync.runSync('DELETE FROM wt'); + const stmt = dbSync.prepareSync(sql); + for (let i = 0; i < n; i++) bind(stmt, i); + stmt.finalize(); + }, + }); + } + + return cases; +} diff --git a/bench/cases/write.js b/bench/cases/write.js new file mode 100644 index 0000000..36f5d42 --- /dev/null +++ b/bench/cases/write.js @@ -0,0 +1,185 @@ +// Write-path cases (Deliverable 13 §2.2): prepared insert, per-call +// prepare, the statement cache, one transaction vs autocommit — the +// single biggest real-world speed lever in SQLite — and multi-statement +// exec. +// +// Every round clears its table first: the DELETE is inside the timed +// region (it has to be, the harness times whole rounds) but excluded from +// the op count, and it is identical across the compared cases, so ratios +// stay apples-to-apples. +// +// The transaction pair runs on a FILE database, not :memory: — on an +// in-memory database a commit is a memcpy and there is no journal, which +// is exactly why run 1 of this suite measured the lever as "within +// noise". The lever is real precisely where durability costs something. +import { rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +/** + * Opens a private file-backed database for the transaction pair. + * + * @param {string} tag filename tag, so the two compared cases never share a file. + * @returns {Promise} env with { db, stmt, path } once open, tabled and prepared. + */ +async function txnFileSetup(tag) { + const { default: sqlite3 } = await import('../../lib/sqlite3.js'); + const path = join(tmpdir(), `node-sqlite3-bench-${tag}-${process.pid}.db`); + for (const suffix of ['', '-journal', '-wal', '-shm']) { + rmSync(path + suffix, { force: true }); + } + const db = new sqlite3.Database(path); + await new Promise((resolve, reject) => { + db.once('open', resolve); + db.once('error', reject); + }); + await db.exec('CREATE TABLE w (c0 INTEGER, c1 REAL, c2 TEXT, c3 BLOB)'); + const stmt = db.prepare('INSERT INTO w VALUES (?, ?, ?, ?)'); + return { db, stmt, path }; +} + +/** + * Closes and removes a transaction-pair file database. + * + * @param {{ db: any, stmt: any, path: string }} env the setup value. + * @returns {Promise} resolves once closed and cleaned up. + */ +async function txnFileTeardown(env) { + await new Promise((resolve) => env.stmt.finalize(resolve)); + await new Promise((resolve) => env.db.close(resolve)); + for (const suffix of ['', '-journal', '-wal', '-shm']) { + rmSync(env.path + suffix, { force: true }); + } +} + +/** + * Builds the write cases. Two connections: the shared plain one, and a + * separate one for the statement-cache case, because `cacheStatements()` + * cannot be turned off and must not leak into the other cases. + * + * @param {any} dbPlain an open, cache-free connection. + * @param {any} dbCached an open connection that this group may cache-enable. + * @returns {import('../harness.js').CaseSpec[]} the write cases. + */ +export function writeCases(dbPlain, dbCached) { + const db = dbPlain; + db.exec('CREATE TABLE w (c0 INTEGER, c1 REAL, c2 TEXT, c3 BLOB)'); + dbCached.exec('CREATE TABLE w (c0 INTEGER, c1 REAL, c2 TEXT, c3 BLOB)'); + const K = 1000; // inserts per round + const buf = Buffer.alloc(64); + + /** @type {import('../harness.js').CaseSpec[]} */ + const cases = [ + { + name: 'write/run: prepared insert ×1,000', + group: 'write', + ops: K, + note: 'each round clears the table first (timed, not counted)', + iter: async (_env, n) => { + const stmt = db.prepare('INSERT INTO w VALUES (?, ?, ?, ?)'); + for (let r = 0; r < n; r++) { + await db.exec('DELETE FROM w'); + for (let i = 0; i < K; i++) { + await stmt.run(i, i + 0.5, `text-value-${i}`, buf); + } + } + await stmt.finalize(); + }, + }, + { + name: 'write/db.run: prepare per call ×1,000', + group: 'write', + ops: K, + note: 'each round clears the table first (timed, not counted)', + iter: async (_env, n) => { + for (let r = 0; r < n; r++) { + await db.exec('DELETE FROM w'); + for (let i = 0; i < K; i++) { + await db.run( + 'INSERT INTO w VALUES (?, ?, ?, ?)', + i, + i + 0.5, + `text-value-${i}`, + buf, + ); + } + } + }, + }, + { + name: 'write/db.run: statement cache ×1,000', + group: 'write', + ops: K, + note: 'each round clears the table first (timed, not counted)', + iter: async (_env, n) => { + dbCached.cacheStatements(); + for (let r = 0; r < n; r++) { + await dbCached.exec('DELETE FROM w'); + for (let i = 0; i < K; i++) { + await dbCached.run( + 'INSERT INTO w VALUES (?, ?, ?, ?)', + i, + i + 0.5, + `text-value-${i}`, + buf, + ); + } + } + }, + }, + { + name: 'write/insert: ×1,000 in one transaction (file db)', + group: 'write', + ops: K, + targetSampleMs: 80, + ratioTo: 'write/insert: ×1,000 autocommit (file db)', + note: 'file-backed: a commit must survive a journal — the lever being measured', + setup: () => txnFileSetup('txn'), + teardown: txnFileTeardown, + iter: async (env, n) => { + for (let r = 0; r < n; r++) { + await env.db.exec('DELETE FROM w'); + await env.db.exec('BEGIN'); + for (let i = 0; i < K; i++) { + await env.stmt.run(i, i + 0.5, `text-value-${i}`, buf); + } + await env.db.exec('COMMIT'); + } + }, + }, + { + name: 'write/insert: ×1,000 autocommit (file db)', + group: 'write', + ops: K, + targetSampleMs: 80, + note: 'file-backed: a commit must survive a journal — the lever being measured', + setup: () => txnFileSetup('autocommit'), + teardown: txnFileTeardown, + iter: async (env, n) => { + for (let r = 0; r < n; r++) { + await env.db.exec('DELETE FROM w'); + for (let i = 0; i < K; i++) { + await env.stmt.run(i, i + 0.5, `text-value-${i}`, buf); + } + } + }, + }, + { + name: 'write/exec: 100-statement script', + group: 'write', + ops: 100, + iter: async (_env, n) => { + const script = Array.from( + { length: 100 }, + (_, i) => + `INSERT INTO w VALUES (${i}, ${i}.5, 's${i}', x'00')`, + ).join(';\n'); + for (let i = 0; i < n; i++) { + await db.exec(`DELETE FROM w;\n${script}`); + } + }, + }, + ]; + + return cases; +} diff --git a/bench/harness.js b/bench/harness.js new file mode 100644 index 0000000..36e8471 --- /dev/null +++ b/bench/harness.js @@ -0,0 +1,356 @@ +// The measurement engine for the v9 benchmark suite. Dependency-free by +// design (Deliverable 13 §2.1): node:perf_hooks for the clock, ~a hundred +// lines of statistics, and nothing else. +// +// The harness is built to refuse rather than misreport. Every case is +// sampled N times and reduced to a median with a bootstrap 95% confidence +// interval; the relative margin of error (RME) is half that interval over +// the median. A case whose RME exceeds the threshold is reported as +// REJECTED with its RME instead of a number that looks trustworthy — the +// failure mode this whole design exists to prevent is a quiet wrong +// number, not a loud missing one. +import { performance } from 'node:perf_hooks'; + +/** + * A small deterministic PRNG (mulberry32) so bootstrap confidence + * intervals are reproducible from the same sample set and seed. + * + * @param {number} seed 32-bit integer seed. + * @returns {() => number} uniform pseudo-random values in [0, 1). + */ +export function mulberry32(seed) { + let a = seed >>> 0; + return function () { + a |= 0; + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** + * Median of a numeric sample set. + * + * @param {number[]} xs samples. + * @returns {number} the median (mean of the two central values for even n). + */ +export function median(xs) { + if (xs.length === 0) throw new Error('median of empty sample set'); + const sorted = [...xs].sort((a, b) => a - b); + const mid = sorted.length >> 1; + return sorted.length % 2 + ? sorted[mid] + : (sorted[mid - 1] + sorted[mid]) / 2; +} + +/** + * Percentile of a numeric sample set with linear interpolation between + * adjacent ranks. + * + * @param {number[]} xs samples. + * @param {number} p percentile in [0, 1]. + * @returns {number} the interpolated percentile value. + */ +export function percentile(xs, p) { + if (xs.length === 0) throw new Error('percentile of empty sample set'); + const sorted = [...xs].sort((a, b) => a - b); + const idx = p * (sorted.length - 1); + const lo = Math.floor(idx); + const hi = Math.ceil(idx); + if (lo === hi) return sorted[lo]; + return sorted[lo] + (sorted[hi] - sorted[lo]) * (idx - lo); +} + +/** + * Mean of a numeric sample set. + * + * @param {number[]} xs samples. + * @returns {number} the arithmetic mean. + */ +export function mean(xs) { + if (xs.length === 0) throw new Error('mean of empty sample set'); + let sum = 0; + for (const x of xs) sum += x; + return sum / xs.length; +} + +/** + * Bootstrap confidence interval for the median, and the relative margin of + * error derived from it. The median is the headline statistic because GC + * and JIT pauses make the mean tail-sensitive; bootstrapping propagates + * the sample spread into an honest interval around it. + * + * RME = (ciHigh - ciLow) / 2 / median, as a fraction of the median. + * + * @param {number[]} xs samples (n >= 2). + * @param {number} resamples bootstrap resample count. + * @param {() => number} rng seeded uniform [0,1) generator. + * @returns {{ rme: number, ciLow: number, ciHigh: number }} interval over the sample values. + */ +export function bootstrapMedianRme(xs, resamples, rng) { + const n = xs.length; + if (n < 2) { + return { rme: Number.POSITIVE_INFINITY, ciLow: xs[0], ciHigh: xs[0] }; + } + const medians = new Array(resamples); + const pick = new Array(n); + for (let r = 0; r < resamples; r++) { + for (let i = 0; i < n; i++) pick[i] = xs[(rng() * n) | 0]; + medians[r] = median(pick); + } + medians.sort((a, b) => a - b); + const ciLow = percentile(medians, 0.025); + const ciHigh = percentile(medians, 0.975); + const med = median(xs); + return { + rme: med > 0 ? (ciHigh - ciLow) / 2 / med : Number.POSITIVE_INFINITY, + ciLow, + ciHigh, + }; +} + +/** + * Full summary of one case's per-operation samples. + * + * @param {number[]} xs per-operation values (ms). + * @param {{ seed?: number, resamples?: number }} [opts] bootstrap controls. + * @returns {{ n: number, median: number, p95: number, min: number, max: number, mean: number, rme: number, ciLow: number, ciHigh: number }} reduced statistics. + */ +export function summarise(xs, opts) { + const rng = mulberry32(opts?.seed ?? 0x5eed1337); + const { rme, ciLow, ciHigh } = bootstrapMedianRme( + xs, + opts?.resamples ?? 2000, + rng, + ); + return { + n: xs.length, + median: median(xs), + p95: percentile(xs, 0.95), + min: Math.min(...xs), + max: Math.max(...xs), + mean: mean(xs), + rme, + ciLow, + ciHigh, + }; +} + +/** @typedef {Object} CaseSpec + * @property {string} name case name, unique across the suite. + * @property {string} group group heading the case is reported under. + * @property {(env: any, n: number) => Promise | void} iter performs `n` logical operations; timed once per sample. Sync paths run a tight loop with no awaits; async paths await each operation, because awaiting is the usage pattern being measured. + * @property {() => Promise | any} [setup] run once before warmup. + * @property {(env: any) => Promise | void} [teardown] run once after sampling. + * @property {number} [ops] logical operations per single `iter` call at n=1 (per-op values divide by n*ops); default 1. + * @property {number} [samples] override the sample count for this case. + * @property {number} [targetSampleMs] override the per-sample duration target — for cases whose cost is dominated by OS-level jitter (journal/fsync churn), longer samples average more of it away. + * @property {boolean} [alloc] also measure allocation per operation (forced-GC deltas). + * @property {(env: any, n: number) => Promise | void} [allocIter] iter variant that stores only its final iteration's output in `env.keep` — the retained-batch convention `measureAlloc` divides by; see its doc comment. + * @property {number} [allocSamples] override the allocation sample count. + * @property {string} [ratioTo] name of another case to publish an A/B ratio against. + * @property {string} [note] one-line caveat printed with the case. + */ + +/** @typedef {Object} HarnessConfig + * @property {number} warmupMs fixed wall-clock warmup budget per case. + * @property {number} targetSampleMs samples are scaled to roughly this duration. + * @property {number} minSampleMs recalibrate if samples come in below this. + * @property {number} samples sample count per case (>= 30 per §2.1). + * @property {number} rmeThresholdPct reject cases whose RME exceeds this. + * @property {number} allocSamples forced-GC allocation sample count per case. + */ + +/** @type {HarnessConfig} */ +export const DEFAULT_CONFIG = { + warmupMs: 500, + targetSampleMs: 20, + minSampleMs: 10, + samples: 32, + rmeThresholdPct: 5, + allocSamples: 16, +}; + +/** + * Warms the iter path for a fixed wall-clock budget with a growing batch, + * and returns the estimated cost of one logical operation. + * + * @param {CaseSpec} spec the case. + * @param {any} env the case's setup value. + * @param {number} budgetMs how long to warm up. + * @param {number} floorOps minimum logical operations to have run. + * @returns {Promise} estimated ms per single operation. + */ +async function warmup(spec, env, budgetMs, floorOps) { + const start = performance.now(); + let ran = 0; + let n = 1; + while (true) { + await spec.iter(env, n); + ran += n; + const elapsed = performance.now() - start; + if (elapsed >= budgetMs && ran >= floorOps) { + return elapsed / ran; + } + // Grow the batch so the budget is reached in O(log) calls even for + // sub-microsecond operations. + n = Math.min(n * 2, 1 << 20); + } +} + +/** + * Measures one case: warm up for a fixed wall-clock budget, auto-scale the + * per-sample batch so each sample is >= 10 ms (making clock resolution + * irrelevant), then take N samples and reduce them with `summarise`. + * + * @param {CaseSpec} spec the case. + * @param {HarnessConfig} cfg harness configuration. + * @returns {Promise<{ spec: CaseSpec, batch: number, perOpMs: ReturnType, sampleMs: number, rejected: boolean }>} the measurement. + */ +export async function measure(spec, cfg) { + const env = spec.setup ? await spec.setup() : {}; + try { + const perOpMs = await warmup(spec, env, cfg.warmupMs, 3); + const target = spec.targetSampleMs ?? cfg.targetSampleMs; + let batch = Math.max(1, Math.round(target / perOpMs)); + const ops = spec.ops ?? 1; + const n = spec.samples ?? cfg.samples; + + /** @param {number} count @returns {Promise} */ + const takeSamples = async (count) => { + const perOp = []; + for (let s = 0; s < count; s++) { + const t0 = performance.now(); + await spec.iter(env, batch); + perOp.push((performance.now() - t0) / (batch * ops)); + } + return perOp; + }; + + let perOp = await takeSamples(Math.min(n, 4)); + + // One recalibration pass: if warm-up made the estimate stale and + // samples came out below the floor, rescale and start over. + const observed = median(perOp) * batch * ops; + if (observed > 0 && observed < cfg.minSampleMs) { + batch = Math.max(1, Math.round((batch * target) / observed)); + perOp = await takeSamples(n); + } else if (perOp.length < n) { + perOp.push(...(await takeSamples(n - perOp.length))); + } + + const stats = summarise(perOp); + return { + spec, + batch, + perOpMs: stats, + sampleMs: stats.median * batch * ops, + rejected: stats.rme * 100 > cfg.rmeThresholdPct, + }; + } finally { + if (spec.teardown) await spec.teardown(env); + } +} + +/** Allocation counters read per sample. `rss` is recorded but expected to + * be too noisy to trust — publishing its rejection is part of the output. */ +const ALLOC_COUNTERS = /** @type {const} */ ([ + 'heapUsed', + 'external', + 'arrayBuffers', + 'rss', +]); + +/** + * Measures allocation per operation via process.memoryUsage() deltas + * around a forced GC (--expose-gc). The convention that makes the delta + * mean something: `allocIter` must store only its FINAL iteration's + * output in `env.keep` (see bench/cases/read.js). The harness drops the + * previous sample's `env.keep` and GCs before reading "before", runs + * `reps` iterations, GCs again with the last output still live, and + * reads "after" — so the delta is exactly one iteration's output. + * Per-op bytes therefore divide by `ops` (one iteration), not `reps`. + * + * External buffers — the blob marshalling path (CellToJS in + * src/convert.cc) — do not live in heapUsed at all; they surface in + * `external` and `arrayBuffers`. Each counter is reduced with the same + * median/RME treatment as time, and a counter whose RME exceeds the + * threshold is reported as too noisy rather than published. + * + * @param {CaseSpec} spec the case. + * @param {HarnessConfig} cfg harness configuration. + * @returns {Promise>} per-counter allocation stats, or a skip reason. + */ +export async function measureAlloc(spec, cfg) { + if (typeof globalThis.gc !== 'function') { + return { skipped: 'allocation needs --expose-gc' }; + } + const env = spec.setup ? await spec.setup() : {}; + try { + const iter = spec.allocIter ?? spec.iter; + + // Warm the alloc path, then calibrate reps like measure() does. + const perOpMs = await warmup(spec, env, 100, 2); + const reps = Math.max(1, Math.round(cfg.targetSampleMs / perOpMs)); + const n = spec.allocSamples ?? cfg.allocSamples; + // One untimed call so the allocIter variant itself is warm. + await iter(env, reps); + + const raw = {}; + for (const k of ALLOC_COUNTERS) raw[k] = []; + for (let s = 0; s < n; s++) { + env.keep = null; + globalThis.gc(); + globalThis.gc(); + const before = process.memoryUsage(); + await iter(env, reps); + globalThis.gc(); + globalThis.gc(); + const after = process.memoryUsage(); + for (const k of ALLOC_COUNTERS) { + // Retained-batch convention: the delta is one iteration's + // retained output, so the divisor is one iteration's ops. + raw[k].push((after[k] - before[k]) / (spec.ops ?? 1)); + } + } + + const out = {}; + for (const k of ALLOC_COUNTERS) { + const stats = summarise(raw[k]); + const zero = raw[k].every((v) => v === 0); + out[k] = { + bytesPerOp: stats.median, + rme: stats.rme, + // `zero` marks a counter that provably moved by nothing — + // a real answer ("no allocation on this counter"), unlike + // `rejected`, which means "moved, but too erratically to + // attach a number to". + zero, + rejected: !zero && stats.rme * 100 > cfg.rmeThresholdPct, + }; + } + return out; + } finally { + if (spec.teardown) await spec.teardown(env); + } +} + +/** + * The noise floor: the same case measured twice in the same run, as a + * relative difference. Any A/B ratio smaller than this is indistinguishable + * from measuring nothing and must not be reported as a result. + * + * @param {{ perOpMs: { median: number } }} a first measurement. + * @param {{ perOpMs: { median: number } }} b second measurement of the same case. + * @returns {{ relativePct: number }} the relative difference, in percent. + */ +export function noiseFloor(a, b) { + const x = a.perOpMs.median; + const y = b.perOpMs.median; + const ref = (x + y) / 2; + return { + relativePct: + ref > 0 ? (Math.abs(y - x) / ref) * 100 : Number.POSITIVE_INFINITY, + }; +} diff --git a/bench/index.js b/bench/index.js new file mode 100644 index 0000000..761f8a4 --- /dev/null +++ b/bench/index.js @@ -0,0 +1,660 @@ +// The v9 benchmark suite entry point (Deliverable 13). +// +// node --expose-gc bench/index.js # run, print table + JSON +// node --expose-gc bench/index.js --filter sync # subset +// node --expose-gc bench/index.js --compare # vs bench/baseline.json (+ better-sqlite3 if installed) +// node --expose-gc bench/index.js --write-baseline # regenerate this environment's baseline entry +// +// Design rule: the harness refuses rather than misreports. Cases whose +// relative margin of error exceeds the gate are REJECTED, ratios smaller +// than the same-run noise floor are marked as noise, and allocation +// counters too noisy to trust are published as exactly that. +import { execFileSync } from 'node:child_process'; +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { cpus } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import sqlite3 from '../lib/sqlite3.js'; +import { buildSuite } from './cases/index.js'; +import { + DEFAULT_CONFIG, + measure, + measureAlloc, + noiseFloor, +} from './harness.js'; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); +const DEFAULT_BASELINE = join(ROOT, 'bench', 'baseline.json'); + +/** + * Parses the CLI flags this suite understands. + * + * @param {string[]} argv arguments after the script path. + * @returns {{ filter: string[] | null, compare: boolean, baselinePath: string, writeBaseline: boolean, baselineFrom: string | null, jsonPath: string | null, strict: boolean, list: boolean }} parsed options. + */ +function parseArgs(argv) { + /** @type {ReturnType} */ + const opts = { + filter: null, + compare: false, + baselinePath: DEFAULT_BASELINE, + writeBaseline: false, + baselineFrom: null, + jsonPath: null, + strict: false, + list: false, + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + // pnpm forwards a bare `--` to the script rather than consuming + // it, so `pnpm run bench:compare -- --json out.json` arrives with + // the separator still in argv. Rejecting it made the CI bench step + // exit 1 on a usage error and measure nothing — under + // continue-on-error, silently. + if (arg === '--') continue; + if (arg === '--compare') opts.compare = true; + else if (arg === '--write-baseline') opts.writeBaseline = true; + else if (arg === '--baseline-from') + opts.baselineFrom = argv[++i] ?? null; + else if (arg === '--strict') opts.strict = true; + else if (arg === '--list') opts.list = true; + else if (arg === '--filter') { + opts.filter = (argv[++i] ?? '') + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + } else if (arg === '--baseline') { + opts.baselinePath = argv[++i] ?? DEFAULT_BASELINE; + } else if (arg === '--json') { + opts.jsonPath = argv[++i] ?? null; + } else { + console.error(`unknown option: ${arg}`); + console.error(USAGE); + process.exit(1); + } + } + return opts; +} + +const USAGE = `usage: node --expose-gc bench/index.js [--filter substr[,substr...]] [--compare] + [--baseline ] [--write-baseline] [--json ] [--strict] [--list] + + --compare compare against the baseline (default bench/baseline.json) + and try to load better-sqlite3 as a mirror (never a + devDependency; npm i --no-save better-sqlite3 first) + --write-baseline write this run's results as the baseline entry for the + current platform/arch — a deliberate, separate act + --baseline-from + merge a recorded run's JSON output as the baseline entry + for ITS recorded environment (promote a CI artifact + without re-running there) + --filter run only cases whose name or group matches a substring + --strict exit non-zero if ANY case is rejected by the RME gate + --json also write the JSON result block to a file + +exit codes: 0 ok · 1 usage · 2 baseline regression(s) · 3 nothing measured`; + +/** + * Collects the pinned environment block (§2.1). + * + * @returns {Record} environment description. + */ +function environment() { + let gitSha = 'unknown'; + let gitDirty = false; + try { + gitSha = execFileSync('git', ['rev-parse', '--short', 'HEAD'], { + cwd: ROOT, + encoding: 'utf8', + }).trim(); + gitDirty = + execFileSync('git', ['status', '--porcelain'], { + cwd: ROOT, + encoding: 'utf8', + }).trim().length > 0; + } catch { + // not a git checkout — keep 'unknown' + } + const pkg = JSON.parse(readFileSync(join(ROOT, 'package.json'), 'utf8')); + return { + node: process.version, + platform: process.platform, + arch: process.arch, + cpuModel: cpus()[0]?.model ?? 'unknown', + cpuCount: cpus().length, + container: existsSync('/.dockerenv') ? 'docker' : 'none', + sqliteVersion: /** @type {any} */ (sqlite3).VERSION, + packageVersion: pkg.version, + gitSha: gitDirty ? `${gitSha}+dirty` : gitSha, + exposeGc: typeof globalThis.gc === 'function', + }; +} + +/** + * Formats a per-operation duration for the human table. + * + * @param {number} ms milliseconds. + * @returns {string} value with unit. + */ +function fmtDuration(ms) { + if (ms < 0.001) return `${(ms * 1e6).toFixed(0)} ns`; + if (ms < 1) return `${(ms * 1000).toFixed(2)} µs`; + if (ms < 1000) return `${ms.toFixed(2)} ms`; + return `${(ms / 1000).toFixed(2)} s`; +} + +/** + * Formats bytes per operation for the allocation table. + * + * @param {number} bytes bytes. + * @returns {string} value with unit. + */ +function fmtBytes(bytes) { + const sign = bytes < 0 ? '-' : '+'; + const abs = Math.abs(bytes); + if (abs >= 1024 * 1024) + return `${sign}${(abs / 1024 / 1024).toFixed(2)} MB`; + if (abs >= 1024) return `${sign}${(abs / 1024).toFixed(1)} KB`; + return `${sign}${abs.toFixed(0)} B`; +} + +/** + * Formats a ratio for the ratios table. + * + * @param {number} r ratio. + * @returns {string} formatted ratio. + */ +function fmtRatio(r) { + if (!Number.isFinite(r)) return '∞'; + return `${r.toFixed(2)}×`; +} + +/** @type {any[]} */ const results = []; +/** @type {{ case: string, alloc: any }[]} */ const allocResults = []; + +/** + * The main run: build fixtures, calibrate, measure every case, print. + * + * @returns {Promise} process exit code. + */ +async function main() { + const opts = parseArgs(process.argv.slice(2)); + const cfg = { ...DEFAULT_CONFIG }; + const env = environment(); + + console.log('@appthreat/sqlite3 benchmark suite'); + console.log( + `${env.node} | ${env.platform} | ${env.arch} | ${env.cpuModel} (${env.cpuCount} cpus)`, + ); + console.log( + `SQLite ${env.sqliteVersion} | package ${env.packageVersion} | git ${env.gitSha} | container: ${env.container}`, + ); + console.log( + `config: ${cfg.samples} samples/case, ${cfg.warmupMs} ms warmup, sample target ${cfg.targetSampleMs} ms (floor ${cfg.minSampleMs} ms), ` + + `RME gate ${cfg.rmeThresholdPct}% (bootstrap 2000×, seeded), alloc samples ${cfg.allocSamples}`, + ); + if (!env.exposeGc) { + console.log( + 'WARNING: --expose-gc is off — allocation measurement will be skipped', + ); + } + console.log(''); + + // Promotion path: --baseline-from merges an already-recorded run and + // exits without measuring anything (used for CI artifacts). + if (opts.baselineFrom) { + promoteResultsToBaseline(opts.baselineFrom, opts.baselinePath); + return 0; + } + + const suite = await buildSuite(sqlite3, { compare: opts.compare }); + for (const skip of suite.skipped) console.log(`skipped: ${skip}`); + if (suite.skipped.length > 0) console.log(''); + + /** @type {any[]} */ + let cases = suite.cases; + if (opts.filter) { + cases = cases.filter( + (c) => + opts.filter?.some( + (f) => c.name.includes(f) || c.group.includes(f), + ) ?? false, + ); + } + + if (opts.list) { + for (const c of cases) console.log(`${c.group.padEnd(16)} ${c.name}`); + await suite.dispose(); + return 0; + } + + // Calibration: the first two cases are the A/A pair. Their same-run + // difference is the noise floor every ratio is checked against. + let floor = { relativePct: Number.POSITIVE_INFINITY }; + let lastGroup = ''; + + for (const spec of cases) { + if (spec.group !== lastGroup) { + lastGroup = spec.group; + console.log( + `\n── ${spec.group} ${'─'.repeat(Math.max(1, 66 - spec.group.length))}`, + ); + } + /** @type {any} */ + let result; + try { + result = await measure(spec, cfg); + } catch (err) { + console.log( + ` ERROR ${spec.name}: ${/** @type {Error} */ (err).message}`, + ); + results.push({ + name: spec.name, + group: spec.group, + error: /** @type {Error} */ (err).message, + }); + continue; + } + results.push(result); + if (result.rejected) { + // The range is disclosed on a rejection so a reader can still + // see the magnitude — but no median is claimed for it. + console.log( + ` REJECTED ${spec.name}: RME ${(result.perOpMs.rme * 100).toFixed(1)}% exceeds ` + + `${cfg.rmeThresholdPct}% — no median reported (samples too noisy to trust; ` + + `observed ${fmtDuration(result.perOpMs.min)}–${fmtDuration(result.perOpMs.max)}/op)`, + ); + } else { + console.log( + ` ${spec.name.padEnd(58)} ${fmtDuration(result.perOpMs.median).padStart(11)}/op` + + ` RME ${(result.perOpMs.rme * 100).toFixed(1)}%` + + ` p95 ${fmtDuration(result.perOpMs.p95)}` + + ` min ${fmtDuration(result.perOpMs.min)}` + + ` ×${result.batch}`, + ); + } + if (spec.note) console.log(` · ${spec.note}`); + + if (spec.alloc) { + const alloc = await measureAlloc(spec, cfg); + allocResults.push({ case: spec.name, alloc }); + } + + if ( + results.length === 2 && + results[0].spec?.name === 'calibration/cached get (A)' + ) { + floor = noiseFloor(results[0], results[1]); + console.log( + `\n noise floor: A vs A = ${floor.relativePct.toFixed(1)}% — ` + + 'any smaller difference is noise, not a result\n', + ); + } + } + + // ── ratios ──────────────────────────────────────────────────────────── + const byName = new Map( + results.filter((r) => r.spec).map((r) => [r.spec.name, r]), + ); + /** @type {{ a: string, b: string, ratio: number, withinNoise: boolean }[]} */ + const ratios = []; + for (const r of results) { + if (!r.spec?.ratioTo || r.rejected) continue; + const target = byName.get(r.spec.ratioTo); + if (!target || target.rejected) continue; + const ratio = target.perOpMs.median / r.perOpMs.median; + const withinNoise = Math.abs(ratio - 1) * 100 < floor.relativePct; + ratios.push({ a: r.spec.name, b: r.spec.ratioTo, ratio, withinNoise }); + } + if (ratios.length > 0) { + console.log( + `\n── ratios (how much faster the case is than its target; must clear the ${floor.relativePct.toFixed(1)}% noise floor to count) ──`, + ); + for (const r of ratios) { + // ratio = target median ÷ case median: >1 means the case is + // faster than its target, <1 means slower. The wording spells + // out which, so an inverted-looking number cannot mislead. + const faster = r.ratio >= 1; + const magnitude = faster ? r.ratio : 1 / r.ratio; + const word = faster ? 'faster' : 'slower'; + if (r.withinNoise) { + console.log( + ` ${r.a.padEnd(58)} ${fmtRatio(magnitude).padStart(8)} ${word} ~ within noise floor — not a result`, + ); + } else { + console.log( + ` ${r.a.padEnd(58)} ${fmtRatio(magnitude).padStart(8)} ${word} than ${r.b}`, + ); + } + } + } + + // ── allocation ──────────────────────────────────────────────────────── + if (allocResults.length > 0) { + console.log( + '\n── allocation per op (forced-GC process.memoryUsage() deltas; heapUsed misses external buffers — watch external/arrayBuffers) ──', + ); + for (const { case: name, alloc } of allocResults) { + console.log(` ${name}`); + if ('skipped' in alloc) { + console.log(` skipped: ${alloc.skipped}`); + continue; + } + for (const [counter, stats] of Object.entries(alloc)) { + const s = /** @type {any} */ (stats); + const line = ` ${counter.padEnd(13)} ${fmtBytes(s.bytesPerOp).padStart(11)}/op`; + if (s.zero) { + console.log(`${line} — nothing allocated on this counter`); + } else if (s.rejected) { + console.log( + `${line} RME ${(s.rme * 100).toFixed(0)}% ✗ too noisy to publish`, + ); + } else { + console.log(`${line} RME ${(s.rme * 100).toFixed(0)}%`); + } + } + } + } + + // ── JSON ───────────────────────────────────────────────────────────── + const doc = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + environment: env, + config: cfg, + noiseFloorPct: Number.isFinite(floor.relativePct) + ? Number(floor.relativePct.toFixed(2)) + : null, + cases: results.map((r) => + r.spec + ? { + name: r.spec.name, + group: r.spec.group, + ops: r.spec.ops ?? 1, + batch: r.batch, + perOpMs: { + median: r.perOpMs.median, + p95: r.perOpMs.p95, + min: r.perOpMs.min, + mean: r.perOpMs.mean, + rme: r.perOpMs.rme, + ciLow: r.perOpMs.ciLow, + ciHigh: r.perOpMs.ciHigh, + n: r.perOpMs.n, + }, + rejected: r.rejected, + error: undefined, + } + : { name: r.name, group: r.group, error: r.error }, + ), + ratios, + allocations: allocResults, + }; + const json = JSON.stringify(doc, null, 2); + if (opts.jsonPath) writeFileSync(opts.jsonPath, `${json}\n`); + console.log('\n── results.json ──'); + console.log(json); + + // ── baseline write / compare ───────────────────────────────────────── + let exitCode = 0; + const sig = `${env.platform}-${env.arch}`; + const measured = results.filter((r) => r.spec && !r.rejected && !r.error); + if (opts.writeBaseline) { + const cases = Object.fromEntries( + measured.map((r) => [ + r.spec.name, + { + medianPerOpMs: r.perOpMs.median, + rme: r.perOpMs.rme, + n: r.perOpMs.n, + }, + ]), + ); + writeBaselineEntry(opts.baselinePath, sig, { + capturedAt: doc.generatedAt, + environment: env, + config: cfg, + noiseFloorPct: doc.noiseFloorPct, + cases, + }); + console.log( + `\nbaseline written: ${sig} → ${measured.length} cases (${opts.baselinePath})`, + ); + } + + if (opts.compare) { + exitCode = compareBaseline( + opts.baselinePath, + sig, + measured, + floor, + cfg, + ); + } + + await suite.dispose(); + + const rejectedCount = results.filter((r) => r.rejected).length; + if (measured.length === 0) { + console.log('\nNOTHING MEASURED: every case was rejected or errored'); + exitCode = exitCode === 2 ? 2 : 3; + } else if (opts.strict && rejectedCount > 0) { + console.log( + `\n--strict: ${rejectedCount} case(s) rejected by the RME gate`, + ); + exitCode = exitCode === 2 ? 2 : 3; + } + + return exitCode; +} + +const BASELINE_NOTE = + 'Per-environment medians captured deliberately via `pnpm run bench:update`. ' + + 'Compare only within one platform-arch signature; ratios travel across ' + + 'platforms, absolute milliseconds do not. See docs/performance.md.'; + +/** + * Upserts one environment entry in the baseline file. + * + * @param {string} path baseline file path. + * @param {string} sig environment signature (platform-arch). + * @param {{ capturedAt: string, environment: Record, config: Record, noiseFloorPct: number | null, cases: Record }} entry the entry to write. + * @returns {void} + */ +function writeBaselineEntry(path, sig, entry) { + /** @type {any} */ + let baseline = { schemaVersion: 1, note: BASELINE_NOTE, environments: {} }; + if (existsSync(path)) { + try { + baseline = JSON.parse(readFileSync(path, 'utf8')); + } catch { + console.error(`baseline file unreadable, starting fresh: ${path}`); + } + } + baseline.environments ??= {}; + baseline.environments[sig] = entry; + writeFileSync(path, `${JSON.stringify(baseline, null, 2)}\n`); +} + +/** + * Merges a recorded results JSON (from --json) into the baseline as the + * entry for the environment it was recorded on — the promote-a-CI-artifact + * path, so a linux-x64 baseline can be captured on a runner without + * anyone hand-editing the file. + * + * @param {string} resultsPath path to a results JSON file. + * @param {string} baselinePath baseline file path. + * @returns {void} + */ +function promoteResultsToBaseline(resultsPath, baselinePath) { + const doc = /** @type {any} */ ( + JSON.parse(readFileSync(resultsPath, 'utf8')) + ); + const e = doc.environment ?? {}; + const sig = `${e.platform}-${e.arch}`; + const cases = {}; + for (const c of doc.cases ?? []) { + if (c.perOpMs && !c.rejected && !c.error) { + cases[c.name] = { + medianPerOpMs: c.perOpMs.median, + rme: c.perOpMs.rme, + n: c.perOpMs.n, + }; + } + } + writeBaselineEntry(baselinePath, sig, { + capturedAt: doc.generatedAt, + environment: e, + config: doc.config, + noiseFloorPct: doc.noiseFloorPct ?? null, + cases, + }); + console.log( + `\nbaseline merged from ${resultsPath}: ${sig} → ${Object.keys(cases).length} cases (${baselinePath})`, + ); +} + +/** + * Compares this run's medians against the baseline entry for the current + * environment signature. FAIL needs Δ > max(10%, 2× the run's noise + * floor) so a noisy run cannot manufacture a regression verdict. + * + * @param {string} path baseline file path. + * @param {string} sig environment signature (platform-arch). + * @param {any[]} measured non-rejected results. + * @param {{ relativePct: number }} floor this run's noise floor. + * @param {any} cfg harness config. + * @returns {number} 2 when regression(s) were found, else 0. + */ +function compareBaseline(path, sig, measured, floor, _cfg) { + if (!existsSync(path)) { + console.log( + `\nbaseline comparison: no baseline file at ${path} — reporting only`, + ); + return 0; + } + /** @type {any} */ + let baseline; + try { + baseline = JSON.parse(readFileSync(path, 'utf8')); + } catch (err) { + console.log( + `\nbaseline comparison: unreadable baseline (${/** @type {Error} */ (err).message})`, + ); + return 0; + } + const entry = baseline.environments?.[sig]; + if (!entry) { + console.log( + `\nbaseline comparison: no entry for ${sig} (have: ${Object.keys(baseline.environments ?? {}).join(', ') || 'none'}) — reporting only.\n` + + 'To make this environment comparable, run `pnpm run bench:update` and commit the file.', + ); + return 0; + } + + const failGate = Math.max( + 10, + 2 * (Number.isFinite(floor.relativePct) ? floor.relativePct : 0), + ); + + // Whole-machine drift, reported but deliberately NOT applied. + // + // The problem it describes is real: the A/A noise floor measures + // variance *within* one process and is blind to the machine being + // globally slower than when the baseline was captured. The first + // committed baseline was captured on an exceptionally quiet run (its + // recorded floor: 0.17%, against 1.6–3.4% for ordinary runs), and + // re-running the unmodified tree against it produced 38 FAIL / 37 WARN + // of 85. That is fixed by capturing baselines from representative runs, + // not by arithmetic here — with a representative baseline the same tree + // reports 0 FAIL. + // + // Dividing the calibration delta out as a drift correction was tried + // and is unsound: `calibration/cached get` reads a row, so it runs the + // same marshalling path most cases do. A real regression slows the + // control too, the "drift" factor absorbs part of the regression, and + // the correction subtracts it from every case. Measured: a 512 B + // per-integer-cell pessimisation in CellToJS read +19–24% FAIL + // uncorrected and collapsed to +6.1% WARN once corrected. A control + // that shares the hot path cannot normalise that path. + // + // So the number is printed as a diagnostic — a large value means the + // two runs are not comparable and the answer is a rerun — and every + // verdict below is taken against the raw baseline. + const calibration = measured + .filter( + (r) => r.spec.group === 'calibration' && entry.cases?.[r.spec.name], + ) + .map((r) => r.perOpMs.median / entry.cases[r.spec.name].medianPerOpMs) + .sort((a, b) => a - b); + const driftPct = calibration.length + ? (calibration[calibration.length >> 1] - 1) * 100 + : Number.NaN; + console.log( + `\n── vs baseline ${sig} (${entry.capturedAt}, git ${entry.environment?.gitSha ?? '?'}) — FAIL at >${failGate.toFixed(0)}%, WARN at >5% ──`, + ); + if (calibration.length) { + console.log( + ` calibration vs baseline: ${driftPct >= 0 ? '+' : ''}${driftPct.toFixed(1)}% ` + + '(diagnostic only, NOT applied to the deltas below — the ' + + 'calibration case shares the marshalling path, so a real ' + + 'regression moves it too)', + ); + if (Math.abs(driftPct) > 10) { + console.log( + ' NOTE: calibration should barely move between runs. This much ' + + 'means either the machine drifted or the change under test ' + + 'reaches the read path — reread the per-case pattern below ' + + 'rather than any single verdict, and rerun on a quiet machine.', + ); + } + } + /** @type {string[]} */ + const failures = []; + /** @type {string[]} */ + const warnings = []; + let unmeasured = 0; + let compared = 0; + for (const r of measured) { + const base = entry.cases?.[r.spec.name]; + if (!base) continue; + compared++; + const delta = + ((r.perOpMs.median - base.medianPerOpMs) / base.medianPerOpMs) * + 100; + const mark = + delta > failGate + ? 'FAIL' + : delta > 5 + ? 'WARN' + : delta < -5 + ? 'improved' + : 'ok'; + if (delta > failGate) failures.push(r.spec.name); + else if (delta > 5) warnings.push(r.spec.name); + console.log( + ` ${String(mark).padEnd(8)} ${r.spec.name.padEnd(58)} ${delta >= 0 ? '+' : ''}${delta.toFixed(1)}%`, + ); + } + for (const [name, base] of Object.entries(entry.cases ?? {})) { + const cur = measured.find((r) => r.spec.name === name); + if (!cur) { + unmeasured++; + console.log( + ` UNMEASURED ${name} (baseline ${/** @type {any} */ (base).medianPerOpMs?.toExponential(2)} ms/op — not run or rejected here)`, + ); + } + } + console.log( + `\nbaseline summary: ${compared} compared, ${failures.length} fail, ${warnings.length} warn, ${unmeasured} unmeasured`, + ); + return failures.length > 0 ? 2 : 0; +} + +main() + .then((code) => process.exit(code)) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/binding.gyp b/binding.gyp index b73e950..11fd9bf 100644 --- a/binding.gyp +++ b/binding.gyp @@ -24,6 +24,19 @@ ["sqlite != 'internal'", { "include_dirs": [ " void`) do not: their arrow parameters +// are types, not declarations. +// - Members inside type-literal bodies (`type X = {...}`) are skipped: +// they are part of a type alias's shape, documented at the alias. +import { readFileSync } from 'node:fs'; + +const FILES = [ + 'lib/sqlite3.d.ts', + 'lib/native.d.ts', + 'lib/augment.d.ts', + 'lib/promises.d.ts', + 'lib/trace.d.ts', + 'lib/pool.d.ts', +]; + +const root = new URL('..', import.meta.url); +const problems = []; + +// Walk upwards from the declaration; return its doc comment, or '' if the +// nearest non-comment line above is not part of a /** */ block. +function docFor(lines, index) { + const out = []; + for (let i = index - 1; i >= 0; i--) { + const t = lines[i].trim(); + if (t.startsWith('/**')) return t === '/**' ? out.join('\n') : t; + if (t.endsWith('*/') || t.startsWith('*')) { + out.unshift(t); + continue; + } + return ''; + } + return ''; +} + +// The parameter list of a call signature that opens on this line, +// including the parts that continue on following lines until the +// parentheses balance. `name(` at member position, or a top-level +// `function`/`constructor` declaration. +function signatureFrom(lines, i) { + const open = lines[i].indexOf('('); + if (open === -1) return null; + let text = lines[i]; + let depth = 0; + let opened = false; + // Close on the parenthesis that rebalances the first one, regardless + // of commas: merging further lines would swallow the next overload. + const step = (line, from) => { + for (let j = from; j < line.length; j++) { + const c = line[j]; + if (c === '(' || c === '<' || c === '[') { + depth++; + opened = true; + } else if (c === ')' || c === '>' || c === ']') { + depth--; + if (opened && depth <= 0) return true; + } + } + return false; + }; + if (step(text, open)) return { text }; + // Continue onto following lines until balanced (or give up after 12). + for (let k = i + 1; k < Math.min(i + 13, lines.length); k++) { + text += ` ${lines[k].trim()}`; + if (step(lines[k], 0)) return { text }; + } + return null; +} + +// Count top-level commas in a balanced parameter list (angles included so +// generic parameter defaults with `` do not confuse depth). +function countParams(list) { + let depth = 0; + let count = 0; + let seen = false; + for (const c of list) { + if (c === '(' || c === '<' || c === '[' || c === '{') depth++; + else if (c === ')' || c === '>' || c === ']' || c === '}') { + depth--; + } else if (c === ',' && depth === 0) count++; + else seen = true; + } + if (count === 0) return seen ? 1 : 0; + return count + 1; +} + +const decl = + /^(?:export\s+)?(?:declare\s+)?(?:abstract\s+)?(?:const|let|var|function|class|interface|type|enum)\s+([A-Za-z_$][\w$]*)/; +const member = + /^ {4}(?:(?:get|set)\s+)?(?:readonly\s+)?(?:override\s+)?([A-Za-z_$][\w$]*)\s*[(:<]/; +const methodLike = /:\s*[^=]*=>/; // function-typed type, not a signature + +for (const file of FILES) { + const lines = readFileSync(new URL(file, root), 'utf8').split('\n'); + const seen = new Set(); + // Depth of `type X = {` literals: members inside them are shape, not + // declarations. + let typeLiteralDepth = 0; + + lines.forEach((line, i) => { + if (/^export type \w+ =.*\{/.test(line)) { + typeLiteralDepth = + (line.match(/\{/g) ?? []).length - + (line.match(/\}/g) ?? []).length; + // The alias is still a declaration; record it so the value + // twin below (`declare const x: x`) shares its docs. + seen.add(line.match(/^export type (\w+)/)[1]); + return; + } + if (typeLiteralDepth > 0) { + for (const c of line) { + if (c === '{') typeLiteralDepth++; + else if (c === '}') typeLiteralDepth--; + } + if (typeLiteralDepth > 0) return; + } + + // A signature (method, function, constructor) needs docs, params + // and a return; a data property needs docs only. Skip overloads + // after the first (later overloads share the family's docs). + const m = line.match(decl) ?? line.match(member); + if (!m) return; + const name = m[1]; + const where = `${file} ${name} (line ${i + 1})`; + if (name === 'constructor' || seen.has(name)) return; + // tsc's declaration emit synthesizes `_base` aliases for + // heritage clauses in the generated entry (e.g. DatabaseClass_base + // for `class DatabaseClass extends NativeDatabase`); there is no + // source-level doc comment they could carry. + if (name.endsWith('_base') && file === 'lib/sqlite3.d.ts') return; + seen.add(name); + + const doc = docFor(lines, i); + if (!doc) { + problems.push(`${where}: missing doc comment`); + return; + } + + const hasParens = /[(:<]/.test(line) && line.includes('('); + const isFnType = methodLike.test(line); + if (hasParens && !isFnType) { + const sig = signatureFrom(lines, i); + if (sig) { + const list = sig.text.slice( + sig.text.indexOf('(') + 1, + sig.text.lastIndexOf(')'), + ); + const params = countParams(list); + const tags = (doc.match(/@param/g) ?? []).length; + if (params > tags) { + problems.push( + `${where}: ${params - tags} @param tag(s) missing (has ${tags}, needs ${params})`, + ); + } + if ( + !/\)\s*:\s*(void|never)\b/.test(sig.text) && + !/@returns/.test(doc) + ) { + problems.push(`${where}: missing @returns tag`); + } + } + } + }); +} + +console.log( + `check-jsdoc: ${problems.length} finding(s) across ${FILES.length} declaration files`, +); +for (const p of problems) console.log(` - ${p}`); +if (problems.length === 0) { + console.log('check-jsdoc: every shipped declaration is documented.'); +} else { + process.exitCode = 1; +} diff --git a/deps/sqlite3.gyp b/deps/sqlite3.gyp index 012cb3a..2b04ab8 100755 --- a/deps/sqlite3.gyp +++ b/deps/sqlite3.gyp @@ -75,10 +75,21 @@ 'SQLITE_ENABLE_FTS5', 'SQLITE_ENABLE_RTREE', 'SQLITE_ENABLE_SESSION', + # The session extension is documented as requiring + # SQLITE_ENABLE_PREUPDATE_HOOK alongside SQLITE_ENABLE_SESSION; + # without it the session sources compile into a non-functional + # state (D08 plan §1). Decision recorded in the D08 handoff. + 'SQLITE_ENABLE_PREUPDATE_HOOK', 'SQLITE_ENABLE_JSON', 'SQLITE_ENABLE_DBSTAT_VTAB=1', 'SQLITE_ENABLE_MATH_FUNCTIONS', 'SQLITE_ENABLE_STAT4', + # Deliverable 07: sqlite3_table_column_metadata and the + # sqlite3_column_{database,table,origin}_name family compile only + # with this define. Decision recorded in the D07 handoff: the + # ~30 KB of extra amalgamation code is accepted in exchange for + # column metadata (stmt.columns) and db.tableInfo(). + 'SQLITE_ENABLE_COLUMN_METADATA', 'SQLITE_DEFAULT_MEMSTATUS=0' ], }, @@ -94,10 +105,14 @@ 'SQLITE_ENABLE_FTS5', 'SQLITE_ENABLE_RTREE', 'SQLITE_ENABLE_SESSION', + # See the direct_dependent_settings copy above (Deliverable 08). + 'SQLITE_ENABLE_PREUPDATE_HOOK', 'SQLITE_ENABLE_JSON', 'SQLITE_ENABLE_DBSTAT_VTAB=1', 'SQLITE_ENABLE_MATH_FUNCTIONS', 'SQLITE_ENABLE_STAT4', + # See the direct_dependent_settings copy above (Deliverable 07). + 'SQLITE_ENABLE_COLUMN_METADATA', 'SQLITE_DEFAULT_MEMSTATUS=0' ], 'conditions': [ diff --git a/docs/concurrency.md b/docs/concurrency.md new file mode 100644 index 0000000..9b447a2 --- /dev/null +++ b/docs/concurrency.md @@ -0,0 +1,207 @@ +# Concurrency: one connection, many connections, workers, and the pool + +One page on what is actually concurrent in this package, what merely +looks concurrent, and which tool to reach for. Everything here is about +`@appthreat/sqlite3` v9. + +## What serialize() and parallelize() really do + +`db.serialize()` does not make anything run in parallel — it is the +opposite. It makes the connection's **queue strictly FIFO**: while it is +in effect, every call through the database queue (statements' prepares, +`exec`, `close`, hook registrations…) waits for everything queued before +it. Under `serialize()` *every* queued call is treated as exclusive, so +it is full serialization of the queue, not merely an ordering of starts. + +`db.parallelize()` (the default) lets non-exclusive work overlap: +statement stepping happens on libuv worker threads while the JS thread +continues, and the driver's queues only enforce what SQLite itself +requires (one write at a time, exclusive operations waiting for a quiet +connection). + +Two things bypass the database queue entirely: + +- **Statement operations** (`stmt.get()`, `stmt.all()`, … on a statement + you hold) never pass through the database queue — they run as soon as + the statement itself is free. This is why `db.serialize()` disables + the statement-cache fast path (`db.run/get/all/…` fall back to fresh + prepares, which *do* go through the queue): a cached statement would + overtake the serialization you asked for. +- **The synchronous methods** (`getSync`/`runSync`/`allSync`) run on the + JS thread and refuse to run unless the connection is fully idle — they + are a fast path for quiet moments, not a way to jump the queue. + +## SQLITE_BUSY, busy_timeout, and WAL + +- SQLite allows **one writer at a time** per database file. A second + write arriving while the first runs fails with `SQLITE_BUSY` — unless + a busy timeout is set, in which case it waits. This driver sets + `busy_timeout = 1000` on open; `configure('busyTimeout', ms)` or + `PRAGMA busy_timeout` changes it. +- In rollback-journal mode (the default for new files created without + WAL), a **reader also blocks the writer**: taking a read lock prevents + the write lock. +- **WAL mode** (`PRAGMA journal_mode = WAL`) is the fix for + reader/writer contention: readers read the committed snapshot without + blocking the writer, and the writer appends to the WAL without + blocking readers. It is a persistent property of the file — set it + once. The tradeoffs: `-wal`/`-shm` sidecar files, and checkpointing + (`db.checkpoint({ mode: 'truncate' })`) to keep the WAL bounded. + +`SQLITE_BUSY` in one sentence: two connections (or a pool — see below) +wanted the same lock at once and nobody had set a timeout long enough to +absorb it. Fix it with WAL plus `busy_timeout`, or by serializing +yourself (one writer connection — which is what the pool does). + +## When to use what + +### One connection (the default) + +A single `Database` object already gives you concurrency between SQLite +and your JS: queries step on worker threads while your code runs. For +most services — one process, mixed read/write, no long queries — one +connection is the right answer, and `db.transaction()` gives you +atomicity. Long-running queries will still delay the *connection*, +because SQLite serializes work per connection. + +### Several connections to one file + +Open one read-write connection per process (or a few), set WAL, set a +generous `busy_timeout`, and let readers multiply. This is the standard +scaling shape for multi-process access. Remember: + +- one writer at a time, always — more writer connections does not mean + more write throughput, only more `SQLITE_BUSY` unless you make writes + single-threaded per process; +- readers see committed data only; a transaction's own uncommitted + writes are visible to its own connection and nobody else. + +### Worker threads: the path handoff + +A `sqlite3*` handle, a `Database` JS object, a prepared statement — none +of these can cross a `worker_threads` boundary. What crosses cheaply is +**the path**: + +```js +const { Worker } = require('node:worker_threads'); +const w = new Worker('./db-worker.js', { + workerData: { filename: 'app.db', mode: sqlite3.OPEN_READONLY }, +}); +``` + +The worker opens its own connection to the same file. With WAL mode this +gives real read concurrency and keeps the main thread free of SQLite +work entirely. This is what most worker use cases actually want, and it +needs no special support — the addon is context-aware and loads cleanly +in every worker (each worker gets its own constructors; nothing +napi-shaped is shared between environments). + +### Worker threads: the bytes handoff + +An **in-memory** database can be moved across threads with one copy: + +```js +// main thread +const bytes = await db.serializeToBytes(); +const movable = bytes.slice().buffer; // plain ArrayBuffer copy +w.postMessage({ bytes: movable }, [movable]); // transfer: zero further copies + +// worker +const { workerData } = require('node:worker_threads'); +const db = await sqlite3.deserializeFromBytes( + new Uint8Array(workerData.bytes), + { resizable: true }, +); +``` + +The `slice()` is required: `serializeToBytes()` returns a view over +SQLite-owned memory, which structured clone refuses to transfer. After +the transfer the worker owns a live, writable in-memory database; the +main thread's copy is gone (detached). This is the mechanism for +snapshotting to a worker for read-heavy analysis without touching the +original. + +Cancelling a query across threads needs no round trip: create a +`db.cancellationToken()` in whichever thread owns the connection, send +its `buffer` (a `SharedArrayBuffer`, shared memory) to the other thread, +and `Atomics.store(new Int32Array(buffer), 0, 1)` there — the running +query aborts with `SQLITE_INTERRUPT`. + +### The pool + +```js +const pool = await sqlite3.pool('app.db', { + readers: 4, // read-only worker connections + walMode: true, // PRAGMA journal_mode = WAL (default) + busyTimeout: 5000, // per connection (default) +}); + +const rows = await pool.read('SELECT * FROM t WHERE a = ?', [1]); +const user = await pool.get('SELECT * FROM t WHERE a = ?', [1]); +await pool.write('INSERT INTO t (a) VALUES (?)', [1]); + +await pool.transaction(async (tx) => { + // pinned to the writer; tx.get sees uncommitted writes + const row = await tx.get('SELECT a FROM t'); + await tx.write('UPDATE t SET a = ?', [row.a + 1]); +}); + +await pool.close(); // drains, closes, terminates — no worker survives +``` + +One writer connection plus N reader connections, each on its own worker +thread. Writes queue on the writer (they were going to serialize inside +SQLite anyway — the pool converts `SQLITE_BUSY` retry loops into +queueing). Reads fan out to the least-busy reader. `pool.transaction()` +pins the whole transaction to the writer and holds it: concurrent +transactions wait, nothing interleaves inside yours, and the pool-facing +write methods refuse from inside a body (they would wait on your own +transaction) — use the `tx` handle. + +What the pool is **not**: it is not the default API, and it is not for +bulk data. Rows are structured-cloned across `postMessage`, so a large +result set pays a full copy, and blob columns come back as plain +`Uint8Array` rather than `Buffer`. For `SELECT *` over a million rows, +use a dedicated worker with the path or bytes handoff instead. Errors +keep their SQLite diagnostics (`code`, `errno`, `primaryCode`) — the +worker re-serializes them explicitly, because structured clone drops an +Error's own properties. + +`pool.close()` is idempotent, drains accepted work (a running +transaction finishes or rolls back), closes every connection and waits +for every worker's exit. `await using pool` works. + +## Terminating a worker + +`worker.terminate()` while a query is in flight used to abort the whole +process: the query's completion is delivered to the addon while V8 is +unwinding the isolate, and every call that enters JS there fails — but +the binding layer's error path is itself a JS call, so the failure +escalates to `FATAL ERROR: napi_throw` rather than an exception you can +catch. The addon now detects that state at the top of each completion +and drops the delivery instead. + +The detection happens when the completion starts, so it does not cover +the case where termination lands **in the middle** of one: a completion +that is still converting rows when the isolate goes down can still abort +the process. In practice that needs several completions queued at once — +a tight loop issuing queries without awaiting them — and it is unchanged +from v8. Closing it fully means status-checking every JS call in the +completion handlers rather than relying on the binding layer's checked +helpers, which is a change of its own. + +The safe pattern, and the one worth preferring regardless, is to shut a +worker down cooperatively — tell it to stop, `await db.close()`, let the +thread exit — and keep `terminate()` for workers that have stopped +responding. `pool.close()` does exactly this; the pool only calls +`terminate()` on the startup-failure path, where no query can be in +flight. + +## A note on cancellation + +All cancellation in this package (`{ signal }` options, the pool, the +cross-thread token above) is **cooperative and interrupt-based**: the +running statement is interrupted (`SQLITE_INTERRUPT`), work that was +queued but not started may simply never run, and an abort that loses the +race with a completing query still rejects and drops the result. None of +it can roll back a committed write — that is what transactions are for. diff --git a/docs/electron.md b/docs/electron.md new file mode 100644 index 0000000..6cd356a --- /dev/null +++ b/docs/electron.md @@ -0,0 +1,254 @@ +# Using @appthreat/sqlite3 in Electron + +`@appthreat/sqlite3` v9 is a [Node-API](https://nodejs.org/api/n-api.html) 10 addon +shipped as prebuilt binaries. **Node-API is ABI-stable across runtimes, so the same +`prebuilds/-/*.node` that Node loads is the one Electron loads — no +rebuild, no `electron-rebuild`, no `--runtime=electron` flags.** + +- **Minimum Electron: 35** (`engines.electron >= 35`). Electron 35 is the first + major whose bundled Node (22.16) exposes Node-API 10; Electron 32–34 bundle + Node 20 (`process.versions.napi === 9`). This number is **verified, not read from + a table**: the shipping prebuild was loaded in Electron 35.7.5 and 44.0.0 and + queried in both. On Electron 34 the load does not fail cleanly — the process + segfaults inside module registration — which is why the binding loader checks + `process.versions.napi` *before* the dlopen and throws an error naming the + floors instead. +- Rebuilds are only ever needed for a **source build** — SQLCipher via + `--sqlite= --sqlite_libname=sqlcipher`, or a custom `sqlite_magic` + header. See [Rebuilds, when they are needed](#rebuilds-when-they-are-needed). + +## Quick start (main process) + +```js +// ESM main process (Electron >= 28) +import { app } from 'electron'; +import sqlite3 from '@appthreat/sqlite3'; + +const db = new sqlite3.Database(`${app.getPath('userData')}/app.db`); +``` + +From a CommonJS main process, use dynamic import — the package is ESM-only and +dual-publishing an ESM/CJS native module invites the dual-package hazard, so there +is deliberately no CJS entry: + +```js +const sqlite3 = (await import('@appthreat/sqlite3')).default; +``` + +Note for ESM main processes: module evaluation is asynchronous relative to app +readiness. `await app.whenReady()` at the **top level of the entry script** can +deadlock when the file is launched through Electron's default-app path +(`electron path/to/file.mjs`); `app.whenReady().then(...)` or a `package.json` +`"main"` entry (the normal app shape) are both fine. + +## Where to put the database code + +Three valid placements; one recommendation. + +### Utility process — recommended + +`utilityProcess.fork` gives the child a full Node environment with its own module +registry — a second environment loading the same `.node` gets its own addon +constructors (this is exactly what the addon's per-environment instance data is +for). The child is crash-isolated from your windows and off the main process's +event loop. + +```js +// main process: db-service-parent.mjs +import { utilityProcess } from 'electron'; + +const child = utilityProcess.fork(new URL('./db-service.mjs', import.meta.url).pathname); +child.on('message', (msg) => { /* replies */ }); + +function query(sql, ...params) { + return new Promise((resolve) => { + const id = nextId(); + const onMessage = (msg) => { + if (msg.id === id) { child.off('message', onMessage); resolve(msg); } + }; + child.on('message', onMessage); + child.postMessage({ id, sql, params }); + }); +} +``` + +```js +// utility process: db-service.mjs +import { app } from 'electron'; +import { join } from 'node:path'; +import sqlite3 from '@appthreat/sqlite3'; + +const db = new sqlite3.Database(join(app.getPath('userData'), 'app.db')); +const parent = process.parentPort; + +parent.on('message', ({ data }) => { + if (data.op === 'quit') return db.close(() => process.exit(0)); + db.get(data.sql, ...data.params, (err, row) => { + // structured clone strips error properties; re-send the essentials + parent.postMessage({ + id: data.id, + ok: !err, + row: row ?? null, + err: err ? { code: err.code, errno: err.errno, message: err.message } : null, + }); + }); +}); +``` + +`test/electron/main.mjs` in this repo is a working version of exactly this +service, run in CI. + +**Pool or utility process?** The v9 connection pool +(`sqlite3.pool()`, see [docs/concurrency.md](concurrency.md)) is the same idea — +many connections, each off the main thread — without any Electron API. Use the +pool inside any single process (main, utility, or plain Node); use a utility +process when you want *process-level* isolation (a native crash cannot take down +your windows) or the database work owned by a service with its own lifecycle. + +### Main process — fine + +Everything works in the main process; the trade-off is that every query competes +with your UI's IPC on one event loop. The v9 API is asynchronous on worker +threads, so queries do not block the loop — but heavy result-set conversion still +costs main-process milliseconds. Prefer `db.iterate()`/`db.stream()` over `all()` +for large reads, or move to a utility process. + +### Renderer — not supported; preload only with `sandbox: false` + +With `contextIsolation: true` and `sandbox: true` — both Electron defaults since +v20 — a renderer cannot load native modules at all, and `@appthreat/sqlite3` +must not be used there. The supported alternative is a `preload` script with +`sandbox: false` exposing narrow functions over `contextBridge`; this is +discouraged (the preload shares the renderer's lifecycle). Keep database work in +the main or a utility process and expose it through IPC. + +## ASAR packaging + +A `.node` inside `app.asar` is handled by Electron in one of two ways, depending +on packaging: + +- **Unpacked at package time** (recommended, and what the major packagers do by + default): the binary sits in `app.asar.unpacked/` next to the archive and is + dlopen'd from there directly. +- **Sealed inside the archive**: current Electron extracts it to a temp file and + loads that — it works, but every launch pays the extraction and you are + depending on a behavior the asar docs do not promise for every platform. + +Configure unpacking explicitly with electron-builder: + +```json +{ + "build": { + "asarUnpack": ["**/node_modules/@appthreat/sqlite3/prebuilds/**"] + } +} +``` + +and with `@electron/packager` / electron-forge's packager config: + +```js +// forge.packagerConfig +{ asar: { unpack: '**/node_modules/@appthreat/sqlite3/prebuilds/**' } } +``` + +Two footguns: + +- `@electron/packager`'s default `asar: true` already unpacks every `*.node`. + Passing an options **object replaces that default** — if you set + `asar: { unpack: }`, add the sqlite3 prebuilds pattern or the + binary ends up sealed inside the archive. +- **pnpm**: the default symlinked `node_modules` layout has historically + confused electron-builder and `@electron/packager` when resolving and + unpacking `.node` files — the packager follows the symlink, or fails to, and + the binary ends up inside the archive or missing entirely. The known + consumer-side workaround is a hoisted layout: + + ```yaml + # .npmrc (pnpm) or pnpm-workspace.yaml + nodeLinker: hoisted + ``` + + If you stay on symlinks, verify the packaged app actually contains + `app.asar.unpacked/**/@appthreat/sqlite3/prebuilds/**`. + +When the binding cannot load, `lib/sqlite3-binding.js` throws an error that names +the resolved package root, notes when that path is inside an `app.asar` archive, +and points at the `asarUnpack` configuration above — instead of the raw +`No native build was found ...` from `node-gyp-build` (which is preserved as +`err.cause`). + +## Bundlers + +If you bundle the main process (electron-forge's Vite and Webpack templates do), +the package must be **external** — a bundler that tries to inline a `.node` +require produces a confusing failure. + +- Vite: `build.rollupOptions.external: ['@appthreat/sqlite3']` (and keep + `node_modules/@appthreat/sqlite3` out of the bundle output). +- Webpack: `externals: { '@appthreat/sqlite3': 'commonjs2 @appthreat/sqlite3' }` + — `commonjs2` is safe even though the package is ESM, because webpack only + uses it to name the require it emits; or use + `externalsPresets: { node: true }`. +- esbuild: `--external:@appthreat/sqlite3`. + +## Database location + +Never write next to the app bundle (read-only on macOS; `Program Files` on +Windows). The correct location is the per-user data directory: + +```js +import { app } from 'electron'; +import { mkdirSync } from 'node:fs'; +import { join } from 'node:path'; + +const dir = app.getPath('userData'); +mkdirSync(dir, { recursive: true }); // not guaranteed to exist yet +const db = new sqlite3.Database(join(dir, 'app.db')); +``` + +In a utility process, `app.getPath('userData')` resolves to the same directory +as in the main process. + +## Rebuilds, when they are needed + +Only for a **source build** — SQLCipher or a custom `sqlite_magic` header. The +default prebuild needs nothing. For a source build against Electron's headers, +use [`@electron/rebuild`](https://github.com/electron/rebuild) (the current +package name; `electron-rebuild` is the old one): + +```bash +# from your app, with this repo checked out (or installed with source build forced) +npm install --build-from-source --sqlite=/usr/local --sqlite_libname=sqlcipher \ + --runtime=electron --target= \ + --dist-url=https://electronjs.org/headers +``` + +or, in an app with `@electron/rebuild` installed: + +```bash +npx @electron/rebuild -f -w @appthreat/sqlite3 \ + -s --sqlite_libname sqlcipher +``` + +This is **not** required for the default build — that is the point of the +Node-API prebuilds. + +## What is tested, and where + +`pnpm run test:electron` runs (against the shipping `prebuilds/`, forced by +`PREBUILDS_ONLY=1`): + +1. **Load + query in the main process** and a **utility-process service** with a + MessagePort round trip and a clean exit — `test/electron/main.mjs`. +2. **The entire Node test suite** (all `test/*.test.js`) inside Electron's + Node/V8 build via `ELECTRON_RUN_AS_NODE` — proving behavioral parity with + Node, not just loadability. + +`pnpm run test:electron:asar` packages a real consumer install (from the packed +tarball) with `@electron/packager` both ways and asserts where the binary +loaded from — slow, macOS/Linux/Windows-local, and ubuntu-only in CI. + +`test/electron/exit-no-close.mjs` is a teardown probe: it opens connections in +the main process *and* a worker, uses the API, and exits without closing +anything — expecting exit status 0 (a teardown segfault reports 139 and is +invisible to in-process assertions). diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 0000000..1c38dd1 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,182 @@ +# Installing @appthreat/sqlite3 + +This is the complete installation guide. Requirements: **Node.js >= 24** +(declared in `engines`). Any package manager works: + +```bash +npm install @appthreat/sqlite3 +pnpm add @appthreat/sqlite3 +yarn add @appthreat/sqlite3 +bun add @appthreat/sqlite3 +``` + +Nothing is downloaded at install time and nothing is compiled at install time +on the platforms below — the prebuilt binaries ship inside the npm tarball +itself. + +## Prebuild coverage + +One binary per platform, built against Node-API (`napi_versions: [10]`), so a +single binary covers every supported Node version. Linux builds carry both +libc flavours side by side, tagged `.glibc.node` / `.musl.node`. + +| Platform | Files in `prebuilds/` | +|----------|----------------------------------------| +| `darwin-arm64` | `@appthreat+sqlite3.node` | +| `darwin-x64` | `@appthreat+sqlite3.node` | +| `linux-arm64` | `@appthreat+sqlite3.glibc.node`, `@appthreat+sqlite3.musl.node` | +| `linux-x64` | `@appthreat+sqlite3.glibc.node`, `@appthreat+sqlite3.musl.node` | +| `win32-arm64` | `@appthreat+sqlite3.node` | +| `win32-x64` | `@appthreat+sqlite3.node` | + +The binding is resolved at **runtime**, not install time: +`lib/sqlite3-binding.js` calls `node-gyp-build(rootDir)` on first import, +which looks in `prebuilds//` first, then falls back to a +`build/Release/` build. (For `--tag-libc` builds the directory stays +`linux-` and the libc is carried by the file suffix.) + +## pnpm 10+ and the blocked install script + +pnpm 10 and later refuse to run a dependency's lifecycle scripts unless the +dependent allowlists it. This package declares `"install": "node-gyp-build"`, +so pnpm prints a notice like: + +``` +[ERR_PNPM_IGNORED_BUILDS] Ignored build scripts: @appthreat/sqlite3@9.0.0 +``` + +**You can ignore that notice.** Verified empirically (pnpm 11.23.0, macOS, +both the published v8 and a packed v9 tarball): with the script blocked, the +module still imports and `sqlite3.VERSION` prints, because a matching prebuild +exists and runtime resolution never needs the install script. + +**No `onlyBuiltDependencies` entry is needed** — we ship prebuilds. + +The one case where you *do* need to allow the script is a **source build**: +no prebuild for your platform, or `--build-from-source`, `--sqlite=`, +SQLCipher, or a custom `sqlite_magic`. (Electron needs none of this: the +Node-API prebuild loads as-is on Electron >= 35 — see +[electron.md](electron.md).) Then the install script must actually run +`node-gyp`. Add to your `pnpm-workspace.yaml`: + +```yaml +onlyBuiltDependencies: + - '@appthreat/sqlite3' +``` + +(or run `pnpm approve-builds` and select `@appthreat/sqlite3`), then trigger +the source build, e.g.: + +```bash +npm_config_build_from_source=true pnpm rebuild @appthreat/sqlite3 +``` + +`pnpm rebuild ` (the builtin, with the package named explicitly — here it +is the right tool) re-runs that dependency's build scripts, which invokes +`node-gyp-build`, which sees the `build_from_source` config and compiles +instead of resolving a prebuild. + +## Source builds + +A source build happens when: + +- your platform has no prebuild in the table above; +- you pass `--build-from-source`; +- you build against an external SQLite or SQLCipher (`--sqlite=`, + `--sqlite_libname=`); +- you set a custom file magic (`--sqlite_magic=`); or +- you build against Electron headers with a custom `--target` (only ever + needed together with a source build; the default prebuild loads in + Electron unchanged). + +Toolchain requirements: + +- **Python 3** (for node-gyp's gyp) +- a **C++17 toolchain**: Xcode CLT on macOS, MSVC (msbuild) on Windows, + gcc/clang elsewhere +- **node-gyp 12.x** — installed automatically as an `optionalDependencies` + entry when your environment needs it; no global install required + +With npm everything works with the classic flags: + +```bash +npm install @appthreat/sqlite3 --build-from-source +``` + +With pnpm, allow the script as shown above and set the config via the +environment (`npm_config_build_from_source=true`), since pnpm does not forward +npm-style `--` flags to dependency scripts. + +### External SQLite, magic, SQLCipher + +```bash +# external sqlite instead of the bundled amalgamation +npm install @appthreat/sqlite3 --build-from-source --sqlite=/usr/local + +# homebrew sqlite on macOS +npm install @appthreat/sqlite3 --build-from-source --sqlite=/usr/local/opt/sqlite/ + +# custom 15-char file magic +npm install @appthreat/sqlite3 --build-from-source --sqlite_magic="MyCustomMagic15" + +# SQLCipher +npm install @appthreat/sqlite3 --build-from-source --sqlite_libname=sqlcipher --sqlite=/usr/ +``` + +For the full SQLCipher/Electron flag set see the +[README](../README.md#sqlcipher-encrypted-databases). + +## Troubleshooting: "No native build was found" + +``` +Error: No native build was found for platform=linux arch=arm64 runtime=node ... +``` + +`node-gyp-build` found neither a matching `prebuilds//` entry nor a +`build/Release/` binding. In order of likelihood: + +1. **pnpm blocked the install script on a platform that needs a source + build.** You saw the `ERR_PNPM_IGNORED_BUILDS` notice and ignored it, but + there is no prebuild for your platform. Add the + `onlyBuiltDependencies` snippet from above, then + `pnpm rebuild @appthreat/sqlite3`. +2. **You are developing this repo** and have no build yet: run + `pnpm install` (the root install script compiles the binding) or + `pnpm run rebuild`. +3. **Stale `prebuilds/` while iterating on C++**: `node-gyp-build` prefers + `prebuilds/` over `build/`, so your `pnpm run rebuild` output is being + shadowed. Delete `prebuilds/` while iterating. +4. **Runtime below the Node-API floor** (Node < 22, Electron < 35): the + binding loader refuses with an error naming the floors rather than + crashing. On Electron the default prebuild needs no rebuild at all; only + a source build against Electron headers uses `--runtime=electron + --target= --dist-url=https://electronjs.org/headers` (see + [electron.md](electron.md)). + +## Development + +This repo is developed with **pnpm >= 11** (pinned exactly in +`packageManager`; `corepack enable` picks it up). Node >= 24 required. + +```bash +pnpm install # strictDepBuilds is on; frozen form: pnpm install --frozen-lockfile +pnpm run rebuild # node-gyp rebuild — always `pnpm run rebuild` +pnpm run test +pnpm run prebuild # prebuildify --napi --strip +pnpm pack # tarball includes prebuilds/ — smoke-test it in a scratch project +``` + +Notes: + +- **Never bare `pnpm rebuild`** in this repo — that is pnpm's builtin for + rebuilding *dependencies*; it silently does not run this repo's `rebuild` + script. The same class of collision is why CI and docs use + `pnpm run . - - \ No newline at end of file diff --git a/test/nw/package.json b/test/nw/package.json deleted file mode 100644 index d1b4aee..0000000 --- a/test/nw/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "nw-demo", - "main": "index.html", - "window": { - "toolbar": false, - "width": 800, - "height": 600 - } -} \ No newline at end of file diff --git a/test/open_close.test.js b/test/open_close.test.js index 4dd2254..a75c1e5 100644 --- a/test/open_close.test.js +++ b/test/open_close.test.js @@ -1,146 +1,181 @@ -import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; -import { ensureExists, deleteFile, fileDoesNotExist, fileExists } from './support/helper.js'; +import assert from 'node:assert'; +import { after, before, describe, it } from 'node:test'; -describe('open/close', function() { - before(function() { +import sqlite3 from '../lib/sqlite3.js'; +import { + deleteFile, + ensureExists, + fileDoesNotExist, + fileExists, +} from './support/helper.js'; + +describe('open/close', function () { + before(function () { ensureExists('test/tmp'); }); - describe('open and close non-existant database', function() { - before(function() { + describe('open and close non-existant database', function () { + before(function () { deleteFile('test/tmp/test_create.db'); }); let db; - it('should open the database', function(done) { + it('should open the database', function (_t, done) { db = new sqlite3.Database('test/tmp/test_create.db', done); }); - it('should close the database', function(done) { + it('should close the database', function (_t, done) { db.close(done); }); - it('should have created the file', function() { + it('should have created the file', function () { fileExists('test/tmp/test_create.db'); }); - after(function() { + after(function () { deleteFile('test/tmp/test_create.db'); }); }); - describe('open and close non-existant shared database', function() { - before(function() { + describe('open and close non-existant shared database', function () { + before(function () { deleteFile('test/tmp/test_create_shared.db'); }); let db; - it('should open the database', function(done) { - db = new sqlite3.Database('file:./test/tmp/test_create_shared.db', sqlite3.OPEN_URI | sqlite3.OPEN_SHAREDCACHE | sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE, done); - }); - - it('should close the database', function(done) { + it('should open the database', function (_t, done) { + db = new sqlite3.Database( + 'file:./test/tmp/test_create_shared.db', + sqlite3.OPEN_URI | + sqlite3.OPEN_SHAREDCACHE | + sqlite3.OPEN_READWRITE | + sqlite3.OPEN_CREATE, + done, + ); + }); + + it('should close the database', function (_t, done) { db.close(done); }); - it('should have created the file', function() { + it('should have created the file', function () { fileExists('test/tmp/test_create_shared.db'); }); - after(function() { + after(function () { deleteFile('test/tmp/test_create_shared.db'); }); }); - - (sqlite3.VERSION_NUMBER < 3008000 ? describe.skip : describe)('open and close shared memory database', function() { - - let db1; - let db2; - - it('should open the first database', function(done) { - db1 = new sqlite3.Database('file:./test/tmp/test_memory.db?mode=memory', sqlite3.OPEN_URI | sqlite3.OPEN_SHAREDCACHE | sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE, done); - }); - - it('should open the second database', function(done) { - db2 = new sqlite3.Database('file:./test/tmp/test_memory.db?mode=memory', sqlite3.OPEN_URI | sqlite3.OPEN_SHAREDCACHE | sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE, done); - }); - - it('first database should set the user_version', function(done) { - db1.exec('PRAGMA user_version=42', done); - }); - - it('second database should get the user_version', function(done) { - db2.get('PRAGMA user_version', function(err, row) { - if (err) throw err; - assert.equal(row.user_version, 42); - done(); + (sqlite3.VERSION_NUMBER < 3008000 ? describe.skip : describe)( + 'open and close shared memory database', + function () { + let db1; + let db2; + + it('should open the first database', function (_t, done) { + db1 = new sqlite3.Database( + 'file:./test/tmp/test_memory.db?mode=memory', + sqlite3.OPEN_URI | + sqlite3.OPEN_SHAREDCACHE | + sqlite3.OPEN_READWRITE | + sqlite3.OPEN_CREATE, + done, + ); }); - }); - - it('should close the first database', function(done) { - db1.close(done); - }); - it('should close the second database', function(done) { - db2.close(done); - }); - }); + it('should open the second database', function (_t, done) { + db2 = new sqlite3.Database( + 'file:./test/tmp/test_memory.db?mode=memory', + sqlite3.OPEN_URI | + sqlite3.OPEN_SHAREDCACHE | + sqlite3.OPEN_READWRITE | + sqlite3.OPEN_CREATE, + done, + ); + }); - it('should not be unable to open an inaccessible database', function(done) { - // NOTE: test assumes that the user is not allowed to create new files - // in /usr/bin. - let db = new sqlite3.Database('/test/tmp/directory-does-not-exist/test.db', function(err) { - if (err && err.errno === sqlite3.CANTOPEN) { - done(); - } else if (err) { - done(err); - } else { - done('Opened database that should be inaccessible'); - } - }); - }); + it('first database should set the user_version', function (_t, done) { + db1.exec('PRAGMA user_version=42', done); + }); + it('second database should get the user_version', function (_t, done) { + db2.get('PRAGMA user_version', function (err, row) { + if (err) throw err; + assert.equal(row.user_version, 42); + done(); + }); + }); - describe('creating database without create flag', function() { - before(function() { - deleteFile('test/tmp/test_readonly.db'); - }); + it('should close the first database', function (_t, done) { + db1.close(done); + }); - it('should fail to open the database', function(done) { - new sqlite3.Database('tmp/test_readonly.db', sqlite3.OPEN_READONLY, function(err) { + it('should close the second database', function (_t, done) { + db2.close(done); + }); + }, + ); + + it('should not be unable to open an inaccessible database', function (_t, done) { + // NOTE: test assumes that the user is not allowed to create new files + // in /usr/bin. + const _db = new sqlite3.Database( + '/test/tmp/directory-does-not-exist/test.db', + function (err) { if (err && err.errno === sqlite3.CANTOPEN) { done(); } else if (err) { done(err); } else { - done('Created database without create flag'); + done('Opened database that should be inaccessible'); } - }); + }, + ); + }); + + describe('creating database without create flag', function () { + before(function () { + deleteFile('test/tmp/test_readonly.db'); + }); + + it('should fail to open the database', function (_t, done) { + new sqlite3.Database( + 'tmp/test_readonly.db', + sqlite3.OPEN_READONLY, + function (err) { + if (err && err.errno === sqlite3.CANTOPEN) { + done(); + } else if (err) { + done(err); + } else { + done('Created database without create flag'); + } + }, + ); }); - it('should not have created the file', function() { + it('should not have created the file', function () { fileDoesNotExist('test/tmp/test_readonly.db'); }); - after(function() { + after(function () { deleteFile('test/tmp/test_readonly.db'); }); }); - describe('open and close memory database queuing', function() { + describe('open and close memory database queuing', function () { let db; - it('should open the database', function(done) { + it('should open the database', function (_t, done) { db = new sqlite3.Database(':memory:', done); }); - it('should close the database', function(done) { + it('should close the database', function (_t, done) { db.close(done); }); - it('shouldn\'t close the database again', function(done) { - db.close(function(err) { + it("shouldn't close the database again", function (_t, done) { + db.close(function (err) { assert.ok(err, 'No error object received on second close'); assert.ok(err.errno === sqlite3.MISUSE); done(); @@ -148,39 +183,41 @@ describe('open/close', function() { }); }); - describe('closing with unfinalized statements', function(done) { - let completed = false; - let completedSecond = false; - let closed = false; + describe('closing with unfinalized statements', function (_t, done) { + const _completed = false; + const _completedSecond = false; + const _closed = false; let db; - before(function() { + before(function () { db = new sqlite3.Database(':memory:', done); }); - it('should create a table', function(done) { - db.run("CREATE TABLE foo (id INT, num INT)", done); + it('should create a table', function (_t, done) { + db.run('CREATE TABLE foo (id INT, num INT)', done); }); let stmt; - it('should prepare/run a statement', function(done) { + it('should prepare/run a statement', function (_t, done) { stmt = db.prepare('INSERT INTO foo VALUES (?, ?)'); stmt.run(1, 2, done); }); - it('should fail to close the database', function(done) { - db.close(function(err) { - assert.ok(err.message, - "SQLITE_BUSY: unable to close due to unfinalised statements"); + it('should fail to close the database', function (_t, done) { + db.close(function (err) { + assert.ok( + err.message, + 'SQLITE_BUSY: unable to close due to unfinalised statements', + ); done(); }); }); - it('should succeed to close the database after finalizing', function(done) { - stmt.run(3, 4, function() { + it('should succeed to close the database after finalizing', function (_t, done) { + stmt.run(3, 4, function () { stmt.finalize(); db.close(done); }); }); }); -}); \ No newline at end of file +}); diff --git a/test/other_objects.test.js b/test/other_objects.test.js index 97cddef..026c1d4 100644 --- a/test/other_objects.test.js +++ b/test/other_objects.test.js @@ -1,25 +1,30 @@ +import assert from 'node:assert'; +import { before, beforeEach, describe, it } from 'node:test'; + import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; -describe('data types', function() { +describe('data types', function () { let db; - before(function(done) { + before(function (_t, done) { db = new sqlite3.Database(':memory:'); - db.run("CREATE TABLE txt_table (txt TEXT)"); - db.run("CREATE TABLE int_table (int INTEGER)"); - db.run("CREATE TABLE flt_table (flt FLOAT)"); + db.run('CREATE TABLE txt_table (txt TEXT)'); + db.run('CREATE TABLE int_table (int INTEGER)'); + db.run('CREATE TABLE flt_table (flt FLOAT)'); db.wait(done); }); - beforeEach(function(done) { - db.exec('DELETE FROM txt_table; DELETE FROM int_table; DELETE FROM flt_table;', done); + beforeEach(function (_t, done) { + db.exec( + 'DELETE FROM txt_table; DELETE FROM int_table; DELETE FROM flt_table;', + done, + ); }); - it('should serialize Date()', function(done) { - let date = new Date(); - db.run("INSERT INTO int_table VALUES(?)", date, function (err) { + it('should serialize Date()', function (_t, done) { + const date = new Date(); + db.run('INSERT INTO int_table VALUES(?)', date, function (err) { if (err) throw err; - db.get("SELECT int FROM int_table", function(err, row) { + db.get('SELECT int FROM int_table', function (err, row) { if (err) throw err; assert.equal(row.int, +date); done(); @@ -27,11 +32,11 @@ describe('data types', function() { }); }); - it('should serialize RegExp()', function(done) { - let regexp = /^f\noo/; - db.run("INSERT INTO txt_table VALUES(?)", regexp, function (err) { + it('should serialize RegExp()', function (_t, done) { + const regexp = /^f\noo/; + db.run('INSERT INTO txt_table VALUES(?)', regexp, function (err) { if (err) throw err; - db.get("SELECT txt FROM txt_table", function(err, row) { + db.get('SELECT txt FROM txt_table', function (err, row) { if (err) throw err; assert.equal(row.txt, String(regexp)); done(); @@ -43,19 +48,19 @@ describe('data types', function() { 4294967296.249, Math.PI, 3924729304762836.5, - new Date().valueOf(), + Date.now(), 912667.394828365, - 2.3948728634826374e+83, - 9.293476892934982e+300, - Infinity, - -9.293476892934982e+300, - -2.3948728634826374e+83, - -Infinity - ].forEach(function(flt) { - it('should serialize float ' + flt, function(done) { - db.run("INSERT INTO flt_table VALUES(?)", flt, function (err) { + 2.3948728634826374e83, + 9.293476892934982e300, + Number.POSITIVE_INFINITY, + -9.293476892934982e300, + -2.3948728634826374e83, + Number.NEGATIVE_INFINITY, + ].forEach(function (flt) { + it(`should serialize float ${flt}`, function (_t, done) { + db.run('INSERT INTO flt_table VALUES(?)', flt, function (err) { if (err) throw err; - db.get("SELECT flt FROM flt_table", function(err, row) { + db.get('SELECT flt FROM flt_table', function (err, row) { if (err) throw err; assert.equal(row.flt, flt); done(); @@ -67,48 +72,93 @@ describe('data types', function() { [ 4294967299, 3924729304762836, - new Date().valueOf(), - 2.3948728634826374e+83, - 9.293476892934982e+300, - Infinity, - -9.293476892934982e+300, - -2.3948728634826374e+83, - -Infinity - ].forEach(function(integer) { - it('should serialize integer ' + integer, function(done) { - db.run("INSERT INTO int_table VALUES(?)", integer, function (err) { + Date.now(), + 2.3948728634826374e83, + 9.293476892934982e300, + Number.POSITIVE_INFINITY, + -9.293476892934982e300, + -2.3948728634826374e83, + Number.NEGATIVE_INFINITY, + ].forEach(function (integer) { + it(`should serialize integer ${integer}`, function (_t, done) { + db.run('INSERT INTO int_table VALUES(?)', integer, function (err) { if (err) throw err; - db.get("SELECT int AS integer FROM int_table", function(err, row) { - if (err) throw err; - assert.equal(row.integer, integer); - done(); - }); + db.get( + 'SELECT int AS integer FROM int_table', + function (err, row) { + if (err) throw err; + assert.equal(row.integer, integer); + done(); + }, + ); }); }); }); - it('should ignore faulty toString', function(done) { + it('should reject faulty toString', function (_t, done) { const faulty = { toString: 23 }; - db.run("INSERT INTO txt_table VALUES(?)", faulty, function (err) { + // v8 bound this as "[object Object]"; v9 rejects non-serialisable + // objects. As a direct argument the object is a named-parameter + // map, so the unknown-parameter error reaches the callback. + db.run('INSERT INTO txt_table VALUES(?)', faulty, function (err) { assert.notEqual(err, undefined); + assert.strictEqual(err.code, 'SQLITE_RANGE'); done(); }); }); - it('should ignore faulty toString in array', function(done) { - const faulty = [[{toString: null}], 1]; - db.all('SELECT * FROM txt_table WHERE txt = ? LIMIT ?', faulty, function (err) { - assert.equal(err, null); - done(); - }); + it('should reject faulty toString in array', function () { + const faulty = [[{ toString: null }], 1]; + // v8 bound the inner object as "[object Object]"; v9 throws a + // TypeError synchronously, naming the parameter index. The + // trailing callback keeps the call in callback mode, where strict + // binding throws synchronously (promise mode rejects instead). + assert.throws( + function () { + db.all( + 'SELECT * FROM txt_table WHERE txt = ? LIMIT ?', + faulty, + function () { + // Never called: the trailing callback only keeps this in + // callback mode, where a bad bind throws synchronously. + }, + ); + }, + function (err) { + assert.ok(err instanceof TypeError); + assert.match(err.message, /Cannot bind parameter 1/); + return true; + }, + ); }); - it('should ignore faulty toString set to function', function(done) { - const faulty = [[{toString: function () {console.log('oh no');}}], 1]; - db.all('SELECT * FROM txt_table WHERE txt = ? LIMIT ?', faulty, function (err) { - assert.equal(err, undefined); - done(); - }); + it('should reject faulty toString set to function', function () { + const faulty = [ + [ + { + toString: function () { + console.log('oh no'); + }, + }, + ], + 1, + ]; + assert.throws( + function () { + db.all( + 'SELECT * FROM txt_table WHERE txt = ? LIMIT ?', + faulty, + function () { + // Never called: the trailing callback only keeps this in + // callback mode, where a bad bind throws synchronously. + }, + ); + }, + function (err) { + assert.ok(err instanceof TypeError); + assert.match(err.message, /Cannot bind parameter 1/); + return true; + }, + ); }); - -}); \ No newline at end of file +}); diff --git a/test/parallel_insert.test.js b/test/parallel_insert.test.js index 710bd2d..81825bb 100644 --- a/test/parallel_insert.test.js +++ b/test/parallel_insert.test.js @@ -1,44 +1,58 @@ +import { after, before, describe, it } from 'node:test'; + import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; -import { ensureExists, deleteFile, fileExists } from './support/helper.js'; +import { deleteFile, ensureExists, fileExists } from './support/helper.js'; -describe('parallel', function() { +describe('parallel', function () { let db; - before(function(done) { + before(function (_t, done) { deleteFile('test/tmp/test_parallel_inserts.db'); ensureExists('test/tmp'); db = new sqlite3.Database('test/tmp/test_parallel_inserts.db', done); }); - let columns = []; + const columns = []; for (let i = 0; i < 128; i++) { - columns.push('id' + i); + columns.push(`id${i}`); } - it('should create the table', function(done) { - db.run("CREATE TABLE foo (" + columns + ")", done); + it('should create the table', function (_t, done) { + db.run(`CREATE TABLE foo (${columns})`, done); }); - it('should insert in parallel', function(done) { - for (let i = 0; i < 1000; i++) { - for (var values = [], j = 0; j < columns.length; j++) { - values.push(i * j); + // 1000 file-backed INSERTs, each its own implicit transaction (journal + // file created and deleted per row). Measured at 38s on GitHub's + // windows-11-arm runner — about 2x over the 20s suite-wide ceiling that + // tools/run-tests.mjs applies, while every other runner finishes in + // single-digit seconds. The override lifts the ceiling for this test + // only; the hang detector still guards the rest of the suite. + it( + 'should insert in parallel', + { + timeout: 180000, + }, + function (_t, done) { + for (let i = 0; i < 1000; i++) { + const values = []; + for (let j = 0; j < columns.length; j++) { + values.push(i * j); + } + db.run(`INSERT INTO foo VALUES (${values})`); } - db.run("INSERT INTO foo VALUES (" + values + ")"); - } - db.wait(done); - }); + db.wait(done); + }, + ); - it('should close the database', function(done) { + it('should close the database', function (_t, done) { db.close(done); }); - it('should verify that the database exists', function() { + it('should verify that the database exists', function () { fileExists('test/tmp/test_parallel_inserts.db'); }); - after(function() { + after(function () { deleteFile('test/tmp/test_parallel_inserts.db'); }); -}); \ No newline at end of file +}); diff --git a/test/patching.test.js b/test/patching.test.js index e948156..d04ee16 100644 --- a/test/patching.test.js +++ b/test/patching.test.js @@ -1,25 +1,29 @@ +import assert from 'node:assert'; +import { after, before, describe, it } from 'node:test'; + import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; -describe('patching', function() { - describe("Database", function() { +describe('patching', function () { + describe('Database', function () { let db; - let originalFunctions = {}; + const originalFunctions = {}; - before(function() { + before(function () { originalFunctions.close = sqlite3.Database.prototype.close; originalFunctions.exec = sqlite3.Database.prototype.exec; originalFunctions.wait = sqlite3.Database.prototype.wait; - originalFunctions.loadExtension = sqlite3.Database.prototype.loadExtension; + originalFunctions.loadExtension = + sqlite3.Database.prototype.loadExtension; originalFunctions.serialize = sqlite3.Database.prototype.serialize; - originalFunctions.parallelize = sqlite3.Database.prototype.parallelize; + originalFunctions.parallelize = + sqlite3.Database.prototype.parallelize; originalFunctions.configure = sqlite3.Database.prototype.configure; originalFunctions.interrupt = sqlite3.Database.prototype.interrupt; }); - it('allow patching native functions', function() { - let myFun = function myFunction() { - return "Success"; + it('allow patching native functions', function () { + const myFun = function myFunction() { + return 'Success'; }; assert.doesNotThrow(() => { @@ -48,37 +52,42 @@ describe('patching', function() { }); db = new sqlite3.Database(':memory:'); - assert.strictEqual(db.close(), "Success"); - assert.strictEqual(db.exec(), "Success"); - assert.strictEqual(db.wait(), "Success"); - assert.strictEqual(db.loadExtension(), "Success"); - assert.strictEqual(db.serialize(), "Success"); - assert.strictEqual(db.parallelize(), "Success"); - assert.strictEqual(db.configure(), "Success"); - assert.strictEqual(db.interrupt(), "Success"); + assert.strictEqual(db.close(), 'Success'); + assert.strictEqual(db.exec(), 'Success'); + assert.strictEqual(db.wait(), 'Success'); + assert.strictEqual(db.loadExtension(), 'Success'); + assert.strictEqual(db.serialize(), 'Success'); + assert.strictEqual(db.parallelize(), 'Success'); + assert.strictEqual(db.configure(), 'Success'); + assert.strictEqual(db.interrupt(), 'Success'); }); - after(function() { - if(db != null) { + after(function () { + if (db != null) { sqlite3.Database.prototype.close = originalFunctions.close; sqlite3.Database.prototype.exec = originalFunctions.exec; sqlite3.Database.prototype.wait = originalFunctions.wait; - sqlite3.Database.prototype.loadExtension = originalFunctions.loadExtension; - sqlite3.Database.prototype.serialize = originalFunctions.serialize; - sqlite3.Database.prototype.parallelize = originalFunctions.parallelize; - sqlite3.Database.prototype.configure = originalFunctions.configure; - sqlite3.Database.prototype.interrupt = originalFunctions.interrupt; + sqlite3.Database.prototype.loadExtension = + originalFunctions.loadExtension; + sqlite3.Database.prototype.serialize = + originalFunctions.serialize; + sqlite3.Database.prototype.parallelize = + originalFunctions.parallelize; + sqlite3.Database.prototype.configure = + originalFunctions.configure; + sqlite3.Database.prototype.interrupt = + originalFunctions.interrupt; db.close(); } }); }); - describe('Statement', function() { + describe('Statement', function () { let db; let statement; - let originalFunctions = {}; + const originalFunctions = {}; - before(function() { + before(function () { originalFunctions.bind = sqlite3.Statement.prototype.bind; originalFunctions.get = sqlite3.Statement.prototype.get; originalFunctions.run = sqlite3.Statement.prototype.run; @@ -88,9 +97,9 @@ describe('patching', function() { originalFunctions.finalize = sqlite3.Statement.prototype.finalize; }); - it('allow patching native functions', function() { - let myFun = function myFunction() { - return "Success"; + it('allow patching native functions', function () { + const myFun = function myFunction() { + return 'Success'; }; assert.doesNotThrow(() => { @@ -116,45 +125,46 @@ describe('patching', function() { }); db = new sqlite3.Database(':memory:'); - statement = db.prepare(""); - assert.strictEqual(statement.bind(), "Success"); - assert.strictEqual(statement.get(), "Success"); - assert.strictEqual(statement.run(), "Success"); - assert.strictEqual(statement.all(), "Success"); - assert.strictEqual(statement.each(), "Success"); - assert.strictEqual(statement.reset(), "Success"); - assert.strictEqual(statement.finalize(), "Success"); + statement = db.prepare(''); + assert.strictEqual(statement.bind(), 'Success'); + assert.strictEqual(statement.get(), 'Success'); + assert.strictEqual(statement.run(), 'Success'); + assert.strictEqual(statement.all(), 'Success'); + assert.strictEqual(statement.each(), 'Success'); + assert.strictEqual(statement.reset(), 'Success'); + assert.strictEqual(statement.finalize(), 'Success'); }); - after(function() { - if(statement != null) { + after(function () { + if (statement != null) { sqlite3.Statement.prototype.bind = originalFunctions.bind; sqlite3.Statement.prototype.get = originalFunctions.get; sqlite3.Statement.prototype.run = originalFunctions.run; sqlite3.Statement.prototype.all = originalFunctions.all; sqlite3.Statement.prototype.each = originalFunctions.each; sqlite3.Statement.prototype.reset = originalFunctions.reset; - sqlite3.Statement.prototype.finalize = originalFunctions.finalize; + sqlite3.Statement.prototype.finalize = + originalFunctions.finalize; } - if(db != null) { + if (db != null) { db.close(); } }); }); - describe('Backup', function() { + describe('Backup', function () { let db; let backup; - let originalFunctions = {}; + const originalFunctions = {}; - before(function() { + before(function () { originalFunctions.step = sqlite3.Backup.prototype.step; originalFunctions.finish = sqlite3.Backup.prototype.finish; }); - it('allow patching native functions', function() { - let myFun = function myFunction() { - return "Success"; + it('allow patching native functions', function () { + const myFun = function myFunction() { + return 'Success'; }; assert.doesNotThrow(() => { @@ -165,20 +175,20 @@ describe('patching', function() { }); db = new sqlite3.Database(':memory:'); - backup = db.backup("somefile", myFun); - assert.strictEqual(backup.step(), "Success"); - assert.strictEqual(backup.finish(), "Success"); + backup = db.backup('somefile', myFun); + assert.strictEqual(backup.step(), 'Success'); + assert.strictEqual(backup.finish(), 'Success'); }); - after(function() { - if(backup != null) { + after(function () { + if (backup != null) { sqlite3.Backup.prototype.step = originalFunctions.step; sqlite3.Backup.prototype.finish = originalFunctions.finish; backup.finish(); } - if(db != null) { + if (db != null) { db.close(); } }); }); -}); \ No newline at end of file +}); diff --git a/test/permission.test.js b/test/permission.test.js new file mode 100644 index 0000000..69ee994 --- /dev/null +++ b/test/permission.test.js @@ -0,0 +1,417 @@ +// The Node permission model under real child processes (Deliverable 11). +// +// --permission cannot be enabled inside an already-running process, and +// the interesting assertions are about the interaction of two flags, so +// every case here spawns a real child (test/support/permission_child.mjs) +// with a real flag combination. The child reports raw observations — +// error codes, messages, outcomes — and this file owns every assertion; +// nothing is stubbed, least of all process.permission. + +import assert from 'node:assert'; +import { spawnSync } from 'node:child_process'; +import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, sep } from 'node:path'; +import { after, before, describe, it } from 'node:test'; + +const repo = join(import.meta.dirname, '..'); +const childScript = join( + import.meta.dirname, + 'support', + 'permission_child.mjs', +); + +// One fixture root per test-file process: inside/ is what the children's +// --allow-fs-* grants cover; the outside/ tree lives under the OS temp +// directory (see the child script for why it must not be under the repo). +// `node --test` may run files in parallel, so the root is unique to this +// file. +const fixtureRoot = join(repo, 'test', 'tmp', `permission-${process.pid}`); +const inside = join(fixtureRoot, 'inside'); +const outside = join(tmpdir(), `permission-outside-permission-${process.pid}`); + +/** + * The flags every scenario child needs: the model, addons (the package + * does not load without them under the model), the repo (the driver's + * own code, its node_modules and its prebuilds/build), and — on Linux — + * /etc/alpine-release, which node-gyp-build's musl detection stats at + * module load; under the permission model that stat is denied and the + * loader crashes before reaching this package's code (observed in the + * alpine container; the file need only be readable, not present). + * + * @returns {string[]} the base flag set for this platform. + */ +function baseFlags() { + const base = ['--permission', '--allow-addons', `--allow-fs-read=${repo}`]; + if (process.platform === 'linux') { + base.push('--allow-fs-read=/etc/alpine-release'); + } + return base; +} + +/** + * Spawns the scenario child with the given permission flags and returns + * its reported steps keyed by name (plus the exit status). + * + * @param {string} scenario the scenario name. + * @param {string[]} extraFlags permission flags beyond the base set. + * @returns {{ steps: Map, status: number, output: string }} the child's observations. + */ +function runChild(scenario, extraFlags = []) { + const res = spawnSync( + process.execPath, + [ + ...baseFlags(), + ...extraFlags.flat(), + childScript, + scenario, + fixtureRoot, + ], + { encoding: 'utf8', timeout: 60000 }, + ); + assert.strictEqual( + res.status, + 0, + `child ${scenario} exited ${res.status}:\n${res.stderr || res.stdout}`, + ); + const steps = new Map(); + for (const line of res.stdout.split('\n')) { + if (line.startsWith('STEP ')) { + steps.set( + JSON.parse(line.slice(5)).name, + JSON.parse(line.slice(5)), + ); + } + } + assert.ok( + steps.size > 0, + `child ${scenario} reported no steps:\n${res.stdout}\n${res.stderr}`, + ); + return { steps, status: res.status, output: res.stdout }; +} + +// Every grant the scenarios need, computed from the fixture paths. The +// wildcard grant is passed in both separator forms — Node's permission +// matching accepts forward slashes everywhere and the platform separator +// on Windows; the redundant form is harmless (multiple --allow-fs-write +// flags accumulate). +const grants = { + insideWrite: [ + `--allow-fs-write=${inside}${sep}*`, + `--allow-fs-write=${inside}/*`, + ], + exactFileOnly: `--allow-fs-write=${join(inside, 'exact-file-only.db')}`, +}; + +before(function () { + rmSync(fixtureRoot, { recursive: true, force: true }); + mkdirSync(inside, { recursive: true }); + mkdirSync(outside, { recursive: true }); + // A zero-byte file is a valid empty database for a read-only open. + writeFileSync(join(inside, 'ro.db'), ''); + writeFileSync(join(inside, 'exact-file-only.db'), ''); +}); + +after(function () { + rmSync(fixtureRoot, { recursive: true, force: true }); + rmSync(outside, { recursive: true, force: true }); +}); + +describe('permission model', function () { + // The guard on the whole file: if Node ever changes the shape (an + // isEnabled return, a different process.permission availability), + // these probes say so instead of every case failing opaquely. + it('process.permission shape under --permission', { + timeout: 60000, + }, function () { + const { steps } = runChild('model-shape'); + const shape = steps.get('shape'); + assert.strictEqual(shape.permissionType, 'object'); + assert.strictEqual(shape.hasType, 'function'); + // Observed on Node 24 and 26: isEnabled does not exist. The + // implementation gates on process.permission's presence, never on + // a method that may not be there. + assert.strictEqual(shape.isEnabledType, 'undefined'); + }); + + it('read-only open inside the allowed fs works', { + timeout: 60000, + }, async function () { + const { steps } = runChild('ro-open-allowed'); + assert.ok(steps.get('read')?.ok, 'read inside allowed fs'); + assert.ok(steps.get('close')?.ok); + }); + + it('writable open inside a read-only-permitted directory is refused, naming the directory', { + timeout: 60000, + }, function () { + // The exact database file IS write-granted; its directory is not + // (no wildcard), so SQLite could not create the -journal file a + // writable database needs. The refusal must name the directory + // and explain the sidecar files, not blame the file. + const { steps } = runChild('rw-open-denied-dir', [ + grants.exactFileOnly, + ]); + const open = steps.get('open'); + assert.strictEqual(open?.ok, false); + assert.strictEqual(open?.code, 'ERR_ACCESS_DENIED'); + assert.strictEqual(open?.permission, 'FileSystemWrite'); + assert.strictEqual(open?.resource, inside, 'must name the directory'); + assert.match( + open?.message ?? '', + /-journal/, + 'must explain why the directory is needed', + ); + assert.match( + open?.message ?? '', + /--allow-fs-write/, + 'must name the remedy', + ); + }); + + it('writable open with the directory granted works', { + timeout: 60000, + }, function () { + const { steps } = runChild('rw-open-allowed', [grants.insideWrite]); + assert.ok(steps.get('write')?.ok, 'write inside granted dir'); + assert.ok(steps.get('close')?.ok); + }); + + it('open outside the allowed fs is refused, naming the path', { + timeout: 60000, + }, function () { + const { steps } = runChild('open-outside'); + const open = steps.get('open'); + assert.strictEqual(open?.ok, false); + assert.strictEqual(open?.code, 'ERR_ACCESS_DENIED'); + assert.strictEqual(open?.permission, 'FileSystemRead'); + assert.strictEqual(open?.resource, join(outside, 'x.db')); + assert.match(open?.message ?? '', /--allow-fs-read/); + }); + + it("opening '' is refused when the temp directory is not writable", { + timeout: 60000, + }, function () { + // '' is SQLite's private temporary database: a real on-disk file + // under the temp directory, not a special name that needs no fs. + const { steps } = runChild('temp-filename'); + const open = steps.get("open ''"); + assert.strictEqual(open?.ok, false); + assert.strictEqual(open?.code, 'ERR_ACCESS_DENIED'); + assert.strictEqual(open?.permission, 'FileSystemWrite'); + assert.strictEqual(open?.resource, tmpdir()); + }); + + it('ATTACH and VACUUM INTO outside the allowed fs are refused by the gate', { + timeout: 60000, + }, function () { + const { steps } = runChild('attach'); + // VACUUM INTO opens its output through an internal ATTACH, so one + // gate covers both SQL-level paths to the filesystem. + assert.strictEqual(steps.get('attach-outside')?.code, 'SQLITE_AUTH'); + assert.strictEqual( + steps.get('vacuum-into-outside')?.code, + 'SQLITE_AUTH', + ); + assert.ok( + steps.get('attach-memory')?.ok, + ':memory: ATTACH is not an fs path', + ); + }); + + it('configure(attachPaths) admits only its permission-checked targets', { + timeout: 60000, + }, function () { + const { steps } = runChild('attach-allowed', [ + grants.insideWrite, + `--allow-fs-read=${join(inside, 'attach-target.db')}`, + ]); + assert.ok( + steps.get('configure')?.ok, + 'configure accepts a permitted target', + ); + assert.ok( + steps.get('attach-allowed-target')?.ok, + 'the allowlisted target attaches', + ); + assert.strictEqual( + steps.get('attach-other-inside')?.code, + 'SQLITE_AUTH', + 'a different target inside the same dir is still denied (exact-match allowlist)', + ); + assert.strictEqual( + steps.get('vacuum-into-allowed-target')?.code, + 'SQLITE_AUTH', + 'VACUUM INTO needs its own allowlist entry — it is not a free pass', + ); + }); + + it('loadExtension is refused unless allowlisted, and SQL load_extension() stays off', { + timeout: 60000, + }, function () { + // The allowlist grant makes the extension path fs.read-permitted, + // so configure('extensionPolicy') accepts it and the load reaches + // the native dlopen (of a file that does not exist — the point is + // which layer refuses, not that the load succeeds). + const { steps } = runChild('load-extension', [ + '--allow-fs-read=/tmp/definitely-not-there.ext', + ]); + const unlisted = steps.get('load-unlisted'); + assert.strictEqual(unlisted?.ok, false); + assert.strictEqual(unlisted?.code, 'ERR_ACCESS_DENIED'); + assert.match(unlisted?.message ?? '', /extensionPolicy/); + assert.match(unlisted?.message ?? '', /--allow-addons/); + // The allowlisted path passes the policy and fails at the native + // dlopen of the missing file — a different, native error, which + // is exactly how the two refusals are told apart. + assert.strictEqual( + steps.get('load-allowlisted')?.code, + 'SQLITE_ERROR', + 'allowlisted path reaches the native load (dlopen of a missing file)', + ); + // The SQL function is off by default in the vendored SQLite + // (observed: probed on 3.53.4 before any change) and stays off. + assert.strictEqual( + steps.get('sql-load-extension-fn')?.code, + 'SQLITE_ERROR', + ); + assert.match( + steps.get('sql-load-extension-fn')?.message ?? '', + /not authorized/, + ); + }); + + it('file: URIs are checked (or refused when unparsable)', { + timeout: 60000, + }, function () { + const { steps } = runChild('uri'); + assert.ok(steps.get('uri-ro-inside')?.ok, 'ro URI inside allowed fs'); + assert.strictEqual(steps.get('uri-outside')?.code, 'ERR_ACCESS_DENIED'); + assert.strictEqual( + steps.get('uri-outside-noquery')?.code, + 'ERR_ACCESS_DENIED', + ); + assert.ok(steps.get('uri-memory')?.ok, 'file::memory: needs no fs'); + // Unparsable forms are refused rather than passed through. + assert.strictEqual( + steps.get('uri-bad-mode')?.code, + 'ERR_ACCESS_DENIED', + ); + assert.match( + steps.get('uri-bad-mode')?.message ?? '', + /mode parameter 'bogus'/, + ); + assert.strictEqual( + steps.get('uri-non-file-scheme')?.code, + 'ERR_ACCESS_DENIED', + ); + }); + + it('backup destinations are checked like opens', { + timeout: 60000, + }, function () { + const { steps } = runChild('backup', [grants.insideWrite]); + const outsideBackup = steps.get('backup-outside'); + assert.strictEqual(outsideBackup?.ok, false); + assert.strictEqual(outsideBackup?.code, 'ERR_ACCESS_DENIED'); + assert.strictEqual( + outsideBackup?.resource, + join(outside, 'b.db'), + 'names the backup destination', + ); + assert.ok( + steps.get('backup-inside')?.ok, + 'granted destination backs up', + ); + }); + + it(':memory: is unaffected by the model', { timeout: 60000 }, function () { + const { steps } = runChild('memory-unaffected'); + assert.deepStrictEqual(steps.get('read')?.value, { a: 1 }); + }); + + it('untrusted hardening composes with the permission model', { + timeout: 60000, + }, function () { + const { steps } = runChild('untrusted-under-permissions'); + assert.ok(steps.get('read')?.ok); + // The ATTACH refusal comes from the LIMIT_ATTACHED=0 ceiling with + // the message naming it — the deny-all gate is behind the limit + // here; both are part of the hardening. + const attach = steps.get('attach-refused'); + assert.strictEqual(attach?.code, 'SQLITE_ERROR'); + assert.match(attach?.message ?? '', /too many attached databases/); + }); + + it('the pool opens its worker connections under the model', { + timeout: 60000, + }, function () { + const { steps } = runChild('worker-pool', [ + '--allow-worker', + grants.insideWrite, + `--allow-fs-read=${join(inside, 'pool.db')}`, + ]); + assert.deepStrictEqual(steps.get('pool-get')?.value, { v: 1 }); + assert.ok(steps.get('pool-write')?.ok); + assert.ok(steps.get('pool-close')?.ok); + }); + + it('exiting without closing is a clean exit 0 (also after a refused open)', { + timeout: 60000, + }, function () { + for (const scenario of ['exit-unclosed', 'exit-after-refusal']) { + const res = spawnSync( + process.execPath, + [ + ...baseFlags(), + ...grants.insideWrite, + childScript, + scenario, + fixtureRoot, + ], + { encoding: 'utf8', timeout: 60000 }, + ); + // 139 would be a teardown segfault, invisible to any + // in-process assertion. + assert.strictEqual( + res.status, + 0, + `${scenario}: stderr:\n${res.stderr}`, + ); + } + }); + + it('with the model off, none of the above changes behaviour (the zero-cost path)', { + timeout: 60000, + }, function () { + // Same child, NO --permission flag: process.permission is + // undefined and every open/attach/load behaves as it did pre-v9. + const res = spawnSync( + process.execPath, + [childScript, 'off-model', fixtureRoot], + { encoding: 'utf8', timeout: 60000 }, + ); + assert.strictEqual(res.status, 0, `stderr:\n${res.stderr}`); + const steps = new Map(); + for (const line of res.stdout.split('\n')) { + if (line.startsWith('STEP ')) { + const parsed = JSON.parse(line.slice(5)); + steps.set(parsed.name, parsed); + } + } + assert.strictEqual(steps.get('shape')?.permissionType, 'undefined'); + assert.ok(steps.get('write')?.ok, 'writable open without any grants'); + // ATTACH outside any grant works — the whole point of the flag. + assert.ok( + steps.get('attach-outside')?.ok, + 'ATTACH is ungated when the model is off', + ); + // loadExtension reaches the native layer (dlopen of a missing + // file), not a policy refusal. + assert.strictEqual( + steps.get('load-extension-reaches-native')?.code, + 'SQLITE_ERROR', + ); + assert.ok(steps.get('close')?.ok); + }); +}); diff --git a/test/pool.test.js b/test/pool.test.js new file mode 100644 index 0000000..ec7762d --- /dev/null +++ b/test/pool.test.js @@ -0,0 +1,410 @@ +// The worker pool (Deliverable 09): read/write routing, write +// serialization, transaction pinning, error diagnostics across the +// postMessage boundary, cancellation through the shared flag, and +// shutdown that leaves no worker behind. +import assert from 'node:assert'; +import { rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { afterEach, describe, it } from 'node:test'; + +import sqlite3 from '../lib/sqlite3.js'; +import { TMP_DIR } from './support/db.js'; + +// A query slow enough to measure overlap against, cheap enough for CI: +// ~2M recursive rows per invocation. +const SLOW_QUERY = + 'WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM c ' + + 'WHERE x < 2000000) SELECT count(*) AS n FROM c'; + +/** Removes a database file and its journal/WAL siblings. */ +function removeDb(file) { + for (const suffix of ['', '-wal', '-shm', '-journal']) { + rmSync(`${file}${suffix}`, { force: true }); + } +} + +describe('pool', function () { + let pool; + let file; + + /** + * Opens a pool over a fresh file with two readers. + * + * @returns {Promise} resolves once the pool is ready. + */ + async function freshPool() { + file = join(TMP_DIR, `pool-test-${process.pid}-${Date.now()}.db`); + removeDb(file); + pool = await sqlite3.pool(file, { readers: 2 }); + await pool.exec('CREATE TABLE t (a INTEGER PRIMARY KEY, b TEXT)'); + } + + afterEach(async function () { + if (pool !== undefined && pool !== null) { + await pool.close(); + pool = /** @type {any} */ (null); + } + if (file !== undefined) { + removeDb(file); + file = /** @type {any} */ (undefined); + } + }); + + it('routes reads to readers and writes to the writer', { + timeout: 30000, + }, async function () { + await freshPool(); + const result = await pool.write('INSERT INTO t (b) VALUES (?)', [ + 'one', + ]); + assert.strictEqual(result.changes, 1); + assert.ok(result.lastID >= 1); + const rows = await pool.read('SELECT b FROM t'); + assert.deepStrictEqual( + rows.map((r) => r.b), + ['one'], + ); + const row = await pool.get('SELECT b FROM t WHERE a = ?', [ + result.lastID, + ]); + assert.strictEqual(row.b, 'one'); + // Repeated gets through the pool stay correct (the cached-get + // re-stepping bug class this deliverable had to route around). + assert.strictEqual( + (await pool.get('SELECT COUNT(*) AS n FROM t')).n, + 1, + ); + assert.strictEqual( + (await pool.get('SELECT COUNT(*) AS n FROM t')).n, + 1, + ); + }); + + it('runs WAL mode and the busy timeout by default', { + timeout: 30000, + }, async function () { + await freshPool(); + const mode = await pool.get('PRAGMA journal_mode'); + assert.strictEqual( + mode.journal_mode.toLowerCase(), + 'wal', + 'pool default enables WAL', + ); + const timeout = await pool.get('PRAGMA busy_timeout'); + assert.strictEqual(timeout.timeout, 5000); + }); + + it('concurrent reads do not queue behind each other', { + timeout: 120000, + }, async function () { + await freshPool(); + // Structural, not wall-time: a trivial read completing while a + // slow one is still in flight proves the two ran on different + // readers (one connection at a time would delay the trivial + // read behind the slow one). The claim is ordering, not speed, + // so it holds on a loaded CI machine too — where a wall-clock + // ratio cannot: with every core saturated, two concurrent + // queries timeshare and cost 2x one, by design of the load. + let slowDone = false; + const slow = pool.read(SLOW_QUERY); + slow.finally(() => { + slowDone = true; + }); + await new Promise((resolve) => setTimeout(resolve, 5)); + const fast = await pool.get('SELECT 1 AS v'); + assert.strictEqual(fast.v, 1); + assert.strictEqual( + slowDone, + false, + 'the slow read is still in flight — the fast one did not queue behind it', + ); + const rows = await slow; + assert.strictEqual(rows[0].n, 2000000); + }); + + it('concurrent writes all land (they queue on the writer)', { + timeout: 60000, + }, async function () { + await freshPool(); + const N = 50; + await Promise.all( + Array.from({ length: N }, (_, i) => + pool.write('INSERT INTO t (b) VALUES (?)', [`row-${i}`]), + ), + ); + const count = await pool.get('SELECT COUNT(*) AS n FROM t'); + assert.strictEqual(count.n, N, 'every write landed exactly once'); + }); + + it('transaction sees its own writes; readers see committed only', { + timeout: 30000, + }, async function () { + await freshPool(); + await pool.write('INSERT INTO t (b) VALUES (?)', ['committed']); + const returned = await pool.transaction(async (tx) => { + await tx.write('INSERT INTO t (b) VALUES (?)', ['uncommitted']); + const inside = await tx.read('SELECT COUNT(*) AS n FROM t'); + const outside = await pool.read('SELECT COUNT(*) AS n FROM t'); + return { inside: inside[0].n, outside: outside[0].n }; + }); + assert.strictEqual(returned.inside, 2, 'tx reads its own writes'); + assert.strictEqual( + returned.outside, + 1, + 'pool.read inside the body sees committed data only', + ); + const after = await pool.get('SELECT COUNT(*) AS n FROM t'); + assert.strictEqual(after.n, 2, 'commit made the write visible'); + }); + + it('rolling back discards the writes', { + timeout: 30000, + }, async function () { + await freshPool(); + await assert.rejects( + pool.transaction(async (tx) => { + await tx.write('INSERT INTO t (b) VALUES (?)', ['doomed']); + throw new Error('body failed'); + }), + /body failed/, + ); + assert.strictEqual( + (await pool.get('SELECT COUNT(*) AS n FROM t')).n, + 0, + 'rollback discarded the insert', + ); + }); + + it('overlapping transactions serialize without losing updates', { + timeout: 60000, + }, async function () { + await freshPool(); + await pool.exec('CREATE TABLE counter (n INTEGER)'); + await pool.write('INSERT INTO counter VALUES (0)'); + // Two transactions at once, each a read-modify-write: the pool + // must serialize them (the second waits for the first) so both + // increments survive — a lost update is the failure mode. + await Promise.all([ + pool.transaction(async (tx) => { + const row = await tx.get('SELECT n FROM counter'); + await tx.write('UPDATE counter SET n = ?', [row.n + 1]); + }), + pool.transaction(async (tx) => { + const row = await tx.get('SELECT n FROM counter'); + await tx.write('UPDATE counter SET n = ?', [row.n + 1]); + }), + ]); + assert.strictEqual( + (await pool.get('SELECT n FROM counter')).n, + 2, + 'both increments survived (transactions serialized, not interleaved)', + ); + }); + + it('pool.write/exec from inside a transaction body refuse instead of deadlocking', { + timeout: 30000, + }, async function () { + await freshPool(); + await assert.rejects( + pool.transaction(async (tx) => { + // tx handle works… + await tx.write('INSERT INTO t (b) VALUES (?)', ['kept']); + // …but the pool-facing writer methods would wait on the + // transaction itself forever. + await pool.write('INSERT INTO t (b) VALUES (?)', ['never']); + }), + /cannot run inside a pool.transaction\(\) body/, + ); + assert.strictEqual( + (await pool.get('SELECT COUNT(*) AS n FROM t')).n, + 0, + 'the refusal unwound the transaction through its rollback', + ); + }); + + it('errors keep code/errno/primaryCode across the boundary', { + timeout: 30000, + }, async function () { + await freshPool(); + await pool.write('INSERT INTO t (a, b) VALUES (1, ?)', ['first']); + await assert.rejects( + pool.write('INSERT INTO t (a, b) VALUES (1, ?)', ['duplicate']), + function (err) { + assert.strictEqual(err.code, 'SQLITE_CONSTRAINT_PRIMARYKEY'); + assert.strictEqual(err.errno, 1555); + assert.strictEqual(err.primaryCode, 'SQLITE_CONSTRAINT'); + assert.match(err.message, /UNIQUE constraint failed/); + return true; + }, + ); + // A syntax error keeps its diagnostics too. + await assert.rejects(pool.read('SELEKT 1'), function (err) { + assert.strictEqual(err.code, 'SQLITE_ERROR'); + assert.strictEqual(err.errno, 1); + return true; + }); + }); + + it('blob columns cross as Uint8Array (documented Buffer difference)', { + timeout: 30000, + }, async function () { + await freshPool(); + await pool.exec('CREATE TABLE blobs (d BLOB)'); + await pool.write('INSERT INTO blobs VALUES (?)', [ + new Uint8Array([1, 2, 3]), + ]); + const row = await pool.get('SELECT d FROM blobs'); + assert.ok(row.d instanceof Uint8Array); + assert.deepStrictEqual(Array.from(row.d), [1, 2, 3]); + assert.ok( + !Buffer.isBuffer(row.d), + 'documented: pool results carry Uint8Array, not Buffer', + ); + }); + + it('BigInt bind values and results survive the round trip', { + timeout: 30000, + }, async function () { + file = join(TMP_DIR, `pool-bigint-${process.pid}-${Date.now()}.db`); + removeDb(file); + pool = await sqlite3.pool(file, { readers: 0, integerMode: 'bigint' }); + await pool.exec('CREATE TABLE big (v INTEGER)'); + const big = 9007199254740993n; + await pool.write('INSERT INTO big VALUES (?)', [big]); + const row = await pool.get('SELECT v AS v FROM big'); + assert.strictEqual(row.v, big); + }); + + it('cancels a running read through the shared flag', { + timeout: 60000, + }, async function () { + await freshPool(); + const controller = new AbortController(); + const reason = new Error('too slow'); + setTimeout(() => controller.abort(reason), 30); + await assert.rejects( + pool.read(SLOW_QUERY, { signal: controller.signal }), + (err) => err === reason, + ); + // The pool keeps working afterwards. + assert.strictEqual( + (await pool.get('SELECT COUNT(*) AS n FROM t')).n, + 0, + ); + }); + + it('close drains in-flight work, is idempotent, and leaves no worker', { + timeout: 60000, + }, async function () { + await freshPool(); + const slow = pool.read(SLOW_QUERY); + const closed = pool.close(); + // The in-flight read still settles — close waits for it. + const rows = await slow; + assert.strictEqual(rows[0].n, 2000000); + await closed; + await pool.close(); // idempotent + assert.ok(pool.closed); + // New work refuses. + await assert.rejects(pool.read('SELECT 1'), /pool is closed/); + // The process exits on its own once the (only) pool is closed — + // no worker survives — which the suite's own exit proves; here + // assert the observable refusal instead. + }); + + // The test above has one operation in flight, which reaches a worker + // immediately. Writes queue on the writer mutex instead, and work + // waiting there had not been registered for the drain — close() saw + // an almost-empty set, shut the workers down, and failed the waiting + // writes with "pool worker exited unexpectedly". Accepted work must + // complete however deep the queue is. + it('close drains writes still queued on the writer', { + timeout: 60000, + }, async function () { + await freshPool(); + const writes = Array.from({ length: 50 }, (_, i) => + pool.write('INSERT INTO t (b) VALUES (?)', [`row-${i}`]), + ); + const closed = pool.close(); + const settled = await Promise.allSettled(writes); + const failed = settled.filter((s) => s.status === 'rejected'); + assert.deepStrictEqual( + failed.map((f) => f.reason?.message), + [], + 'every accepted write completes across close()', + ); + await closed; + + // And they are actually in the file, not merely resolved. + const check = await sqlite3.open(file); + assert.strictEqual((await check.get('SELECT count(*) n FROM t')).n, 50); + await check.close(); + }); + + // Same gap on the transaction path, which takes the writer mutex + // directly rather than through the shared helper. + it('close waits for a transaction that is still queued', { + timeout: 60000, + }, async function () { + await freshPool(); + const blocker = pool.write('INSERT INTO t (b) VALUES (?)', ['first']); + const queued = pool.transaction(async (tx) => { + await tx.write('INSERT INTO t (b) VALUES (?)', ['in-tx']); + }); + const closed = pool.close(); + await blocker; + await queued; + await closed; + + const check = await sqlite3.open(file); + assert.strictEqual((await check.get('SELECT count(*) n FROM t')).n, 2); + await check.close(); + }); + + it('await using closes the pool', { timeout: 30000 }, async function () { + file = join(TMP_DIR, `pool-dispose-${process.pid}-${Date.now()}.db`); + removeDb(file); + { + const p = await sqlite3.pool(file, { readers: 1 }); + await p.exec('CREATE TABLE t (a)'); + await using poolDisposed = p; + await poolDisposed.write('INSERT INTO t VALUES (1)'); + } + assert.ok(true, 'await using disposed without hanging'); + }); + + it('refuses :memory: and unknown options loudly', { + timeout: 30000, + }, async function () { + await assert.rejects( + sqlite3.pool(':memory:'), + /in-memory one cannot be shared across workers/, + ); + await assert.rejects( + sqlite3.pool('/tmp/x.db', { readers: -1 }), + /readers.*non-negative integer/, + ); + await assert.rejects( + sqlite3.pool('/tmp/x.db', { nope: true }), + /unknown option 'nope'/, + ); + await assert.rejects(sqlite3.pool(''), /non-empty filename/); + }); + + it('readers: 0 routes reads to the writer', { + timeout: 30000, + }, async function () { + file = join(TMP_DIR, `pool-wonly-${process.pid}-${Date.now()}.db`); + removeDb(file); + pool = await sqlite3.pool(file, { readers: 0 }); + await pool.exec('CREATE TABLE t (a)'); + await pool.write('INSERT INTO t VALUES (1)'); + assert.strictEqual((await pool.read('SELECT a FROM t'))[0].a, 1); + // And the in-transaction refusal applies (reads would wait on + // the writer the body holds). + await assert.rejects( + pool.transaction(async () => pool.read('SELECT 1')), + /cannot run inside a pool.transaction\(\) body/, + ); + }); +}); diff --git a/test/prepare.test.js b/test/prepare.test.js index ae2d990..4a16542 100644 --- a/test/prepare.test.js +++ b/test/prepare.test.js @@ -1,166 +1,225 @@ +import assert from 'node:assert'; +import { after, before, describe, it } from 'node:test'; + import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; -describe('prepare', function() { - describe('invalid SQL', function() { +describe('prepare', function () { + describe('invalid SQL', function () { let db; - before(function(done) { db = new sqlite3.Database(':memory:', done); }); + before(function (_t, done) { + db = new sqlite3.Database(':memory:', done); + }); - let stmt; - it('should fail preparing a statement with invalid SQL', function(done) { - stmt = db.prepare('CRATE TALE foo text bar)', function(err, statement) { - if (err && err.errno == sqlite3.ERROR && - err.message === 'SQLITE_ERROR: near "CRATE": syntax error') { - done(); - } - else throw err; - }); + let _stmt; + it('should fail preparing a statement with invalid SQL', function (_t, done) { + _stmt = db.prepare( + 'CRATE TALE foo text bar)', + function (err, _statement) { + if ( + err && + err.errno === sqlite3.ERROR && + err.message === + 'SQLITE_ERROR: near "CRATE": syntax error' + ) { + done(); + } else throw err; + }, + ); }); - after(function(done) { db.close(done); }); + after(function (_t, done) { + db.close(done); + }); }); - describe('simple prepared statement', function() { + describe('simple prepared statement', function () { let db; - before(function(done) { db = new sqlite3.Database(':memory:', done); }); + before(function (_t, done) { + db = new sqlite3.Database(':memory:', done); + }); - it('should prepare, run and finalize the statement', function(done) { - db.prepare("CREATE TABLE foo (text bar)") - .run() + it('should prepare, run and finalize the statement', function (_t, done) { + db.prepare('CREATE TABLE foo (text bar)') + .run(function (err) { + if (err) throw err; + }) .finalize(done); }); - after(function(done) { db.close(done); }); + after(function (_t, done) { + db.close(done); + }); }); - describe('inserting and retrieving rows', function() { + describe('inserting and retrieving rows', function () { let db; - before(function(done) { db = new sqlite3.Database(':memory:', done); }); + before(function (_t, done) { + db = new sqlite3.Database(':memory:', done); + }); let inserted = 0; let retrieved = 0; // We insert and retrieve that many rows. - let count = 1000; + const count = 1000; - it('should create the table', function(done) { - db.prepare("CREATE TABLE foo (txt text, num int, flt float, blb blob)").run().finalize(done); + it('should create the table', function (_t, done) { + db.prepare( + 'CREATE TABLE foo (txt text, num int, flt float, blb blob)', + ) + .run(function (err) { + if (err) throw err; + }) + .finalize(done); }); - it('should insert ' + count + ' rows', function(done) { + it(`should insert ${count} rows`, function (_t, done) { for (let i = 0; i < count; i++) { - db.prepare("INSERT INTO foo VALUES(?, ?, ?, ?)").run( - 'String ' + i, - i, - i * Math.PI, - // null (SQLite sets this implicitly) - function(err) { + db.prepare('INSERT INTO foo VALUES(?, ?, ?, ?)') + .run( + `String ${i}`, + i, + i * Math.PI, + // The 4th parameter is bound explicitly: v9 rejects + // a parameter-count mismatch instead of silently + // binding the missing ones as NULL. + null, + function (err) { + if (err) throw err; + inserted++; + }, + ) + .finalize(function (err) { if (err) throw err; - inserted++; - } - ).finalize(function(err) { - if (err) throw err; - if (inserted == count) done(); - }); + if (inserted === count) done(); + }); } }); - it('should prepare a statement and run it ' + (count + 5) + ' times', function(done) { - let stmt = db.prepare("SELECT txt, num, flt, blb FROM foo ORDER BY num", function(err) { - if (err) throw err; - assert.equal(stmt.sql, 'SELECT txt, num, flt, blb FROM foo ORDER BY num'); - }); - - for (let i = 0; i < count + 5; i++) (function(i) { - stmt.get(function(err, row) { + it(`should prepare a statement and run it ${count + 5} times`, function (_t, done) { + const stmt = db.prepare( + 'SELECT txt, num, flt, blb FROM foo ORDER BY num', + function (err) { if (err) throw err; + assert.equal( + stmt.sql, + 'SELECT txt, num, flt, blb FROM foo ORDER BY num', + ); + }, + ); + + for (let i = 0; i < count + 5; i++) + (function (i) { + stmt.get(function (err, row) { + if (err) throw err; - if (retrieved >= 1000) { - assert.equal(row, undefined); - } else { - assert.equal(row.txt, 'String ' + i); - assert.equal(row.num, i); - assert.equal(row.flt, i * Math.PI); - assert.equal(row.blb, null); - } + if (retrieved >= 1000) { + assert.equal(row, undefined); + } else { + assert.equal(row.txt, `String ${i}`); + assert.equal(row.num, i); + assert.equal(row.flt, i * Math.PI); + assert.equal(row.blb, null); + } - retrieved++; - }); - })(i); + retrieved++; + }); + })(i); stmt.finalize(done); }); - it('should have retrieved ' + (count + 5) + ' rows', function() { + it(`should have retrieved ${count + 5} rows`, function () { assert.equal(count + 5, retrieved, "Didn't retrieve all rows"); }); - - after(function(done) { db.close(done); }); + after(function (_t, done) { + db.close(done); + }); }); - describe('inserting with accidental undefined', function() { + describe('inserting with accidental undefined', function () { let db; - before(function(done) { db = new sqlite3.Database(':memory:', done); }); + before(function (_t, done) { + db = new sqlite3.Database(':memory:', done); + }); let inserted = 0; let retrieved = 0; - it('should create the table', function(done) { - db.prepare("CREATE TABLE foo (num int)").run().finalize(done); + it('should create the table', function (_t, done) { + db.prepare('CREATE TABLE foo (num int)') + .run(function (err) { + if (err) throw err; + }) + .finalize(done); }); - it('should insert two rows', function(done) { - db.prepare('INSERT INTO foo VALUES(4)').run(function(err) { - if (err) throw err; - inserted++; - }).run(undefined, function (err) { - // The second time we pass undefined as a parameter. This is - // a mistake, but it should either throw an error or be ignored, - // not silently fail to run the statement. - if (err) throw err; - inserted++; - }).finalize(function(err) { - if (err) throw err; - if (inserted == 2) done(); - }); + it('should insert two rows', function (_t, done) { + db.prepare('INSERT INTO foo VALUES(4)') + .run(function (err) { + if (err) throw err; + inserted++; + }) + .run(undefined, function (err) { + // The second time we pass undefined as a parameter. This is + // a mistake, but it should either throw an error or be ignored, + // not silently fail to run the statement. + if (err) throw err; + inserted++; + }) + .finalize(function (err) { + if (err) throw err; + if (inserted === 2) done(); + }); }); - it('should retrieve the data', function(done) { - let stmt = db.prepare("SELECT num FROM foo", function(err) { + it('should retrieve the data', function (_t, done) { + const stmt = db.prepare('SELECT num FROM foo', function (err) { if (err) throw err; }); - for (let i = 0; i < 2; i++) (function(i) { - stmt.get(function(err, row) { - if (err) throw err; - assert(row); - assert.equal(row.num, 4); - retrieved++; - }); - })(i); + for (let i = 0; i < 2; i++) + (function (_i) { + stmt.get(function (err, row) { + if (err) throw err; + assert(row); + assert.equal(row.num, 4); + retrieved++; + }); + })(i); stmt.finalize(done); }); - it('should have retrieved two rows', function() { + it('should have retrieved two rows', function () { assert.equal(2, retrieved, "Didn't retrieve all rows"); }); - after(function(done) { db.close(done); }); + after(function (_t, done) { + db.close(done); + }); }); - describe('retrieving reset() function', function() { + describe('retrieving reset() function', function () { let db; - before(function(done) { db = new sqlite3.Database('test/support/prepare.db', sqlite3.OPEN_READONLY, done); }); + before(function (_t, done) { + db = new sqlite3.Database( + 'test/support/prepare.db', + sqlite3.OPEN_READONLY, + done, + ); + }); let retrieved = 0; - it('should retrieve the same row over and over again', function(done) { - let stmt = db.prepare("SELECT txt, num, flt, blb FROM foo ORDER BY num"); + it('should retrieve the same row over and over again', function (_t, done) { + const stmt = db.prepare( + 'SELECT txt, num, flt, blb FROM foo ORDER BY num', + ); for (let i = 0; i < 10; i++) { stmt.reset(); - stmt.get(function(err, row) { + stmt.get(function (err, row) { if (err) throw err; assert.equal(row.txt, 'String 0'); assert.equal(row.num, 0); @@ -172,53 +231,76 @@ describe('prepare', function() { stmt.finalize(done); }); - it('should have retrieved 10 rows', function() { + it('should have retrieved 10 rows', function () { assert.equal(10, retrieved, "Didn't retrieve all rows"); }); - after(function(done) { db.close(done); }); + after(function (_t, done) { + db.close(done); + }); }); - describe('multiple get() parameter binding', function() { + describe('multiple get() parameter binding', function () { let db; - before(function(done) { db = new sqlite3.Database('test/support/prepare.db', sqlite3.OPEN_READONLY, done); }); + before(function (_t, done) { + db = new sqlite3.Database( + 'test/support/prepare.db', + sqlite3.OPEN_READONLY, + done, + ); + }); let retrieved = 0; - it('should retrieve particular rows', function(done) { - let stmt = db.prepare("SELECT txt, num, flt, blb FROM foo WHERE num = ?"); + it('should retrieve particular rows', function (_t, done) { + const stmt = db.prepare( + 'SELECT txt, num, flt, blb FROM foo WHERE num = ?', + ); - for (let i = 0; i < 10; i++) (function(i) { - stmt.get(i * 10 + 1, function(err, row) { - if (err) throw err; - let val = i * 10 + 1; - assert.equal(row.txt, 'String ' + val); - assert.equal(row.num, val); - assert.equal(row.flt, val * Math.PI); - assert.equal(row.blb, null); - retrieved++; - }); - })(i); + for (let i = 0; i < 10; i++) + (function (i) { + stmt.get(i * 10 + 1, function (err, row) { + if (err) throw err; + const val = i * 10 + 1; + assert.equal(row.txt, `String ${val}`); + assert.equal(row.num, val); + assert.equal(row.flt, val * Math.PI); + assert.equal(row.blb, null); + retrieved++; + }); + })(i); stmt.finalize(done); }); - it('should have retrieved 10 rows', function() { + it('should have retrieved 10 rows', function () { assert.equal(10, retrieved, "Didn't retrieve all rows"); }); - after(function(done) { db.close(done); }); + after(function (_t, done) { + db.close(done); + }); }); - describe('prepare() parameter binding', function() { + describe('prepare() parameter binding', function () { let db; - before(function(done) { db = new sqlite3.Database('test/support/prepare.db', sqlite3.OPEN_READONLY, done); }); + before(function (_t, done) { + db = new sqlite3.Database( + 'test/support/prepare.db', + sqlite3.OPEN_READONLY, + done, + ); + }); let retrieved = 0; - it('should retrieve particular rows', function(done) { - db.prepare("SELECT txt, num, flt, blb FROM foo WHERE num = ? AND txt = ?", 10, 'String 10') - .get(function(err, row) { + it('should retrieve particular rows', function (_t, done) { + db.prepare( + 'SELECT txt, num, flt, blb FROM foo WHERE num = ? AND txt = ?', + 10, + 'String 10', + ) + .get(function (err, row) { if (err) throw err; assert.equal(row.txt, 'String 10'); assert.equal(row.num, 10); @@ -229,26 +311,37 @@ describe('prepare', function() { .finalize(done); }); - it('should have retrieved 1 row', function() { + it('should have retrieved 1 row', function () { assert.equal(1, retrieved, "Didn't retrieve all rows"); }); - after(function(done) { db.close(done); }); + after(function (_t, done) { + db.close(done); + }); }); - describe('all()', function() { + describe('all()', function () { let db; - before(function(done) { db = new sqlite3.Database('test/support/prepare.db', sqlite3.OPEN_READONLY, done); }); + before(function (_t, done) { + db = new sqlite3.Database( + 'test/support/prepare.db', + sqlite3.OPEN_READONLY, + done, + ); + }); let retrieved = 0; - let count = 1000; - - it('should retrieve particular rows', function(done) { - db.prepare("SELECT txt, num, flt, blb FROM foo WHERE num < ? ORDER BY num", count) - .all(function(err, rows) { + const count = 1000; + + it('should retrieve particular rows', function (_t, done) { + db.prepare( + 'SELECT txt, num, flt, blb FROM foo WHERE num < ? ORDER BY num', + count, + ) + .all(function (err, rows) { if (err) throw err; for (let i = 0; i < rows.length; i++) { - assert.equal(rows[i].txt, 'String ' + i); + assert.equal(rows[i].txt, `String ${i}`); assert.equal(rows[i].num, i); assert.equal(rows[i].flt, i * Math.PI); assert.equal(rows[i].blb, null); @@ -258,32 +351,44 @@ describe('prepare', function() { .finalize(done); }); - it('should have retrieved all rows', function() { + it('should have retrieved all rows', function () { assert.equal(count, retrieved, "Didn't retrieve all rows"); }); - after(function(done) { db.close(done); }); + after(function (_t, done) { + db.close(done); + }); }); - describe('all()', function() { + describe('all()', function () { let db; - before(function(done) { db = new sqlite3.Database('test/support/prepare.db', sqlite3.OPEN_READONLY, done); }); + before(function (_t, done) { + db = new sqlite3.Database( + 'test/support/prepare.db', + sqlite3.OPEN_READONLY, + done, + ); + }); - it('should retrieve particular rows', function(done) { - db.prepare("SELECT txt, num, flt, blb FROM foo WHERE num > 5000") - .all(function(err, rows) { + it('should retrieve particular rows', function (_t, done) { + db.prepare('SELECT txt, num, flt, blb FROM foo WHERE num > 5000') + .all(function (err, rows) { if (err) throw err; assert.ok(rows.length === 0); }) .finalize(done); }); - after(function(done) { db.close(done); }); + after(function (_t, done) { + db.close(done); + }); }); - describe('high concurrency', function() { + describe('high concurrency', function () { let db; - before(function(done) { db = new sqlite3.Database(':memory:', done); }); + before(function (_t, done) { + db = new sqlite3.Database(':memory:', done); + }); function randomString() { let str = ''; @@ -294,35 +399,47 @@ describe('prepare', function() { } // Generate random data. - let data = []; - let length = Math.floor(Math.random() * 1000) + 200; + const data = []; + const length = Math.floor(Math.random() * 1000) + 200; for (let i = 0; i < length; i++) { - data.push([ randomString(), i, i * Math.random(), null ]); + data.push([randomString(), i, i * Math.random(), null]); } let inserted = 0; let retrieved = 0; - it('should create the table', function(done) { - db.prepare("CREATE TABLE foo (txt text, num int, flt float, blb blob)").run().finalize(done); + it('should create the table', function (_t, done) { + db.prepare( + 'CREATE TABLE foo (txt text, num int, flt float, blb blob)', + ) + .run(function (err) { + if (err) throw err; + }) + .finalize(done); }); - it('should insert all values', function(done) { + it('should insert all values', function (_t, done) { for (let i = 0; i < data.length; i++) { - let stmt = db.prepare("INSERT INTO foo VALUES(?, ?, ?, ?)"); - stmt.run(data[i][0], data[i][1], data[i][2], data[i][3], function(err) { - if (err) throw err; - inserted++; - }).finalize(function(err) { + const stmt = db.prepare('INSERT INTO foo VALUES(?, ?, ?, ?)'); + stmt.run( + data[i][0], + data[i][1], + data[i][2], + data[i][3], + function (err) { + if (err) throw err; + inserted++; + }, + ).finalize(function (err) { if (err) throw err; - if (inserted == data.length) done(); + if (inserted === data.length) done(); }); } }); - it('should retrieve all values', function(done) { - db.prepare("SELECT txt, num, flt, blb FROM foo") - .all(function(err, rows) { + it('should retrieve all values', function (_t, done) { + db.prepare('SELECT txt, num, flt, blb FROM foo') + .all(function (err, rows) { if (err) throw err; for (let i = 0; i < rows.length; i++) { @@ -336,7 +453,6 @@ describe('prepare', function() { // Mark the data row as already retrieved. data[rows[i].num] = true; retrieved++; - } assert.equal(retrieved, data.length); @@ -345,83 +461,111 @@ describe('prepare', function() { .finalize(done); }); - after(function(done) { db.close(done); }); + after(function (_t, done) { + db.close(done); + }); }); - - describe('test Database#get()', function() { + describe('test Database#get()', function () { let db; - before(function(done) { db = new sqlite3.Database('test/support/prepare.db', sqlite3.OPEN_READONLY, done); }); + before(function (_t, done) { + db = new sqlite3.Database( + 'test/support/prepare.db', + sqlite3.OPEN_READONLY, + done, + ); + }); let retrieved = 0; - it('should get a row', function(done) { - db.get("SELECT txt, num, flt, blb FROM foo WHERE num = ? AND txt = ?", 10, 'String 10', function(err, row) { - if (err) throw err; - assert.equal(row.txt, 'String 10'); - assert.equal(row.num, 10); - assert.equal(row.flt, 10 * Math.PI); - assert.equal(row.blb, null); - retrieved++; - done(); - }); + it('should get a row', function (_t, done) { + db.get( + 'SELECT txt, num, flt, blb FROM foo WHERE num = ? AND txt = ?', + 10, + 'String 10', + function (err, row) { + if (err) throw err; + assert.equal(row.txt, 'String 10'); + assert.equal(row.num, 10); + assert.equal(row.flt, 10 * Math.PI); + assert.equal(row.blb, null); + retrieved++; + done(); + }, + ); }); - it('should have retrieved all rows', function() { + it('should have retrieved all rows', function () { assert.equal(1, retrieved, "Didn't retrieve all rows"); }); - after(function(done) { db.close(done); }); + after(function (_t, done) { + db.close(done); + }); }); - describe('Database#run() and Database#all()', function() { + describe('Database#run() and Database#all()', function () { let db; - before(function(done) { db = new sqlite3.Database(':memory:', done); }); + before(function (_t, done) { + db = new sqlite3.Database(':memory:', done); + }); let inserted = 0; let retrieved = 0; // We insert and retrieve that many rows. - let count = 1000; + const count = 1000; - it('should create the table', function(done) { - db.run("CREATE TABLE foo (txt text, num int, flt float, blb blob)", done); + it('should create the table', function (_t, done) { + db.run( + 'CREATE TABLE foo (txt text, num int, flt float, blb blob)', + done, + ); }); - it('should insert ' + count + ' rows', function(done) { + it(`should insert ${count} rows`, function (_t, done) { for (let i = 0; i < count; i++) { - db.run("INSERT INTO foo VALUES(?, ?, ?, ?)", - 'String ' + i, + db.run( + 'INSERT INTO foo VALUES(?, ?, ?, ?)', + `String ${i}`, i, i * Math.PI, - // null (SQLite sets this implicitly) - function(err) { + // The 4th parameter is bound explicitly: v9 rejects + // a parameter-count mismatch instead of silently + // binding the missing ones as NULL. + null, + function (err) { if (err) throw err; inserted++; - if (inserted == count) done(); - } + if (inserted === count) done(); + }, ); } }); - it('should retrieve all rows', function(done) { - db.all("SELECT txt, num, flt, blb FROM foo ORDER BY num", function(err, rows) { - if (err) throw err; - for (let i = 0; i < rows.length; i++) { - assert.equal(rows[i].txt, 'String ' + i); - assert.equal(rows[i].num, i); - assert.equal(rows[i].flt, i * Math.PI); - assert.equal(rows[i].blb, null); - retrieved++; - } + it('should retrieve all rows', function (_t, done) { + db.all( + 'SELECT txt, num, flt, blb FROM foo ORDER BY num', + function (err, rows) { + if (err) throw err; + for (let i = 0; i < rows.length; i++) { + assert.equal(rows[i].txt, `String ${i}`); + assert.equal(rows[i].num, i); + assert.equal(rows[i].flt, i * Math.PI); + assert.equal(rows[i].blb, null); + retrieved++; + } - assert.equal(retrieved, count); - assert.equal(retrieved, inserted); + assert.equal(retrieved, count); + assert.equal(retrieved, inserted); - done(); - }); + done(); + }, + ); }); - after(function(done) { db.close(done); }); + after(function (_t, done) { + db.close(done); + }); }); -}); \ No newline at end of file +}); diff --git a/test/profile.test.js b/test/profile.test.js index 6284301..5d63a82 100644 --- a/test/profile.test.js +++ b/test/profile.test.js @@ -1,56 +1,55 @@ +import assert from 'node:assert'; +import { after, before, describe, it } from 'node:test'; + import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; -describe('profiling', function() { +describe('profiling', function () { let create = false; let select = false; let db; - before(function(done) { + before(function (_t, done) { db = new sqlite3.Database(':memory:', done); - db.on('profile', function(sql, nsecs) { - assert.ok(typeof nsecs === "number"); + db.on('profile', function (sql, nsecs) { + assert.ok(typeof nsecs === 'number'); if (sql.match(/^SELECT/)) { assert.ok(!select); - assert.equal(sql, "SELECT * FROM foo"); + assert.equal(sql, 'SELECT * FROM foo'); select = true; - } - else if (sql.match(/^CREATE/)) { + } else if (sql.match(/^CREATE/)) { assert.ok(!create); - assert.equal(sql, "CREATE TABLE foo (id int)"); + assert.equal(sql, 'CREATE TABLE foo (id int)'); create = true; - } - else { + } else { assert.ok(false); } }); }); - it('should profile a create table', function(done) { + it('should profile a create table', function (_t, done) { assert.ok(!create); - db.run("CREATE TABLE foo (id int)", function(err) { + db.run('CREATE TABLE foo (id int)', function (err) { if (err) throw err; - setImmediate(function() { + setImmediate(function () { assert.ok(create); done(); }); }); }); - - it('should profile a select', function(done) { + it('should profile a select', function (_t, done) { assert.ok(!select); - db.run("SELECT * FROM foo", function(err) { + db.run('SELECT * FROM foo', function (err) { if (err) throw err; - setImmediate(function() { + setImmediate(function () { assert.ok(select); done(); }, 0); }); }); - after(function(done) { + after(function (_t, done) { db.close(done); }); -}); \ No newline at end of file +}); diff --git a/test/progress.test.js b/test/progress.test.js new file mode 100644 index 0000000..7aa7669 --- /dev/null +++ b/test/progress.test.js @@ -0,0 +1,288 @@ +import assert from 'node:assert'; +import { afterEach, beforeEach, describe, it } from 'node:test'; +import { Worker } from 'node:worker_threads'; + +import sqlite3 from '../lib/sqlite3.js'; + +// Progress handler and cancellation token (Deliverable 07). The token is +// the recommended form: an atomic flag in a SharedArrayBuffer, polled by +// the native handler — zero JS per check, and cancellable from any +// thread. The JS callback form round-trips per invocation and is gated +// out of the synchronous methods like collations are. + +const RECURSIVE = ` +WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x + 1 FROM c) +SELECT sum(x) FROM c +`; + +/** + * Waits for the connection's in-flight work to drain, up to `budget` ms. + * Returns either way: the caller asserts on db.pending, so a genuine + * failure to drain still fails the test — just without a fixed sleep + * deciding it. + */ +async function waitForDrain(db, budget = 5000) { + const deadline = Date.now() + budget; + while (db.pending !== 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + +async function openDb() { + const db = new sqlite3.Database(':memory:'); + await new Promise((resolve, reject) => { + db.once('open', resolve); + db.once('error', reject); + }); + return db; +} + +describe('progress handler', function () { + /** @type {sqlite3.Database} */ + let db; + + beforeEach(async function () { + db = await openDb(); + }); + + afterEach(async function () { + await db.close(); + }); + + it('a cancellation token aborts a long query and the connection survives', { + timeout: 15000, + }, async function () { + const token = db.cancellationToken(); + assert.strictEqual(token.cancelled, false); + + const started = Date.now(); + // Cancel shortly after the query starts running. + setTimeout(() => token.cancel(), 20); + + await assert.rejects( + db.all(RECURSIVE), + (err) => err.code === 'SQLITE_INTERRUPT', + ); + const elapsed = Date.now() - started; + // Unbounded, this query runs for hundreds of milliseconds; the + // token must stop it well before that. Generous bound so a slow + // runner fails on correctness, not on scheduling. + assert.ok(elapsed < 5000, `abort took ${elapsed}ms`); + + assert.strictEqual(token.cancelled, true); + // The connection is fully usable again: pending drained to 0 and + // a fresh query completes normally. + assert.strictEqual(db.pending, 0); + const row = await db.get('SELECT 41 + 1 AS v'); + assert.strictEqual(row.v, 42); + }); + + it('a token cancelled from a worker thread aborts the query', { + timeout: 20000, + }, async function () { + // The whole point of the SharedArrayBuffer: the main thread is + // busy awaiting the query, so the flag has to be set from another + // thread with no JS involvement on this connection. + const token = db.cancellationToken(); + const workerSource = ` + const { parentPort, workerData } = require('node:worker_threads'); + const flag = new Int32Array(workerData.sab); + setTimeout(() => { + Atomics.store(flag, 0, 1); + parentPort.postMessage('set'); + }, 20); + `; + const worker = new Worker(workerSource, { + eval: true, + workerData: { sab: token.buffer }, + }); + const flagged = new Promise((resolve) => { + worker.once('message', resolve); + }); + + await assert.rejects( + db.all(RECURSIVE), + (err) => err.code === 'SQLITE_INTERRUPT', + ); + assert.strictEqual(await flagged, 'set'); + await worker.terminate(); + + // The connection survived the cross-thread abort. + const row = await db.get('SELECT 7 AS v'); + assert.strictEqual(row.v, 7); + assert.strictEqual(db.pending, 0); + }); + + it('the token’s signal integrates with the { signal } option', { + timeout: 15000, + }, async function () { + const token = db.cancellationToken(); + const reason = new Error('stopped by user'); + const started = Date.now(); + setTimeout(() => token.cancel(reason), 20); + + await assert.rejects( + db.all(RECURSIVE, { signal: token.signal }), + (err) => err === reason, + ); + assert.ok(Date.now() - started < 5000); + // The signal rejects immediately, independently of the query, so + // unlike the other pending checks here the interrupted work is + // still unwinding. Poll rather than sleeping a fixed 25ms: on a + // loaded runner that sleep is not enough, and the assertion then + // fails on scheduling instead of on the thing it is testing. + await waitForDrain(db); + assert.strictEqual(db.pending, 0); + }); + + it('an unused token costs nothing observable and queries still complete', { + timeout: 15000, + }, async function () { + const token = db.cancellationToken(); + const rows = await db.all('SELECT 1 AS a UNION ALL SELECT 2'); + assert.deepStrictEqual( + rows.map((r) => r.a), + [1, 2], + ); + token.destroy(); + // Still works after destroy. + const row = await db.get('SELECT 3 AS v'); + assert.strictEqual(row.v, 3); + }); + + it('reset() lets a token be reused', { timeout: 15000 }, async function () { + const token = db.cancellationToken(); + token.cancel(); + token.reset(); + assert.strictEqual(token.cancelled, false); + const row = await db.get('SELECT 1 AS v'); + assert.strictEqual(row.v, 1); + }); + + it('the JavaScript callback form aborts on a truthy return', { + timeout: 15000, + }, async function () { + let calls = 0; + db.progress(1000, () => { + calls++; + return calls > 3; + }); + await assert.rejects( + db.all(RECURSIVE), + (err) => err.code === 'SQLITE_INTERRUPT', + ); + assert.ok(calls >= 4, `callback ran ${calls} times`); + // The handler stays installed until removed. + assert.strictEqual(db.pending, 0); + db.progress(); + const row = await db.get('SELECT 5 AS v'); + assert.strictEqual(row.v, 5); + }); + + it('a throwing progress callback aborts the query and carries the cause', { + timeout: 15000, + }, async function () { + const boom = new Error('progress exploded'); + db.progress(1000, () => { + throw boom; + }); + await assert.rejects(db.all(RECURSIVE), (err) => { + assert.strictEqual(err.code, 'SQLITE_INTERRUPT'); + assert.strictEqual(err.cause, boom); + return true; + }); + db.progress(); + }); + + it('sync methods refuse while a JS progress callback is registered', async function () { + db.progress(1000, () => false); + // getSync hits the sync-prepare gate first, prepareSync directly. + assert.throws( + () => db.getSync('SELECT 1'), + /while a JavaScript progress callback/, + ); + assert.throws( + () => db.prepareSync('SELECT 1'), + /while a JavaScript progress callback/, + ); + db.progress(); + // Removing the handler re-enables the sync path. + assert.strictEqual(db.getSync('SELECT 1 AS v').v, 1); + }); + + it('sync methods work with the token form installed', async function () { + const token = db.cancellationToken(); + assert.strictEqual(db.getSync('SELECT 2 AS v').v, 2); + token.destroy(); + }); + + it('validates the period and callback types', async function () { + assert.throws( + () => db.progress(0, () => false), + /period must be a positive integer/, + ); + assert.throws( + () => /** @type {any} */ (db).progress(10, 'nope'), + /callback must be a function/, + ); + assert.throws( + () => db.cancellationToken({ period: 0 }), + /period must be a positive integer/, + ); + }); + + it('cancelling with no query running affects the next one only if still set', { + timeout: 15000, + }, async function () { + const token = db.cancellationToken(); + token.cancel(); + await assert.rejects( + db.all(RECURSIVE), + (err) => err.code === 'SQLITE_INTERRUPT', + ); + token.destroy(); + }); + + // SQLite has one progress slot per connection, so a second token + // replaces the first. destroy() on the displaced token must not + // disarm its replacement: it used to, and the runaway query below + // then never aborted — the connection wedged and the process would + // not exit. + it('a displaced token’s destroy() does not disarm its replacement', { + timeout: 15000, + }, async function () { + const first = db.cancellationToken(); + const second = db.cancellationToken(); + + first.destroy(); + second.cancel(); + + await assert.rejects( + db.all(RECURSIVE), + (err) => err.code === 'SQLITE_INTERRUPT', + ); + second.destroy(); + assert.deepStrictEqual(await db.all('SELECT 1 AS v'), [{ v: 1 }]); + }); + + // Same slot, other direction: progress(fn) takes it from the token, + // so the token's destroy() must leave the callback installed. + it('a token displaced by progress() does not disarm the callback', { + timeout: 15000, + }, async function () { + const token = db.cancellationToken(); + let calls = 0; + db.progress(1000, () => { + calls++; + return calls > 50; + }); + token.destroy(); + + await assert.rejects( + db.all(RECURSIVE), + (err) => err.code === 'SQLITE_INTERRUPT', + ); + assert.ok(calls > 50, `progress callback ran (${calls} calls)`); + db.progress(); + }); +}); diff --git a/test/promises.test.js b/test/promises.test.js new file mode 100644 index 0000000..fd5c07d --- /dev/null +++ b/test/promises.test.js @@ -0,0 +1,349 @@ +import assert from 'node:assert'; +import { describe, it } from 'node:test'; + +import sqlite3 from '../lib/sqlite3.js'; + +async function openDb() { + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (a INT, b TEXT)'); + return db; +} + +describe('promise API', function () { + describe('open()', function () { + it('resolves an open database', async function () { + const db = await sqlite3.open(':memory:'); + assert.strictEqual(db.open, true); + assert.ok(db instanceof sqlite3.Database); + await db.close(); + }); + + it('honours open flags', async function () { + const db = await sqlite3.open( + 'test/tmp/promises-flags.db', + sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE, + ); + assert.strictEqual(db.open, true); + await db.close(); + }); + + it('rejects on open failure', async function () { + await assert.rejects( + sqlite3.open('test/tmp/no-such-dir-03x/foo.db'), + function (err) { + assert.strictEqual(err.primaryCode, 'SQLITE_CANTOPEN'); + return true; + }, + ); + }); + }); + + describe('dual-mode resolution values', function () { + it('run resolves { lastID, changes, lastIDBigInt }', async function () { + const db = await openDb(); + const r = await db.run('INSERT INTO t VALUES (?, ?)', 1, 'one'); + assert.strictEqual(r.lastID, 1); + assert.strictEqual(r.changes, 1); + assert.strictEqual(r.lastIDBigInt, 1n); + assert.deepStrictEqual(Object.keys(r).sort(), [ + 'changes', + 'lastID', + 'lastIDBigInt', + ]); + await db.close(); + }); + + it('get resolves the row or undefined', async function () { + const db = await openDb(); + await db.run('INSERT INTO t VALUES (?, ?)', 1, 'one'); + const row = await db.get('SELECT * FROM t WHERE a = ?', 1); + assert.deepStrictEqual(row, { a: 1, b: 'one' }); + const none = await db.get('SELECT * FROM t WHERE a = ?', 99); + assert.strictEqual(none, undefined); + await db.close(); + }); + + it('all resolves an array of rows', async function () { + const db = await openDb(); + await db.run('INSERT INTO t VALUES (?, ?)', 1, 'one'); + await db.run('INSERT INTO t VALUES (?, ?)', 2, 'two'); + const rows = await db.all('SELECT a FROM t ORDER BY a'); + assert.deepStrictEqual(rows, [{ a: 1 }, { a: 2 }]); + await db.close(); + }); + + it('map resolves the mapped object', async function () { + const db = await openDb(); + await db.run('INSERT INTO t VALUES (?, ?)', 1, 'one'); + await db.run('INSERT INTO t VALUES (?, ?)', 2, 'two'); + const mapped = await db.map('SELECT a, b FROM t'); + assert.deepStrictEqual(mapped, { 1: 'one', 2: 'two' }); + await db.close(); + }); + + it('exec, close, wait resolve undefined', async function () { + const db = await openDb(); + assert.strictEqual(await db.exec('CREATE TABLE u (i)'), undefined); + assert.strictEqual(await db.wait(), undefined); + assert.strictEqual(await db.close(), undefined); + assert.strictEqual(db.open, false); + }); + + it('statement methods resolve like their database forms', async function () { + const db = await openDb(); + const stmt = db.prepare('INSERT INTO t VALUES (?, ?)'); + const r = await stmt.run(3, 'three'); + assert.strictEqual(r.changes, 1); + // get on an INSERT returns no row: use a fresh select instead. + const sel = db.prepare('SELECT b FROM t WHERE a = ?'); + assert.deepStrictEqual(await sel.get(3), { b: 'three' }); + assert.strictEqual((await sel.all()).length, 1); + const sel2 = db.prepare('SELECT a, b FROM t'); + assert.deepStrictEqual(await sel2.map(), { 3: 'three' }); + assert.strictEqual(await sel.reset(), undefined); + assert.strictEqual(await stmt.finalize(), undefined); + assert.strictEqual(await sel.finalize(), undefined); + assert.strictEqual(await sel2.finalize(), undefined); + await db.close(); + }); + + it('backup step resolves completion, finish resolves undefined', async function () { + const db = await openDb(); + await db.run('INSERT INTO t VALUES (?, ?)', 1, 'one'); + const backup = db.backup('test/tmp/promises-backup.db'); + let completed = false; + while (!completed) { + completed = await backup.step(16); + } + assert.strictEqual(completed, true); + assert.strictEqual(await backup.finish(), undefined); + await db.close(); + }); + }); + + describe('run result value semantics', function () { + it('captures lastID at settle time, not lazily', async function () { + const db = await openDb(); + const r1 = await db.run('INSERT INTO t VALUES (?, ?)', 1, 'one'); + await db.run('INSERT INTO t VALUES (?, ?)', 2, 'two'); + // A lazy read of the underlying (reused) statement would now + // report the second insert's rowid; the snapshot must not. + assert.strictEqual(r1.lastID, 1); + await db.close(); + }); + + it('keeps the unsafe-rowid RangeError lazy in number mode', async function () { + const db = await openDb(); + const r = await db.run( + "INSERT INTO t (rowid, a, b) VALUES (9007199254740993, 1, 'big')", + ); + // Resolving is fine; only reading lastID throws. + assert.strictEqual(r.lastIDBigInt, 9007199254740993n); + assert.throws(function () { + r.lastID; + }, RangeError); + await db.close(); + }); + + it('lastID is a BigInt in bigint and mixed modes', async function () { + const db = await openDb(); + db.configure('integerMode', 'mixed'); + const r = await db.run( + "INSERT INTO t (rowid, a, b) VALUES (9007199254740993, 1, 'big')", + ); + assert.strictEqual(r.lastID, 9007199254740993n); + db.configure('integerMode', 'bigint'); + const r2 = await db.run( + "INSERT INTO t (rowid, a, b) VALUES (2, 2, 'small')", + ); + assert.strictEqual(r2.lastID, 2n); + await db.close(); + }); + }); + + describe('callback mode is unchanged', function () { + it('returns this and stays chainable', async function () { + const db = await openDb(); + let ran = 0; + const out = db.run( + 'INSERT INTO t VALUES (?, ?)', + 1, + 'one', + function (err) { + if (err) throw err; + ran++; + // Issued from the run's callback: two separate + // statements in parallel mode have no cross-statement + // FIFO guarantee (statement operations bypass the + // database queue), so the read must follow the write + // explicitly rather than by assumption. + assert.strictEqual( + db.get('SELECT a FROM t', function (err2, row) { + if (err2) throw err2; + assert.deepStrictEqual(row, { a: 1 }); + }), + db, + ); + }, + ); + assert.strictEqual(out, db); + await db.wait(); + assert.strictEqual(ran, 1); + await db.close(); + }); + + it('statement methods with callbacks return the statement (finalize the database)', async function () { + const db = await openDb(); + const stmt = db.prepare('INSERT INTO t VALUES (?, ?)'); + const out = stmt.run(1, 'one', function (err) { + if (err) throw err; + }); + assert.strictEqual(out, stmt); + // finalize has always returned the database, for chaining. + const fin = stmt.finalize(function (err) { + if (err) throw err; + }); + assert.strictEqual(fin, db); + await db.close(); + }); + + it('prepare keeps its synchronous statement return', async function () { + const db = await openDb(); + const stmt = db.prepare('SELECT 1 AS one'); + assert.ok(stmt instanceof sqlite3.Statement); + const row = await new Promise(function (resolve, reject) { + stmt.get(function (err, r) { + if (err) reject(err); + else resolve(r); + }); + }); + assert.deepStrictEqual(row, { one: 1 }); + await stmt.finalize(); + await db.close(); + }); + }); + + describe('rejections', function () { + it('carry code, errno and primaryCode', async function () { + const db = await openDb(); + await db.exec('CREATE TABLE u (x INT UNIQUE)'); + await db.run('INSERT INTO u VALUES (1)'); + await assert.rejects( + db.run('INSERT INTO u VALUES (1)'), + function (err) { + assert.strictEqual(err.code, 'SQLITE_CONSTRAINT_UNIQUE'); + assert.strictEqual(err.primaryCode, 'SQLITE_CONSTRAINT'); + assert.strictEqual(err.errno, sqlite3.CONSTRAINT_UNIQUE); + return true; + }, + ); + await db.close(); + }); + + it('reject instead of throwing synchronously on bad binds', async function () { + const db = await openDb(); + const p = db.run('INSERT INTO t VALUES (?)', [{ a: 1 }]); + assert.ok(p instanceof Promise); + await assert.rejects(p, TypeError); + // The connection survived: the orphaned statement was finalized. + const rows = await db.all('SELECT * FROM t'); + assert.deepStrictEqual(rows, []); + await db.close(); + }); + + it('reject on statement methods too', async function () { + const db = await openDb(); + const stmt = db.prepare('INSERT INTO t VALUES (?, ?)'); + await assert.rejects(stmt.run(1, { nope: 1 }), TypeError); + await stmt.finalize(); + await db.close(); + }); + }); + + describe('each() is callback-only', function () { + it('throws a TypeError without callbacks, pointing at iterate()', async function () { + const db = await openDb(); + assert.throws( + function () { + db.each('SELECT 1'); + }, + function (err) { + assert.ok(err instanceof TypeError); + assert.match(err.message, /iterate\(\)/); + return true; + }, + ); + const stmt = db.prepare('SELECT 1'); + assert.throws(function () { + stmt.each(); + }, TypeError); + await stmt.finalize(); + await db.close(); + }); + + it('still streams rows with callbacks', async function () { + const db = await openDb(); + await db.run('INSERT INTO t VALUES (?, ?)', 1, 'one'); + const rows = []; + await new Promise(function (resolve, reject) { + db.each( + 'SELECT a FROM t', + function (err, row) { + if (err) return reject(err); + rows.push(row); + }, + function (err) { + if (err) reject(err); + else resolve(); + }, + ); + }); + assert.deepStrictEqual(rows, [{ a: 1 }]); + await db.close(); + }); + }); + + describe('verbose() augments promise rejections', function () { + it('stack contains the calling frame', async function () { + const db = await openDb(); + sqlite3.verbose(); + await assert.rejects( + db.all('SELECT * FROM promises_no_such_table'), + function (err) { + assert.ok( + err.stack.includes('promises.test.js'), + `stack should mention the test file: ${err.stack}`, + ); + assert.match(err.stack, /Database#all/); + return true; + }, + ); + await db.close(); + }); + }); + + // Statement#map is the one JS-side method that reshapes its callback's + // arguments, so it is the one a refactor can silently change. On error + // it must hand back the error alone — an empty object there reads as a + // successful empty result. + describe('Statement#map error contract', function () { + it('passes the error alone, with no result object', async function () { + const db = await openDb(); + const stmt = db.prepare('SELECT a FROM t'); + await db.exec('DROP TABLE t'); + // Called through a reference: `stmt.map(...)` written out + // trips Biome's Array#map rule. + const mapRows = stmt.map.bind(stmt); + const args = await new Promise(function (resolve) { + mapRows(function (...called) { + resolve(called); + }); + }); + assert.strictEqual(args.length, 2); + assert.strictEqual(args[0].code, 'SQLITE_ERROR'); + assert.strictEqual(args[1], undefined); + await stmt.finalize(); + await db.close(); + }); + }); +}); diff --git a/test/rerun.test.js b/test/rerun.test.js index aa64fe4..e08596e 100644 --- a/test/rerun.test.js +++ b/test/rerun.test.js @@ -1,22 +1,26 @@ +import assert from 'node:assert'; +import { after, before, describe, it } from 'node:test'; + import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; -describe('rerunning statements', function() { +describe('rerunning statements', function () { let db; - before(function(done) { db = new sqlite3.Database(':memory:', done); }); + before(function (_t, done) { + db = new sqlite3.Database(':memory:', done); + }); - let count = 10; + const count = 10; let inserted = 0; let retrieved = 0; - it('should create the table', function(done) { - db.run("CREATE TABLE foo (id int)", done); + it('should create the table', function (_t, done) { + db.run('CREATE TABLE foo (id int)', done); }); - it('should insert repeatedly, reusing the same statement', function(done) { - let stmt = db.prepare("INSERT INTO foo VALUES(?)"); + it('should insert repeatedly, reusing the same statement', function (_t, done) { + const stmt = db.prepare('INSERT INTO foo VALUES(?)'); for (let i = 5; i < count; i++) { - stmt.run(i, function(err) { + stmt.run(i, function (err) { if (err) throw err; inserted++; }); @@ -24,27 +28,35 @@ describe('rerunning statements', function() { stmt.finalize(done); }); - it('should retrieve repeatedly, resuing the same statement', function(done) { - let collected = []; - let stmt = db.prepare("SELECT id FROM foo WHERE id = ?"); + it('should retrieve repeatedly, resuing the same statement', function (_t, done) { + const collected = []; + const stmt = db.prepare('SELECT id FROM foo WHERE id = ?'); for (let i = 0; i < count; i++) { - stmt.get(i, function(err, row) { + stmt.get(i, function (err, row) { if (err) throw err; if (row) collected.push(row); }); } - stmt.finalize(function(err) { + stmt.finalize(function (err) { if (err) throw err; retrieved += collected.length; - assert.deepEqual(collected, [ { id: 5 }, { id: 6 }, { id: 7 }, { id: 8 }, { id: 9 } ]); + assert.deepEqual(collected, [ + { id: 5 }, + { id: 6 }, + { id: 7 }, + { id: 8 }, + { id: 9 }, + ]); done(); }); }); - it('should have inserted and retrieved the right amount', function() { + it('should have inserted and retrieved the right amount', function () { assert.equal(inserted, 5); assert.equal(retrieved, 5); }); - after(function(done) { db.close(done); }); -}); \ No newline at end of file + after(function (_t, done) { + db.close(done); + }); +}); diff --git a/test/scheduling.test.js b/test/scheduling.test.js index 884049f..783fec1 100644 --- a/test/scheduling.test.js +++ b/test/scheduling.test.js @@ -1,44 +1,66 @@ -import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; +import assert from 'node:assert'; +import { describe, it } from 'node:test'; -describe('scheduling', function() { - it('scheduling after the database was closed', function(done) { - let db = new sqlite3.Database(':memory:'); - db.on('error', function(err) { - assert.ok(err.message && err.message.indexOf("SQLITE_MISUSE: Database handle is closed") > -1); - done(); - }); +import sqlite3 from '../lib/sqlite3.js'; +describe('scheduling', function () { + it('scheduling after the database was closed', function (_t, done) { + const db = new sqlite3.Database(':memory:'); + // A callback-less call is promise mode since v9, so the failure is + // a rejection rather than an 'error' event (the callback form is + // covered by the next test). db.close(); - db.run("CREATE TABLE foo (id int)"); + db.run('CREATE TABLE foo (id int)').then( + function () { + assert.fail('expected the run to fail'); + }, + function (err) { + assert.ok( + err.message && + err.message.indexOf( + 'SQLITE_MISUSE: Database handle is closed', + ) > -1, + ); + done(); + }, + ); }); - - it('scheduling a query with callback after the database was closed', function(done) { - let db = new sqlite3.Database(':memory:'); - db.on('error', function(err) { + it('scheduling a query with callback after the database was closed', function (_t, done) { + const db = new sqlite3.Database(':memory:'); + db.on('error', function (_err) { assert.ok(false, 'Event was accidentally triggered'); }); db.close(); - db.run("CREATE TABLE foo (id int)", function(err) { - assert.ok(err.message && err.message.indexOf("SQLITE_MISUSE: Database handle is closed") > -1); + db.run('CREATE TABLE foo (id int)', function (err) { + assert.ok( + err.message && + err.message.indexOf( + 'SQLITE_MISUSE: Database handle is closed', + ) > -1, + ); done(); }); }); - it('running a query after the database was closed', function(done) { - let db = new sqlite3.Database(':memory:'); + it('running a query after the database was closed', function (_t, done) { + const db = new sqlite3.Database(':memory:'); - let stmt = db.prepare("SELECT * FROM sqlite_master", function(err) { + const stmt = db.prepare('SELECT * FROM sqlite_master', function (err) { if (err) throw err; - db.close(function(err) { + db.close(function (err) { assert.ok(err); - assert.ok(err.message && err.message.indexOf("SQLITE_BUSY: unable to close due to") > -1); + assert.ok( + err.message && + err.message.indexOf( + 'SQLITE_BUSY: unable to close due to', + ) > -1, + ); // Running a statement now should not fail. stmt.run(done); }); }); }); -}); \ No newline at end of file +}); diff --git a/test/serialization.test.js b/test/serialization.test.js index fd16863..c7667cf 100644 --- a/test/serialization.test.js +++ b/test/serialization.test.js @@ -1,34 +1,39 @@ -import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; +import assert from 'node:assert'; +import { after, before, describe, it } from 'node:test'; +import sqlite3 from '../lib/sqlite3.js'; -describe('serialize() and parallelize()', function() { +describe('serialize() and parallelize()', function () { let db; - before(function(done) { db = new sqlite3.Database(':memory:', done); }); + before(function (_t, done) { + db = new sqlite3.Database(':memory:', done); + }); let inserted1 = 0; let inserted2 = 0; let retrieved = 0; - let count = 1000; + const count = 1000; - it('should toggle', function(done) { + it('should toggle', function (_t, done) { db.serialize(); - db.run("CREATE TABLE foo (txt text, num int, flt float, blb blob)"); + db.run('CREATE TABLE foo (txt text, num int, flt float, blb blob)'); db.parallelize(done); }); - it('should insert rows', function() { - let stmt1 = db.prepare("INSERT INTO foo VALUES(?, ?, ?, ?)"); - let stmt2 = db.prepare("INSERT INTO foo VALUES(?, ?, ?, ?)"); + it('should insert rows', function () { + const stmt1 = db.prepare('INSERT INTO foo VALUES(?, ?, ?, ?)'); + const stmt2 = db.prepare('INSERT INTO foo VALUES(?, ?, ?, ?)'); for (let i = 0; i < count; i++) { - // Interleaved inserts with two statements. - stmt1.run('String ' + i, i, i * Math.PI, function(err) { + // Interleaved inserts with two statements. The 4th parameter + // is bound explicitly: v9 rejects a parameter-count mismatch + // instead of silently binding the missing ones as NULL. + stmt1.run(`String ${i}`, i, i * Math.PI, null, function (err) { if (err) throw err; inserted1++; }); i++; - stmt2.run('String ' + i, i, i * Math.PI, function(err) { + stmt2.run(`String ${i}`, i, i * Math.PI, null, function (err) { if (err) throw err; inserted2++; }); @@ -37,68 +42,84 @@ describe('serialize() and parallelize()', function() { stmt2.finalize(); }); - it('should have inserted all the rows after synchronizing with serialize()', function(done) { + it('should have inserted all the rows after synchronizing with serialize()', function (_t, done) { db.serialize(); - db.all("SELECT txt, num, flt, blb FROM foo ORDER BY num", function(err, rows) { - if (err) throw err; - for (let i = 0; i < rows.length; i++) { - assert.equal(rows[i].txt, 'String ' + i); - assert.equal(rows[i].num, i); - assert.equal(rows[i].flt, i * Math.PI); - assert.equal(rows[i].blb, null); - retrieved++; - } + db.all( + 'SELECT txt, num, flt, blb FROM foo ORDER BY num', + function (err, rows) { + if (err) throw err; + for (let i = 0; i < rows.length; i++) { + assert.equal(rows[i].txt, `String ${i}`); + assert.equal(rows[i].num, i); + assert.equal(rows[i].flt, i * Math.PI); + assert.equal(rows[i].blb, null); + retrieved++; + } - assert.equal(count, inserted1 + inserted2, "Didn't insert all rows"); - assert.equal(count, retrieved, "Didn't retrieve all rows"); - done(); - }); + assert.equal( + count, + inserted1 + inserted2, + "Didn't insert all rows", + ); + assert.equal(count, retrieved, "Didn't retrieve all rows"); + done(); + }, + ); }); - after(function(done) { db.close(done); }); + after(function (_t, done) { + db.close(done); + }); }); -describe('serialize(fn)', function() { +describe('serialize(fn)', function () { let db; - before(function(done) { db = new sqlite3.Database(':memory:', done); }); + before(function (_t, done) { + db = new sqlite3.Database(':memory:', done); + }); let inserted = 0; let retrieved = 0; - let count = 1000; + const count = 1000; - it('should call the callback', function(done) { - db.serialize(function() { - db.run("CREATE TABLE foo (txt text, num int, flt float, blb blob)"); + it('should call the callback', function (_t, done) { + db.serialize(function () { + db.run('CREATE TABLE foo (txt text, num int, flt float, blb blob)'); - let stmt = db.prepare("INSERT INTO foo VALUES(?, ?, ?, ?)"); + const stmt = db.prepare('INSERT INTO foo VALUES(?, ?, ?, ?)'); for (let i = 0; i < count; i++) { - stmt.run('String ' + i, i, i * Math.PI, function(err) { + // 4th parameter bound explicitly; v9 rejects mismatches. + stmt.run(`String ${i}`, i, i * Math.PI, null, function (err) { if (err) throw err; inserted++; }); } stmt.finalize(); - db.all("SELECT txt, num, flt, blb FROM foo ORDER BY num", function(err, rows) { - if (err) throw err; - for (let i = 0; i < rows.length; i++) { - assert.equal(rows[i].txt, 'String ' + i); - assert.equal(rows[i].num, i); - assert.equal(rows[i].flt, i * Math.PI); - assert.equal(rows[i].blb, null); - retrieved++; - } - done(); - }); + db.all( + 'SELECT txt, num, flt, blb FROM foo ORDER BY num', + function (err, rows) { + if (err) throw err; + for (let i = 0; i < rows.length; i++) { + assert.equal(rows[i].txt, `String ${i}`); + assert.equal(rows[i].num, i); + assert.equal(rows[i].flt, i * Math.PI); + assert.equal(rows[i].blb, null); + retrieved++; + } + done(); + }, + ); }); }); - - it('should have inserted and retrieved all rows', function() { + it('should have inserted and retrieved all rows', function () { assert.equal(count, inserted, "Didn't insert all rows"); assert.equal(count, retrieved, "Didn't retrieve all rows"); }); - after(function(done) { db.close(done); }); -}); \ No newline at end of file + after(function (_t, done) { + db.close(done); + }); +}); diff --git a/test/serialize_bytes.test.js b/test/serialize_bytes.test.js new file mode 100644 index 0000000..6bb0a21 --- /dev/null +++ b/test/serialize_bytes.test.js @@ -0,0 +1,230 @@ +// serializeToBytes / deserializeFromBytes (Deliverable 08): round trips +// of real data (the marshalling corpus: integers including unsafe ones, +// floats, text, blobs, nulls), the copy semantics, corrupt input, the +// readonly/resizable options, and the naming discipline (serialize means +// FIFO ordering, the byte form is serializeToBytes). +import assert from 'node:assert'; +import { describe, it } from 'node:test'; + +import sqlite3 from '../lib/sqlite3.js'; + +describe('serializeToBytes / deserializeFromBytes', function () { + it('round-trips a database with every value shape', async function () { + const db = await sqlite3.open(':memory:'); + await db.exec(` + CREATE TABLE t ( + id INTEGER PRIMARY KEY, + n REAL, + s TEXT, + b BLOB + ); + `); + const big = 9007199254740993n; // 2^53 + 1, unsafe as a number + await db.run( + 'INSERT INTO t VALUES (?, ?, ?, ?)', + 1, + 1.5, + 'héllo', + new Uint8Array([1, 2, 3, 0, 255]), + ); + await db.run('INSERT INTO t VALUES (?, ?, ?, ?)', 2, null, null, null); + await db.run( + 'INSERT INTO t VALUES (?, ?, ?, ?)', + 3, + -0.0, + '', + new Uint8Array(0), + ); + // An unsafe integer rowid, stored exactly through the bigint + // bind. + await db.run('INSERT INTO t (id, n) VALUES (?, ?)', big, 0); + db.configure('integerMode', 'mixed'); + + const bytes = await db.serializeToBytes(); + assert.ok(bytes instanceof Uint8Array, 'returns a Uint8Array'); + assert.ok(!Buffer.isBuffer(bytes), 'not a Buffer'); + assert.ok(bytes.length > 0); + // A serialized database starts with the SQLite magic. + assert.strictEqual( + Buffer.from(bytes.buffer, bytes.byteOffset, 16).toString('latin1'), + 'SQLite format 3\u0000', + ); + + const copy = await sqlite3.deserializeFromBytes(bytes); + copy.configure('integerMode', 'mixed'); + const rows = await copy.all('SELECT * FROM t ORDER BY id'); + assert.strictEqual(rows.length, 4); + // (Blobs read back as Buffer — the established read-side type.) + assert.deepStrictEqual( + rows.map((r) => [typeof r.id, r.n, r.s, r.b?.length]), + [ + ['number', 1.5, 'héllo', 5], + ['number', null, null, undefined], + ['number', 0, '', 0], // -0.0 stores as 0 + ['bigint', 0, null, undefined], + ], + ); + assert.deepStrictEqual([...rows[0].b], [1, 2, 3, 0, 255]); + // The unsafe rowid survived the round trip exactly. + assert.strictEqual(rows[3].id, big); + await copy.close(); + await db.close(); + }); + + it('round-trips a 10k-row database with identical queries', async function () { + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)'); + await db.transaction(async () => { + const stmt = db.prepare('INSERT INTO t VALUES (?, ?)'); + for (let i = 1; i <= 10000; i++) { + await stmt.run(i, `row-${i}`); + } + await stmt.finalize(); + }); + + const bytes = await db.serializeToBytes(); + const copy = await sqlite3.deserializeFromBytes(bytes); + assert.strictEqual( + (await copy.get('SELECT count(*) c FROM t')).c, + 10000, + ); + assert.deepStrictEqual( + await copy.all('SELECT * FROM t WHERE id % 997 = 0 ORDER BY id'), + await db.all('SELECT * FROM t WHERE id % 997 = 0 ORDER BY id'), + ); + assert.deepStrictEqual( + await copy.all('SELECT v FROM t ORDER BY RANDOM() LIMIT 0'), + [], + ); + await copy.close(); + await db.close(); + }); + + it('deserializeFromBytes copies — the input stays usable afterwards', async function () { + const db = await sqlite3.open(':memory:'); + await db.exec( + "CREATE TABLE t (v TEXT); INSERT INTO t VALUES ('before')", + ); + const bytes = await db.serializeToBytes(); + await db.close(); + + const copy = await sqlite3.deserializeFromBytes(bytes); + // Mutating the source bytes must not affect the deserialized db. + bytes.fill(0); + assert.strictEqual((await copy.get('SELECT v FROM t')).v, 'before'); + await copy.close(); + }); + + it('deserializeFromBytes on corrupt bytes rejects with SQLITE_NOTADB', { + timeout: 30000, + }, async function () { + const garbage = new Uint8Array(4096); + for (let i = 0; i < garbage.length; i++) garbage[i] = (i * 7) & 0xff; + await assert.rejects( + () => sqlite3.deserializeFromBytes(garbage), + (err) => { + assert.strictEqual(err.code, 'SQLITE_NOTADB'); + assert.strictEqual(err.errno, sqlite3.NOTADB); + return true; + }, + ); + // Truncated-but-valid-header input also fails, not crashes. + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (v)'); + const bytes = await db.serializeToBytes(); + await db.close(); + await assert.rejects( + () => sqlite3.deserializeFromBytes(bytes.subarray(0, 100)), + (err) => err instanceof Error, + ); + }); + + it('readOnly rejects writes; resizable lets the copy grow', async function () { + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (v)'); + const bytes = await db.serializeToBytes(); + await db.close(); + + const ro = await sqlite3.deserializeFromBytes(bytes, { + readOnly: true, + }); + await assert.rejects( + () => ro.run("INSERT INTO t VALUES ('x')"), + (err) => err.primaryCode === 'SQLITE_READONLY', + ); + await ro.close(); + + const rw = await sqlite3.deserializeFromBytes(bytes, { + resizable: true, + }); + for (let i = 0; i < 200; i++) { + await rw.run('INSERT INTO t VALUES (?)', `row-${i}`); + } + assert.strictEqual((await rw.get('SELECT count(*) c FROM t')).c, 200); + // Growth survives a second round trip. + const grown = await rw.serializeToBytes(); + const again = await sqlite3.deserializeFromBytes(grown); + assert.strictEqual( + (await again.get('SELECT count(*) c FROM t')).c, + 200, + ); + await rw.close(); + await again.close(); + }); + + it('a deserialized db is a normal connection (events, close, transactions)', async function () { + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (v)'); + const bytes = await db.serializeToBytes(); + await db.close(); + + const copy = await sqlite3.deserializeFromBytes(bytes); + const commits = []; + copy.on('commit', () => commits.push(1)); + await copy.transaction(async () => { + await copy.run("INSERT INTO t VALUES ('in tx')"); + }); + await new Promise((resolve) => setTimeout(resolve, 25)); + assert.strictEqual(commits.length, 1); + assert.strictEqual((await copy.get('SELECT count(*) c FROM t')).c, 1); + await copy.close(); + }); + + it('serializeToBytes takes the schema name', async function () { + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (v)'); + // serializing a nonexistent schema is an error, not a crash + await assert.rejects( + () => db.serializeToBytes('nosuch'), + (err) => err instanceof Error, + ); + await db.close(); + }); + + it('accepts ArrayBuffer and DataView inputs, honouring offsets', async function () { + const db = await sqlite3.open(':memory:'); + await db.exec("CREATE TABLE t (v); INSERT INTO t VALUES ('data')"); + const bytes = await db.serializeToBytes(); + await db.close(); + + // The bytes embedded at an offset inside a larger buffer. + const padded = new Uint8Array(bytes.length + 16); + padded.set(bytes, 16); + const asDataView = new DataView(padded.buffer, 16, bytes.length); + const copy = await sqlite3.deserializeFromBytes(asDataView); + assert.strictEqual((await copy.get('SELECT v FROM t')).v, 'data'); + await copy.close(); + }); + + it('keeps the FIFO serialize() name distinct from serializeToBytes()', async function () { + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (v)'); + // serialize() is still the FIFO-ordering control, returning this. + assert.strictEqual(db.serialize(), db); + assert.strictEqual(db.parallelize(), db); + // serializeToBytes is the byte snapshot, returning bytes. + const bytes = await db.serializeToBytes(); + assert.ok(bytes instanceof Uint8Array); + await db.close(); + }); +}); diff --git a/test/session.test.js b/test/session.test.js new file mode 100644 index 0000000..8b7a2de --- /dev/null +++ b/test/session.test.js @@ -0,0 +1,624 @@ +// Sessions and changesets (Deliverable 08): capture, harvest (changeset +// vs patchset), apply with each conflict policy, invert/concat/iterate, +// the preupdate event, and the lifetime rules (close idempotency, a +// session left open at close(), the shared preupdate-hook slot). +import assert from 'node:assert'; +import { execFile } from 'node:child_process'; +import { afterEach, beforeEach, describe, it } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import sqlite3 from '../lib/sqlite3.js'; + +const SETUP = 'CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)'; + +/** Opens a memory database with the shared schema. */ +async function openDb() { + const db = await sqlite3.open(':memory:'); + await db.exec(SETUP); + return db; +} + +describe('sessions and changesets', function () { + /** @type {sqlite3.Database} */ + let db; + + beforeEach(async function () { + db = await openDb(); + }); + + afterEach(async function () { + await db.close(); + }); + + it('captures insert, update and delete into a changeset', async function () { + await db.exec("INSERT INTO t VALUES (1, 'a'), (2, 'b')"); + const session = db.session(); + await db.exec("UPDATE t SET v = 'B' WHERE id = 2"); + await db.exec("INSERT INTO t VALUES (3, 'c')"); + await db.exec('DELETE FROM t WHERE id = 1'); + const changeset = await session.changeset(); + await session.close(); + + assert.ok(changeset instanceof Uint8Array, 'a Uint8Array'); + assert.ok(!Buffer.isBuffer(changeset), 'not a Buffer'); + assert.ok(changeset.length > 0); + + const ops = [...sqlite3.iterateChangeset(changeset)]; + assert.strictEqual(ops.length, 3); + assert.deepStrictEqual( + ops.map((op) => op.op), + ['delete', 'update', 'insert'], + ); + // DELETE carries the full old row. + assert.deepStrictEqual(ops[0].oldRow, [1, 'a']); + assert.strictEqual(ops[0].newRow, undefined); + // UPDATE carries old (pk + modified) and new (modified only). + assert.deepStrictEqual(ops[1].oldRow, [2, 'b']); + assert.deepStrictEqual(ops[1].newRow, [null, 'B']); + // INSERT carries only the new row. + assert.strictEqual(ops[2].oldRow, undefined); + assert.deepStrictEqual(ops[2].newRow, [3, 'c']); + // The pk positions travel with the change. + assert.deepStrictEqual(ops[0].primaryKey, [true, false]); + }); + + it('records only the attached table', async function () { + await db.exec('CREATE TABLE other (id INTEGER PRIMARY KEY)'); + await db.exec('INSERT INTO t VALUES (1, NULL)'); + await db.exec('INSERT INTO other VALUES (1)'); + const session = db.session({ table: 't' }); + await db.exec("INSERT INTO t VALUES (2, 'x')"); + await db.exec('INSERT INTO other VALUES (2)'); + const changeset = await session.changeset(); + await session.close(); + const tables = [ + ...new Set( + [...sqlite3.iterateChangeset(changeset)].map((op) => op.table), + ), + ]; + assert.deepStrictEqual(tables, ['t']); + }); + + it('applies a changeset and leaves both databases identical', async function () { + await db.exec("INSERT INTO t VALUES (1, 'a'), (2, 'b')"); + const before = await db.serializeToBytes(); + + const session = db.session(); + await db.exec("UPDATE t SET v = 'B' WHERE id = 2"); + await db.exec("INSERT INTO t VALUES (3, 'c')"); + await db.exec('DELETE FROM t WHERE id = 1'); + const changeset = await session.changeset(); + await session.close(); + + const target = await sqlite3.deserializeFromBytes(before); + await target.applyChangeset(changeset); + + assert.deepStrictEqual( + await target.all('SELECT * FROM t ORDER BY id'), + await db.all('SELECT * FROM t ORDER BY id'), + ); + // Identical content, not just rows: the schema tables must + // match too. (Byte-equality of the serializations is too strict + // after an apply — SQLite's header change counter legitimately + // differs between a db written directly and one built by + // replaying a changeset.) + assert.deepStrictEqual( + await target.all( + 'SELECT type, name, sql FROM sqlite_schema ORDER BY name', + ), + await db.all( + 'SELECT type, name, sql FROM sqlite_schema ORDER BY name', + ), + ); + await target.close(); + }); + + it('rolls the whole apply back when the policy is abort', async function () { + const session = db.session(); + await db.exec("INSERT INTO t VALUES (1, 'a')"); + await db.exec("INSERT INTO t VALUES (2, 'b')"); + const changeset = await session.changeset(); + await session.close(); + + // The target already contains id=1: applying the first insert + // conflicts, and 'abort' must undo the whole apply — id=2 (which + // would have applied cleanly) must not survive either. + const target = await openDb(); + await target.exec("INSERT INTO t VALUES (1, 'target')"); + await assert.rejects( + () => target.applyChangeset(changeset), + (err) => { + assert.strictEqual(err.code, 'SQLITE_ABORT'); + return true; + }, + ); + const rows = await target.all('SELECT * FROM t ORDER BY id'); + assert.deepStrictEqual(rows, [{ id: 1, v: 'target' }]); + await target.close(); + }); + + it('skips conflicting changes with the omit policy', async function () { + const session = db.session(); + await db.exec("INSERT INTO t VALUES (1, 'recorded')"); + const changeset = await session.changeset(); + await session.close(); + + const target = await openDb(); + await target.exec("INSERT INTO t VALUES (1, 'existing')"); + await target.applyChangeset(changeset, { conflict: 'omit' }); + const row = await target.get('SELECT v FROM t WHERE id = 1'); + assert.strictEqual(row.v, 'existing'); + await target.close(); + }); + + it('overwrites conflicting rows with the replace policy', async function () { + const session = db.session(); + await db.exec("INSERT INTO t VALUES (1, 'recorded')"); + const changeset = await session.changeset(); + await session.close(); + + const target = await openDb(); + await target.exec("INSERT INTO t VALUES (1, 'existing')"); + await target.applyChangeset(changeset, { conflict: 'replace' }); + const row = await target.get('SELECT v FROM t WHERE id = 1'); + assert.strictEqual(row.v, 'recorded'); + await target.close(); + }); + + it('runs the JS conflict handler with materialised rows', async function () { + const session = db.session(); + await db.exec("INSERT INTO t VALUES (2, 'recorded')"); + const changeset = await session.changeset(); + await session.close(); + + const target = await openDb(); + await target.exec("INSERT INTO t VALUES (2, 'existing')"); + /** @type {import('../lib/native.js').ChangesetConflict[]} */ + const seen = []; + await target.applyChangeset(changeset, { + conflict: (info) => { + seen.push(info); + assert.strictEqual(info.op, 'insert'); + assert.strictEqual(info.table, 't'); + assert.strictEqual(info.conflict, 'conflict'); + assert.deepStrictEqual(info.conflictRow.map(String), [ + '2', + 'existing', + ]); + assert.deepStrictEqual( + info.newRow.map((v) => (v === null ? null : String(v))), + ['2', 'recorded'], + ); + return 'replace'; + }, + }); + assert.strictEqual(seen.length, 1); + const row = await target.get('SELECT v FROM t WHERE id = 2'); + assert.strictEqual(row.v, 'recorded'); + await target.close(); + }); + + // Nothing serialises applies, so two with JS handlers can overlap. + // The gate that keeps main-thread sqlite calls deferring while an + // apply holds the connection mutex must therefore count applies, not + // flag them: cleared by whichever finished first, it declared the + // connection safe while the second was still inside + // sqlite3changeset_apply, and the next main-thread sqlite call + // blocked on a mutex whose owner was waiting on this thread. That is + // a hard deadlock — this test hangs rather than fails when it + // regresses, which is why it carries an explicit timeout. + it('two overlapping applies with JS handlers keep the deferral honest', { + timeout: 30000, + }, async function () { + const ROWS = 400; + /** Builds a changeset inserting ROWS rows tagged with `seed`. */ + async function changesetOf(seed) { + const src = await openDb(); + const session = src.session(); + await src.exec('BEGIN'); + for (let i = 1; i <= ROWS; i++) { + await src.run('INSERT INTO t VALUES (?, ?)', [ + i, + `${seed}${i}`, + ]); + } + await src.exec('COMMIT'); + const bytes = await session.changeset(); + await session.close(); + await src.close(); + return bytes; + } + + const first = await changesetOf('a'); + const second = await changesetOf('b'); + + // Every row conflicts, so both applies call their JS handler + // hundreds of times and overlap for a good while. + await db.exec('BEGIN'); + for (let i = 1; i <= ROWS; i++) { + await db.run('INSERT INTO t VALUES (?, ?)', [i, 'orig']); + } + await db.exec('COMMIT'); + + const applyFirst = db.applyChangeset(first, { + conflict: () => 'replace', + }); + const applySecond = db.applyChangeset(second, { + conflict: () => 'replace', + }); + + // The moment one apply finishes, issue a main-thread sqlite call + // while the other is still running. On the unfixed build this + // never returns. + const configured = applyFirst.then(() => { + db.configure('busyTimeout', 1234); + }); + + await Promise.all([applyFirst, applySecond, configured]); + // The connection is still usable afterwards. + const row = await db.get('SELECT count(*) AS n FROM t'); + assert.strictEqual(row.n, ROWS); + }); + + it('aborts and reports when the JS conflict handler throws', async function () { + const session = db.session(); + await db.exec("INSERT INTO t VALUES (2, 'recorded')"); + const changeset = await session.changeset(); + await session.close(); + + const target = await openDb(); + await target.exec("INSERT INTO t VALUES (2, 'existing')"); + await assert.rejects( + () => + target.applyChangeset(changeset, { + conflict: () => { + throw new Error('decide better'); + }, + }), + (err) => { + assert.match(err.message, /decide better/); + assert.strictEqual(err.code, 'SQLITE_ABORT'); + return true; + }, + ); + const row = await target.get('SELECT v FROM t WHERE id = 2'); + assert.strictEqual(row.v, 'existing'); + await target.close(); + }); + + it('excludes tables the filter refuses', async function () { + await db.exec('CREATE TABLE audit (id INTEGER PRIMARY KEY)'); + await db.exec('INSERT INTO t VALUES (1, NULL)'); + await db.exec('INSERT INTO audit VALUES (1)'); + const session = db.session(); + await db.exec("INSERT INTO t VALUES (2, 'x')"); + await db.exec('INSERT INTO audit VALUES (2)'); + const changeset = await session.changeset(); + await session.close(); + + const target = await openDb(); + await target.exec('CREATE TABLE audit (id INTEGER PRIMARY KEY)'); + /** @type {string[]} */ + const offered = []; + await target.applyChangeset(changeset, { + filter: (table) => { + offered.push(table); + return table !== 'audit'; + }, + }); + assert.deepStrictEqual(offered.sort(), ['audit', 't']); + assert.strictEqual( + (await target.get('SELECT count(*) c FROM audit')).c, + 0, + ); + assert.strictEqual((await target.get('SELECT count(*) c FROM t')).c, 1); + await target.close(); + }); + + it('inverts a changeset so applying the inverse undoes it', async function () { + await db.exec("INSERT INTO t VALUES (1, 'a')"); + const before = await db.serializeToBytes(); + const session = db.session(); + await db.exec("INSERT INTO t VALUES (2, 'b')"); + await db.exec("UPDATE t SET v = 'A' WHERE id = 1"); + const changeset = await session.changeset(); + await session.close(); + + const inverse = sqlite3.invertChangeset(changeset); + assert.ok(inverse instanceof Uint8Array); + const ops = [...sqlite3.iterateChangeset(inverse)]; + assert.deepStrictEqual( + ops.map((op) => op.op), + ['update', 'delete'], + ); + + // Apply to a copy of the pre-session state, then the inverse on + // top: the result must be the pre-session bytes again. + const target = await sqlite3.deserializeFromBytes(before); + await target.applyChangeset(changeset); + assert.strictEqual((await target.get('SELECT count(*) c FROM t')).c, 2); + await target.applyChangeset(inverse); + // Content-identical to the pre-session state (byte equality is + // out for the change-counter reason above). + assert.deepStrictEqual( + await target.all('SELECT * FROM t ORDER BY id'), + [{ id: 1, v: 'a' }], + ); + await target.close(); + }); + + it('concatenates two changesets', async function () { + const sessionA = db.session(); + await db.exec("INSERT INTO t VALUES (1, 'a')"); + const csA = await sessionA.changeset(); + + const sessionB = db.session(); + await db.exec("INSERT INTO t VALUES (2, 'b')"); + const csB = await sessionB.changeset(); + await sessionA.close(); + await sessionB.close(); + + const combined = sqlite3.concatChangeset(csA, csB); + const target = await openDb(); + await target.applyChangeset(combined); + assert.deepStrictEqual( + await target.all('SELECT * FROM t ORDER BY id'), + [ + { id: 1, v: 'a' }, + { id: 2, v: 'b' }, + ], + ); + await target.close(); + }); + + it('rejects garbage instead of undefined behaviour', async function () { + const garbage = new Uint8Array(64).fill(0x5a); + assert.throws( + () => sqlite3.invertChangeset(garbage), + (err) => err instanceof Error, + ); + assert.throws( + () => sqlite3.concatChangeset(garbage, garbage), + (err) => err instanceof Error, + ); + assert.throws( + () => [...sqlite3.iterateChangeset(garbage)], + (err) => err instanceof Error, + ); + }); + + it('honours byteOffset on typed-array inputs', async function () { + await db.exec("INSERT INTO t VALUES (1, 'a')"); + const session = db.session(); + await db.exec("INSERT INTO t VALUES (2, 'b')"); + const changeset = await session.changeset(); + await session.close(); + + // Embed the changeset at an offset inside a larger buffer. + const padded = new Uint8Array(changeset.length + 8); + padded.set(changeset, 8); + const view = padded.subarray(8); + const inverse = sqlite3.invertChangeset(view); + assert.strictEqual(inverse.length, changeset.length); + }); + + it('distinguishes a patchset from a changeset', async function () { + await db.exec("INSERT INTO t VALUES (1, 'a'), (2, 'b')"); + const session = db.session(); + await db.exec("UPDATE t SET v = 'B' WHERE id = 1"); + const changeset = await session.changeset(); + const patchset = await session.patchset(); + await session.close(); + + // A changeset stores old+new for an update; a patchset only the + // new values plus the primary key, so it is smaller and cannot + // detect data conflicts on non-key columns. + assert.ok(patchset.length < changeset.length); + const csOps = [...sqlite3.iterateChangeset(changeset)]; + const psOps = [...sqlite3.iterateChangeset(patchset)]; + assert.deepStrictEqual(csOps[0].oldRow, [1, 'a']); + // The patchset's "old" side carries only the primary key. + assert.deepStrictEqual(psOps[0].oldRow, [1, null]); + assert.deepStrictEqual(psOps[0].newRow, [null, 'B']); + }); + + it('close is idempotent and later calls fail with MISUSE', async function () { + const session = db.session(); + await session.close(); + assert.strictEqual(session.closed, true); + await session.close(); // benign no-op + await assert.rejects( + () => session.changeset(), + (err) => { + assert.strictEqual(err.code, 'SQLITE_MISUSE'); + return true; + }, + ); + }); + + it('a session left open at close() neither crashes nor leaks', { + timeout: 30000, + }, async function () { + const standalone = await openDb(); + await standalone.exec("INSERT INTO t VALUES (1, 'a')"); + const session = standalone.session(); + await standalone.exec("INSERT INTO t VALUES (2, 'b')"); + await standalone.close(); + // The connection closed the session underneath; the wrapper is + // inert and says so. + assert.strictEqual(session.closed, true); + await assert.rejects( + () => session.changeset(), + (err) => { + assert.strictEqual(err.code, 'SQLITE_MISUSE'); + return true; + }, + ); + // The process still exits cleanly afterwards (a leaked session + // handle would wedge sqlite3_close or hang teardown). + }); + + it('marking changes indirect flags them in the changeset', async function () { + const session = db.session({ indirect: true }); + await db.exec("INSERT INTO t VALUES (1, 'a')"); + const changeset = await session.changeset(); + await session.close(); + const ops = [...sqlite3.iterateChangeset(changeset)]; + assert.strictEqual(ops[0].indirect, true); + }); + + it('using after `using` disposes the session', async function () { + { + using session = db.session(); + await db.exec("INSERT INTO t VALUES (1, 'a')"); + const changeset = await session.changeset(); + assert.ok(changeset.length > 0); + } + // Disposed: further work fails. + const session2 = db.session(); + await session2.close(); + }); +}); + +describe('environment teardown', function () { + // The addon's class constructors must live in per-environment + // instance data, not in file statics: a static Napi::Reference is + // destroyed at process exit, after the environment is gone, so its + // napi_delete_reference lands on a dead env. That segfaulted at exit + // on musl while glibc and macOS tolerated it — a crash no assertion + // inside the process can observe, so the child's exit status is the + // test. + it('exits cleanly with a database and session left open', { + timeout: 15000, + }, async function () { + const child = execFile( + process.execPath, + [ + fileURLToPath( + new URL( + './support/teardown_exit_child.mjs', + import.meta.url, + ), + ), + ], + { cwd: process.cwd() }, + ); + let stdout = ''; + child.stdout?.on('data', (c) => (stdout += c)); + const code = await new Promise((resolve) => { + child.on('close', resolve); + }); + assert.strictEqual(code, 0, `child exited with ${code}`); + assert.match(stdout, /CHILD-EXITING/); + }); +}); + +describe('preupdate events', function () { + /** @type {sqlite3.Database} */ + let db; + + beforeEach(async function () { + db = await openDb(); + }); + + afterEach(async function () { + await db.close(); + }); + + it('INSERT gives newRow only, DELETE oldRow only, UPDATE both', async function () { + /** @type {import('../lib/native.js').PreupdateEventInfo[]} */ + const events = []; + db.on('preupdate', (info) => events.push(info)); + await db.exec("INSERT INTO t VALUES (1, 'a')"); + await db.exec("UPDATE t SET v = 'b' WHERE id = 1"); + await db.exec('DELETE FROM t WHERE id = 1'); + await new Promise((resolve) => setTimeout(resolve, 25)); + + assert.strictEqual(events.length, 3); + const [insert, update, remove] = events; + assert.strictEqual(insert.op, 'insert'); + assert.strictEqual(insert.oldRow, null); + assert.deepStrictEqual(insert.newRow, [1, 'a']); + assert.strictEqual(insert.oldRowid, null); + assert.strictEqual(insert.rowid, 1); + + assert.strictEqual(update.op, 'update'); + assert.deepStrictEqual(update.oldRow, [1, 'a']); + assert.deepStrictEqual(update.newRow, [1, 'b']); + assert.strictEqual(update.oldRowid, 1); + assert.strictEqual(update.rowid, 1); + + assert.strictEqual(remove.op, 'delete'); + assert.deepStrictEqual(remove.oldRow, [1, 'b']); + assert.strictEqual(remove.newRow, null); + assert.strictEqual(remove.oldRowid, 1); + }); + + it('oldRowid differs from rowid on a rowid-changing update', async function () { + /** @type {import('../lib/native.js').PreupdateEventInfo[]} */ + const events = []; + db.on('preupdate', (info) => events.push(info)); + await db.exec("INSERT INTO t (rowid, v) VALUES (10, 'x')"); + await db.exec('UPDATE t SET rowid = 20 WHERE rowid = 10'); + await new Promise((resolve) => setTimeout(resolve, 25)); + + const moved = events.find( + (info) => info.op === 'update' && info.oldRowid !== info.rowid, + ); + assert.ok(moved, 'a rowid-changing update event exists'); + assert.strictEqual(moved.oldRowid, 10); + assert.strictEqual(moved.rowid, 20); + }); + + it('fires for the writes a changeset apply performs', async function () { + const source = await openDb(); + await source.exec("INSERT INTO t VALUES (1, 'a')"); + const session = source.session(); + await source.exec("INSERT INTO t VALUES (2, 'b')"); + const changeset = await session.changeset(); + await session.close(); + + /** @type {import('../lib/native.js').PreupdateEventInfo[]} */ + const events = []; + db.on('preupdate', (info) => events.push(info)); + await db.applyChangeset(changeset); + await new Promise((resolve) => setTimeout(resolve, 25)); + assert.strictEqual( + events.filter((info) => info.op === 'insert').length, + 1, + ); + await source.close(); + }); + + it('shares one hook slot with sessions and refuses loudly both ways', async function () { + /** @type {import('../lib/native.js').PreupdateEventInfo[]} */ + const heard = []; + const listener = (info) => heard.push(info); + db.on('preupdate', listener); + assert.throws(() => db.session(), /single preupdate hook/); + assert.strictEqual(heard.length, 0); // nothing fired yet + db.removeListener('preupdate', listener); + + // The other direction: a session is open, so the registration + // fails (reported on the connection's 'error' event) instead of + // silently stopping the session's recording. + const session = db.session(); + await db.exec("INSERT INTO t VALUES (1, 'a')"); // session records + const errors = []; + db.on('error', (err) => errors.push(err)); + const heardDuringSession = []; + db.on('preupdate', (info) => heardDuringSession.push(info)); + await new Promise((resolve) => setTimeout(resolve, 25)); + assert.strictEqual(errors.length, 1); + // The rejected registration took no events (the session kept the + // slot), so nothing arrived on the listener that lost it. + assert.strictEqual(heardDuringSession.length, 0); + assert.match(errors[0].message, /single preupdate hook/); + + // And the session still records — the failed registration did + // not take the slot from it. + const changeset = await session.changeset(); + assert.strictEqual([...sqlite3.iterateChangeset(changeset)].length, 1); + await session.close(); + }); +}); diff --git a/test/state_machine.test.js b/test/state_machine.test.js new file mode 100644 index 0000000..93a19bc --- /dev/null +++ b/test/state_machine.test.js @@ -0,0 +1,307 @@ +// Regression tests for the native state machine (Deliverable 05): the +// Backup call guard, the db.state accessor, the statement cache's +// serialize guard, and finalize-on-GC safety. + +import assert from 'node:assert'; +import { spawn } from 'node:child_process'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import sqlite3 from '../lib/sqlite3.js'; + +// Runs the throwing-backup scenario in a child process; see the helper's +// header comment for why this is not done in-process. +describe('backup call guard', function () { + it('a throwing step callback does not wedge the connection', { + timeout: 30000, + }, async function () { + // Absolute path and a joined cwd, both built with node:path: the + // previous form stripped the trailing directory with /\/test$/, + // which never matches a Windows path, leaving cwd inside test/ so + // the relative argument resolved to test\test\support\… and the + // child died with MODULE_NOT_FOUND. Nothing caught it because the + // suite does not run on Windows in CI — only the Electron job does. + const child = spawn( + process.execPath, + [join(import.meta.dirname, 'support', 'throwing_backup_child.mjs')], + { cwd: join(import.meta.dirname, '..') }, + ); + let out = ''; + child.stdout.on('data', (chunk) => { + out += chunk; + }); + child.stderr.on('data', (chunk) => { + out += chunk; + }); + const code = await new Promise((resolve) => { + child.on('close', resolve); + }); + assert.strictEqual(code, 0, `child exited ${code}; output:\n${out}`); + assert.match(out, /GETSYNC_OK/); + assert.match(out, /CLOSE_OK/); + assert.match(out, /UNCAUGHT:step callback boom/); + }); +}); + +describe('db.state', function () { + it('is a frozen snapshot of exactly the six scheduling fields', async function () { + const db = await sqlite3.open(':memory:'); + const state = db.state; + assert.deepStrictEqual(Object.keys(state).sort(), [ + 'closing', + 'locked', + 'open', + 'pending', + 'queued', + 'serialized', + ]); + assert.ok(Object.isFrozen(state)); + // A fresh read must reflect changes, not a cached snapshot. + db.serialize(); + assert.strictEqual(db.state.serialized, true); + assert.notStrictEqual(state.serialized, db.state.serialized); + db.parallelize(); + assert.strictEqual(db.state.serialized, false); + await db.close(); + }); + + it('reflects an idle open connection', async function () { + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (a INT)'); + // locked is honest since DbState/exclusiveHeld: false once the + // exclusive exec completed (it used to be sticky history). + assert.deepStrictEqual(db.state, { + open: true, + closing: false, + locked: false, + serialized: false, + pending: 0, + queued: 0, + }); + await db.close(); + }); + + it('shows an exclusive operation in flight immediately after exec()', async function () { + const db = await sqlite3.open(':memory:'); + // exec() with an idle queue dispatches synchronously: locked is + // set and pending incremented before exec() returns. + const done = new Promise((resolve) => + db.exec('CREATE TABLE t (a INT)', resolve), + ); + assert.strictEqual(db.state.locked, true); + assert.strictEqual(db.state.pending, 1); + assert.strictEqual(db.state.queued, 0); + await done; + // Honest release: locked drops when the exclusive call completes + // (it used to stay true until the next dispatch). + assert.deepStrictEqual(db.state, { + open: true, + closing: false, + locked: false, + serialized: false, + pending: 0, + queued: 0, + }); + await db.close(); + }); + + it('shows queued work while a statement operation is in flight', async function () { + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (a INT)'); + // A statement run is in flight (pending > 0, bypassing the + // database queue), and an exclusive exec behind it is queued. + const ran = new Promise((resolve) => + db.run('INSERT INTO t VALUES (1)', resolve), + ); + const executed = new Promise((resolve) => + db.exec('SELECT * FROM t', resolve), + ); + const state = db.state; + assert.ok(state.pending >= 1, `pending ${state.pending} >= 1`); + assert.ok(state.queued >= 1, `queued ${state.queued} >= 1`); + await ran; + await executed; + await db.close(); + }); + + it('shows a close queued behind in-flight work, then settled', async function () { + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (a INT)'); + const ran = new Promise((resolve) => + db.run('INSERT INTO t VALUES (1)', resolve), + ); + const closed = db.close(); + // The close is waiting behind the in-flight statement work: + // queued, but `closing` only turns true once the close itself + // starts (which requires pending == 0), so it is not observable + // in this window. + const queued = db.state; + assert.strictEqual(queued.open, true); + assert.strictEqual(queued.closing, false); + assert.ok(queued.queued >= 1, `queued ${queued.queued} >= 1`); + assert.ok(queued.pending >= 1, `pending ${queued.pending} >= 1`); + await ran; + await closed; + // After the close completed: fully settled, locked included (the + // old tombstone used to read true here by design). + assert.deepStrictEqual(db.state, { + open: false, + closing: false, + locked: false, + serialized: false, + pending: 0, + queued: 0, + }); + }); + + it('sees a fetch in flight while a statement iterates', async function () { + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (a INT); INSERT INTO t VALUES (1),(2)'); + const iterator = db.iterate('SELECT a FROM t ORDER BY a'); + const first = iterator.next(); + assert.ok(db.state.pending >= 1); + await first; + // Between fetches nothing is in flight. + assert.strictEqual(db.state.pending, 0); + await iterator.return(); + await db.close(); + }); + + it('serialize() reached via a saved prototype reference still disables the cache fast path', async function () { + // The _serialized drift bug: a saved reference to the native + // serialize bypassed the JS mirror, so the statement cache kept + // taking the fast path while the connection was serialized, + // silently breaking FIFO. There is no mirror anymore: the native + // flag is the state. + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (a INT)'); + db.cacheStatements(); + // Prime the cache so a fast-path take is possible at all. + await db.run('INSERT INTO t VALUES (1)'); + const nativeSerialize = sqlite3.Database.prototype.serialize; + nativeSerialize.call(db); + assert.strictEqual(db.state.serialized, true); + assert.strictEqual(db.serialized, true); + // While serialized, a cached call must not overtake queued + // database work: two ops issued now must complete in order. + const order = []; + await Promise.all([ + new Promise((resolve) => + db.exec('SELECT 1', () => { + order.push('exec'); + resolve(); + }), + ), + new Promise((resolve) => + db.run('INSERT INTO t VALUES (2)', () => { + order.push('run'); + resolve(); + }), + ), + ]); + assert.deepStrictEqual(order, ['exec', 'run']); + db.parallelize(); + assert.strictEqual(db.state.serialized, false); + await db.close(); + }); +}); + +describe('finalize-on-GC safety net', function () { + // Both tests need a real GC collection to drive the native + // finalizers; without --expose-gc they would pass vacuously, so they + // skip instead. + it('a collected unfinalized statement stops blocking close()', async function (t) { + if (typeof globalThis.gc !== 'function') { + return t.skip('requires --expose-gc'); + } + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (a INT)'); + let stmt = db.prepare('INSERT INTO t VALUES (?)'); + await stmt.run(1); + // Forget the statement without finalize(). + stmt = null; + globalThis.gc(); + // Let the collection (and thus the destructor's finalize) land. + await new Promise((resolve) => setImmediate(resolve)); + await db.close(); + }); + + it('statements and backups whose construction threw are safe to collect', async function (t) { + if (typeof globalThis.gc !== 'function') { + return t.skip('requires --expose-gc'); + } + // Before Deliverable 05 the constructors left `db` uninitialised + // when validation failed, so collecting the abandoned wrapper + // segfaulted at GC time (verified: exit 139 on release/v9). + assert.throws(() => new sqlite3.Statement({}, 'SELECT 1')); + assert.throws( + () => new sqlite3.Backup({}, 'f.db', 'main', 'main', true), + ); + globalThis.gc(); + await new Promise((resolve) => setImmediate(resolve)); + // Surviving to here is the assertion; a regression segfaults the + // process outright. + }); +}); + +// A prepare can fail with calls already queued behind it, and those calls +// are what the caller is actually waiting on. When the prepare had no +// callback of its own -- every promise-mode entry point -- the failure +// used to be reported only on the statement's 'error' event while the +// queue was discarded in silence, so nothing ever settled and the caller +// hung. sqlite3_interrupt() aborts a prepare exactly as readily as a +// step, so any abort landing in that window wedged the connection; the +// abort tests timed out on CI's slower runners for this reason, where the +// window is wide enough to hit almost every time. +describe('a prepare that fails with work queued behind it', function () { + it('settles the queued call rather than dropping it', { + timeout: 10000, + }, async function () { + const db = await sqlite3.open(':memory:'); + // No prepare callback, and the call below is queued in the same + // tick, so it is still waiting when the prepare fails. + const stmt = db.prepare('SELECT * FROM no_such_table'); + /** @type {Error[]} */ + const events = []; + stmt.on('error', function (err) { + events.push(err); + }); + const err = await new Promise(function (resolve) { + stmt.all(function (e) { + resolve(e); + }); + }); + assert.strictEqual(err.code, 'SQLITE_ERROR'); + assert.match(err.message, /no such table: no_such_table/); + // The statement still reports its own failure: that event is the + // documented surface for a prepare given no callback. + assert.strictEqual(events.length, 1); + assert.deepStrictEqual(db.getSync('SELECT 1 AS x'), { x: 1 }); + await db.close(); + }); + + it('settles a queued each() through its completion handler', { + timeout: 10000, + }, async function () { + const db = await sqlite3.open(':memory:'); + const stmt = db.prepare('SELECT * FROM no_such_table'); + stmt.on('error', function () { + // Absorbed: this test is about the queued each(), not the event. + }); + let rows = 0; + const err = await new Promise(function (resolve) { + stmt.each( + function () { + rows++; + }, + function (e) { + resolve(e); + }, + ); + }); + assert.strictEqual(err.code, 'SQLITE_ERROR'); + // The per-row callback must not be handed the error as a row. + assert.strictEqual(rows, 0); + await db.close(); + }); +}); diff --git a/test/statement_cache.test.js b/test/statement_cache.test.js index 39e3060..385b1e8 100644 --- a/test/statement_cache.test.js +++ b/test/statement_cache.test.js @@ -1,5 +1,7 @@ +import assert from 'node:assert'; +import { afterEach, beforeEach, describe, it } from 'node:test'; + import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; import { deleteFile } from './support/helper.js'; // The statement cache is opt-in: db.cacheStatements() makes run/get/all/ @@ -9,25 +11,30 @@ import { deleteFile } from './support/helper.js'; // - serialize() keeps strict FIFO ordering (cache defers to uncached path) // - close() flushes cached statements: no SQLITE_BUSY, no lost callbacks // - prepare errors surface like the uncached path and don't poison the cache -describe('statement cache', function() { +describe('statement cache', function () { let db; - beforeEach(function(done) { - db = new sqlite3.Database(':memory:', function(err) { + beforeEach(function (_t, done) { + db = new sqlite3.Database(':memory:', function (err) { assert.ifError(err); - db.exec('CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER, b TEXT, c BLOB)', done); + db.exec( + 'CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER, b TEXT, c BLOB)', + done, + ); }); }); - afterEach(function(done) { - db.close(function() { done(); }); + afterEach(function (_t, done) { + db.close(function () { + done(); + }); }); - it('is off by default and does not change behavior', function(done) { + it('is off by default and does not change behavior', function (_t, done) { assert.strictEqual(db._stmtCache, undefined); let n = 0; for (let i = 0; i < 5; i++) { - db.run('INSERT INTO t (a) VALUES (?)', i, function(err) { + db.run('INSERT INTO t (a) VALUES (?)', i, function (err) { assert.ifError(err); // Parallel completion order is unspecified; lastID must // still be a valid rowid of this batch. @@ -37,38 +44,41 @@ describe('statement cache', function() { } }); - it('reuses statements with per-call bind values', function(done) { + it('reuses statements with per-call bind values', function (_t, done) { db.cacheStatements(); const vals = []; - for (let i = 0; i < 25; i++) vals.push('v' + i + '-' + 'x'.repeat(40)); + for (let i = 0; i < 25; i++) vals.push(`v${i}-${'x'.repeat(40)}`); let n = 0; - vals.forEach(function(v, i) { - db.run('INSERT INTO t (a, b) VALUES (?, ?)', i, v, function(err) { + vals.forEach(function (v, i) { + db.run('INSERT INTO t (a, b) VALUES (?, ?)', i, v, function (err) { assert.ifError(err); if (++n === vals.length) { - db.all('SELECT a, b FROM t ORDER BY id', function(err, rows) { - assert.ifError(err); - assert.strictEqual(rows.length, vals.length); - rows.forEach(function(row, i) { - assert.strictEqual(row.a, i); - assert.strictEqual(row.b, vals[i]); - }); - done(); - }); + db.all( + 'SELECT a, b FROM t ORDER BY id', + function (err, rows) { + assert.ifError(err); + assert.strictEqual(rows.length, vals.length); + rows.forEach(function (row, i) { + assert.strictEqual(row.a, i); + assert.strictEqual(row.b, vals[i]); + }); + done(); + }, + ); } }); }); }); - it('re-runs statements bound with no parameters', function(done) { + it('re-runs statements bound with no parameters', function (_t, done) { db.cacheStatements(); for (let i = 0; i < 5; i++) { db.run("INSERT INTO t (b) VALUES ('fixed')"); } // Parallel mode never guaranteed cross-statement visibility; // drain first (same pattern as parallel_insert.test.js). - db.wait(function() { - db.get('SELECT COUNT(*) AS n FROM t', function(err, row) { + db.wait(function () { + db.get('SELECT COUNT(*) AS n FROM t', function (err, row) { assert.ifError(err); assert.strictEqual(row.n, 5); done(); @@ -76,96 +86,166 @@ describe('statement cache', function() { }); }); - it('supports get/all/each/map through the cache', function(done) { + it('re-runs a cached get() with no bind parameters from its first row', function (_t, done) { + // db.get(sql) with no parameters used to re-step the previous + // call's cursor: the second call returned undefined while the + // first row sat unconsumed, and the unreset statement pinned the + // connection's WAL read snapshot. The cached path now forces a + // fresh execution (an empty bind array marks "bindings + // supplied"). Fails on release/v9: the second get yields + // undefined. + db.cacheStatements(); + for (let i = 0; i < 3; i++) { + db.run('INSERT INTO t (a) VALUES (?)', i); + } + db.wait(function () { + db.get('SELECT COUNT(*) AS n FROM t', function (err, row) { + assert.ifError(err); + assert.strictEqual(row.n, 3, 'first call'); + db.get('SELECT COUNT(*) AS n FROM t', function (err2, row2) { + assert.ifError(err2); + assert.notStrictEqual(row2, undefined, 'second call'); + assert.strictEqual(row2.n, 3, 'second call re-runs'); + // The promise-mode form of the same path. + db.get('SELECT MAX(a) AS m FROM t').then(function (row3) { + assert.strictEqual(row3.m, 2); + done(); + }, done); + }); + }); + }); + }); + + it('cached getSync re-runs without parameters too', function (_t, done) { + db.cacheStatements(); + for (let i = 0; i < 3; i++) { + db.runSync('INSERT INTO t (a) VALUES (?)', i); + } + assert.strictEqual(db.getSync('SELECT COUNT(*) AS n FROM t').n, 3); + assert.strictEqual(db.getSync('SELECT COUNT(*) AS n FROM t').n, 3); + done(); + }); + + it('supports get/all/each/map through the cache', function (_t, done) { db.cacheStatements(); const buf = Buffer.from('blob-bytes'); const ins = db.prepare('INSERT INTO t (a, b, c) VALUES (?, ?, ?)'); let n = 0; for (let i = 0; i < 4; i++) { - ins.run(i, 'row' + i, buf, function() { + ins.run(i, `row${i}`, buf, function () { if (++n === 4) ins.finalize(check); }); } function check() { - db.get('SELECT a, b, c FROM t WHERE a = ?', 2, function(err, row) { + db.get('SELECT a, b, c FROM t WHERE a = ?', 2, function (err, row) { assert.ifError(err); assert.strictEqual(row.b, 'row2'); assert.ok(buf.equals(row.c)); - db.all('SELECT a FROM t ORDER BY a', function(err, rows) { + db.all('SELECT a FROM t ORDER BY a', function (err, rows) { assert.ifError(err); - assert.deepStrictEqual(rows.map(r => r.a), [0, 1, 2, 3]); + assert.deepStrictEqual( + rows.map((r) => r.a), + [0, 1, 2, 3], + ); let seen = 0; - db.each('SELECT a FROM t ORDER BY a', function(err, row) { - assert.ifError(err); - assert.strictEqual(row.a, seen++); - }, function(err, count) { - assert.ifError(err); - assert.strictEqual(count, 4); - db.map('SELECT a, b FROM t ORDER BY a', function(err, result) { + db.each( + 'SELECT a FROM t ORDER BY a', + function (err, row) { assert.ifError(err); - assert.deepStrictEqual(Object.keys(result), ['0', '1', '2', '3']); - assert.strictEqual(result['2'], 'row2'); - done(); - }); - }); + assert.strictEqual(row.a, seen++); + }, + function (err, count) { + assert.ifError(err); + assert.strictEqual(count, 4); + db.map( + 'SELECT a, b FROM t ORDER BY a', + function (err, result) { + assert.ifError(err); + assert.deepStrictEqual( + Object.keys(result), + ['0', '1', '2', '3'], + ); + assert.strictEqual(result['2'], 'row2'); + done(); + }, + ); + }, + ); }); }); } }); - it('keeps named-parameter and object binds per call', function(done) { + it('keeps named-parameter and object binds per call', function (_t, done) { db.cacheStatements(); - db.run('INSERT INTO t (a, b) VALUES ($a, $b)', {$a: 1, $b: 'one'}, function(err) { - assert.ifError(err); - db.run('INSERT INTO t (a, b) VALUES ($a, $b)', {$a: 2, $b: 'two'}, function(err) { + db.run( + 'INSERT INTO t (a, b) VALUES ($a, $b)', + { $a: 1, $b: 'one' }, + function (err) { assert.ifError(err); - db.all('SELECT a, b FROM t ORDER BY a', function(err, rows) { - assert.ifError(err); - assert.strictEqual(rows.length, 2); - assert.strictEqual(rows[0].b, 'one'); - assert.strictEqual(rows[1].b, 'two'); - done(); - }); - }); - }); + db.run( + 'INSERT INTO t (a, b) VALUES ($a, $b)', + { $a: 2, $b: 'two' }, + function (err) { + assert.ifError(err); + db.all( + 'SELECT a, b FROM t ORDER BY a', + function (err, rows) { + assert.ifError(err); + assert.strictEqual(rows.length, 2); + assert.strictEqual(rows[0].b, 'one'); + assert.strictEqual(rows[1].b, 'two'); + done(); + }, + ); + }, + ); + }, + ); }); - it('keeps FIFO ordering on the same cached statement in one tick', function(done) { + it('keeps FIFO ordering on the same cached statement in one tick', function (_t, done) { db.cacheStatements(); const N = 100; for (let i = 0; i < N; i++) { db.run('INSERT INTO t (a) VALUES (?)', i); } - db.wait(function() { - db.all('SELECT a FROM t ORDER BY id', function(err, rows) { + db.wait(function () { + db.all('SELECT a FROM t ORDER BY id', function (err, rows) { assert.ifError(err); assert.strictEqual(rows.length, N); - rows.forEach(function(row, i) { - assert.strictEqual(row.a, i, 'FIFO broken at ' + i); + rows.forEach(function (row, i) { + assert.strictEqual(row.a, i, `FIFO broken at ${i}`); }); done(); }); }); }); - it('defers to the uncached path under serialize() and keeps FIFO order', function(done) { + it('defers to the uncached path under serialize() and keeps FIFO order', function (_t, done) { db.cacheStatements(); const order = []; - db.serialize(function() { + db.serialize(function () { for (let i = 0; i < 30; i++) { - db.run('INSERT INTO t (a) VALUES (?)', i, function() { order.push(i); }); + db.run('INSERT INTO t (a) VALUES (?)', i, function () { + order.push(i); + }); } }); - db.wait(function() { + db.wait(function () { // serialize() must complete callbacks in issue order for (let i = 0; i < 30; i++) { - assert.strictEqual(order[i], i, 'serialize order broken at ' + i); + assert.strictEqual( + order[i], + i, + `serialize order broken at ${i}`, + ); } - db.parallelize(function() { - db.all('SELECT a FROM t ORDER BY id', function(err, rows) { + db.parallelize(function () { + db.all('SELECT a FROM t ORDER BY id', function (err, rows) { assert.ifError(err); assert.strictEqual(rows.length, 30); - rows.forEach(function(row, i) { + rows.forEach(function (row, i) { assert.strictEqual(row.a, i); }); done(); @@ -174,13 +254,13 @@ describe('statement cache', function() { }); }); - it('survives LRU eviction pressure with a tiny cache', function(done) { + it('survives LRU eviction pressure with a tiny cache', function (_t, done) { db.cacheStatements(1); const sqls = []; - for (let i = 0; i < 10; i++) sqls.push('SELECT ' + i + ' AS v'); + for (let i = 0; i < 10; i++) sqls.push(`SELECT ${i} AS v`); let n = 0; - sqls.forEach(function(sql, i) { - db.get(sql, function(err, row) { + sqls.forEach(function (sql, i) { + db.get(sql, function (err, row) { assert.ifError(err); assert.strictEqual(row.v, i); if (++n === sqls.length) done(); @@ -188,14 +268,14 @@ describe('statement cache', function() { }); }); - it('reports prepare errors and recovers', function(done) { + it('reports prepare errors and recovers', function (_t, done) { db.cacheStatements(); - db.run('NOT VALID SQL AT ALL', function(err) { + db.run('NOT VALID SQL AT ALL', function (err) { assert.ok(err); assert.strictEqual(err.code, 'SQLITE_ERROR'); - db.run('INSERT INTO t (a) VALUES (?)', 1, function(err) { + db.run('INSERT INTO t (a) VALUES (?)', 1, function (err) { assert.ifError(err); - db.get('SELECT COUNT(*) AS n FROM t', function(err, row) { + db.get('SELECT COUNT(*) AS n FROM t', function (err, row) { assert.ifError(err); assert.strictEqual(row.n, 1); done(); @@ -204,32 +284,48 @@ describe('statement cache', function() { }); }); - it('reports runtime errors per call without poisoning the cache', function(done) { + it('reports runtime errors per call without poisoning the cache', function (_t, done) { db.cacheStatements(); - db.run('CREATE TABLE IF NOT EXISTS u (x INTEGER UNIQUE)', function(err) { - assert.ifError(err); - db.run('INSERT INTO u (x) VALUES (?)', 1, function(err) { + db.run( + 'CREATE TABLE IF NOT EXISTS u (x INTEGER UNIQUE)', + function (err) { assert.ifError(err); - db.run('INSERT INTO u (x) VALUES (?)', 1, function(err) { - assert.ok(err); - assert.strictEqual(err.code, 'SQLITE_CONSTRAINT'); - db.run('INSERT INTO u (x) VALUES (?)', 2, function(err) { - assert.ifError(err); - done(); + db.run('INSERT INTO u (x) VALUES (?)', 1, function (err) { + assert.ifError(err); + db.run('INSERT INTO u (x) VALUES (?)', 1, function (err) { + assert.ok(err); + // v9 reports the extended code; the primary code + // moved to err.primaryCode. + assert.strictEqual( + err.code, + 'SQLITE_CONSTRAINT_UNIQUE', + ); + assert.strictEqual( + err.primaryCode, + 'SQLITE_CONSTRAINT', + ); + db.run( + 'INSERT INTO u (x) VALUES (?)', + 2, + function (err) { + assert.ifError(err); + done(); + }, + ); }); }); - }); - }); + }, + ); }); - it('coexists with user-held prepared statements', function(done) { + it('coexists with user-held prepared statements', function (_t, done) { db.cacheStatements(); const mine = db.prepare('INSERT INTO t (a) VALUES (?)'); - db.run('INSERT INTO t (a) VALUES (?)', 100, function(err) { + db.run('INSERT INTO t (a) VALUES (?)', 100, function (err) { assert.ifError(err); - mine.run(200, function(err) { + mine.run(200, function (err) { assert.ifError(err); - db.get('SELECT COUNT(*) AS n FROM t', function(err, row) { + db.get('SELECT COUNT(*) AS n FROM t', function (err, row) { assert.ifError(err); assert.strictEqual(row.n, 2); mine.finalize(done); @@ -238,65 +334,86 @@ describe('statement cache', function() { }); }); - describe('close interaction', function() { + describe('close interaction', function () { const FILE = 'test/tmp/test_stmt_cache_close.db'; - beforeEach(function(done) { + beforeEach(function (_t, done) { deleteFile(FILE); - db.close(function() { - db = new sqlite3.Database(FILE, function(err) { + db.close(function () { + db = new sqlite3.Database(FILE, function (err) { assert.ifError(err); - db.exec('CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER)', done); + db.exec( + 'CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER)', + done, + ); }); }); }); - it('flushes cached statements on close (no SQLITE_BUSY)', function(done) { + it('flushes cached statements on close (no SQLITE_BUSY)', function (_t, done) { db.cacheStatements(); for (let i = 0; i < 20; i++) { db.run('INSERT INTO t (a) VALUES (?)', i); } - db.close(function(err) { + db.close(function (err) { assert.ifError(err); - db = new sqlite3.Database(FILE, sqlite3.OPEN_READONLY, function(err2) { - assert.ifError(err2); - db.get('SELECT COUNT(*) AS n FROM t', function(e, row) { - assert.ifError(e); - assert.strictEqual(row.n, 20); - done(); - }); - }); + db = new sqlite3.Database( + FILE, + sqlite3.OPEN_READONLY, + function (err2) { + assert.ifError(err2); + db.get( + 'SELECT COUNT(*) AS n FROM t', + function (e, row) { + assert.ifError(e); + assert.strictEqual(row.n, 20); + done(); + }, + ); + }, + ); }); }); - it('delivers queued cached callbacks before close completes', function(done) { + it('delivers queued cached callbacks before close completes', function (_t, done) { db.cacheStatements(); let callbacks = 0; for (let i = 0; i < 50; i++) { - db.run('INSERT INTO t (a) VALUES (?)', i, function() { callbacks++; }); + db.run('INSERT INTO t (a) VALUES (?)', i, function () { + callbacks++; + }); } // Same-tick close: the review cycle's killer case. Every // callback must fire, close must not error. - db.close(function(err) { + db.close(function (err) { assert.ifError(err); assert.strictEqual(callbacks, 50, 'cached callbacks were lost'); - db = new sqlite3.Database(FILE, sqlite3.OPEN_READONLY, function(err2) { - assert.ifError(err2); - db.get('SELECT COUNT(*) AS n FROM t', function(e, row) { - assert.ifError(e); - assert.strictEqual(row.n, 50); - done(); - }); - }); + db = new sqlite3.Database( + FILE, + sqlite3.OPEN_READONLY, + function (err2) { + assert.ifError(err2); + db.get( + 'SELECT COUNT(*) AS n FROM t', + function (e, row) { + assert.ifError(e); + assert.strictEqual(row.n, 50); + done(); + }, + ); + }, + ); }); }); - it('closes cleanly with cache enabled but empty', function(done) { + it('closes cleanly with cache enabled but empty', function (_t, done) { db.cacheStatements(); - db.close(function(err) { + db.close(function (err) { assert.ifError(err); // keep afterEach happy: already closed - db = new sqlite3.Database(':memory:', function() { done(); }); + db = new sqlite3.Database(':memory:', function () { + done(); + }); }); }); }); @@ -305,21 +422,23 @@ describe('statement cache', function() { // The cached path skips the prepare, and statement operations never travel // through the database queue. Nothing may therefore overtake an exclusive // operation (exec/close/wait/loadExtension) just because its SQL was cached. -describe('statement cache ordering vs exclusive operations', function() { - it('does not let a cached statement overtake exec()', function(done) { +describe('statement cache ordering vs exclusive operations', function () { + it('does not let a cached statement overtake exec()', function (_t, done) { const order = []; const db = new sqlite3.Database(':memory:'); db.cacheStatements(); - db.run('CREATE TABLE t (i)', function() { + db.run('CREATE TABLE t (i)', function () { // Prime the cache for this exact SQL. - db.run('INSERT INTO t VALUES (1)', function() { - db.exec('INSERT INTO t VALUES (2); INSERT INTO t VALUES (3);', - function(err) { + db.run('INSERT INTO t VALUES (1)', function () { + db.exec( + 'INSERT INTO t VALUES (2); INSERT INTO t VALUES (3);', + function (err) { assert.ifError(err); order.push('exec'); - }); + }, + ); // Cache hit, issued after exec: must still run after it. - db.run('INSERT INTO t VALUES (1)', function(err) { + db.run('INSERT INTO t VALUES (1)', function (err) { assert.ifError(err); order.push('cached run'); assert.deepStrictEqual(order, ['exec', 'cached run']); @@ -329,20 +448,20 @@ describe('statement cache ordering vs exclusive operations', function() { }); }); - it('rejects a cached statement issued after close()', function(done) { + it('rejects a cached statement issued after close()', function (_t, done) { const db = new sqlite3.Database(':memory:'); db.cacheStatements(); - db.run('CREATE TABLE t (i)', function() { - db.run('INSERT INTO t VALUES (1)', function() { - setImmediate(function() { + db.run('CREATE TABLE t (i)', function () { + db.run('INSERT INTO t VALUES (1)', function () { + setImmediate(function () { let closed = false; - db.close(function(err) { + db.close(function (err) { assert.ifError(err); closed = true; }); // close() must be requested synchronously, not deferred: // this cache hit has to land behind it and fail. - db.run('INSERT INTO t VALUES (1)', function(err) { + db.run('INSERT INTO t VALUES (1)', function (err) { assert.ok(err, 'run after close() must not succeed'); assert.ok(closed, 'close should have completed first'); done(); @@ -352,16 +471,17 @@ describe('statement cache ordering vs exclusive operations', function() { }); }); - it('rejects a cached statement issued after close() from a callback', - function(done) { + it('rejects a cached statement issued after close() from a callback', function (_t, done) { const db = new sqlite3.Database(':memory:'); db.cacheStatements(); - db.run('CREATE TABLE t (i)', function() { - db.run('INSERT INTO t VALUES (1)', function() { + db.run('CREATE TABLE t (i)', function () { + db.run('INSERT INTO t VALUES (1)', function () { // Still inside a statement callback: the statement is locked // and db->pending is non-zero, so the cache flush queues. - db.close(function(err) { assert.ifError(err); }); - db.run('INSERT INTO t VALUES (1)', function(err) { + db.close(function (err) { + assert.ifError(err); + }); + db.run('INSERT INTO t VALUES (1)', function (err) { assert.ok(err, 'run after close() must not succeed'); done(); }); diff --git a/test/support/bindpaths.js b/test/support/bindpaths.js new file mode 100644 index 0000000..c361d5b --- /dev/null +++ b/test/support/bindpaths.js @@ -0,0 +1,345 @@ +// Entry-point driver for marshalling tests (Deliverable 02). Runs one +// bind+read through every public query path — async and sync, Database +// and Statement — so the two native marshalling implementations +// (src/statement.cc Work_* vs the *Sync fast path) cannot drift apart +// unnoticed. + +/** + * Wraps a callback-style call so a synchronous throw is reported as + * `{ threw }` instead of escaping. + * + * @param {() => void} fn + * @returns {Promise<{ threw?: Error }>} + */ +function captureSyncThrow(fn) { + return new Promise((resolve) => { + try { + fn(); + } catch (err) { + resolve({ threw: err }); + } + }); +} + +/** + * Builds one driver per query path. Each driver binds `params` to + * `SELECT ? AS v` (or the given single-placeholder SQL) and resolves + * with `{ threw }` (synchronous failure), `{ err }` (callback failure), + * or `{ v }` (the value read back; bind-only paths resolve `{}` on + * success). + * + * @param {import('../../lib/sqlite3.js').Database} db + * @returns {{ name: string, reads: boolean, run: (value: unknown) => Promise<{threw?: Error, err?: Error, v?: unknown}> }[]} + */ +export function bindPaths(db) { + const select = 'SELECT ? AS v'; + + // The sync paths below require a fully idle connection, and the + // async paths above cannot promise that at the moment their await + // resolves: a bind-rejected value throws synchronously out of the + // db-level wrappers, abandoning that call's in-flight async + // prepare (db.pending stays elevated for another turn), and those + // wrappers' internal finalize is deliberately fire-and-forget. + // db.wait() schedules an exclusive call that runs only once + // pending == 0, so awaiting it puts every sync path on the idle + // side of the gate — the same discipline the stmt drivers apply + // by resolving from inside stmt.finalize's callback. + const whenIdle = () => new Promise((resolve) => db.wait(resolve)); + + const paths = [ + { + name: 'db.get', + reads: true, + run: (value) => + new Promise((resolve) => { + try { + db.get(select, [value], (err, row) => + resolve(err ? { err } : { v: row?.v }), + ); + } catch (err) { + resolve({ threw: err }); + } + }), + }, + { + name: 'db.all', + reads: true, + run: (value) => + new Promise((resolve) => { + try { + db.all(select, [value], (err, rows) => + resolve(err ? { err } : { v: rows[0]?.v }), + ); + } catch (err) { + resolve({ threw: err }); + } + }), + }, + { + name: 'db.each', + reads: true, + run: (value) => + new Promise((resolve) => { + let seen; + let itemErr; + try { + db.each( + select, + [value], + (err, row) => { + if (err) { + // e.g. the 'number'-mode RangeError + itemErr = itemErr || err; + return; + } + seen = row?.v; + }, + (err) => + resolve( + err || itemErr + ? { err: err || itemErr } + : { v: seen }, + ), + ); + } catch (err) { + resolve({ threw: err }); + } + }), + }, + { + name: 'db.map', + reads: true, + // map() keys its result by the first column and takes the + // second as the value, so a two-column shape is required. + // The read-back is reported as the stringified key. + run: (value) => + new Promise((resolve) => { + try { + db.map( + 'SELECT ? AS v, 0 AS extra', + [value], + (err, map) => + resolve( + err + ? { err } + : { + v: Object.keys(map ?? {})[0], + stringified: true, + }, + ), + ); + } catch (err) { + resolve({ threw: err }); + } + }), + }, + { + name: 'db.run', + reads: false, + run: (value) => + new Promise((resolve) => { + try { + db.run(select, [value], (err) => + resolve(err ? { err } : {}), + ); + } catch (err) { + resolve({ threw: err }); + } + }), + }, + { + name: 'stmt.get', + reads: true, + run: (value) => + new Promise((resolve) => { + let stmt; + try { + stmt = db.prepare(select); + } catch (err) { + resolve({ threw: err }); + return; + } + try { + stmt.get([value], (err, row) => { + // Resolve only once finalize completed, so + // the next path sees a fully idle database. + stmt.finalize(() => + resolve(err ? { err } : { v: row?.v }), + ); + }); + } catch (err) { + stmt.finalize(() => resolve({ threw: err })); + } + }), + }, + { + name: 'stmt.all', + reads: true, + run: (value) => + new Promise((resolve) => { + let stmt; + try { + stmt = db.prepare(select); + } catch (err) { + resolve({ threw: err }); + return; + } + try { + stmt.all([value], (err, rows) => { + stmt.finalize(() => + resolve(err ? { err } : { v: rows[0]?.v }), + ); + }); + } catch (err) { + stmt.finalize(() => resolve({ threw: err })); + } + }), + }, + { + name: 'stmt.each', + reads: true, + run: (value) => + new Promise((resolve) => { + let stmt; + try { + stmt = db.prepare(select); + } catch (err) { + resolve({ threw: err }); + return; + } + let seen; + let itemErr; + try { + stmt.each( + [value], + (err, row) => { + if (err) { + itemErr = itemErr || err; + return; + } + seen = row?.v; + }, + (err) => { + stmt.finalize(() => + resolve( + err || itemErr + ? { err: err || itemErr } + : { v: seen }, + ), + ); + }, + ); + } catch (err) { + stmt.finalize(() => resolve({ threw: err })); + } + }), + }, + { + name: 'stmt.run', + reads: false, + run: (value) => + new Promise((resolve) => { + let stmt; + try { + stmt = db.prepare(select); + } catch (err) { + resolve({ threw: err }); + return; + } + try { + stmt.run([value], (err) => { + stmt.finalize(() => resolve(err ? { err } : {})); + }); + } catch (err) { + stmt.finalize(() => resolve({ threw: err })); + } + }), + }, + { + name: 'db.getSync', + reads: true, + run: async (value) => { + await whenIdle(); + try { + return { v: db.getSync(select, [value])?.v }; + } catch (err) { + return { threw: err }; + } + }, + }, + { + name: 'db.allSync', + reads: true, + run: async (value) => { + await whenIdle(); + try { + return { v: db.allSync(select, [value])[0]?.v }; + } catch (err) { + return { threw: err }; + } + }, + }, + { + name: 'db.runSync', + reads: false, + run: async (value) => { + await whenIdle(); + try { + db.runSync(select, [value]); + return {}; + } catch (err) { + return { threw: err }; + } + }, + }, + { + name: 'stmt.getSync', + reads: true, + run: async (value) => { + await whenIdle(); + try { + const stmt = db.prepareSync(select); + const out = { v: stmt.getSync([value])?.v }; + stmt.finalize(); + return out; + } catch (err) { + return { threw: err }; + } + }, + }, + { + name: 'stmt.allSync', + reads: true, + run: async (value) => { + await whenIdle(); + try { + const stmt = db.prepareSync(select); + const out = { v: stmt.allSync([value])[0]?.v }; + stmt.finalize(); + return out; + } catch (err) { + return { threw: err }; + } + }, + }, + { + name: 'stmt.runSync', + reads: false, + run: async (value) => { + await whenIdle(); + try { + const stmt = db.prepareSync(select); + stmt.runSync([value]); + stmt.finalize(); + return {}; + } catch (err) { + return { threw: err }; + } + }, + }, + ]; + + return paths; +} + +export { captureSyncThrow }; diff --git a/test/support/corpus.js b/test/support/corpus.js new file mode 100644 index 0000000..caf341c --- /dev/null +++ b/test/support/corpus.js @@ -0,0 +1,216 @@ +// Shared value corpus for marshalling tests (Deliverable 02 §4). One +// place decides what "every interesting JS value" means, so the bind, +// column, and sync-path suites (D02/D06/D08) all assert the same +// contract. `sqliteType` is the *expected* storage class after a +// round-trip. Entries with `rejected: true` must be refused by bind +// (TypeError by default, RangeError for out-of-range BigInts), never +// silently coerced. +// +// Notes on the boundary entries: +// - 2**53+1, 2**63-1 and -(2**63) as *numbers* are not all exactly +// representable (2**63-1 rounds up to 2**63); they are included +// precisely because that neighbourhood is where silent-coercion bugs +// live. The double 2**63 clamps to INT64_MAX on bind. +// - `undefined` binds as NULL (Deliverable 02 decision): object shorthand +// { $x: obj.maybeMissing } is a common call shape, and typo'd property +// names are caught by the named-parameter and arity checks instead. +// - -0 binds as INTEGER 0: SQLite has no signed integer zero, and v8 +// pinned the same behaviour (test/marshalling.test.js). + +/** @returns {Buffer} a deterministic blob of `n` bytes */ +function blob(n) { + const b = Buffer.alloc(n); + for (let i = 0; i < n; i++) b[i] = (i * 251) % 256; + return b; +} + +/** @returns {Uint8Array} a deterministic view of `n` bytes */ +function u8(n) { + return new Uint8Array(blob(n)); +} + +/** + * Builds typed-array/DataView/ArrayBuffer views over one deterministic + * 16-byte buffer so byteOffset handling is observable. + * + * @returns {{ plain: Uint8Array, offset: Uint8Array, dataview: DataView, arraybuffer: ArrayBuffer, shared: Uint8Array, u16: Uint16Array }} + */ +function views() { + const bytes = blob(16); + const plain = new Uint8Array(bytes); + const offset = new Uint8Array(bytes.buffer, 4, 8); + const dataview = new DataView(bytes.buffer, 2, 6); + const sab = new SharedArrayBuffer(16); + new Uint8Array(sab).set(bytes); + const shared = new Uint8Array(sab); + const u16 = new Uint16Array( + bytes.buffer.slice(0), // own copy: byte length must be even + 0, + 8, + ); + return { plain, offset, dataview, arraybuffer: bytes.buffer, shared, u16 }; +} + +const FIXED_DATE = new Date('2026-08-25T12:34:56.789Z'); + +export const corpus = [ + // Safe integers. + { label: 'zero', value: 0, sqliteType: 'INTEGER' }, + { label: 'one', value: 1, sqliteType: 'INTEGER' }, + { label: 'negative one', value: -1, sqliteType: 'INTEGER' }, + { label: 'int32 max', value: 2 ** 31 - 1, sqliteType: 'INTEGER' }, + { label: 'int32 min', value: -(2 ** 31), sqliteType: 'INTEGER' }, + { + label: 'max safe integer', + value: Number.MAX_SAFE_INTEGER, + sqliteType: 'INTEGER', + }, + // int64 boundaries beyond the safe-integer range. + { label: '2**53 + 1', value: 2 ** 53 + 1, sqliteType: 'INTEGER' }, + { + label: '2**63 - 1 (rounds to 2**63 as a double)', + value: 2 ** 63 - 1, + sqliteType: 'INTEGER', + }, + { label: '-(2**63)', value: -(2 ** 63), sqliteType: 'INTEGER' }, + // BigInts: exact, no clamping. + { label: 'bigint one', value: 1n, sqliteType: 'INTEGER' }, + { + label: 'bigint 2**53 + 1', + value: 9007199254740993n, + sqliteType: 'INTEGER', + }, + { + label: 'bigint 2**63 - 1', + value: 9223372036854775807n, + sqliteType: 'INTEGER', + }, + { + label: 'bigint -(2**63)', + value: -(2n ** 63n), + sqliteType: 'INTEGER', + }, + // Floats. + { label: 'fraction', value: Math.PI, sqliteType: 'REAL' }, + { label: 'negative fraction', value: -0.5, sqliteType: 'REAL' }, + { label: 'large exponent', value: 1.5e300, sqliteType: 'REAL' }, + // -0 is INTEGER 0: no signed zero exists in SQLite integers. + { label: 'negative zero', value: -0, sqliteType: 'INTEGER' }, + // SQLite converts NaN to NULL on bind_double; Infinity stays REAL. + { label: 'NaN', value: Number.NaN, sqliteType: 'NULL' }, + { label: 'Infinity', value: Number.POSITIVE_INFINITY, sqliteType: 'REAL' }, + { label: '-Infinity', value: Number.NEGATIVE_INFINITY, sqliteType: 'REAL' }, + // Strings. + { label: 'empty string', value: '', sqliteType: 'TEXT' }, + { label: 'string with NUL byte', value: 'a\0b', sqliteType: 'TEXT' }, + { + label: 'lone high surrogate', + value: 'before \uD800 after', + sqliteType: 'TEXT', + }, + { + label: 'lone low surrogate', + value: 'before \uDC00 after', + sqliteType: 'TEXT', + }, + { label: 'non-BMP text', value: 'sqlite \u{1F5A0} v9', sqliteType: 'TEXT' }, + // Blobs at page-boundary-adjacent sizes and 1 MiB. + { label: 'empty blob', value: blob(0), sqliteType: 'BLOB' }, + { label: '64-byte blob', value: blob(64), sqliteType: 'BLOB' }, + { label: '4095-byte blob', value: blob(4095), sqliteType: 'BLOB' }, + { label: '4096-byte blob', value: blob(4096), sqliteType: 'BLOB' }, + { label: '4097-byte blob', value: blob(4097), sqliteType: 'BLOB' }, + { label: '1 MiB blob', value: blob(1024 * 1024), sqliteType: 'BLOB' }, + // Typed-array views bind as blobs of their exact byte range. + { label: 'Uint8Array', value: u8(32), sqliteType: 'BLOB' }, + { + label: 'Uint8Array with byteOffset', + value: views().offset, + sqliteType: 'BLOB', + }, + { + label: 'DataView with byteOffset', + value: views().dataview, + sqliteType: 'BLOB', + }, + { label: 'ArrayBuffer', value: views().arraybuffer, sqliteType: 'BLOB' }, + { + label: 'Uint8Array over SharedArrayBuffer', + value: views().shared, + sqliteType: 'BLOB', + }, + { + label: 'Uint16Array (raw bytes)', + value: views().u16, + sqliteType: 'BLOB', + }, + // Misc bindable types. + { label: 'true', value: true, sqliteType: 'INTEGER' }, + { label: 'false', value: false, sqliteType: 'INTEGER' }, + { label: 'null', value: null, sqliteType: 'NULL' }, + { + label: 'undefined (binds as NULL)', + value: undefined, + sqliteType: 'NULL', + }, + { label: 'Date (epoch ms)', value: FIXED_DATE, sqliteType: 'REAL' }, + { label: 'RegExp (toString)', value: /corpus[0-9]+/i, sqliteType: 'TEXT' }, + // Values that bind must refuse rather than coerce. + { + label: 'plain object', + value: { nope: 1 }, + sqliteType: null, + rejected: true, + }, + { label: 'array', value: [1, 2, 3], sqliteType: null, rejected: true }, + { + label: 'Symbol', + value: Symbol('corpus'), + sqliteType: null, + rejected: true, + }, + { + label: 'function', + value: () => { + /* intentionally does nothing */ + }, + sqliteType: null, + rejected: true, + }, + { + label: 'Map', + value: new Map([['a', 1]]), + sqliteType: null, + rejected: true, + }, + { + label: 'class instance', + value: new (class Widget { + constructor() { + this.size = 1; + } + })(), + sqliteType: null, + rejected: true, + }, + { + label: 'bigint 2**63 (out of int64 range)', + value: 2n ** 63n, + sqliteType: null, + rejected: true, + rejection: 'RangeError', + }, + { + label: 'bigint -(2**63)-1 (out of int64 range)', + value: -(2n ** 63n) - 1n, + sqliteType: null, + rejected: true, + rejection: 'RangeError', + }, +]; + +/** The subset expected to bind successfully. */ +export const bindableValues = corpus.filter((e) => !e.rejected); + +/** The subset bind must refuse. */ +export const rejectedValues = corpus.filter((e) => e.rejected === true); diff --git a/test/support/createdb-electron.js b/test/support/createdb-electron.js deleted file mode 100644 index a060079..0000000 --- a/test/support/createdb-electron.js +++ /dev/null @@ -1,8 +0,0 @@ -import { app } from 'electron'; -import createdb from './createdb.js'; - -createdb(function () { - setTimeout(function () { - app.quit(); - }, 20000); -}); \ No newline at end of file diff --git a/test/support/createdb.js b/test/support/createdb.js index daa6ab0..cb83ea5 100755 --- a/test/support/createdb.js +++ b/test/support/createdb.js @@ -1,51 +1,56 @@ #!/usr/bin/env node -import { existsSync, statSync } from 'fs'; -import { fileURLToPath } from 'url'; -import { dirname, join } from 'path'; +import { existsSync, statSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + import sqlite3 from '../../lib/sqlite3.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); function randomString() { - let str = ''; - let chars = 'abcdefghijklmnopqrstuvwxzyABCDEFGHIJKLMNOPQRSTUVWXZY0123456789 '; - for (let i = Math.random() * 100; i > 0; i--) { - str += chars[Math.floor(Math.random() * chars.length)]; - } - return str; + let str = ''; + const chars = + 'abcdefghijklmnopqrstuvwxzyABCDEFGHIJKLMNOPQRSTUVWXZY0123456789 '; + for (let i = Math.random() * 100; i > 0; i--) { + str += chars[Math.floor(Math.random() * chars.length)]; + } + return str; } function createdb(callback) { - const count = 1000000; - const db_path = join(__dirname, 'big.db'); - // Make sure the file exists and is also valid. - if (existsSync(db_path) && statSync(db_path).size !== 0) { - console.log('okay: database already created (' + db_path + ')'); - if (callback) callback(); - } else { - console.log("Creating test database... This may take several minutes."); - let db = new sqlite3.Database(db_path); - db.serialize(function() { - db.run("CREATE TABLE foo (id INT, txt TEXT)"); - db.run("BEGIN TRANSACTION"); - let stmt = db.prepare("INSERT INTO foo VALUES(?, ?)"); - for (let i = 0; i < count; i++) { - stmt.run(i, randomString()); - } - stmt.finalize(); - db.run("COMMIT TRANSACTION", [], function () { - db.close(callback); - }); - }); - } + const count = 1000000; + const db_path = join(__dirname, 'big.db'); + // Make sure the file exists and is also valid. + if (existsSync(db_path) && statSync(db_path).size !== 0) { + console.log(`okay: database already created (${db_path})`); + if (callback) callback(); + } else { + console.log('Creating test database... This may take several minutes.'); + const db = new sqlite3.Database(db_path); + db.serialize(function () { + db.run('CREATE TABLE foo (id INT, txt TEXT)'); + db.run('BEGIN TRANSACTION'); + const stmt = db.prepare('INSERT INTO foo VALUES(?, ?)'); + for (let i = 0; i < count; i++) { + stmt.run(i, randomString()); + } + stmt.finalize(); + db.run('COMMIT TRANSACTION', [], function () { + db.close(callback); + }); + }); + } } if (import.meta.url === `file://${process.argv[1]}`.replaceAll('\\', '/')) { - createdb(); -} else if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { - createdb(); + createdb(); +} else if ( + process.argv[1] && + fileURLToPath(import.meta.url) === process.argv[1] +) { + createdb(); } -export default createdb; \ No newline at end of file +export default createdb; diff --git a/test/support/db.js b/test/support/db.js new file mode 100644 index 0000000..811f017 --- /dev/null +++ b/test/support/db.js @@ -0,0 +1,67 @@ +// Shared database helpers for new tests. Guarantee close() and temp-file +// cleanup so a failing assertion cannot leak handles or files into +// test/tmp/ — the legacy suites open-code this and several leak. +import { randomUUID } from 'node:crypto'; +import { mkdirSync, rmSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import sqlite3 from '../../lib/sqlite3.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +export const TMP_DIR = join(__dirname, '..', 'tmp'); + +/** + * Runs `fn` with an open `Database`, closing it afterwards whether `fn` + * succeeds or throws. + * + * @param {(db: import('../../lib/sqlite3.js').Database) => Promise | void} fn + * @param {{ filename?: string, mode?: number }} [options] + * @returns {Promise} + * @throws whatever `fn` threw, or the close error when close fails. + * @example + * await withDb(async (db) => { + * db.exec('CREATE TABLE t (i)'); + * }); + */ +export async function withDb(fn, { filename = ':memory:', mode } = {}) { + mkdirSync(TMP_DIR, { recursive: true }); + const db = + mode === undefined + ? new sqlite3.Database(filename) + : new sqlite3.Database(filename, mode); + let error; + try { + await fn(db); + } catch (err) { + error = err; + } + await new Promise((resolve, reject) => { + db.close((err) => (err ? reject(err) : resolve())); + }).catch((closeErr) => { + if (!error) error = closeErr; + }); + if (error) throw error; +} + +/** + * Runs `fn` on a database backed by a unique file under `test/tmp/`, + * removing the file afterwards whether `fn` succeeds or throws. + * + * @param {(db: import('../../lib/sqlite3.js').Database) => Promise | void} fn + * @param {{ mode?: number }} [options] + * @returns {Promise} + * @throws whatever `fn` threw. + * @example + * await withTempDb(async (db) => { + * db.exec('CREATE TABLE t (i)'); + * }); + */ +export async function withTempDb(fn, { mode } = {}) { + const filename = join(TMP_DIR, `helper-${randomUUID()}.db`); + try { + await withDb(fn, { filename, mode }); + } finally { + rmSync(filename, { force: true }); + } +} diff --git a/test/support/function_exit_child.mjs b/test/support/function_exit_child.mjs new file mode 100644 index 0000000..a7df28a --- /dev/null +++ b/test/support/function_exit_child.mjs @@ -0,0 +1,37 @@ +// Child process for the function-registration process-exit test: a +// registered function's ThreadSafeFunction must not keep the event loop +// alive, so a process that registers (and uses) functions without closing +// the database still exits. Prints CHILD-EXITING right before returning to +// the event loop for the last time; the parent asserts exit code 0. +import sqlite3 from '../../lib/sqlite3.js'; + +const db = new sqlite3.Database(':memory:'); +db.function('double', (x) => x * 2); +db.collation('rev', (a, b) => b.localeCompare(a)); +db.aggregate('total', { + start: () => 0, + step: (acc, v) => acc + v, + result: (acc) => acc, +}); + +db.exec('CREATE TABLE t (a INT); INSERT INTO t VALUES (1), (2), (3)', (err) => { + if (err) { + console.error('CHILD-ERROR', err); + process.exit(1); + } + db.get('SELECT total(a) AS v FROM t', (err2, row) => { + if (err2 || row.v !== 6) { + console.error('CHILD-ERROR', err2 ?? row); + process.exit(1); + } + db.get('SELECT double(21) AS v', (err3, row2) => { + if (err3 || row2.v !== 42) { + console.error('CHILD-ERROR', err3 ?? row2); + process.exit(1); + } + // No close(): the connection is deliberately left open. The + // process must exit on its own. + console.log('CHILD-EXITING'); + }); + }); +}); diff --git a/test/support/helper.js b/test/support/helper.js index 80b8862..c64c68c 100644 --- a/test/support/helper.js +++ b/test/support/helper.js @@ -1,31 +1,41 @@ -import fs from 'fs'; +import fs from 'node:fs'; export function deleteFile(name) { try { fs.unlinkSync(name); - } catch(err) { - if (err.errno !== process.ENOENT && err.code !== 'ENOENT' && err.syscall !== 'unlink') { + } catch (err) { + if ( + err.errno !== process.ENOENT && + err.code !== 'ENOENT' && + err.syscall !== 'unlink' + ) { throw err; } } } -export function ensureExists(name, cb) { - if (!fs.existsSync(name)) { - fs.mkdirSync(name); - } +export function ensureExists(name, _cb) { + // recursive, not existsSync-then-mkdir: node --test runs each file in + // its own process, and two before-hooks creating test/tmp at the same + // moment raced the check (EEXIST cancelled a whole suite once). mkdir + // -p semantics are race-free and tolerate the existing directory. + fs.mkdirSync(name, { recursive: true }); } export function fileDoesNotExist(name) { try { fs.statSync(name); - } catch(err) { - if (err.errno !== process.ENOENT && err.code !== 'ENOENT' && err.syscall !== 'unlink') { + } catch (err) { + if ( + err.errno !== process.ENOENT && + err.code !== 'ENOENT' && + err.syscall !== 'unlink' + ) { throw err; } } -}; +} export function fileExists(name) { fs.statSync(name); -}; \ No newline at end of file +} diff --git a/test/support/permission_child.mjs b/test/support/permission_child.mjs new file mode 100644 index 0000000..157a64c --- /dev/null +++ b/test/support/permission_child.mjs @@ -0,0 +1,330 @@ +// Child-process scenario runner for test/permission.test.js. +// +// The Node permission model cannot be enabled inside an already-running +// process, and every interesting assertion here is about the interaction +// of two *flags*, so each scenario runs as a real child with real +// --permission flags chosen by the parent. The child reports raw +// observations (error codes, messages, outcomes) as one JSON line per +// step; the parent owns the assertions — nothing here decides pass or +// fail. +// +// Usage: node permission_child.mjs +// The fixture root (under the repo, created by the parent) holds the +// inside/ and outside/ trees; the parent's --allow-fs-* grants are +// computed from it. + +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import sqlite3 from '../../lib/sqlite3.js'; + +const [, , scenario, fixtureRoot] = process.argv; +const inside = path.join(fixtureRoot, 'inside'); +// The outside tree deliberately lives under the OS temp directory, NOT +// inside the repo: the children are granted fs.read of the repo (they +// must read the driver itself), so an "outside" fixture under the repo +// would be inside the grant and prove nothing. +const outside = path.join( + tmpdir(), + `permission-outside-${path.basename(fixtureRoot)}`, +); + +/** @type {unknown[]} */ +const report = []; +const say = (entry) => { + report.push(entry); + console.log(`STEP ${JSON.stringify(entry)}`); +}; + +/** + * Makes a step value JSON-safe: run results carry BigInts (`lastID`), + * which JSON.stringify refuses. + * + * @param {unknown} value the raw value. + * @returns {unknown} a serializable stand-in. + */ +function jsonSafe(value) { + if (typeof value === 'bigint') return `${value}n`; + if (Array.isArray(value)) return value.map(jsonSafe); + if (value !== null && typeof value === 'object') { + /** @type {Record} */ + const out = {}; + for (const [k, v] of Object.entries(value)) out[k] = jsonSafe(v); + return out; + } + return value; +} + +/** + * Runs one step and reports the raw outcome (value or error + * code/message), never judging it. + * + * @param {string} name the step name. + * @param {() => unknown} fn the action. + */ +async function step(name, fn) { + try { + const value = await fn(); + say({ name, ok: true, value: jsonSafe(value ?? null) }); + } catch (err) { + const e = + /** @type {Error & { code?: string, permission?: string, resource?: string }} */ ( + err + ); + say({ + name, + ok: false, + code: e.code ?? null, + permission: e.permission ?? null, + resource: e.resource ?? null, + message: e.message, + }); + } +} + +switch (scenario) { + case 'model-shape': + say({ + name: 'shape', + permissionType: typeof process.permission, + hasType: typeof process.permission?.has, + isEnabledType: typeof process.permission?.isEnabled, + }); + break; + + case 'ro-open-allowed': { + const db = await sqlite3.open(path.join(inside, 'ro.db'), { + mode: sqlite3.OPEN_READONLY, + }); + await step('read', () => db.get('SELECT 1 AS v')); + await step('close', () => db.close()); + break; + } + + case 'rw-open-denied-dir': { + // The exact file is write-granted but its directory is not: the + // journal/WAL check is what must refuse, naming the directory. + const target = path.join(inside, 'exact-file-only.db'); + await step('open', () => sqlite3.open(target)); + break; + } + + case 'rw-open-allowed': { + const db = await sqlite3.open(path.join(inside, 'w.db')); + await step('write', () => db.exec('CREATE TABLE IF NOT EXISTS t (x)')); + await step('close', () => db.close()); + break; + } + + case 'open-outside': { + await step('open', () => + sqlite3.open(path.join(outside, 'x.db'), { + mode: sqlite3.OPEN_READONLY, + }), + ); + break; + } + + case 'temp-filename': { + await step("open ''", () => sqlite3.open('')); + break; + } + + case 'attach': { + const db = await sqlite3.open(':memory:'); + await step('attach-outside', () => + db.exec(`ATTACH '${path.join(outside, 'y.db')}' AS y`), + ); + await step('vacuum-into-outside', () => + db.exec(`VACUUM INTO '${path.join(outside, 'z.db')}'`), + ); + await step('attach-memory', () => db.exec("ATTACH ':memory:' AS m")); + await step('close', () => db.close()); + break; + } + + case 'attach-allowed': { + const db = await sqlite3.open(':memory:'); + const target = path.join(inside, 'attach-target.db'); + await step('configure', () => db.configure('attachPaths', [target])); + await step('attach-allowed-target', () => + db.exec(`ATTACH '${target}' AS ok`), + ); + await step('attach-other-inside', () => + db.exec(`ATTACH '${path.join(inside, 'other.db')}' AS nope`), + ); + await step('vacuum-into-allowed-target', () => + db.exec(`VACUUM INTO '${path.join(inside, 'vac.db')}'`), + ); + await step('close', () => db.close()); + break; + } + + case 'load-extension': { + const db = await sqlite3.open(':memory:'); + await step('load-unlisted', () => + db.loadExtension('/tmp/definitely-not-there.ext'), + ); + await step('configure-allow', () => + db.configure('extensionPolicy', { + allow: ['/tmp/definitely-not-there.ext'], + }), + ); + // Allowlisted: the policy lets it through, so the failure is the + // native dlopen of a missing file — a different error than the + // policy refusal, which is the observable distinction. + await step('load-allowlisted', () => + db.loadExtension('/tmp/definitely-not-there.ext'), + ); + await step('sql-load-extension-fn', () => + db.exec("SELECT load_extension('/tmp/x')"), + ); + await step('close', () => db.close()); + break; + } + + case 'uri': { + await step('uri-ro-inside', () => + sqlite3.open(`file:${path.join(inside, 'ro.db')}?mode=ro`, { + mode: sqlite3.OPEN_READONLY | sqlite3.OPEN_URI, + }), + ); + await step('uri-outside', () => + sqlite3.open(`file:${path.join(outside, 'x.db')}?mode=ro`, { + mode: sqlite3.OPEN_READONLY | sqlite3.OPEN_URI, + }), + ); + await step('uri-outside-noquery', () => + sqlite3.open(`file:${path.join(outside, 'x.db')}`, { + mode: sqlite3.OPEN_READONLY | sqlite3.OPEN_URI, + }), + ); + await step('uri-memory', () => + sqlite3.open('file::memory:', { + mode: sqlite3.OPEN_READWRITE | sqlite3.OPEN_URI, + }), + ); + await step('uri-bad-mode', () => + sqlite3.open(`file:${path.join(inside, 'ro.db')}?mode=bogus`, { + mode: sqlite3.OPEN_READONLY | sqlite3.OPEN_URI, + }), + ); + await step('uri-non-file-scheme', () => + sqlite3.open('http://host/x.db', { + mode: sqlite3.OPEN_READONLY | sqlite3.OPEN_URI, + }), + ); + break; + } + + case 'backup': { + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE b (x)'); + await step( + 'backup-outside', + () => + new Promise((resolve, reject) => { + const backup = db.backup(path.join(outside, 'b.db')); + backup.step(-1, (err) => (err ? reject(err) : resolve())); + backup.on('error', reject); + }), + ); + await step( + 'backup-inside', + () => + new Promise((resolve, reject) => { + const backup = db.backup(path.join(inside, 'b.db')); + backup.on('error', reject); + backup.step(-1, () => { + backup.finish(() => resolve()); + }); + }), + ); + await step('close', () => db.close()); + break; + } + + case 'memory-unaffected': { + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE m (a); INSERT INTO m VALUES (1)'); + await step('read', () => db.get('SELECT a FROM m')); + await step('close', () => db.close()); + break; + } + + case 'untrusted-under-permissions': { + const db = await sqlite3.open(path.join(inside, 'ro.db'), { + mode: sqlite3.OPEN_READONLY, + untrusted: true, + }); + await step('read', () => db.get('SELECT 1 AS v')); + await step('attach-refused', () => db.exec("ATTACH ':memory:' AS m")); + await step('close', () => db.close()); + break; + } + + case 'exit-unclosed': { + // Opens, reads, and exits without closing anything. The exit code + // is the assertion (139 would be a segfault at teardown). + const db = await sqlite3.open(path.join(inside, 'ro.db'), { + mode: sqlite3.OPEN_READONLY, + }); + await db.get('SELECT 1 AS v'); + say({ name: 'unclosed-live', ok: true }); + process.exit(0); + break; + } + + case 'exit-after-refusal': { + try { + await sqlite3.open(path.join(outside, 'x.db')); + } catch { + // Refused; nothing was opened, nothing to close. + } + say({ name: 'refused-and-alive', ok: true }); + process.exit(0); + break; + } + + case 'off-model': { + // Runs with NO --permission flag: the zero-cost path. Behaviour + // must be identical to pre-v9. + say({ + name: 'shape', + permissionType: typeof process.permission, + }); + const db = await sqlite3.open(path.join(inside, 'w.db')); + await step('write', () => db.exec('CREATE TABLE IF NOT EXISTS t (x)')); + await step('attach-outside', () => + db.exec(`ATTACH '${path.join(outside, 'y.db')}' AS y`), + ); + await step('load-extension-reaches-native', () => + db.loadExtension('/tmp/definitely-not-there.ext'), + ); + await step('close', () => db.close()); + break; + } + + case 'worker-pool': { + // The pool opens its connections inside worker threads: the + // wrapper's checks run there too (workers see the same + // permission model; the parent grants this file's dir for the + // writer's read-write open). + const p = await sqlite3.pool(path.join(inside, 'pool.db'), { + readers: 0, + }); + await step('pool-get', () => p.get('SELECT 1 AS v')); + await step('pool-write', () => + p.write('CREATE TABLE IF NOT EXISTS pw (x)'), + ); + await step('pool-close', () => p.close()); + break; + } + + default: + console.error(`unknown scenario ${scenario}`); + process.exit(2); +} + +console.log('CHILD_DONE'); +process.exit(0); diff --git a/test/support/teardown_exit_child.mjs b/test/support/teardown_exit_child.mjs new file mode 100644 index 0000000..caaebf6 --- /dev/null +++ b/test/support/teardown_exit_child.mjs @@ -0,0 +1,37 @@ +// Opens a connection, uses the changeset iterator, and deliberately +// never closes anything, so the wrappers are torn down by the +// environment rather than by an explicit close. +// +// The addon's class constructors have to live in per-environment +// instance data for that to be safe: node-addon-api deletes instance +// data while the environment is still alive, whereas a file-static +// Napi::Reference is destroyed at process exit, after the environment is +// gone, and its napi_delete_reference then lands on a dead env. That +// segfaulted at exit on musl (glibc and macOS tolerated it), so this +// child asserts almost nothing and exists for its exit status. +// +// The connection is left with nothing else holding a reference on it: a +// session or blob kept open changes the finalization order and hides the +// crash, so this stays deliberately plain. +import sqlite3 from '../../lib/sqlite3.js'; + +const setup = await sqlite3.open(':memory:'); +await setup.exec('CREATE TABLE t (id INTEGER PRIMARY KEY, v)'); +const session = setup.session(); +await setup.run('INSERT INTO t VALUES (1, ?)', ['x']); +const changeset = await session.changeset(); +await session.close(); + +// Touch the changeset iterator: its constructor was the reference that +// outlived the environment. +const ops = [...sqlite3.iterateChangeset(changeset)]; +if (ops.length !== 1) { + console.log('UNEXPECTED-OPS', ops.length); + process.exit(2); +} +await setup.close(); + +// Now the case that actually reproduces it: a live connection at exit. +new sqlite3.Database(':memory:'); + +console.log('CHILD-EXITING'); diff --git a/test/support/throwing_backup_child.mjs b/test/support/throwing_backup_child.mjs new file mode 100644 index 0000000..94f221f --- /dev/null +++ b/test/support/throwing_backup_child.mjs @@ -0,0 +1,76 @@ +// Child scenario for the Backup CallGuard regression test +// (test/state_machine.test.js). Runs in its own process because a +// deliberate throw from a native async callback surfaces as an +// uncaughtException, which node:test attributes to the running test even +// when a user listener absorbs it. +// +// Signals on stdout (one per line): +// GETSYNC_OK / GETSYNC_FAIL: — the sync fast path after the throw +// CLOSE_OK / CLOSE_TIMEOUT — db.close() completing afterwards +// Exit status is 0 only when both succeeded. +import path from 'node:path'; + +import sqlite3 from '../../lib/sqlite3.js'; + +let sawThrow = false; +process.on('uncaughtException', (err) => { + sawThrow = true; + console.log(`UNCAUGHT:${err.message}`); +}); + +const exit = setTimeout(() => { + console.log('CLOSE_TIMEOUT'); + process.exit(1); +}, 15000); +exit.unref?.(); + +const db = await sqlite3.open(':memory:'); +await db.exec('CREATE TABLE t (a INT); INSERT INTO t VALUES (1),(2)'); + +const backup = db.backup( + path.join( + import.meta.dirname, + '..', + 'tmp', + `backup-throw-${process.pid}.db`, + ), +); +backup.step(-1, function () { + throw new Error('step callback boom'); +}); + +// Wait for the backup step to actually finish, rather than assuming it +// fits in a fixed sleep. The point of this scenario is what the *next* +// call sees after a throwing step callback, so the step's async work has +// to have landed first — otherwise getSync legitimately reports "database +// is busy" and the test fails for a reason that has nothing to do with +// the call guard it exists to check. A fixed 100 ms raced on loaded +// machines: on a CPU-starved ubuntu-22.04 runner (and reproducibly in a +// 1-CPU container under load) the step was still running at 100 ms, +// failing ~15 runs in 20 on this commit and on its parent alike. +const deadline = Date.now() + 10000; +while (Date.now() < deadline && !(sawThrow && db.state.pending === 0)) { + await new Promise((resolve) => setTimeout(resolve, 10)); +} + +try { + const row = db.getSync('SELECT 1 AS x'); + if (row && row.x === 1) { + console.log('GETSYNC_OK'); + } else { + console.log(`GETSYNC_FAIL:unexpected row ${JSON.stringify(row)}`); + process.exit(1); + } +} catch (err) { + console.log(`GETSYNC_FAIL:${err.message}`); + process.exit(1); +} + +try { + await db.close(); + console.log('CLOSE_OK'); + process.exit(0); +} catch (err) { + console.log(`CLOSE_FAIL:${err.message}`); + process.exit(1); +} diff --git a/test/sync.test.js b/test/sync.test.js index 71ce4f5..dc591f2 100644 --- a/test/sync.test.js +++ b/test/sync.test.js @@ -1,32 +1,42 @@ +import assert from 'node:assert'; +import { spawnSync } from 'node:child_process'; +import { afterEach, beforeEach, describe, it } from 'node:test'; + import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; // Opt-in synchronous fast path. prepareSync/getSync/runSync/allSync skip // the threadpool when — and only when — the database is fully idle. These // tests pin correctness, type fidelity, and the busy/error semantics. -describe('sync api', function() { +describe('sync api', function () { let db; - beforeEach(function(done) { - db = new sqlite3.Database(':memory:', function(err) { + beforeEach(function (_t, done) { + db = new sqlite3.Database(':memory:', function (err) { assert.ifError(err); - db.exec('CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER, b TEXT, c BLOB, d REAL)', done); + db.exec( + 'CREATE TABLE t (id INTEGER PRIMARY KEY, a INTEGER, b TEXT, c BLOB, d REAL)', + done, + ); }); }); - afterEach(function(done) { - db.close(function() { done(); }); + afterEach(function (_t, done) { + db.close(function () { + done(); + }); }); - describe('statement-level', function() { - it('prepareSync returns a usable statement', function() { + describe('statement-level', function () { + it('prepareSync returns a usable statement', function () { const stmt = db.prepareSync('SELECT ? AS v'); assert.strictEqual(stmt.getSync(42).v, 42); stmt.finalize(); }); - it('getSync returns rows with full type fidelity', function() { - const ins = db.prepareSync('INSERT INTO t (a, b, c, d) VALUES (?, ?, ?, ?)'); + it('getSync returns rows with full type fidelity', function () { + const ins = db.prepareSync( + 'INSERT INTO t (a, b, c, d) VALUES (?, ?, ?, ?)', + ); const buf = Buffer.from([0x00, 0xff, 0x10, 0x20]); ins.runSync(7, 'héllo ✓', buf, 2.5); ins.finalize(); @@ -41,22 +51,30 @@ describe('sync api', function() { sel.finalize(); }); - it('getSync binds array, positional and named params', function() { + it('getSync binds array, positional and named params', function () { const stmt = db.prepareSync('SELECT ?1 AS a, ?2 AS b'); - assert.deepStrictEqual({a: 1, b: 2}, (() => { const r = stmt.getSync([1, 2]); return {a: r.a, b: r.b}; })()); + assert.deepStrictEqual( + { a: 1, b: 2 }, + (() => { + const r = stmt.getSync([1, 2]); + return { a: r.a, b: r.b }; + })(), + ); assert.strictEqual(stmt.getSync(5, 6).a, 5); stmt.finalize(); const named = db.prepareSync('SELECT $x AS x, :y AS y'); - const r = named.getSync({$x: 1, ':y': 2}); + const r = named.getSync({ $x: 1, ':y': 2 }); assert.strictEqual(r.x, 1); assert.strictEqual(r.y, 2); named.finalize(); }); - it('getSync returns undefined when exhausted and rows while stepping', function() { + it('getSync returns undefined when exhausted and rows while stepping', function () { const ins = db.prepareSync('INSERT INTO t (a) VALUES (?)'); - ins.runSync(1); ins.runSync(2); ins.runSync(3); + ins.runSync(1); + ins.runSync(2); + ins.runSync(3); ins.finalize(); const sel = db.prepareSync('SELECT a FROM t ORDER BY a'); @@ -68,9 +86,10 @@ describe('sync api', function() { sel.finalize(); }); - it('getSync re-executes when re-bound after exhaustion', function() { + it('getSync re-executes when re-bound after exhaustion', function () { const ins = db.prepareSync('INSERT INTO t (a) VALUES (?)'); - ins.runSync(1); ins.runSync(2); + ins.runSync(1); + ins.runSync(2); ins.finalize(); const sel = db.prepareSync('SELECT a FROM t WHERE a = ?'); @@ -80,7 +99,7 @@ describe('sync api', function() { sel.finalize(); }); - it('runSync sets lastID/changes on the statement and returns it', function() { + it('runSync sets lastID/changes on the statement and returns it', function () { const stmt = db.prepareSync('INSERT INTO t (a) VALUES (?)'); for (let i = 1; i <= 3; i++) { const ret = stmt.runSync(i); @@ -95,7 +114,7 @@ describe('sync api', function() { stmt.finalize(); }); - it('allSync returns arrays, empty and non-empty', function() { + it('allSync returns arrays, empty and non-empty', function () { const sel = db.prepareSync('SELECT a FROM t ORDER BY a'); assert.deepStrictEqual(sel.allSync(), []); sel.finalize(); @@ -105,15 +124,21 @@ describe('sync api', function() { ins.finalize(); const sel2 = db.prepareSync('SELECT a FROM t ORDER BY a'); - assert.deepStrictEqual(sel2.allSync().map(r => r.a), [1, 2, 3, 4]); + assert.deepStrictEqual( + sel2.allSync().map((r) => r.a), + [1, 2, 3, 4], + ); // Reusable afterwards. - assert.deepStrictEqual(sel2.allSync().map(r => r.a), [1, 2, 3, 4]); + assert.deepStrictEqual( + sel2.allSync().map((r) => r.a), + [1, 2, 3, 4], + ); sel2.finalize(); }); - it('allSync handles large results', function() { + it('allSync handles large results', function () { const ins = db.prepareSync('INSERT INTO t (b) VALUES (?)'); - for (let i = 0; i < 2000; i++) ins.runSync('row-' + i); + for (let i = 0; i < 2000; i++) ins.runSync(`row-${i}`); ins.finalize(); const sel = db.prepareSync('SELECT id, b FROM t ORDER BY id'); const rows = sel.allSync(); @@ -122,37 +147,57 @@ describe('sync api', function() { sel.finalize(); }); - it('throws sqlite errors with errno and code', function() { + it('throws sqlite errors with errno and code', function () { // prepare_v2 reports missing tables at prepare time. - assert.throws(function() { db.prepareSync('SELECT * FROM nonexistent_table'); }, - function(err) { + assert.throws( + function () { + db.prepareSync('SELECT * FROM nonexistent_table'); + }, + function (err) { assert.strictEqual(err.code, 'SQLITE_ERROR'); assert.strictEqual(err.errno, 1); return true; - }); + }, + ); const ins = db.prepareSync('INSERT INTO t (id) VALUES (?)'); ins.runSync(1); - assert.throws(function() { ins.runSync(1); }, function(err) { - assert.strictEqual(err.code, 'SQLITE_CONSTRAINT'); - return true; - }); + assert.throws( + function () { + ins.runSync(1); + }, + function (err) { + // v9 reports the extended code; the primary code + // moved to err.primaryCode. INTEGER PRIMARY KEY + // conflicts report CONSTRAINT_PRIMARYKEY. + assert.strictEqual( + err.code, + 'SQLITE_CONSTRAINT_PRIMARYKEY', + ); + assert.strictEqual(err.primaryCode, 'SQLITE_CONSTRAINT'); + return true; + }, + ); ins.finalize(); }); - it('prepareSync throws on invalid SQL', function() { - assert.throws(function() { db.prepareSync('NO SUCH SYNTAX'); }, - /SQLITE_ERROR|syntax error/); + it('prepareSync throws on invalid SQL', function () { + assert.throws(function () { + db.prepareSync('NO SUCH SYNTAX'); + }, /SQLITE_ERROR|syntax error/); }); - it('rejects callback arguments', function() { + it('rejects callback arguments', function () { const stmt = db.prepareSync('SELECT ? AS v'); - assert.throws(function() { stmt.getSync(function() {}); }, - /callback/i); + assert.throws(function () { + stmt.getSync(function () { + /* any callback must be rejected */ + }); + }, /callback/i); stmt.finalize(); }); - it('works repeatedly on the same statement without lockup', function() { + it('works repeatedly on the same statement without lockup', function () { const stmt = db.prepareSync('SELECT ? AS v'); for (let i = 0; i < 1000; i++) { assert.strictEqual(stmt.getSync(i).v, i); @@ -161,91 +206,112 @@ describe('sync api', function() { }); }); - describe('busy gating', function() { - it('throws while async work is in flight', function(done) { - db.run('INSERT INTO t (a) VALUES (?)', 1, function(err) { + describe('busy gating', function () { + it('throws while async work is in flight', function (_t, done) { + db.run('INSERT INTO t (a) VALUES (?)', 1, function (err) { assert.ifError(err); done(); }); - assert.throws(function() { + assert.throws(function () { db.getSync('SELECT COUNT(*) AS n FROM t'); }, /busy/); }); - it('throws inside an async completion callback (bookkeeping pending)', function(done) { + it('throws inside an async completion callback (bookkeeping pending)', function (_t, done) { // STATEMENT_END runs after user callbacks, so the completing // op itself still counts as in-flight. Deferred calls see the // drained state. This pins the documented semantics. - db.run('INSERT INTO t (a) VALUES (?)', 1, function(err) { + db.run('INSERT INTO t (a) VALUES (?)', 1, function (err) { assert.ifError(err); - assert.throws(function() { + assert.throws(function () { db.getSync('SELECT COUNT(*) AS n FROM t'); }, /busy/); - setImmediate(function() { - assert.strictEqual(db.getSync('SELECT COUNT(*) AS n FROM t').n, 1); + setImmediate(function () { + assert.strictEqual( + db.getSync('SELECT COUNT(*) AS n FROM t').n, + 1, + ); done(); }); }); }); - it('works again once the database drains', function(done) { - db.run('INSERT INTO t (a) VALUES (?)', 1, function(err) { + it('works again once the database drains', function (_t, done) { + db.run('INSERT INTO t (a) VALUES (?)', 1, function (err) { assert.ifError(err); - setImmediate(function() { - assert.strictEqual(db.getSync('SELECT COUNT(*) AS n FROM t').n, 1); + setImmediate(function () { + assert.strictEqual( + db.getSync('SELECT COUNT(*) AS n FROM t').n, + 1, + ); done(); }); }); }); - it('throws under serialize() with queued work', function() { - db.serialize(function() { - db.run('INSERT INTO t (a) VALUES (1)', function(err) { + it('throws under serialize() with queued work', function () { + db.serialize(function () { + db.run('INSERT INTO t (a) VALUES (1)', function (err) { assert.ifError(err); }); - assert.throws(function() { + assert.throws(function () { db.runSync('INSERT INTO t (a) VALUES (2)'); }, /busy/); }); }); - it('throws on a finalized statement', function() { + it('throws on a finalized statement', function () { const stmt = db.prepareSync('SELECT 1 AS v'); stmt.finalize(); - assert.throws(function() { stmt.getSync(); }, /finalized/); + assert.throws(function () { + stmt.getSync(); + }, /finalized/); }); }); - describe('database-level', function() { - it('getSync/runSync/allSync without a cache', function() { - const info = db.runSync('INSERT INTO t (a, b) VALUES (?, ?)', 1, 'one'); + describe('database-level', function () { + it('getSync/runSync/allSync without a cache', function () { + const info = db.runSync( + 'INSERT INTO t (a, b) VALUES (?, ?)', + 1, + 'one', + ); assert.strictEqual(info.lastID, 1); assert.strictEqual(info.changes, 1); - assert.strictEqual(db.getSync('SELECT b FROM t WHERE a = ?', 1).b, 'one'); - assert.strictEqual(db.getSync('SELECT b FROM t WHERE a = ?', 99), undefined); + assert.strictEqual( + db.getSync('SELECT b FROM t WHERE a = ?', 1).b, + 'one', + ); + assert.strictEqual( + db.getSync('SELECT b FROM t WHERE a = ?', 99), + undefined, + ); assert.strictEqual(db.allSync('SELECT a FROM t').length, 1); const info2 = db.runSync('UPDATE t SET b = ?', 'uno'); assert.strictEqual(info2.changes, 1); }); - it('getSync/runSync/allSync with the statement cache reuse statements', function() { + it('getSync/runSync/allSync with the statement cache reuse statements', function () { db.cacheStatements(); for (let i = 0; i < 50; i++) { db.runSync('INSERT INTO t (a) VALUES (?)', i); } assert.strictEqual(db.getSync('SELECT COUNT(*) AS n FROM t').n, 50); - assert.deepStrictEqual(db.allSync('SELECT COUNT(*) AS n FROM t').map(r => r.n), [50]); + assert.deepStrictEqual( + db.allSync('SELECT COUNT(*) AS n FROM t').map((r) => r.n), + [50], + ); // Two distinct SQL strings: the SELECT is shared between the // getSync and allSync calls. assert.strictEqual(db._stmtCache.size, 2); }); - it('sync and async calls interleave correctly when drained', function(done) { - db.run('INSERT INTO t (a) VALUES (1)', function(err) { + it('sync and async calls interleave correctly when drained', function (_t, done) { + db.run('INSERT INTO t (a) VALUES (1)', function (err) { assert.ifError(err); - setImmediate(function() { + setImmediate(function () { db.runSync('INSERT INTO t (a) VALUES (2)'); - db.get('SELECT COUNT(*) AS n FROM t', function(err, row) { + db.get('SELECT COUNT(*) AS n FROM t', function (err, row) { assert.ifError(err); assert.strictEqual(row.n, 2); done(); @@ -254,13 +320,14 @@ describe('sync api', function() { }); }); - it('close() still works with cache populated by sync calls', function(done) { - const db2 = new sqlite3.Database(':memory:', function(err) { + it('close() still works with cache populated by sync calls', function (_t, done) { + const db2 = new sqlite3.Database(':memory:', function (err) { assert.ifError(err); - db2.exec('CREATE TABLE u (x INTEGER)', function() { + db2.exec('CREATE TABLE u (x INTEGER)', function () { db2.cacheStatements(); - for (let i = 0; i < 10; i++) db2.runSync('INSERT INTO u (x) VALUES (?)', i); - db2.close(function(err2) { + for (let i = 0; i < 10; i++) + db2.runSync('INSERT INTO u (x) VALUES (?)', i); + db2.close(function (err2) { assert.ifError(err2); done(); }); @@ -274,30 +341,37 @@ describe('sync api', function() { // bookkeeping must still run, or `locked` stays set and db->pending stays // elevated forever -- which would make the idle gate unsatisfiable and // permanently disable the sync fast path on that connection. -describe('sync fast path after a throwing callback', function() { - it('stays usable when a query callback throws', function(done) { +describe('sync fast path after a throwing callback', function () { + it('stays usable when a query callback throws', function (_t, done) { const db = new sqlite3.Database(':memory:'); - const mochaHandlers = process.listeners('uncaughtException'); + const savedHandlers = process.listeners('uncaughtException'); process.removeAllListeners('uncaughtException'); let restored = false; - const restore = function() { + const restore = function () { if (restored) return; restored = true; process.removeAllListeners('uncaughtException'); - for (const h of mochaHandlers) process.on('uncaughtException', h); + for (const h of savedHandlers) process.on('uncaughtException', h); }; - process.once('uncaughtException', function(err) { + process.once('uncaughtException', function (err) { assert.strictEqual(err.message, 'boom from callback'); // The connection must not be wedged by the throw above. - setTimeout(function() { + setTimeout(function () { restore(); try { - assert.deepStrictEqual(db.getSync('SELECT 1 AS v'), { v: 1 }); - assert.strictEqual(db.runSync('INSERT INTO t VALUES (2)').changes, 1); + assert.deepStrictEqual(db.getSync('SELECT 1 AS v'), { + v: 1, + }); + assert.strictEqual( + db.runSync('INSERT INTO t VALUES (2)').changes, + 1, + ); assert.deepStrictEqual( - db.getSync('SELECT COUNT(*) AS n FROM t'), { n: 2 }); + db.getSync('SELECT COUNT(*) AS n FROM t'), + { n: 2 }, + ); } catch (e) { return done(e); } @@ -305,30 +379,36 @@ describe('sync fast path after a throwing callback', function() { }, 50); }); - db.run('CREATE TABLE t (i)', function() { - db.run('INSERT INTO t VALUES (1)', function() { + db.run('CREATE TABLE t (i)', function () { + db.run('INSERT INTO t VALUES (1)', function () { throw new Error('boom from callback'); }); }); }); }); -// Without cacheStatements() the sync methods prepare a transient statement -// per call. It must be finalized, or every call leaks a prepared statement -// and close() fails with SQLITE_BUSY. -describe('sync fast path without the statement cache', function() { - it('does not leak prepared statements', function(done) { +// The sync methods cache their prepared statements, so those outlive the +// call. close() must drain that cache, or it fails with SQLITE_BUSY. +describe('sync fast path statement lifetime', function () { + it('does not leak prepared statements', function (_t, done) { const db = new sqlite3.Database(':memory:'); - db.run('CREATE TABLE t (i)', function() { - setImmediate(function() { + db.run('CREATE TABLE t (i)', function () { + setImmediate(function () { for (let i = 0; i < 20; i++) { - assert.deepStrictEqual(db.getSync('SELECT 1 AS v'), { v: 1 }); + assert.deepStrictEqual(db.getSync('SELECT 1 AS v'), { + v: 1, + }); + assert.strictEqual( + db.runSync('INSERT INTO t VALUES (?)', i).changes, + 1, + ); assert.strictEqual( - db.runSync('INSERT INTO t VALUES (?)', i).changes, 1); - assert.strictEqual(db.allSync('SELECT i FROM t').length, i + 1); + db.allSync('SELECT i FROM t').length, + i + 1, + ); } - // Fails with SQLITE_BUSY if any transient statement leaked. - db.close(function(err) { + // Fails with SQLITE_BUSY if the cache was not drained. + db.close(function (err) { assert.ifError(err); done(); }); @@ -336,3 +416,983 @@ describe('sync fast path without the statement cache', function() { }); }); }); + +// The sync read paths are the ones under optimisation pressure: they are +// being reshaped to convert rows straight from the sqlite3_stmt instead of +// materialising an intermediate C++ copy of the whole result set. These +// tests pin the observable semantics that refactor must preserve — the +// row shape, the marshalled types, and the exact wording of the errors, +// none of which was covered before. The error text matters twice over: +// the string it names a column with is built per cell on the hot path, +// so any change to how it is produced is a change to this message. +describe('sync read paths: shape, types and error text', function () { + /** @type {import('../lib/sqlite3.js').Database} */ + let db; + + beforeEach(async function () { + db = await sqlite3.open(':memory:'); + await db.exec( + 'CREATE TABLE m (i INTEGER, r REAL, t TEXT, b BLOB, n INTEGER)', + ); + await db.run( + 'INSERT INTO m VALUES (?, ?, ?, ?, ?)', + 42, + 1.5, + 'héllo', + Buffer.from([1, 2, 3]), + null, + ); + }); + + afterEach(async function () { + await db.close(); + }); + + it('getSync marshals every storage class and keeps insertion order', function () { + const row = db.getSync('SELECT i, r, t, b, n FROM m'); + assert.deepStrictEqual(Object.keys(row), ['i', 'r', 't', 'b', 'n']); + assert.strictEqual(row.i, 42); + assert.strictEqual(row.r, 1.5); + assert.strictEqual(row.t, 'héllo'); + assert.ok(Buffer.isBuffer(row.b)); + assert.deepStrictEqual([...row.b], [1, 2, 3]); + assert.strictEqual(row.n, null); + }); + + it('rows are plain objects on Object.prototype', function () { + // Not a null-prototype object: `row.hasOwnProperty(...)`, + // `instanceof Object` and util.inspect output all depend on this, + // and node:sqlite's choice of a null prototype is NOT ours to copy + // without a major-version note. + const row = db.getSync('SELECT i FROM m'); + assert.strictEqual(Object.getPrototypeOf(row), Object.prototype); + }); + + it('allSync returns one object per row, sharing the column names', function () { + db.runSync('INSERT INTO m (i) VALUES (7)'); + const rows = db.allSync('SELECT i FROM m ORDER BY i'); + assert.strictEqual(rows.length, 2); + assert.deepStrictEqual( + rows.map((r) => r.i), + [7, 42], + ); + assert.deepStrictEqual(Object.keys(rows[0]), ['i']); + }); + + it('duplicate column names collapse to the last value, as JS objects do', function () { + const row = db.getSync('SELECT 1 AS dup, 2 AS dup'); + assert.deepStrictEqual(Object.keys(row), ['dup']); + assert.strictEqual(row.dup, 2); + }); + + it('a zero-row query yields undefined from getSync and [] from allSync', function () { + assert.strictEqual( + db.getSync('SELECT i FROM m WHERE i = 999'), + undefined, + ); + assert.deepStrictEqual(db.allSync('SELECT i FROM m WHERE i = 999'), []); + }); + + it('an unsafe integer names the column it came from, by result name', async function () { + await db.exec('CREATE TABLE big (v INTEGER)'); + await db.run('INSERT INTO big VALUES (?)', 9007199254740993n); + // The message is asserted in full: it is the only consumer of the + // per-cell column description, so it is what proves that + // description is still correct however it comes to be built. + assert.throws( + () => db.getSync('SELECT v FROM big'), + (err) => + err instanceof RangeError && + err.message === + "Integer 9007199254740993 in column 'v' is outside the safe " + + 'integer range (-(2^53-1) .. 2^53-1); ' + + "configure('integerMode', 'bigint' | 'mixed') to read it exactly", + ); + // An alias renames it; an expression names itself. Both come from + // sqlite3_column_name, so both must survive the same way. + assert.throws( + () => db.getSync('SELECT v AS renamed FROM big'), + /in column 'renamed' is outside/, + ); + assert.throws( + () => db.getSync('SELECT v + 0 FROM big'), + /in column 'v \+ 0' is outside/, + ); + // And the column is named correctly when it is not the first one. + assert.throws( + () => db.getSync("SELECT 'a' AS first, v AS second FROM big"), + /in column 'second' is outside/, + ); + }); + + it('allSync reports the offending column from a later row, not the first', async function () { + // The failure is raised while converting row 2, after row 1 has + // already been built — a single-pass implementation must not lose + // the column identity by then. + await db.exec('CREATE TABLE big (v INTEGER)'); + await db.run('INSERT INTO big VALUES (?)', 1n); + await db.run('INSERT INTO big VALUES (?)', 9007199254740993n); + assert.throws( + () => db.allSync('SELECT v FROM big ORDER BY v'), + /Integer 9007199254740993 in column 'v' is outside/, + ); + }); + + it('bigint and mixed integer modes read the same rows without throwing', async function () { + await db.exec('CREATE TABLE big (v INTEGER)'); + await db.run('INSERT INTO big VALUES (?)', 9007199254740993n); + db.configure('integerMode', 'bigint'); + assert.strictEqual( + db.getSync('SELECT v FROM big').v, + 9007199254740993n, + ); + db.configure('integerMode', 'mixed'); + assert.strictEqual( + db.getSync('SELECT v FROM big').v, + 9007199254740993n, + ); + // In mixed mode a safe value stays a number. + assert.strictEqual(db.getSync('SELECT 5 AS v').v, 5); + }); + + it('a wide row keeps every column distinct', function () { + const cols = Array.from({ length: 40 }, (_, i) => `${i} AS c${i}`); + const row = db.getSync(`SELECT ${cols.join(', ')}`); + assert.strictEqual(Object.keys(row).length, 40); + assert.strictEqual(row.c0, 0); + assert.strictEqual(row.c39, 39); + }); + + it('text and blobs survive at the sizes that switch copy strategy', function () { + // 4 KiB is the zero-copy boundary for blobs (src/convert.cc); both + // sides of it must round-trip byte-for-byte. + for (const size of [1, 4095, 4096, 65536]) { + const buf = Buffer.alloc(size, 0xab); + const row = db.getSync('SELECT ? AS b', buf); + assert.strictEqual(row.b.length, size, `blob ${size}`); + assert.ok(row.b.equals(buf), `blob ${size} contents`); + const text = 'ü'.repeat(size); + assert.strictEqual( + db.getSync('SELECT ? AS t', text).t, + text, + `text ${size}`, + ); + } + }); +}); + +// The `{ rowMode: 'array' }` opt-in on the sync read paths: one array per +// row instead of an object. The default row shape is pinned above and must +// not change; these pin the array shape, which bulk readers (CSV export, +// ETL) opt into. +describe('sync read paths: rowMode array', function () { + /** @type {import('../lib/sqlite3.js').Database} */ + let db; + + beforeEach(async function () { + db = await sqlite3.open(':memory:'); + await db.exec( + 'CREATE TABLE m (i INTEGER, r REAL, t TEXT, b BLOB, n INTEGER)', + ); + await db.run( + 'INSERT INTO m VALUES (?, ?, ?, ?, ?)', + 42, + 1.5, + 'héllo', + Buffer.from([1, 2, 3]), + null, + ); + }); + + afterEach(async function () { + await db.close(); + }); + + it('getSync and allSync return arrays with full type fidelity', async function () { + await db.run('INSERT INTO m (i) VALUES (7)'); + const rows = db.allSync('SELECT i, r, t, b, n FROM m ORDER BY i', { + rowMode: 'array', + }); + assert.strictEqual(rows.length, 2); + assert.ok(Array.isArray(rows[0])); + assert.deepStrictEqual(rows[0], [7, null, null, null, null]); + assert.deepStrictEqual([...rows[1].slice(0, 2)], [42, 1.5]); + assert.strictEqual(rows[1][2], 'héllo'); + assert.ok(Buffer.isBuffer(rows[1][3])); + assert.deepStrictEqual([...rows[1][3]], [1, 2, 3]); + assert.strictEqual(rows[1][4], null); + + const row = db.getSync('SELECT i, r FROM m WHERE i = 7', { + rowMode: 'array', + }); + assert.deepStrictEqual(row, [7, null]); + }); + + it('duplicate column names keep every value, unlike object mode', function () { + assert.deepStrictEqual( + db.getSync('SELECT 1 AS dup, 2 AS dup', { rowMode: 'array' }), + [1, 2], + ); + // The object mode default still collapses (pinned above too). + assert.deepStrictEqual( + Object.keys(db.getSync('SELECT 1 AS dup, 2 AS dup')), + ['dup'], + ); + }); + + it('zero-row queries return [] and undefined, like object mode', function () { + assert.deepStrictEqual( + db.allSync('SELECT i FROM m WHERE i = 999', { rowMode: 'array' }), + [], + ); + assert.strictEqual( + db.getSync('SELECT i FROM m WHERE i = 999', { rowMode: 'array' }), + undefined, + ); + }); + + it('repeated database-level calls are independent queries, not cursor steps', function () { + db.cacheStatements(); + for (const mode of [{ rowMode: 'array' }, { rowMode: 'array' }]) { + const row = db.getSync('SELECT i FROM m', mode); + assert.deepStrictEqual(row, [42]); + } + assert.deepStrictEqual( + db.allSync('SELECT i FROM m', { rowMode: 'array' }).length, + 1, + ); + }); + + it('statement-level calls mix modes on one statement', function () { + const stmt = db.prepareSync('SELECT i FROM m'); + assert.deepStrictEqual(stmt.getSync({ rowMode: 'array' }), [42]); + assert.strictEqual(stmt.getSync(), undefined); // cursor exhausted + assert.deepStrictEqual(stmt.allSync(), [{ i: 42 }]); + assert.deepStrictEqual(stmt.allSync({ rowMode: 'array' }), [[42]]); + stmt.finalize(); + }); + + it('named binds and the options bag coexist', function () { + const stmt = db.prepareSync('SELECT $x AS x, :y AS y'); + assert.deepStrictEqual( + stmt.getSync({ $x: 1, ':y': 2 }, { rowMode: 'array' }), + [1, 2], + ); + assert.deepStrictEqual( + db.getSync('SELECT $x AS x', { $x: 5 }, { rowMode: 'array' }), + [5], + ); + stmt.finalize(); + }); + + it('rejects a rowMode that is not object or array', function () { + assert.throws( + () => db.getSync('SELECT i FROM m', { rowMode: 'bogus' }), + (err) => + err instanceof TypeError && + err.message === "rowMode must be 'object' or 'array'", + ); + assert.throws( + () => db.allSync('SELECT i FROM m', { rowMode: 7 }), + TypeError, + ); + }); + + it('the integer-mode RangeError names the column in array mode too', async function () { + await db.exec('CREATE TABLE big (v INTEGER)'); + await db.run('INSERT INTO big VALUES (?)', 9007199254740993n); + // Same wording as object mode: the column description is shared. + assert.throws( + () => db.getSync('SELECT v FROM big', { rowMode: 'array' }), + (err) => + err instanceof RangeError && + err.message === + "Integer 9007199254740993 in column 'v' is outside the safe " + + 'integer range (-(2^53-1) .. 2^53-1); ' + + "configure('integerMode', 'bigint' | 'mixed') to read it exactly", + ); + // And from a later row of allSync (the single-pass loop must not + // lose the column identity by then). + await db.run('INSERT INTO big VALUES (?)', 1n); + assert.throws( + () => + db.allSync('SELECT v FROM big ORDER BY v', { + rowMode: 'array', + }), + /Integer 9007199254740993 in column 'v' is outside/, + ); + // bigint mode reads the same rows as arrays without throwing. + db.configure('integerMode', 'bigint'); + assert.deepStrictEqual( + db.getSync('SELECT v FROM big', { rowMode: 'array' }), + [9007199254740993n], + ); + }); + + it('a schema change on a cached statement rebuilds the row arrays', async function () { + // SELECT * over a table that is dropped and recreated with a + // different shape: the cached statement transparently re-prepares + // on the next step, and the sync paths must follow the live + // statement's new result shape instead of reusing the cached keys. + db.cacheStatements(); + await db.exec('CREATE TABLE u (a INTEGER)'); + await db.run('INSERT INTO u VALUES (1)'); + assert.deepStrictEqual( + db.allSync('SELECT * FROM u', { rowMode: 'array' }), + [[1]], + ); + await db.exec('DROP TABLE u; CREATE TABLE u (a INTEGER, b TEXT)'); + await db.runSync("INSERT INTO u VALUES (2, 'x')"); + assert.deepStrictEqual( + db.getSync('SELECT * FROM u', { rowMode: 'array' }), + [2, 'x'], + ); + assert.deepStrictEqual(Object.keys(db.getSync('SELECT * FROM u')), [ + 'a', + 'b', + ]); + }); +}); + +// The row factory (lib/sqlite3.js makeRowFactory + Statement::RowFactoryForShape) +// builds each row by calling a generated function instead of storing each +// column from C++. It is a pure optimisation, so every one of these +// assertions describes behaviour that predates it and must survive it — +// the generated source embeds the column names, which is exactly where a +// fast path can start disagreeing with the slow one. +describe('row factory: generated rows match the store loop', function () { + /** @type {import('../lib/sqlite3.js').Database} */ + let db; + + beforeEach(async function () { + db = await sqlite3.open(':memory:'); + }); + + afterEach(async function () { + await db.close(); + }); + + /** + * Reads one row through every path that builds rows, so a fast path + * cannot disagree with a slow one unnoticed. + * @param {string} sql the query. + * @returns {Promise[]>} one row per path. + */ + async function everyPath(sql) { + const rows = [ + db.getSync(sql), + db.allSync(sql)[0], + (await db.all(sql))[0], + await db.get(sql), + ]; + const each = []; + await new Promise((resolve, reject) => { + db.each( + sql, + (err, row) => (err ? reject(err) : each.push(row)), + (err) => (err ? reject(err) : resolve(undefined)), + ); + }); + rows.push(each[0]); + return /** @type {Record[]} */ (rows); + } + + it('escapes quotes, backslashes and newlines in column names', async function () { + // These names are interpolated into generated source; an escaping + // bug here is a syntax error at best and a wrong row at worst. + const sql = + 'SELECT 1 AS "a\'b", 2 AS "c""d", 3 AS "e\\f", 4 AS "g' + + String.fromCharCode(10) + + 'h"'; + for (const row of await everyPath(sql)) { + assert.deepStrictEqual(Object.keys(row), [ + "a'b", + 'c"d', + 'e\\f', + 'g\nh', + ]); + assert.deepStrictEqual(Object.values(row), [1, 2, 3, 4]); + } + }); + + it('keeps non-ASCII and empty column names intact', async function () { + const sql = 'SELECT 1 AS "héllo—✓", 2 AS ""'; + for (const row of await everyPath(sql)) { + assert.deepStrictEqual(Object.keys(row), ['héllo—✓', '']); + assert.strictEqual(row['héllo—✓'], 1); + assert.strictEqual(row[''], 2); + } + }); + + it('treats a __proto__ column exactly as the store loop did', async function () { + // An object literal assigns the prototype for this key rather than + // creating an own property — which is also what a property store + // did, so the observable result is unchanged. Pinned because the + // two mechanisms agreeing here is load-bearing, not obvious. + const sql = 'SELECT 1 AS "__proto__", 2 AS keep'; + for (const row of await everyPath(sql)) { + assert.strictEqual(Object.hasOwn(row, '__proto__'), false); + assert.strictEqual(Object.getPrototypeOf(row), Object.prototype); + assert.strictEqual(row.keep, 2); + } + }); + + it('collapses duplicate column names to the last value', async function () { + const sql = 'SELECT 1 AS dup, 2 AS other, 3 AS dup'; + for (const row of await everyPath(sql)) { + assert.deepStrictEqual(row, { dup: 3, other: 2 }); + } + }); + + it('rebuilds the row shape after a re-prepare', async function () { + // The factory bakes in the column names, so a schema change that + // re-prepares the statement behind sqlite3_step must invalidate it. + await db.exec('CREATE TABLE s (a INTEGER)'); + await db.run('INSERT INTO s VALUES (1)'); + assert.deepStrictEqual(db.allSync('SELECT * FROM s'), [{ a: 1 }]); + await db.exec('DROP TABLE s'); + await db.exec('CREATE TABLE s (b INTEGER, c INTEGER)'); + await db.run('INSERT INTO s VALUES (2, 3)'); + assert.deepStrictEqual(db.allSync('SELECT * FROM s'), [{ b: 2, c: 3 }]); + }); + + it('falls back to the store loop beyond the factory column limit', async function () { + // kMaxFactoryColumns is 256; a wider result must still be correct. + const width = 300; + const cols = Array.from( + { length: width }, + (_, i) => `${i} AS c${i}`, + ).join(','); + const row = db.getSync(`SELECT ${cols}`); + assert.strictEqual(Object.keys(row).length, width); + assert.strictEqual(row.c0, 0); + assert.strictEqual(row.c299, 299); + assert.deepStrictEqual(await db.get(`SELECT ${cols}`), row); + }); + + it('still raises the integer-mode RangeError from a factory row', async function () { + await db.exec('CREATE TABLE big (v INTEGER)'); + await db.run('INSERT INTO big VALUES (9007199254740993)'); + assert.throws(() => db.allSync('SELECT v FROM big'), { + name: 'RangeError', + message: /column 'v'/, + }); + await assert.rejects(db.all('SELECT v FROM big'), { + name: 'RangeError', + message: /column 'v'/, + }); + }); + + it('preserves every value type through the factory', async function () { + await db.exec( + 'CREATE TABLE t (i INTEGER, r REAL, s TEXT, b BLOB, n INTEGER)', + ); + await db.run( + 'INSERT INTO t VALUES (?, ?, ?, ?, ?)', + 7, + 1.5, + 'héllo', + Buffer.from([1, 2, 3]), + null, + ); + for (const row of await everyPath('SELECT * FROM t')) { + assert.strictEqual(row.i, 7); + assert.strictEqual(row.r, 1.5); + assert.strictEqual(row.s, 'héllo'); + assert.ok(Buffer.isBuffer(row.b)); + assert.deepStrictEqual( + [.../** @type {Buffer} */ (row.b)], + [1, 2, 3], + ); + assert.strictEqual(row.n, null); + } + }); +}); + +describe('implicit sync statement cache', function () { + /** @type {import('../lib/sqlite3.js').Database} */ + let db; + + beforeEach(async function () { + db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (a INTEGER)'); + await db.run('INSERT INTO t VALUES (1)'); + }); + + afterEach(async function () { + await db.close(); + }); + + it('reuses the prepared statement across identical sync calls', function () { + db.getSync('SELECT a FROM t'); + db.getSync('SELECT a FROM t'); + assert.strictEqual(db._syncStmtCache.size, 1); + }); + + it('re-runs a cached parameterless query from its first row', function () { + // The cache makes the statement outlive the call, so a second + // getSync must restart rather than step a spent cursor. + assert.deepStrictEqual(db.getSync('SELECT a FROM t'), { a: 1 }); + assert.deepStrictEqual(db.getSync('SELECT a FROM t'), { a: 1 }); + }); + + it('is invalidated by registering a user function', async function () { + db.getSync('SELECT a FROM t'); + assert.strictEqual(db._syncStmtCache.size, 1); + db.function('noop', (x) => x); + assert.strictEqual(db._syncStmtCache.size, 0); + }); + + it('does not enable the opt-in async statement cache', function () { + db.getSync('SELECT a FROM t'); + assert.strictEqual(db._stmtCache, undefined); + }); + + it('is emptied by close, leaving no unfinalized statements', async function () { + const fresh = await sqlite3.open(':memory:'); + fresh.getSync('SELECT 1 AS a'); + assert.strictEqual(fresh._syncStmtCache.size, 1); + await fresh.close(); + assert.strictEqual(fresh._syncStmtCache.size, 0); + }); + + it('evicts beyond its capacity without leaking statements', function () { + for (let i = 0; i < 80; i++) db.getSync(`SELECT ${i} AS a`); + assert.ok(db._syncStmtCache.size <= 64); + }); +}); + +describe('row factory: realms that forbid code generation', function () { + it('falls back to the store loop and returns identical rows', function () { + // The generated row builder needs `new Function`. A CSP'd Electron + // renderer or this flag forbids it, and the addon must degrade to + // building rows column by column rather than fail. Run out of + // process because the restriction is per-isolate. + // Already a file: URL — import it as one. Going via .pathname and + // back through pathToFileURL doubles the drive letter on Windows + // ("D:\D:\a\...", because "/D:/a/..." reads as a relative path) + // and drops percent-encoding on any path containing spaces. + const lib = new URL('../lib/sqlite3.js', import.meta.url).href; + const script = ` + import mod from '${lib}'; + const sqlite3 = mod.verbose ? mod : mod.default; + try { new Function('return 1'); console.log('CODEGEN=allowed'); } + catch { console.log('CODEGEN=blocked'); } + const db = new sqlite3.Database(':memory:'); + await new Promise((r, j) => db.run('SELECT 1', (e) => e ? j(e) : r())); + db.runSync('CREATE TABLE t (a INTEGER, b TEXT, c BLOB)'); + db.runSync("INSERT INTO t VALUES (1, 'x', x'0102')"); + const row = db.allSync('SELECT * FROM t')[0]; + console.log('OBJ=' + JSON.stringify(db.allSync('SELECT * FROM t'))); + console.log('ARR=' + JSON.stringify( + db.allSync('SELECT * FROM t', { rowMode: 'array' }))); + console.log('ASYNC=' + JSON.stringify(await db.all('SELECT * FROM t'))); + console.log('PROTO=' + (Object.getPrototypeOf(row) === Object.prototype)); + console.log('BUFFER=' + Buffer.isBuffer(row.c)); + db.close(); + `; + const run = (extraArgs) => + spawnSync( + process.execPath, + [...extraArgs, '--input-type=module', '-e', script], + { encoding: 'utf8' }, + ); + + const blocked = run(['--disallow-code-generation-from-strings']); + assert.strictEqual(blocked.status, 0, blocked.stderr); + assert.match(blocked.stdout, /CODEGEN=blocked/); + + const allowed = run([]); + assert.strictEqual(allowed.status, 0, allowed.stderr); + assert.match(allowed.stdout, /CODEGEN=allowed/); + + // The rows themselves must not depend on which path built them. + const rowsOf = (out) => + out + .split(String.fromCharCode(10)) + .filter((line) => /^(OBJ|ARR|ASYNC|PROTO|BUFFER)=/.test(line)); + assert.deepStrictEqual(rowsOf(blocked.stdout), rowsOf(allowed.stdout)); + assert.match(blocked.stdout, /PROTO=true/); + assert.match(blocked.stdout, /BUFFER=true/); + }); +}); + +// The synchronous paths bind straight onto the statement instead of +// building a Values::Field per parameter (src/convert.cc BindValueDirect). +// That is a second implementation of the bind semantics, so these tests +// hold it against the first one: for every value shape and every failure, +// the sync paths must agree with the asynchronous Field path exactly. +describe('sync bind agrees with the async bind path', function () { + /** @type {import('../lib/sqlite3.js').Database} */ + let db; + + beforeEach(async function () { + db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (v)'); + }); + + afterEach(async function () { + await db.close(); + }); + + /** + * Round-trips one value through both bind implementations. + * @param {unknown} value the value to bind. + * @returns {Promise<{sync: unknown, async: unknown}>} both readings. + */ + async function bothPaths(value) { + await db.run('DELETE FROM t'); + db.runSync('INSERT INTO t VALUES (?)', value); + const sync = db.getSync('SELECT v FROM t').v; + await db.run('DELETE FROM t'); + await db.run('INSERT INTO t VALUES (?)', value); + const asyncRead = (await db.get('SELECT v FROM t')).v; + return { sync, async: asyncRead }; + } + + const cases = [ + ['integer', 42], + ['negative integer', -7], + ['zero', 0], + ['large safe integer', 9007199254740991], + ['float', 1.5], + ['NaN', Number.NaN], + ['Infinity', Number.POSITIVE_INFINITY], + ['string', 'hello'], + ['empty string', ''], + ['unicode string', 'héllo—✓'], + ['string with NUL', 'a\u0000b'], + ['true', true], + ['false', false], + ['null', null], + ['bigint', 123n], + ['negative bigint', -123n], + ]; + + for (const [label, value] of cases) { + it(`binds ${label} identically`, async function () { + const { sync, async: asyncValue } = await bothPaths(value); + assert.deepStrictEqual(sync, asyncValue); + }); + } + + it('binds an empty Buffer as an empty blob, not NULL', async function () { + const { sync, async: asyncValue } = await bothPaths(Buffer.alloc(0)); + assert.ok(Buffer.isBuffer(sync), 'sync bound NULL, not a blob'); + assert.strictEqual(/** @type {Buffer} */ (sync).length, 0); + assert.deepStrictEqual(sync, asyncValue); + }); + + it('binds Buffers, typed arrays, DataViews and ArrayBuffers alike', async function () { + const bytes = [1, 2, 3, 4]; + const views = [ + Buffer.from(bytes), + new Uint8Array(bytes), + new DataView(new Uint8Array(bytes).buffer), + new Uint8Array(bytes).buffer, + ]; + for (const view of views) { + const { sync, async: asyncValue } = await bothPaths(view); + assert.deepStrictEqual([.../** @type {Buffer} */ (sync)], bytes); + assert.deepStrictEqual(sync, asyncValue); + } + }); + + it('binds a Date as epoch milliseconds', async function () { + const date = new Date(1700000000000); + const { sync, async: asyncValue } = await bothPaths(date); + assert.strictEqual(sync, 1700000000000); + assert.strictEqual(sync, asyncValue); + }); + + it('binds a byteOffset view without the whole backing buffer', async function () { + const backing = new Uint8Array([9, 9, 1, 2, 3, 9]); + const view = backing.subarray(2, 5); + const { sync } = await bothPaths(view); + assert.deepStrictEqual([.../** @type {Buffer} */ (sync)], [1, 2, 3]); + }); + + /** + * Captures the error message from each path for the same call. + * @param {unknown[]} params the bind parameters. + * @param {string} [sql] the statement to bind against. + * @returns {Promise<{sync: string, async: string}>} both messages. + */ + async function bothErrors(params, sql = 'INSERT INTO t VALUES (?)') { + let syncMessage = '(no error)'; + try { + db.runSync(sql, ...params); + } catch (err) { + syncMessage = /** @type {Error} */ (err).message; + } + let asyncMessage = '(no error)'; + try { + await db.run(sql, ...params); + } catch (err) { + asyncMessage = /** @type {Error} */ (err).message; + } + return { sync: syncMessage, async: asyncMessage }; + } + + it('reports too few parameters identically', async function () { + const { sync, async: asyncMessage } = await bothErrors( + [1], + 'INSERT INTO t SELECT ? UNION ALL SELECT ?', + ); + assert.match( + sync, + /supplied 1 parameter\(s\) but the statement takes 2/, + ); + assert.strictEqual(sync, asyncMessage); + }); + + it('reports too many parameters identically', async function () { + const { sync, async: asyncMessage } = await bothErrors([1, 2, 3]); + assert.match( + sync, + /supplied 3 parameter\(s\) but the statement takes 1/, + ); + assert.strictEqual(sync, asyncMessage); + }); + + it('reports an unknown named parameter identically', async function () { + const { sync, async: asyncMessage } = await bothErrors( + [{ $nope: 1 }], + 'INSERT INTO t VALUES ($v)', + ); + assert.match(sync, /unknown named parameter "\$nope"/); + assert.strictEqual(sync, asyncMessage); + }); + + it('reports an unsupported type identically', async function () { + // A bare object is the named-parameters shape, not a value; nest + // it in the array form so it is bound as one. + const { sync, async: asyncMessage } = await bothErrors([[{ a: 1 }]]); + assert.match(sync, /Cannot bind parameter 1: unsupported type Object/); + assert.strictEqual(sync, asyncMessage); + }); + + it('reports an out-of-range BigInt identically', async function () { + const huge = 2n ** 64n; + const { sync, async: asyncMessage } = await bothErrors([huge]); + assert.match(sync, /BigInt .* outside the signed 64-bit integer range/); + assert.strictEqual(sync, asyncMessage); + }); + + it('names the offending parameter by position', async function () { + const { sync } = await bothErrors( + [1, Symbol('x')], + 'INSERT INTO t VALUES (?), (?)', + ); + assert.match(sync, /Cannot bind parameter 2:/); + }); + + it('names a failing named parameter by name', async function () { + const { sync } = await bothErrors( + [{ $v: Symbol('x') }], + 'INSERT INTO t VALUES ($v)', + ); + assert.match(sync, /Cannot bind parameter \$v:/); + }); + + it('accepts array, positional and named shapes alike', function () { + db.runSync('DELETE FROM t'); + db.runSync('INSERT INTO t VALUES (?)', 1); + db.runSync('INSERT INTO t VALUES (?)', [2]); + db.runSync('INSERT INTO t VALUES ($v)', { $v: 3 }); + assert.deepStrictEqual( + db.allSync('SELECT v FROM t ORDER BY v').map((r) => r.v), + [1, 2, 3], + ); + }); + + it('leaves no partial binding behind after a failed bind', function () { + db.runSync('DELETE FROM t'); + assert.throws(() => + db.runSync('INSERT INTO t VALUES (?), (?)', 1, Symbol('x')), + ); + // The statement must be re-runnable with a valid call afterwards. + db.runSync('INSERT INTO t VALUES (?), (?)', 4, 5); + assert.deepStrictEqual( + db.allSync('SELECT v FROM t ORDER BY v').map((r) => r.v), + [4, 5], + ); + }); + + // Named binding classifies each key as a position or a name, and + // decides whether the argument is a parameter map at all. Both bind + // implementations share that code, and its fast paths are only valid + // where they agree with the general one — so the odd spellings are + // pinned here rather than left to the common case. + + /** + * Binds a named-parameter map through both implementations. + * @param {object} params the named parameters. + * @param {string} sql the statement to bind against. + * @returns {Promise<{sync: unknown, async: unknown}>} both readings. + */ + async function bothNamed(params, sql) { + await db.run('DELETE FROM t'); + db.runSync(sql, params); + const sync = db.getSync('SELECT v FROM t').v; + await db.run('DELETE FROM t'); + await db.run(sql, params); + const asyncRead = (await db.get('SELECT v FROM t')).v; + return { sync, async: asyncRead }; + } + + it('binds a named parameter identically on both paths', async function () { + const { sync, async: asyncValue } = await bothNamed( + { $v: 'x' }, + 'INSERT INTO t VALUES ($v)', + ); + assert.strictEqual(sync, 'x'); + assert.strictEqual(sync, asyncValue); + }); + + it('binds the :name and @name sigils too', async function () { + for (const sigil of [':', '@']) { + const { sync, async: asyncValue } = await bothNamed( + { [`${sigil}v`]: 5 }, + `INSERT INTO t VALUES (${sigil}v)`, + ); + assert.strictEqual(sync, 5); + assert.strictEqual(sync, asyncValue); + } + }); + + it('reads an integer key as a bind position', async function () { + const { sync, async: asyncValue } = await bothNamed( + { 1: 'by index' }, + 'INSERT INTO t VALUES (?)', + ); + assert.strictEqual(sync, 'by index'); + assert.strictEqual(sync, asyncValue); + }); + + // Keys that could read as a number must not take the "obviously a + // name" shortcut: each of these coerces to an integer, so it selects + // a position exactly as it always has. + for (const key of [' 1', '1.0', '+1']) { + it(`treats the key ${JSON.stringify(key)} as position 1`, async function () { + const { sync, async: asyncValue } = await bothNamed( + { [key]: 'numeric-ish' }, + 'INSERT INTO t VALUES (?)', + ); + assert.strictEqual(sync, 'numeric-ish'); + assert.strictEqual(sync, asyncValue); + }); + } + + // Likewise for the ones that coerce to an out-of-range position: + // they must still fail, and fail the same way on both paths. + for (const key of ['0x10', '', '1e2']) { + it(`fails alike for the out-of-range key ${JSON.stringify(key)}`, async function () { + const { sync, async: asyncMessage } = await bothErrors([ + { [key]: 'v' }, + ]); + assert.notStrictEqual(sync, '(no error)'); + assert.strictEqual(sync, asyncMessage); + }); + } + + // The key is read into a fixed stack buffer, with a fallback for + // anything longer. Straddle that boundary so the fallback is real. + for (const length of [120, 126, 127, 128, 200]) { + it(`binds a ${length}-byte parameter name`, async function () { + const name = `$${'a'.repeat(length - 1)}`; + const { sync, async: asyncValue } = await bothNamed( + { [name]: length }, + `INSERT INTO t VALUES (${name})`, + ); + assert.strictEqual(sync, length); + assert.strictEqual(sync, asyncValue); + }); + } + + it('reports an unknown named parameter identically', async function () { + const { sync, async: asyncMessage } = await bothErrors( + [{ $nope: 1 }], + 'INSERT INTO t VALUES ($v)', + ); + assert.match(sync, /unknown named parameter "\$nope"/); + assert.strictEqual(sync, asyncMessage); + }); + + it('accepts a null-prototype object as a parameter map', async function () { + const params = Object.create(null); + params.$v = 'no proto'; + const { sync, async: asyncValue } = await bothNamed( + params, + 'INSERT INTO t VALUES ($v)', + ); + assert.strictEqual(sync, 'no proto'); + assert.strictEqual(sync, asyncValue); + }); + + it('accepts a class instance as a parameter map', async function () { + class Params { + constructor() { + this.$v = 'from a class'; + } + } + const { sync, async: asyncValue } = await bothNamed( + new Params(), + 'INSERT INTO t VALUES ($v)', + ); + assert.strictEqual(sync, 'from a class'); + assert.strictEqual(sync, asyncValue); + }); + + it('still binds a lone Date, RegExp or Buffer positionally', async function () { + // These are objects, but they are values rather than parameter + // maps — the distinction the map check exists to draw. + const date = new Date(1700000000000); + assert.strictEqual((await bothPaths(date)).sync, 1700000000000); + + // A RegExp binds as a value (its text), not as an empty map — + // which is what it would look like if read as named parameters. + const re = await bothPaths(/x/); + assert.strictEqual(typeof re.sync, 'string'); + assert.strictEqual(re.sync, re.async); + + const buf = Buffer.from([1, 2, 3]); + assert.deepStrictEqual( + [.../** @type {Buffer} */ ((await bothPaths(buf)).sync)], + [1, 2, 3], + ); + }); + + it('re-steps a parameterless statement without rebinding', async function () { + const statement = db.prepare('INSERT INTO t VALUES (7)'); + await db.wait(); + statement.runSync(); + statement.runSync(); + assert.strictEqual(db.getSync('SELECT count(*) AS n FROM t').n, 2); + statement.finalize(); + }); + + it('keeps a previous binding when re-run with no arguments', async function () { + const statement = db.prepare('INSERT INTO t VALUES (?)'); + await db.wait(); + statement.runSync(11); + statement.runSync(); + assert.deepStrictEqual( + db.allSync('SELECT v FROM t').map((r) => r.v), + [11, 11], + ); + statement.finalize(); + }); + + it('ignores an all-undefined call against a parameterless statement', function () { + // Historical call shape: generic wrappers forwarding an absent + // value must not trip the arity check. + db.runSync('INSERT INTO t VALUES (7)', undefined); + assert.strictEqual(db.getSync('SELECT count(*) AS n FROM t').n, 1); + }); + + it('binds undefined as NULL when the statement takes a parameter', function () { + db.runSync('INSERT INTO t VALUES (?)', undefined); + assert.strictEqual(db.getSync('SELECT v FROM t').v, null); + }); +}); diff --git a/test/throwing_completion.test.js b/test/throwing_completion.test.js new file mode 100644 index 0000000..78a5ece --- /dev/null +++ b/test/throwing_completion.test.js @@ -0,0 +1,169 @@ +import assert from 'node:assert'; +import { describe, it } from 'node:test'; + +import sqlite3 from '../lib/sqlite3.js'; + +// A throwing completion callback on a database-level exclusive operation +// (open/exec/close/loadExtension) must not wedge the connection. The +// completions end by draining the database queue (Process()); the JS +// callback fires first, and when it throws TRY_CATCH_CALL returns early — +// so the drain must run from a guard (Database::ProcessGuard, the same +// discipline as Statement::CallGuard). Without it, everything queued +// behind the exclusive call stays queued forever and every later call on +// the connection never settles. +// +// The pending exception from the throwing callback surfaces as an +// uncaughtException at the next tick boundary, so each test follows the +// test/sync.test.js "sync fast path after a throwing callback" pattern: +// detach node:test's uncaught handlers, capture the throw, assert the +// connection stayed live, restore the handlers. +function withCapturedThrow(message, run) { + return new Promise((resolve, reject) => { + const savedHandlers = process.listeners('uncaughtException'); + process.removeAllListeners('uncaughtException'); + + let restored = false; + const restore = () => { + if (restored) return; + restored = true; + process.removeAllListeners('uncaughtException'); + for (const h of savedHandlers) process.on('uncaughtException', h); + }; + + process.once('uncaughtException', (err) => { + if (!(err instanceof Error) || err.message !== message) { + restore(); + return reject( + new Error(`unexpected uncaught exception: ${err?.message}`), + ); + } + // Give the drained queue a moment: the work queued behind the + // exclusive call needs a worker round trip to settle. + setTimeout(() => { + restore(); + resolve(); + }, 150); + }); + + run().catch((err) => { + restore(); + reject(err); + }); + }); +} + +describe('throwing completion callbacks', () => { + it('exec: a throwing completion callback does not wedge the connection', (_t, done) => { + const db = new sqlite3.Database(':memory:'); + let settled = false; + + withCapturedThrow('boom from exec', async () => { + // Queued behind the exec: this is what the guard must + // dispatch when the exec completion callback throws. + db.exec('CREATE TABLE t (i)', () => { + throw new Error('boom from exec'); + }); + db.get('SELECT COUNT(*) AS n FROM sqlite_master', (err, row) => { + // n === 1: the table the exec created — proves both that + // the exec ran and that this query settled. + if (!err) settled = row && row.n === 1; + }); + }) + .then(() => { + assert.strictEqual( + settled, + true, + 'the query queued behind the throwing exec never settled', + ); + db.close(done); + }) + .catch((err) => done(err)); + }); + + it('loadExtension: a throwing completion callback does not wedge the connection', (_t, done) => { + const db = new sqlite3.Database(':memory:', (err) => { + assert.ifError(err); + let settled = false; + + withCapturedThrow('boom from loadExtension', async () => { + // The load fails (no such file) and the error branch + // fires the throwing callback — the same TRY_CATCH_CALL + // early return as the success path. + db.loadExtension('/nonexistent/ext.dylib', () => { + throw new Error('boom from loadExtension'); + }); + db.get('SELECT 1 AS v', (err2, row) => { + if (!err2) settled = row && row.v === 1; + }); + }) + .then(() => { + assert.strictEqual( + settled, + true, + 'the query queued behind the throwing loadExtension never settled', + ); + db.close(done); + }) + .catch((err3) => done(err3)); + }); + }); + + it('open: a throwing open callback does not wedge the connection', (_t, done) => { + let settled = false; + + withCapturedThrow('boom from open', async () => { + const db = new sqlite3.Database(':memory:', () => { + throw new Error('boom from open'); + }); + // Queued behind the open by construction: it was scheduled + // while the connection was still Opening. + db.get('SELECT 1 AS v', (err, row) => { + if (!err) settled = row && row.v === 1; + }); + setTimeout(() => { + db.close(() => { + // nothing to assert; just release the handle + }); + }, 100); + }) + .then(() => { + assert.strictEqual( + settled, + true, + 'the query queued behind the throwing open never settled', + ); + done(); + }) + .catch((err) => done(err)); + }); + + it('close: a throwing close callback fails the work queued behind it', (_t, done) => { + const db = new sqlite3.Database(':memory:', (err) => { + assert.ifError(err); + let settled = false; + + withCapturedThrow('boom from close', async () => { + // Queued behind the close: scheduled while the close is + // still Closing, so it can only be failed by the drain + // after the close completes. + db.close(() => { + throw new Error('boom from close'); + }); + db.get('SELECT 1 AS v', (err2) => { + // The connection is closed: the call must settle + // with the closed-database error, not hang forever. + settled = err2 && err2.code === 'SQLITE_MISUSE'; + }); + }) + .then(() => { + assert.strictEqual( + settled, + true, + 'the query queued behind the throwing close never settled', + ); + done(); + }) + .catch((err3) => done(err3)); + }); + }); +}); diff --git a/test/trace.test.js b/test/trace.test.js index d8e9687..c3c7e39 100644 --- a/test/trace.test.js +++ b/test/trace.test.js @@ -1,34 +1,34 @@ +import assert from 'node:assert'; +import { describe, it } from 'node:test'; + import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; -describe('tracing', function() { - it('Database tracing', function(done) { - let db = new sqlite3.Database(':memory:'); +describe('tracing', function () { + it('Database tracing', function (_t, done) { + const db = new sqlite3.Database(':memory:'); let create = false; let select = false; - db.on('trace', function(sql) { + db.on('trace', function (sql) { if (sql.match(/^SELECT/)) { assert.ok(!select); - assert.equal(sql, "SELECT * FROM foo"); + assert.equal(sql, 'SELECT * FROM foo'); select = true; - } - else if (sql.match(/^CREATE/)) { + } else if (sql.match(/^CREATE/)) { assert.ok(!create); - assert.equal(sql, "CREATE TABLE foo (id int)"); + assert.equal(sql, 'CREATE TABLE foo (id int)'); create = true; - } - else { + } else { assert.ok(false); } }); - db.serialize(function() { - db.run("CREATE TABLE foo (id int)"); - db.run("SELECT * FROM foo"); + db.serialize(function () { + db.run('CREATE TABLE foo (id int)'); + db.run('SELECT * FROM foo'); }); - db.close(function(err) { + db.close(function (err) { if (err) throw err; assert.ok(create); assert.ok(select); @@ -36,32 +36,34 @@ describe('tracing', function() { }); }); + it('test disabling tracing #1', function (_t, done) { + const db = new sqlite3.Database(':memory:'); - it('test disabling tracing #1', function(done) { - let db = new sqlite3.Database(':memory:'); - - db.on('trace', function(sql) {}); + db.on('trace', function (_sql) { + /* no-op listener */ + }); db.removeAllListeners('trace'); - db._events['trace'] = function(sql) { + db._events['trace'] = function (_sql) { assert.ok(false); }; - db.run("CREATE TABLE foo (id int)"); + db.run('CREATE TABLE foo (id int)'); db.close(done); }); + it('test disabling tracing #2', function (_t, done) { + const db = new sqlite3.Database(':memory:'); - it('test disabling tracing #2', function(done) { - let db = new sqlite3.Database(':memory:'); - - let trace = function(sql) {}; + const trace = function (_sql) { + /* no-op listener */ + }; db.on('trace', trace); db.removeListener('trace', trace); - db._events['trace'] = function(sql) { + db._events['trace'] = function (_sql) { assert.ok(false); }; - db.run("CREATE TABLE foo (id int)"); + db.run('CREATE TABLE foo (id int)'); db.close(done); }); -}); \ No newline at end of file +}); diff --git a/test/trace_profile.test.js b/test/trace_profile.test.js index 3c1228a..a65c07f 100644 --- a/test/trace_profile.test.js +++ b/test/trace_profile.test.js @@ -1,111 +1,144 @@ +import assert from 'node:assert'; +import { afterEach, beforeEach, describe, it } from 'node:test'; + import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; // Pins the trace/profile API semantics across the sqlite3_trace_v2 // migration: expanded SQL (bind values inlined) and timing payloads. -describe('trace/profile payload semantics', function() { +describe('trace/profile payload semantics', function () { let db; - beforeEach(function(done) { - db = new sqlite3.Database(':memory:', function(err) { + beforeEach(function (_t, done) { + db = new sqlite3.Database(':memory:', function (err) { assert.ifError(err); db.exec('CREATE TABLE t (a INTEGER, b TEXT)', done); }); }); - afterEach(function(done) { + afterEach(function (_t, done) { db.close(done); }); - it('trace emits expanded SQL for positional params', function(done) { + it('trace emits expanded SQL for positional params', function (_t, done) { const seen = []; - db.on('trace', function(sql) { + db.on('trace', function (sql) { seen.push(sql); }); - db.run('INSERT INTO t VALUES (?, ?)', 42, 'hello', function(err) { + db.run('INSERT INTO t VALUES (?, ?)', 42, 'hello', function (err) { assert.ifError(err); - setTimeout(function() { - assert.ok(seen.some(s => s === "INSERT INTO t VALUES (42, 'hello')"), - 'expanded insert not found in: ' + JSON.stringify(seen)); + setTimeout(function () { + assert.ok( + seen.some( + (s) => s === "INSERT INTO t VALUES (42, 'hello')", + ), + `expanded insert not found in: ${JSON.stringify(seen)}`, + ); done(); }, 50); }); }); - it('trace emits expanded SQL for array params', function(done) { + it('trace emits expanded SQL for array params', function (_t, done) { const seen = []; - db.on('trace', function(sql) { + db.on('trace', function (sql) { seen.push(sql); }); - db.get('SELECT ? AS v', [7], function(err, row) { + db.get('SELECT ? AS v', [7], function (err, row) { assert.ifError(err); assert.strictEqual(row.v, 7); - setTimeout(function() { - assert.ok(seen.some(s => s === 'SELECT 7 AS v'), - 'expanded select not found in: ' + JSON.stringify(seen)); + setTimeout(function () { + assert.ok( + seen.some((s) => s === 'SELECT 7 AS v'), + `expanded select not found in: ${JSON.stringify(seen)}`, + ); done(); }, 50); }); }); - it('trace emits expanded SQL for named params', function(done) { + it('trace emits expanded SQL for named params', function (_t, done) { const seen = []; - db.on('trace', function(sql) { + db.on('trace', function (sql) { seen.push(sql); }); - db.get('SELECT $a AS a, :b AS b', {$a: 1, ':b': 2}, function(err) { + db.get('SELECT $a AS a, :b AS b', { $a: 1, ':b': 2 }, function (err) { assert.ifError(err); - setTimeout(function() { - assert.ok(seen.some(s => s === 'SELECT 1 AS a, 2 AS b'), - 'expanded named-param select not found in: ' + JSON.stringify(seen)); + setTimeout(function () { + assert.ok( + seen.some((s) => s === 'SELECT 1 AS a, 2 AS b'), + 'expanded named-param select not found in: ' + + JSON.stringify(seen), + ); done(); }, 50); }); }); - it('profile emits expanded SQL and non-negative ms', function(done) { + it('profile emits expanded SQL and non-negative ms', function (_t, done) { const seen = []; - db.on('profile', function(sql, ms) { + db.on('profile', function (sql, ms) { assert.equal(typeof ms, 'number'); assert.ok(ms >= 0, 'negative duration'); seen.push(sql); }); - db.run('INSERT INTO t VALUES (?, ?)', 1, 'x', function(err) { + db.run('INSERT INTO t VALUES (?, ?)', 1, 'x', function (err) { assert.ifError(err); - setTimeout(function() { - assert.ok(seen.some(s => s === "INSERT INTO t VALUES (1, 'x')"), - 'profiled insert not found in: ' + JSON.stringify(seen)); + setTimeout(function () { + assert.ok( + seen.some((s) => s === "INSERT INTO t VALUES (1, 'x')"), + `profiled insert not found in: ${JSON.stringify(seen)}`, + ); done(); }, 50); }); }); - it('trace and profile work simultaneously', function(done) { + it('trace and profile work simultaneously', function (_t, done) { const traces = []; const profiles = []; - db.on('trace', function(sql) { traces.push(sql); }); - db.on('profile', function(sql) { profiles.push(sql); }); - db.run('INSERT INTO t VALUES (?, ?)', 5, 'five', function(err) { + db.on('trace', function (sql) { + traces.push(sql); + }); + db.on('profile', function (sql) { + profiles.push(sql); + }); + db.run('INSERT INTO t VALUES (?, ?)', 5, 'five', function (err) { assert.ifError(err); - setTimeout(function() { - assert.ok(traces.some(s => s === "INSERT INTO t VALUES (5, 'five')"), - 'trace payload missing: ' + JSON.stringify(traces)); - assert.ok(profiles.some(s => s === "INSERT INTO t VALUES (5, 'five')"), - 'profile payload missing: ' + JSON.stringify(profiles)); + setTimeout(function () { + assert.ok( + traces.some( + (s) => s === "INSERT INTO t VALUES (5, 'five')", + ), + `trace payload missing: ${JSON.stringify(traces)}`, + ); + assert.ok( + profiles.some( + (s) => s === "INSERT INTO t VALUES (5, 'five')", + ), + `profile payload missing: ${JSON.stringify(profiles)}`, + ); done(); }, 50); }); }); - it('profile fires for statement-heavy exec too', function(done) { + it('profile fires for statement-heavy exec too', function (_t, done) { const profiles = []; - db.on('profile', function(sql) { profiles.push(sql); }); - db.exec("INSERT INTO t VALUES (1, 'a'); INSERT INTO t VALUES (2, 'b');", function(err) { - assert.ifError(err); - setTimeout(function() { - assert.ok(profiles.length >= 2, 'expected >=2 profile events, got ' + profiles.length); - done(); - }, 50); + db.on('profile', function (sql) { + profiles.push(sql); }); + db.exec( + "INSERT INTO t VALUES (1, 'a'); INSERT INTO t VALUES (2, 'b');", + function (err) { + assert.ifError(err); + setTimeout(function () { + assert.ok( + profiles.length >= 2, + `expected >=2 profile events, got ${profiles.length}`, + ); + done(); + }, 50); + }, + ); }); }); diff --git a/test/transaction.test.js b/test/transaction.test.js new file mode 100644 index 0000000..f2d35e5 --- /dev/null +++ b/test/transaction.test.js @@ -0,0 +1,270 @@ +import assert from 'node:assert'; +import fs from 'node:fs'; +import { after, before, describe, it } from 'node:test'; + +import sqlite3 from '../lib/sqlite3.js'; + +const FILE = 'test/tmp/transaction-03.db'; + +describe('transaction', function () { + let db; + before(async function () { + db = await sqlite3.open(FILE); + await db.exec('CREATE TABLE IF NOT EXISTS t (a INT)'); + }); + + after(async function () { + await db.close(); + fs.unlinkSync(FILE); + }); + + it('commits on success and resolves the body value', async function () { + await db.exec('DELETE FROM t'); + const out = await db.transaction(async (tx) => { + assert.strictEqual(tx, db); + await tx.run('INSERT INTO t VALUES (1)'); + await tx.run('INSERT INTO t VALUES (2)'); + return tx.all('SELECT a FROM t ORDER BY a'); + }); + assert.deepStrictEqual( + out.map((r) => r.a), + [1, 2], + ); + const rows = await db.all('SELECT a FROM t ORDER BY a'); + assert.deepStrictEqual( + rows.map((r) => r.a), + [1, 2], + ); + }); + + it('rolls back on throw and rethrows the original error', async function () { + await db.exec('DELETE FROM t'); + await assert.rejects( + db.transaction(async (tx) => { + await tx.run('INSERT INTO t VALUES (1)'); + throw new Error('body boom'); + }), + /body boom/, + ); + const rows = await db.all('SELECT a FROM t'); + assert.deepStrictEqual(rows, []); + }); + + it('surfaces both errors as an AggregateError when rollback fails too', async function () { + await db.exec('DELETE FROM t'); + const realExec = db.exec; + let sabotaged = false; + db.exec = function (sql, ...rest) { + if (sabotaged && sql === 'ROLLBACK') { + return Promise.reject(new Error('rollback boom')); + } + return realExec.call(this, sql, ...rest); + }; + try { + await assert.rejects( + db.transaction(async (tx) => { + sabotaged = true; + await tx.run('INSERT INTO t VALUES (1)'); + throw new Error('body boom'); + }), + function (err) { + assert.ok(err instanceof AggregateError); + assert.strictEqual(err.errors.length, 2); + assert.match(err.errors[0].message, /body boom/); + assert.match(err.errors[1].message, /rollback boom/); + return true; + }, + ); + } finally { + db.exec = realExec; + // Clean up the still-open transaction left by the sabotage. + await realExec.call(db, 'ROLLBACK'); + } + }); + + it('nested transactions use savepoints automatically', async function () { + await db.exec('DELETE FROM t'); + await db.transaction(async (tx) => { + await tx.run('INSERT INTO t VALUES (1)'); + await tx + .transaction(async (inner) => { + await inner.run('INSERT INTO t VALUES (2)'); + // Only the savepoint rolls back. + throw new Error('inner boom'); + }) + .catch(function (err) { + assert.match(err.message, /inner boom/); + }); + await tx.run('INSERT INTO t VALUES (3)'); + }); + const rows = await db.all('SELECT a FROM t ORDER BY a'); + // 2 was rolled back to the savepoint; 1 and 3 committed. + assert.deepStrictEqual( + rows.map((r) => r.a), + [1, 3], + ); + }); + + it('{ savepoint: true } nests via SAVEPOINT at the top level', async function () { + await db.exec('DELETE FROM t'); + await db + .transaction( + async (tx) => { + await tx.run('INSERT INTO t VALUES (9)'); + throw new Error('sp boom'); + }, + { savepoint: true }, + ) + .catch(function (err) { + assert.match(err.message, /sp boom/); + }); + const rows = await db.all('SELECT a FROM t'); + assert.deepStrictEqual(rows, []); + }); + + it('supports immediate and exclusive modes', async function () { + await db.exec('DELETE FROM t'); + await db.transaction( + async (tx) => { + await tx.run('INSERT INTO t VALUES (1)'); + }, + { mode: 'immediate' }, + ); + await db.transaction( + async (tx) => { + await tx.run('INSERT INTO t VALUES (2)'); + }, + { mode: 'exclusive' }, + ); + const rows = await db.all('SELECT a FROM t ORDER BY a'); + assert.deepStrictEqual( + rows.map((r) => r.a), + [1, 2], + ); + }); + + it('rejects invalid modes and non-function bodies', async function () { + await assert.rejects( + db.transaction( + async () => { + /* body never runs: the mode is rejected first */ + }, + { mode: 'sideways' }, + ), + TypeError, + ); + await assert.rejects(db.transaction('not a function'), TypeError); + }); + + it('{ serialize: true } runs the body in serialize mode', async function () { + await db.exec('DELETE FROM t'); + let sawSerialized; + await db.transaction( + async (tx) => { + sawSerialized = db.state.serialized; + await tx.run('INSERT INTO t VALUES (1)'); + }, + { serialize: true }, + ); + assert.strictEqual(sawSerialized, true); + assert.strictEqual(db.state.serialized, false); + const rows = await db.all('SELECT a FROM t'); + assert.strictEqual(rows.length, 1); + }); + + it('a concurrent second transaction rejects instead of silently nesting', async function () { + await db.exec('DELETE FROM t'); + // Both bodies hold the connection open across an await, so the + // second transaction's BEGIN lands while the first is open. With + // the old connection-wide depth counter it silently rode inside + // the first as a savepoint: its "commit" was a RELEASE that the + // first transaction's rollback would have undone. + const slow = db.transaction(async (tx) => { + await tx.run('INSERT INTO t VALUES (1)'); + await new Promise((resolve) => setTimeout(resolve, 50)); + }); + const overlapping = db.transaction(async (tx) => { + await tx.run('INSERT INTO t VALUES (2)'); + }); + const results = await Promise.allSettled([slow, overlapping]); + const rejected = results.filter((r) => r.status === 'rejected'); + assert.strictEqual( + rejected.length, + 1, + `expected exactly one rejection, got ${JSON.stringify( + results.map((r) => r.status), + )}`, + ); + assert.match( + /** @type {PromiseRejectedResult} */ (rejected[0]).reason.message, + /already active on this connection/, + ); + // The survivor's work is intact and the connection is healthy. + const rows = await db.all('SELECT a FROM t'); + assert.ok(rows.length === 0 || rows.length === 1); + await db.run('INSERT INTO t VALUES (3)'); + }); + + it('rejects with SQLITE_BUSY when a second connection holds the write lock', { + timeout: 30000, + }, async function () { + const db1 = await sqlite3.open(FILE); + const db2 = await sqlite3.open(FILE); + try { + await db1.exec('BEGIN IMMEDIATE'); + await db1.run('INSERT INTO t VALUES (100)'); + await assert.rejects( + db2.transaction( + async (tx) => { + await tx.run('INSERT INTO t VALUES (200)'); + }, + { mode: 'immediate' }, + ), + function (err) { + assert.strictEqual(err.primaryCode, 'SQLITE_BUSY'); + return true; + }, + ); + await db1.exec('ROLLBACK'); + } finally { + await db1.close(); + await db2.close(); + } + }); + + // Nesting depth belongs to a connection, not to the async flow it runs + // in. A transaction on one connection that happens to sit inside a + // transaction on another is still that connection's first, so it must + // issue a real BEGIN: taking the savepoint path there would discard + // the requested mode and the write lock that comes with it. + it("keeps its own BEGIN mode inside another connection's transaction", async function () { + const outer = await sqlite3.open(':memory:'); + const inner = await sqlite3.open(':memory:'); + try { + await inner.exec('CREATE TABLE t (x)'); + /** @type {string[]} */ + const sql = []; + inner.on('trace', function (statement) { + sql.push(statement); + }); + await outer.transaction(async function () { + await inner.transaction( + async function () { + await inner.run('INSERT INTO t VALUES (1)'); + }, + { mode: 'immediate' }, + ); + }); + assert.ok( + sql.includes('BEGIN IMMEDIATE'), + `inner connection should have begun its own transaction, saw ${JSON.stringify(sql)}`, + ); + assert.deepStrictEqual(await inner.all('SELECT x FROM t'), [ + { x: 1 }, + ]); + } finally { + await outer.close(); + await inner.close(); + } + }); +}); diff --git a/test/unicode.test.js b/test/unicode.test.js index f55c221..efa99d3 100644 --- a/test/unicode.test.js +++ b/test/unicode.test.js @@ -1,16 +1,20 @@ -import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; +import assert from 'node:assert'; +import { after, before, describe, it } from 'node:test'; -describe('unicode', function() { - let first_values = [], - trailing_values = [], - chars = [], - subranges = new Array(2), - len = subranges.length, - db, - i; +import sqlite3 from '../lib/sqlite3.js'; - before(function(done) { db = new sqlite3.Database(':memory:', done); }); +describe('unicode', function () { + const first_values = []; + const trailing_values = []; + const _chars = []; + const subranges = new Array(2); + const len = subranges.length; + let db; + let i; + + before(function (_t, done) { + db = new sqlite3.Database(':memory:', done); + }); for (i = 0x20; i < 0x80; i++) { first_values.push(i); @@ -37,28 +41,47 @@ describe('unicode', function() { } function random_choice(arr) { - return arr[Math.random() * arr.length | 0]; + return arr[(Math.random() * arr.length) | 0]; } function random_utf8() { - let first = random_choice(first_values); + const first = random_choice(first_values); if (first < 0x80) { return String.fromCharCode(first); - } else if (first < 0xe0) { - return String.fromCharCode((first & 0x1f) << 0x6 | random_choice(trailing_values) & 0x3f); - } else if (first == 0xe0) { - return String.fromCharCode(((first & 0xf) << 0xc) | ((random_choice(subranges[0]) & 0x3f) << 6) | random_choice(trailing_values) & 0x3f); - } else if (first == 0xed) { - return String.fromCharCode(((first & 0xf) << 0xc) | ((random_choice(subranges[1]) & 0x3f) << 6) | random_choice(trailing_values) & 0x3f); - } else if (first < 0xf0) { - return String.fromCharCode(((first & 0xf) << 0xc) | ((random_choice(trailing_values) & 0x3f) << 6) | random_choice(trailing_values) & 0x3f); + } + if (first < 0xe0) { + return String.fromCharCode( + ((first & 0x1f) << 0x6) | + (random_choice(trailing_values) & 0x3f), + ); + } + if (first === 0xe0) { + return String.fromCharCode( + ((first & 0xf) << 0xc) | + ((random_choice(subranges[0]) & 0x3f) << 6) | + (random_choice(trailing_values) & 0x3f), + ); + } + if (first === 0xed) { + return String.fromCharCode( + ((first & 0xf) << 0xc) | + ((random_choice(subranges[1]) & 0x3f) << 6) | + (random_choice(trailing_values) & 0x3f), + ); + } + if (first < 0xf0) { + return String.fromCharCode( + ((first & 0xf) << 0xc) | + ((random_choice(trailing_values) & 0x3f) << 6) | + (random_choice(trailing_values) & 0x3f), + ); } } function randomString() { - let str = '', - i; + let str = ''; + let i; for (i = Math.random() * 300; i > 0; i--) { str += random_utf8(); @@ -67,10 +90,9 @@ describe('unicode', function() { return str; } - // Generate random data. - let data = []; - let length = Math.floor(Math.random() * 1000) + 200; + const data = []; + const length = Math.floor(Math.random() * 1000) + 200; for (let i = 0; i < length; i++) { data.push(randomString()); } @@ -78,14 +100,14 @@ describe('unicode', function() { let inserted = 0; let retrieved = 0; - it('should create the table', function(done) { - db.run("CREATE TABLE foo (id int, txt text)", done); + it('should create the table', function (_t, done) { + db.run('CREATE TABLE foo (id int, txt text)', done); }); - it('should insert all values', function(done) { - let stmt = db.prepare("INSERT INTO foo VALUES(?, ?)"); + it('should insert all values', function (_t, done) { + const stmt = db.prepare('INSERT INTO foo VALUES(?, ?)'); for (let i = 0; i < data.length; i++) { - stmt.run(i, data[i], function(err) { + stmt.run(i, data[i], function (err) { if (err) throw err; inserted++; }); @@ -93,8 +115,8 @@ describe('unicode', function() { stmt.finalize(done); }); - it('should retrieve all values', function(done) { - db.all("SELECT txt FROM foo ORDER BY id", function(err, rows) { + it('should retrieve all values', function (_t, done) { + db.all('SELECT txt FROM foo ORDER BY id', function (err, rows) { if (err) throw err; for (let i = 0; i < rows.length; i++) { @@ -105,10 +127,12 @@ describe('unicode', function() { }); }); - it('should have inserted and retrieved the correct amount', function() { + it('should have inserted and retrieved the correct amount', function () { assert.equal(inserted, length); assert.equal(retrieved, length); }); - after(function(done) { db.close(done); }); -}); \ No newline at end of file + after(function (_t, done) { + db.close(done); + }); +}); diff --git a/test/untrusted.test.js b/test/untrusted.test.js new file mode 100644 index 0000000..d1a39b9 --- /dev/null +++ b/test/untrusted.test.js @@ -0,0 +1,344 @@ +// Untrusted database files (Deliverable 11 §2.3): the `untrusted: true` +// open option applies the hostile-file hardening recipe — defensive mode, +// untrusted schema, writable_schema off, extension loading permanently +// disabled, conservative run-time limits and a deny-all ATTACH gate — and +// the fixture below exercises each switch the way a hostile file would. + +import assert from 'node:assert'; +import { mkdirSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { after, before, describe, it } from 'node:test'; + +import sqlite3 from '../lib/sqlite3.js'; + +const dir = join(import.meta.dirname, 'tmp'); +const hostile = join(dir, `untrusted-hostile-${process.pid}.db`); +const malformed = join(dir, `untrusted-malformed-${process.pid}.db`); + +/** + * Builds a fresh one-table fixture with a trusted connection. + * + * @param {string} file where to write it. + * @returns {Promise} resolves once written and closed. + */ +async function makeFixture(file) { + rmSync(file, { force: true }); + const plan = await sqlite3.open(file, { + mode: sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE, + }); + await plan.exec('CREATE TABLE innocent (x)'); + await plan.close(); +} + +before(async function () { + mkdirSync(dir, { recursive: true }); + await makeFixture(hostile); + // Not a database at all. + const junk = Buffer.alloc(4096); + for (let i = 0; i < junk.length; i++) junk[i] = (i * 31) & 0xff; + writeFileSync(malformed, junk); +}); + +after(function () { + rmSync(hostile, { force: true }); + rmSync(malformed, { force: true }); + rmSync(join(dir, `untrusted-tamper-plain-${process.pid}.db`), { + force: true, + }); + rmSync(join(dir, `untrusted-tamper-careful-${process.pid}.db`), { + force: true, + }); +}); + +describe('untrusted database files', function () { + it('opens the file read-only and reads fine', async function () { + const db = await sqlite3.open(hostile, { + mode: sqlite3.OPEN_READONLY, + untrusted: true, + }); + const row = await db.get('SELECT count(*) AS n FROM innocent'); + assert.strictEqual(row.n, 0); + await db.close(); + }); + + it('applies the hardening switches', async function () { + const db = await sqlite3.open(hostile, { + mode: sqlite3.OPEN_READONLY, + untrusted: true, + }); + assert.strictEqual( + await db.dbConfig(sqlite3.DBCONFIG_DEFENSIVE), + true, + 'defensive mode is on', + ); + assert.strictEqual( + await db.dbConfig(sqlite3.DBCONFIG_TRUSTED_SCHEMA), + false, + 'the schema is not trusted', + ); + assert.strictEqual( + await db.dbConfig(sqlite3.DBCONFIG_WRITABLE_SCHEMA), + false, + 'writable_schema is off', + ); + await db.close(); + }); + + it('refuses the schema tamper a plain connection allows', async function () { + const tamper = + "PRAGMA writable_schema=ON; UPDATE sqlite_master SET sql='CREATE TABLE evil (pwned)' WHERE name='innocent'"; + // The plain connection is the control: without the hardening the + // rewrite goes through (this is the classic hostile-file lever). + // It gets its own fixture because a successful tamper leaves the + // schema genuinely inconsistent — that corruption is the point. + const plainFile = join(dir, `untrusted-tamper-plain-${process.pid}.db`); + await makeFixture(plainFile); + const plain = await sqlite3.open(plainFile, { + mode: sqlite3.OPEN_READWRITE, + }); + await assert.doesNotReject( + plain.exec(tamper), + 'control: a plain connection must allow the tamper for this contrast to mean anything', + ); + await plain.close(); + + const carefulFile = join( + dir, + `untrusted-tamper-careful-${process.pid}.db`, + ); + await makeFixture(carefulFile); + const careful = await sqlite3.open(carefulFile, { + mode: sqlite3.OPEN_READWRITE, + untrusted: true, + }); + // The untrusted connection refuses (writable_schema DBCONFIG 0 + // turns the PRAGMA into a no-op, so the update hits sqlite's + // hard protection of sqlite_master). + await assert.rejects(careful.exec(tamper), (err) => + /may not be modified|readonly database/.test( + /** @type {Error} */ (err).message, + ), + ); + // And the schema really is untouched. + const row = await careful.get( + "SELECT count(*) AS n FROM sqlite_master WHERE name='evil'", + ); + assert.strictEqual(row.n, 0); + await careful.close(); + }); + + it('denies ATTACH and VACUUM INTO', async function () { + const db = await sqlite3.open(hostile, { + mode: sqlite3.OPEN_READONLY, + untrusted: true, + }); + await assert.rejects( + db.exec("ATTACH ':memory:' AS m"), + (err) => + /** @type {Error & { code?: string }} */ ( + err.code === 'SQLITE_ERROR' || + /** @type {Error & { code?: string }} */ (err).code === + 'SQLITE_AUTH' + ) && + /too many attached databases|not authorized/.test( + /** @type {Error} */ (err).message, + ), + ); + await assert.rejects( + db.exec(`VACUUM INTO '${join(dir, 'untrusted-vac.db')}'`), + (err) => + /too many attached databases|not authorized|authorization denied|readonly/.test( + /** @type {Error} */ (err).message, + ), + ); + await db.close(); + rmSync(join(dir, 'untrusted-vac.db'), { force: true }); + }); + + it('permanently disables extension loading', async function () { + const db = await sqlite3.open(hostile, { + mode: sqlite3.OPEN_READONLY, + untrusted: true, + }); + await assert.rejects(db.loadExtension('/nonexistent.ext'), (err) => + /untrusted|permanently disabled/.test( + /** @type {Error} */ (err).message, + ), + ); + // And it cannot be re-enabled from SQL either: load_extension() + // is not authorized on the vendored SQLite (observed default). + await assert.rejects( + db.exec("SELECT load_extension('/nonexistent.ext')"), + (err) => /not authorized/.test(/** @type {Error} */ (err).message), + ); + // configure('attachPaths') is refused too: the deny-all gate is + // part of the hardening, not a starting point. + assert.throws( + () => db.configure('attachPaths', ['/tmp/x.db']), + /untrusted connections cannot allow ATTACH/, + ); + assert.throws( + () => + db.configure('extensionPolicy', { + allow: ['/tmp/x.ext'], + }), + /permanently disabled/, + ); + await db.close(); + }); + + it('applies conservative run-time limits', async function () { + const db = await sqlite3.open(hostile, { + mode: sqlite3.OPEN_READONLY, + untrusted: true, + }); + // 200-deep arithmetic nesting: over the EXPR_DEPTH ceiling of 100. + const deep = `SELECT ${'1+('.repeat(200)}1${')'.repeat(200)}`; + await assert.rejects(db.exec(deep), (err) => + /Expression tree is too large/.test( + /** @type {Error} */ (err).message, + ), + ); + // Compound SELECT beyond the ceiling. + const union = `SELECT 1 ${'UNION ALL SELECT 1 '.repeat(600)}`; + await assert.rejects(db.exec(union), (err) => + /too many terms in compound SELECT/.test( + /** @type {Error} */ (err).message, + ), + ); + // Normal queries are unaffected. + assert.strictEqual((await db.get('SELECT 41+1 AS v')).v, 42); + await db.close(); + }); + + it('a malformed file errors gracefully, without crashing', async function () { + const db = await sqlite3.open(malformed, { + mode: sqlite3.OPEN_READONLY, + untrusted: true, + }); + // The open of a non-database file succeeds lazily; the read is + // what reports NOTADB — an error, never a crash. + await assert.rejects( + db.get('SELECT count(*) AS n FROM sqlite_master'), + (err) => + /** @type {Error & { code?: string }} */ (err).code === + 'SQLITE_NOTADB', + ); + await db.close(); + }); + + it('validates the option shape', function () { + assert.throws( + () => + new sqlite3.Database(':memory:', { + untrusted: 'yes', + }), + /untrusted.*must be a boolean/, + ); + assert.throws( + () => + new sqlite3.Database(':memory:', { + mode: 'rw', + }), + /mode.*must be a number/, + ); + assert.throws( + () => new sqlite3.Database(':memory:', 'nonsense'), + /expects a mode number, an options object or a callback/, + ); + }); + + it('the constructor form accepts options in either slot', async function () { + // (filename, options) and (filename, mode, options) — the latter + // is what sqlite3.open produces internally. + const a = new sqlite3.Database(hostile, { + mode: sqlite3.OPEN_READONLY, + untrusted: true, + }); + await a.close(); + const b = new sqlite3.Database(hostile, sqlite3.OPEN_READONLY, { + untrusted: true, + }); + await b.close(); + }); +}); + +// The ATTACH gate matches target filenames lexically, and two of its +// early spellings-based shortcuts were fail-*open*: each let an ATTACH +// create a real file outside the permission-checked allowlist. Both are +// pinned here because the failure is silent — the ATTACH succeeds and the +// file simply appears. +describe('ATTACH gate spelling rules', function () { + const probe = join(dir, `gate-probe-${process.pid}`); + + before(function () { + rmSync(probe, { force: true, recursive: true }); + mkdirSync(probe, { recursive: true }); + }); + after(function () { + rmSync(probe, { force: true, recursive: true }); + }); + + /** + * Opens a gated connection whose allowlist holds exactly one path. + * + * @param {string} allowed the single permitted ATTACH target. + * @param {number} [mode] open flags; defaults to in-memory read/write. + * @returns {Promise} the connection. + */ + async function gated(allowed, mode) { + const db = await sqlite3.open(':memory:', { + mode: mode ?? sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE, + }); + db.configure('attachPaths', [allowed]); + return db; + } + + it('denies URI memory spellings on a connection without OPEN_URI', async function () { + // Without SQLITE_OPEN_URI (the default) SQLite reads 'file:…' as + // an ordinary *filename*, so treating these as in-memory admitted + // a real file: ATTACH 'file::memory:' created a file of that + // literal name in the process cwd, outside the allowlist. + const db = await gated(join(probe, 'allowed.db')); + for (const target of ['file::memory:', 'file:x.db?mode=memory']) { + await assert.rejects( + db.exec(`ATTACH DATABASE '${target}' AS z`), + (err) => + /** @type {Error & { code?: string }} */ (err).code === + 'SQLITE_AUTH', + `${target} must be denied without OPEN_URI`, + ); + } + await db.close(); + }); + + it('allows URI memory spellings when the connection did open with OPEN_URI', async function () { + // There they really are in-memory, so denying them would be a + // gratuitous narrowing rather than a safety property. + const db = await gated( + join(probe, 'allowed.db'), + sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE | sqlite3.OPEN_URI, + ); + for (const target of ['file::memory:', 'file:x.db?mode=memory']) { + await db.exec(`ATTACH DATABASE '${target}' AS z; DETACH z`); + } + await db.close(); + }); + + it('does not treat a backslash as a separator on POSIX', async function () { + // 'dir\x.db' and 'dir/x.db' are two different files on POSIX, and + // only the latter was permission-checked. Normalising separators + // here (correct on Windows) widened the allowlist to a file that + // was never checked, and the ATTACH created it. + if (process.platform === 'win32') return; + const db = await gated(join(probe, 'sub', 'ok.db')); + await assert.rejects( + db.exec(`ATTACH DATABASE '${join(probe, 'sub')}\\ok.db' AS z`), + (err) => + /** @type {Error & { code?: string }} */ (err).code === + 'SQLITE_AUTH', + ); + assert.deepStrictEqual(readdirSync(probe), []); + await db.close(); + }); +}); diff --git a/test/update_hook.test.js b/test/update_hook.test.js index b33c455..c427c52 100644 --- a/test/update_hook.test.js +++ b/test/update_hook.test.js @@ -1,19 +1,24 @@ +import assert from 'node:assert'; +import { afterEach, beforeEach, describe, it } from 'node:test'; + import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; -describe('update_hook', function() { +describe('update_hook', function () { let db; - beforeEach(function(done) { - db = new sqlite3.Database(':memory:', function(err) { + beforeEach(function (_t, done) { + db = new sqlite3.Database(':memory:', function (err) { if (err) return done(err); - db.run("CREATE TABLE update_hooks_test (id int PRIMARY KEY, value text)", done); + db.run( + 'CREATE TABLE update_hooks_test (id int PRIMARY KEY, value text)', + done, + ); }); }); - it('emits insert event on inserting data to table', function(done) { - db.addListener('change', function(eventType, database, table, rowId) { + it('emits insert event on inserting data to table', function (_t, done) { + db.addListener('change', function (eventType, database, table, rowId) { assert.equal(eventType, 'insert'); assert.equal(database, 'main'); assert.equal(table, 'update_hooks_test'); @@ -22,54 +27,81 @@ describe('update_hook', function() { return done(); }); - db.run("INSERT INTO update_hooks_test VALUES (1, 'value')", function(err) { - if (err) return done(err); - }); - }); - - it('emits update event on row modification in table', function(done) { - db.run("INSERT INTO update_hooks_test VALUES (2, 'value'), (3, 'value4')", function(err) { - if (err) return done(err); - - db.addListener('change', function(eventType, database, table, rowId) { - assert.equal(eventType, 'update'); - assert.equal(database, 'main'); - assert.equal(table, 'update_hooks_test'); - assert.equal(rowId, 1); - - db.all("SELECT * FROM update_hooks_test WHERE rowid = ?", rowId, function(err, rows) { - assert.deepEqual(rows, [{ id: 2, value: 'new_val' }]); - - return done(err); - }); - }); - - db.run("UPDATE update_hooks_test SET value = 'new_val' WHERE id = 2", function(err) { + db.run( + "INSERT INTO update_hooks_test VALUES (1, 'value')", + function (err) { if (err) return done(err); - }); - }); + }, + ); }); - it('emits delete event on row was deleted from table', function(done) { - db.run("INSERT INTO update_hooks_test VALUES (2, 'value')", function(err) { - if (err) return done(err); - - db.addListener('change', function(eventType, database, table, rowId) { - assert.equal(eventType, 'delete'); - assert.equal(database, 'main'); - assert.equal(table, 'update_hooks_test'); - assert.equal(rowId, 1); + it('emits update event on row modification in table', function (_t, done) { + db.run( + "INSERT INTO update_hooks_test VALUES (2, 'value'), (3, 'value4')", + function (err) { + if (err) return done(err); - return done(); - }); + db.addListener( + 'change', + function (eventType, database, table, rowId) { + assert.equal(eventType, 'update'); + assert.equal(database, 'main'); + assert.equal(table, 'update_hooks_test'); + assert.equal(rowId, 1); + + db.all( + 'SELECT * FROM update_hooks_test WHERE rowid = ?', + rowId, + function (err, rows) { + assert.deepEqual(rows, [ + { id: 2, value: 'new_val' }, + ]); + + return done(err); + }, + ); + }, + ); + + db.run( + "UPDATE update_hooks_test SET value = 'new_val' WHERE id = 2", + function (err) { + if (err) return done(err); + }, + ); + }, + ); + }); - db.run("DELETE FROM update_hooks_test WHERE id = 2", function(err) { + it('emits delete event on row was deleted from table', function (_t, done) { + db.run( + "INSERT INTO update_hooks_test VALUES (2, 'value')", + function (err) { if (err) return done(err); - }); - }); + + db.addListener( + 'change', + function (eventType, database, table, rowId) { + assert.equal(eventType, 'delete'); + assert.equal(database, 'main'); + assert.equal(table, 'update_hooks_test'); + assert.equal(rowId, 1); + + return done(); + }, + ); + + db.run( + 'DELETE FROM update_hooks_test WHERE id = 2', + function (err) { + if (err) return done(err); + }, + ); + }, + ); }); - afterEach(function(done) { + afterEach(function (_t, done) { db.close(done); }); -}); \ No newline at end of file +}); diff --git a/test/upsert.test.js b/test/upsert.test.js index b4a80d1..6cd2717 100644 --- a/test/upsert.test.js +++ b/test/upsert.test.js @@ -1,27 +1,38 @@ +import assert from 'node:assert'; +import { before, describe, it } from 'node:test'; + import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; -describe('query properties', function() { +describe('query properties', function () { let db; - before(function(done) { + before(function (_t, done) { db = new sqlite3.Database(':memory:'); - db.run("CREATE TABLE foo (id INT PRIMARY KEY, count INT)", done); + db.run('CREATE TABLE foo (id INT PRIMARY KEY, count INT)', done); }); - (sqlite3.VERSION_NUMBER < 3024000 ? it.skip : it)('should upsert', function(done) { - let stmt = db.prepare("INSERT INTO foo VALUES(?, ?)"); - stmt.run(1, 1, function(err) { // insert 1 - if (err) throw err; - let upsert_stmt = db.prepare("INSERT INTO foo VALUES(?, ?) ON CONFLICT (id) DO UPDATE SET count = count + excluded.count"); - upsert_stmt.run(1, 2, function(err) { // add 2 + (sqlite3.VERSION_NUMBER < 3024000 ? it.skip : it)( + 'should upsert', + function (_t, done) { + const stmt = db.prepare('INSERT INTO foo VALUES(?, ?)'); + stmt.run(1, 1, function (err) { + // insert 1 if (err) throw err; - let select_stmt = db.prepare("SELECT count FROM foo WHERE id = ?"); - select_stmt.get(1, function(err, row) { + const upsert_stmt = db.prepare( + 'INSERT INTO foo VALUES(?, ?) ON CONFLICT (id) DO UPDATE SET count = count + excluded.count', + ); + upsert_stmt.run(1, 2, function (err) { + // add 2 if (err) throw err; - assert.equal(row.count, 3); // equals 3 + const select_stmt = db.prepare( + 'SELECT count FROM foo WHERE id = ?', + ); + select_stmt.get(1, function (err, row) { + if (err) throw err; + assert.equal(row.count, 3); // equals 3 + }); }); }); - }); - db.wait(done); - }); -}); \ No newline at end of file + db.wait(done); + }, + ); +}); diff --git a/test/verbose.test.js b/test/verbose.test.js index dbdb997..af689d0 100644 --- a/test/verbose.test.js +++ b/test/verbose.test.js @@ -1,40 +1,42 @@ +import assert from 'node:assert'; +import { describe, it } from 'node:test'; + import sqlite3 from '../lib/sqlite3.js'; -import assert from 'assert'; -let invalid_sql = 'update non_existent_table set id=1'; +const invalid_sql = 'update non_existent_table set id=1'; -let originalMethods = { +const originalMethods = { Database: {}, Statement: {}, }; function backupOriginalMethods() { - for (let obj in originalMethods) { - for (let attr in sqlite3[obj].prototype) { + for (const obj in originalMethods) { + for (const attr in sqlite3[obj].prototype) { originalMethods[obj][attr] = sqlite3[obj].prototype[attr]; } } } function resetVerbose() { - for (let obj in originalMethods) { - for (let attr in originalMethods[obj]) { + for (const obj in originalMethods) { + for (const attr in originalMethods[obj]) { sqlite3[obj].prototype[attr] = originalMethods[obj][attr]; } } } -describe('verbose', function() { - it('Shoud add trace info to error when verbose is called', function(done) { - let db = new sqlite3.Database(':memory:'); +describe('verbose', function () { + it('Shoud add trace info to error when verbose is called', function (_t, done) { + const db = new sqlite3.Database(':memory:'); backupOriginalMethods(); sqlite3.verbose(); - db.run(invalid_sql, function(err) { + db.run(invalid_sql, function (err) { assert(err instanceof Error); assert( err.stack.indexOf(`Database#run('${invalid_sql}'`) > -1, - `Stack shoud contain trace info, stack = ${err.stack}` + `Stack shoud contain trace info, stack = ${err.stack}`, ); done(); @@ -42,17 +44,17 @@ describe('verbose', function() { }); }); - it('Shoud not add trace info to error when verbose is not called', function(done) { - let db = new sqlite3.Database(':memory:'); + it('Shoud not add trace info to error when verbose is not called', function (_t, done) { + const db = new sqlite3.Database(':memory:'); - db.run(invalid_sql, function(err) { + db.run(invalid_sql, function (err) { assert(err instanceof Error); assert( err.stack.indexOf(invalid_sql) === -1, - `Stack shoud not contain trace info, stack = ${err.stack}` + `Stack shoud not contain trace info, stack = ${err.stack}`, ); done(); }); }); -}); \ No newline at end of file +}); diff --git a/test/wal.test.js b/test/wal.test.js new file mode 100644 index 0000000..c2e0a31 --- /dev/null +++ b/test/wal.test.js @@ -0,0 +1,198 @@ +import assert from 'node:assert'; +import { rmSync, statSync } from 'node:fs'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, it } from 'node:test'; + +/** Removes a database file and its journal/WAL siblings. */ +function removeDb(file) { + for (const suffix of ['', '-wal', '-shm', '-journal']) { + rmSync(`${file}${suffix}`, { force: true }); + } +} + +import sqlite3 from '../lib/sqlite3.js'; +import { TMP_DIR, withDb } from './support/db.js'; + +// WAL control (Deliverable 07): the 'wal' event fires per writing commit +// with a frame count, checkpoint() reports plausible frame counts in +// every mode, and TRUNCATE actually shrinks the -wal file. + +const FILE = join(TMP_DIR, 'wal-hook-test.db'); + +async function walDb() { + removeDb(FILE); + const db = new sqlite3.Database(FILE); + await new Promise((resolve, reject) => { + db.once('open', resolve); + db.once('error', reject); + }); + const mode = await db.get('PRAGMA journal_mode=WAL'); + assert.strictEqual( + mode.journal_mode.toLowerCase(), + 'wal', + 'test db must be in WAL mode', + ); + await db.exec('CREATE TABLE IF NOT EXISTS t (a)'); + return db; +} + +describe('wal', function () { + /** @type {sqlite3.Database} */ + let db; + + beforeEach(async function () { + db = await walDb(); + }); + + afterEach(async function () { + await db.close(); + removeDb(FILE); + }); + + it('the wal event fires with a database name and frame count', async function () { + /** @type {[string, number][]} */ + const events = []; + db.on('wal', (database, pages) => events.push([database, pages])); + await db.run('INSERT INTO t VALUES (1)'); + await new Promise((resolve) => setTimeout(resolve, 25)); + assert.ok(events.length >= 1, `wal events: ${events.length}`); + for (const [database, pages] of events) { + assert.strictEqual(database, 'main'); + assert.ok( + Number.isInteger(pages) && pages >= 0, + `pages must be a non-negative integer, got ${pages}`, + ); + } + // Frames accumulate until a checkpoint. + const last = events[events.length - 1][1]; + assert.ok(last >= 1, `at least one frame after a write, got ${last}`); + }); + + it('wal events stop after removeListener', async function () { + let count = 0; + const listener = () => count++; + db.on('wal', listener); + await db.run('INSERT INTO t VALUES (2)'); + await new Promise((resolve) => setTimeout(resolve, 25)); + const afterFirst = count; + db.removeListener('wal', listener); + await db.run('INSERT INTO t VALUES (3)'); + await new Promise((resolve) => setTimeout(resolve, 25)); + assert.ok(afterFirst >= 1); + assert.strictEqual(count, afterFirst, 'no events after removal'); + }); +}); + +describe('checkpoint', function () { + it('reports plausible frame counts in every mode', async function () { + const file = join(TMP_DIR, 'wal-checkpoint-modes.db'); + removeDb(file); + await withDb( + async (db) => { + const mode = await db.get('PRAGMA journal_mode=WAL'); + assert.strictEqual(mode.journal_mode.toLowerCase(), 'wal'); + await db.exec('CREATE TABLE c (a)'); + await db.run('INSERT INTO c VALUES (1)'); + await db.run('INSERT INTO c VALUES (2)'); + + for (const name of ['passive', 'full', 'restart', 'truncate']) { + const result = await db.checkpoint({ mode: name }); + assert.strictEqual( + result.busy, + false, + `${name} should not be busy`, + ); + assert.ok( + Number.isInteger(result.logFrames) && + result.logFrames >= 0, + `${name} logFrames: ${result.logFrames}`, + ); + assert.ok( + Number.isInteger(result.checkpointedFrames) && + result.checkpointedFrames >= 0, + `${name} checkpointedFrames: ${result.checkpointedFrames}`, + ); + } + }, + { filename: file }, + ); + removeDb(file); + }); + + it('TRUNCATE shrinks the -wal file', async function () { + const file = join(TMP_DIR, 'wal-truncate.db'); + removeDb(file); + await withDb( + async (db) => { + await db.get('PRAGMA journal_mode=WAL'); + await db.exec('CREATE TABLE t (a BLOB)'); + for (let i = 0; i < 20; i++) { + await db.run( + 'INSERT INTO t VALUES (?)', + Buffer.alloc(4096, 1), + ); + } + const walFile = `${file}-wal`; + const before = statSync(walFile).size; + assert.ok( + before > 0, + 'WAL file must have frames before the checkpoint', + ); + + const result = await db.checkpoint({ mode: 'truncate' }); + assert.strictEqual(result.busy, false); + const after = statSync(walFile).size; + assert.ok(after < before, `WAL shrank: ${before} -> ${after}`); + }, + { filename: file }, + ); + removeDb(file); + }); + + it('accepts the mode as a string and the db name in options', async function () { + const file = join(TMP_DIR, 'wal-options.db'); + removeDb(file); + await withDb( + async (db) => { + await db.get('PRAGMA journal_mode=WAL'); + await db.exec('CREATE TABLE t (a)'); + await db.run('INSERT INTO t VALUES (1)'); + const r1 = await db.checkpoint('truncate'); + assert.strictEqual(r1.busy, false); + const r2 = await db.checkpoint({ mode: 'passive', db: 'main' }); + assert.strictEqual(r2.busy, false); + // Callback form returns the database itself. + const self = await new Promise((resolve, reject) => { + const returned = db.checkpoint(function (err, result) { + if (err) reject(err); + else resolve(result); + }); + assert.strictEqual(returned, db); + }); + assert.ok(typeof self.busy === 'boolean'); + }, + { filename: file }, + ); + removeDb(file); + }); + + it('rejects an unknown mode', async function () { + await withDb(async (db) => { + await assert.rejects( + /** @type {any} */ (db).checkpoint({ mode: 'sideways' }), + /mode must be/, + ); + }); + }); + + it('is quiet on a non-WAL database', async function () { + // A rollback-journal database has no WAL; sqlite reports a + // successful no-op rather than an error. + await withDb(async (db) => { + const result = await db.checkpoint(); + assert.strictEqual(result.busy, false); + assert.strictEqual(result.logFrames, -1); + assert.strictEqual(result.checkpointedFrames, -1); + }); + }); +}); diff --git a/test/worker.test.js b/test/worker.test.js new file mode 100644 index 0000000..cf4ca46 --- /dev/null +++ b/test/worker.test.js @@ -0,0 +1,452 @@ +// Worker-thread safety (Deliverable 09). The addon loads per +// environment; every worker is its own napi env, so these tests are the +// proof that nothing napi-shaped is shared across environments — the +// per-env constructor block (AddonData) is what makes the 8-worker +// cycle pass, and a regression to a file static fails it (that bug +// class segfaults at env teardown on musl and silently misbehaves +// everywhere else). +import assert from 'node:assert'; +import { randomUUID } from 'node:crypto'; +import { rmSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { describe, it } from 'node:test'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { Worker } from 'node:worker_threads'; + +import sqlite3 from '../lib/sqlite3.js'; +import { TMP_DIR } from './support/db.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const DRIVER_URL = pathToFileURL(join(__dirname, '../lib/sqlite3.js')).href; + +/** Removes a database file and its journal/WAL siblings. */ +function removeDb(file) { + for (const suffix of ['', '-wal', '-shm', '-journal']) { + rmSync(`${file}${suffix}`, { force: true }); + } +} + +/** + * Spawns an eval-mode worker that loads the driver and runs `body`. + * + * The body receives the loaded driver, a `report(value)` function whose + * values arrive through `next()`, and a JSON-serialized `payload` + * (bodies are stringified into the worker, so they cannot close over + * test-file variables). When the body completes, the worker unrefs its + * parent port: the thread then exits naturally once idle — with + * whatever objects the body left open, which is exactly the teardown + * these tests want to exercise. + * + * @template [P=undefined] + * @param {(sqlite3: typeof import('../lib/sqlite3.js').default, + * report: (value: any) => void, payload: P) => void | Promise} body + * @param {P} [payload] JSON-serializable value handed to the body. + * @returns {{ worker: Worker, + * next: () => Promise, + * exited: Promise }} the worker, a report reader and the + * exit-code promise. + */ +function driverWorker(body, payload) { + const hasPayload = payload !== undefined; + const worker = new Worker( + ` + const { parentPort } = require('node:worker_threads'); + const payload = ${hasPayload ? JSON.stringify(payload) : 'undefined'}; + import(${JSON.stringify(DRIVER_URL)}).then(async (m) => { + const report = (value) => + parentPort.postMessage({ kind: 'report', value }); + try { + await (${body.toString()})(m.default, report, payload); + } catch (err) { + parentPort.postMessage({ + kind: 'fatal', + message: err && err.message ? err.message : String(err), + }); + } + // Let the thread die naturally once the body is done, so the + // napi environment tears down around whatever is still open. + parentPort.unref(); + }); + `, + { eval: true }, + ); + + /** @type {Promise[]} */ + const queued = []; + /** @type {{ resolve: (v: any) => void, reject: (e: Error) => void }[]} */ + const waiters = []; + worker.on('message', (msg) => { + if (msg.kind === 'report') { + const waiter = waiters.shift(); + if (waiter) waiter.resolve(msg.value); + else queued.push(Promise.resolve(msg.value)); + } else if (msg.kind === 'fatal') { + const err = new Error(`worker body failed: ${msg.message}`); + const waiter = waiters.shift(); + if (waiter) waiter.reject(err); + else queued.push(Promise.reject(err)); + } + }); + worker.on('error', (err) => { + const waiter = waiters.shift(); + if (waiter) waiter.reject(err); + else queued.push(Promise.reject(err)); + }); + const exited = new Promise((resolve) => { + worker.once('exit', (code) => resolve(code)); + }); + /** + * Awaits the next reported value. + * + * @returns {Promise} the reported value. + */ + function next() { + if (queued.length > 0) return queued.shift(); + return new Promise((resolve, reject) => { + waiters.push({ resolve, reject }); + }); + } + return { worker, next, exited }; +} + +describe('worker threads', function () { + it('loads the addon in 8 workers at once, 1000 queries each, 20 cycles', { + timeout: 180000, + }, async function () { + const CYCLES = 20; + const WORKERS = 8; + const QUERIES = 1000; + for (let cycle = 0; cycle < CYCLES; cycle++) { + const results = await Promise.all( + Array.from({ length: WORKERS }, (_, id) => { + const { next, exited } = driverWorker( + async (driver, report, spec) => { + const QUERIES = spec.queries; + const db = await driver.open(':memory:'); + await db.exec( + 'CREATE TABLE t (i INTEGER, sq INTEGER)', + ); + const insert = db.prepare( + 'INSERT INTO t VALUES (?, ?)', + ); + for (let i = 0; i < QUERIES; i++) { + await insert.run(i, i * i); + } + await insert.finalize(); + const row = await db.get( + 'SELECT COUNT(*) AS n, SUM(sq) AS total FROM t', + ); + await db.close(); + report({ n: row.n, total: row.total }); + }, + { queries: QUERIES }, + ); + return next().then((r) => ({ id, r, exited })); + }), + ); + // Sum of squares for i = 0..999. + const expectedTotal = + ((QUERIES - 1) * QUERIES * (2 * QUERIES - 1)) / 6; + for (const { id, r } of results) { + assert.strictEqual( + r.n, + QUERIES, + `cycle ${cycle} worker ${id} row count`, + ); + assert.strictEqual( + r.total, + expectedTotal, + `cycle ${cycle} worker ${id} sum of squares`, + ); + } + // Every worker of the cycle must have exited cleanly. + for (const { exited } of results) { + assert.strictEqual(await exited, 0); + } + } + }); + + it('worker exits cleanly without closing anything (env teardown with live objects)', { + timeout: 30000, + }, async function () { + // The D08 musl crash shape: a napi environment torn down + // with a live connection and statements. Inside a worker + // this is a normal exit, and it must be a clean one. + const { next, exited } = driverWorker(async (driver, report) => { + const db = await driver.open(':memory:'); + await db.exec('CREATE TABLE t (a)'); + await db.run('INSERT INTO t VALUES (1)'); + // Deliberately no close, no finalize: the environment + // tears down around the live handle. + report('alive'); + await new Promise((resolve) => setTimeout(resolve, 20)); + }); + assert.strictEqual(await next(), 'alive'); + assert.strictEqual( + await exited, + 0, + 'worker must exit cleanly with a live connection', + ); + }); + + it('terminating a worker mid-query leaves the parent healthy', { + timeout: 120000, + }, async function () { + const { worker, next } = driverWorker(async (driver, report) => { + const db = await driver.open(':memory:'); + await db.exec('CREATE TABLE t (a)'); + report('ready'); + // Runs long enough to still be running when the terminate + // lands, cheap enough that teardown (which waits for this + // work on the threadpool) completes quickly even under + // heavy contention — the cost must not depend on losing a + // race (REVIEW-LOG D05 addendum). + await db + .all( + 'WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM c WHERE x < 5000000) SELECT count(*) FROM c', + ) + .catch(() => { + // Terminated, not rejected — this worker never + // reports success either way. + }); + report('query-completed'); + }); + // Wait for the query to be running, then kill the thread. + assert.strictEqual(await next(), 'ready'); + await new Promise((resolve) => setTimeout(resolve, 100)); + await worker.terminate(); + + // The parent's own connection must be unaffected. + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (a)'); + await db.run('INSERT INTO t VALUES (42)'); + assert.strictEqual( + (await db.get('SELECT a FROM t')).a, + 42, + 'parent connection still works after worker.terminate()', + ); + await db.close(); + }); + + it('main thread and a worker share one WAL file: writes are visible both ways', { + timeout: 60000, + }, async function () { + // Unique per run: concurrent stress runs of the suite share + // test/tmp/, and a fixed name would have twenty processes + // creating and deleting one database out from under each other + // (the observed "no such table" under 20-way stress). + const FILE = join( + TMP_DIR, + `worker-wal-share-${process.pid}-${randomUUID()}.db`, + ); + removeDb(FILE); + try { + const db = await sqlite3.open(FILE); + assert.strictEqual( + ( + await db.get('PRAGMA journal_mode=WAL') + ).journal_mode.toLowerCase(), + 'wal', + ); + await db.exec('CREATE TABLE t (a)'); + await db.run('INSERT INTO t VALUES (1)'); + + const { worker, next } = driverWorker( + async (driver, report, file) => { + // A readonly open of a WAL database needs the -shm + // to exist (or be creatable); under extreme + // filesystem contention that race can transiently + // lose with SQLITE_CANTOPEN. Back off and retry — + // observed only with 20 suite runs plus container + // builds hammering one volume, never otherwise. + let reader; + for (let attempt = 0; ; attempt++) { + try { + reader = await driver.open( + file, + driver.OPEN_READONLY, + ); + break; + } catch (err) { + if ( + err.code !== 'SQLITE_CANTOPEN' || + attempt >= 4 + ) { + throw err; + } + await new Promise((resolve) => + setTimeout(resolve, 25 * (attempt + 1)), + ); + } + } + const seen = await reader.all('SELECT a FROM t ORDER BY a'); + // The same worker also writes through its own + // read-write connection… + const writer = await driver.open(file); + await writer.run('INSERT INTO t VALUES (2)'); + // …and its reader sees the commit once made. + const after = await reader.all( + 'SELECT a FROM t ORDER BY a', + ); + await reader.close(); + await writer.close(); + report({ seen, after }); + }, + FILE, + ); + worker.unref(); + const reported = await next(); + assert.deepStrictEqual( + reported.seen.map((row) => row.a), + [1], + 'worker reader sees the main-thread write', + ); + assert.deepStrictEqual( + reported.after.map((row) => row.a), + [1, 2], + 'worker reader sees its own writer commit', + ); + // The worker's report follows its commit and the close of + // both its connections, so the write is durable in the WAL. + // Under extreme contention one same-connection read has + // landed on a stale snapshot (never unloaded); poll a FRESH + // connection — the visibility claim — bounded, then assert + // the final state either way. + let seen; + for (let i = 0; i < 200; i++) { + const check = await sqlite3.open(FILE); + seen = await check.all('SELECT a FROM t ORDER BY a'); + await check.close(); + if (seen.length === 2) break; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.deepStrictEqual( + seen.map((row) => row.a), + [1, 2], + 'main thread sees the worker write', + ); + await db.close(); + } finally { + removeDb(FILE); + } + }); + + it('moves an in-memory database to a worker via serialize/deserialize bytes', { + timeout: 30000, + }, async function () { + const db = await sqlite3.open(':memory:'); + await db.exec('CREATE TABLE t (a INTEGER, b TEXT)'); + await db.run('INSERT INTO t VALUES (?, ?)', [7, 'seven']); + await db.run('INSERT INTO t VALUES (?, ?)', [8, 'eight']); + const bytes = await db.serializeToBytes(); + await db.close(); + + const worker = new Worker( + ` + const { parentPort } = require('node:worker_threads'); + import(${JSON.stringify(DRIVER_URL)}).then(async (m) => { + const report = (value) => + parentPort.postMessage({ kind: 'report', value }); + parentPort.on('message', async (msg) => { + if (msg.kind !== 'bytes') return; + try { + const revived = + await m.default.deserializeFromBytes( + new Uint8Array(msg.bytes), + { resizable: true }, + ); + const rows = await revived.all( + 'SELECT * FROM t ORDER BY a', + ); + await revived.run('INSERT INTO t VALUES (?, ?)', [ + 9, + 'nine', + ]); + const count = await revived.get( + 'SELECT COUNT(*) AS n FROM t', + ); + await revived.close(); + report({ rows, count: count.n }); + parentPort.unref(); + } catch (err) { + report({ + fatal: + err && err.message + ? err.message + : String(err), + }); + } + }); + }); + `, + { eval: true }, + ); + /** @type {Promise} */ + const reported = new Promise((resolve, reject) => { + worker.on('message', (msg) => { + if (msg.kind === 'report') resolve(msg.value); + }); + worker.on('error', reject); + }); + // serializeToBytes' Uint8Array is a view over SQLite-owned + // external memory, which structured clone refuses to + // transfer — copy it into a plain ArrayBuffer (the one copy + // the handoff costs) and transfer that. + const movable = bytes.slice().buffer; + worker.postMessage({ kind: 'bytes', bytes: movable }, [movable]); + const result = await reported; + assert.ok(!result.fatal, `worker failed: ${result.fatal}`); + assert.deepStrictEqual(result.rows, [ + { a: 7, b: 'seven' }, + { a: 8, b: 'eight' }, + ]); + assert.strictEqual(result.count, 3, 'revived db is writable'); + assert.ok( + movable.byteLength === 0, + 'transferred buffer detached in the parent', + ); + }); + + it('cancels a worker query from the main thread via the shared token buffer', { + timeout: 30000, + }, async function () { + const { worker, next } = driverWorker(async (driver, report) => { + const db = await driver.open(':memory:'); + await db.exec('CREATE TABLE t (a)'); + const token = db.cancellationToken(); + // Hand the raw SharedArrayBuffer to the parent before + // starting the runaway query. + report({ sab: token.buffer }); + try { + await db.all( + 'WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM c WHERE x < 100000000) SELECT count(*) FROM c', + ); + report({ aborted: false }); + } catch (err) { + report({ aborted: true, code: err.code }); + } + }); + worker.unref(); + const armed = await next(); + assert.ok( + armed.sab instanceof SharedArrayBuffer, + 'token buffer crosses to the main thread', + ); + const flag = new Int32Array(armed.sab); + // The query is now running in the worker; set the flag from + // this thread. + await new Promise((resolve) => setTimeout(resolve, 50)); + Atomics.store(flag, 0, 1); + const outcome = await next(); + assert.strictEqual( + outcome.aborted, + true, + 'cross-thread cancel aborts the worker query', + ); + assert.strictEqual( + outcome.code, + 'SQLITE_INTERRUPT', + 'aborted query reports SQLITE_INTERRUPT', + ); + }); +}); diff --git a/tools/BinaryBuilder.Dockerfile b/tools/BinaryBuilder.Dockerfile index 328a201..3e39ad6 100644 --- a/tools/BinaryBuilder.Dockerfile +++ b/tools/BinaryBuilder.Dockerfile @@ -10,18 +10,24 @@ RUN if case $VARIANT in "alpine"*) true;; *) false;; esac; then apk add build-ba WORKDIR /usr/src/build COPY . . -RUN npm install --ignore-scripts +# Install the exact pnpm version pinned in packageManager; an unpinned pnpm +# resolves to whatever is latest at build time. pnpm 11 publishes static +# linux binaries (@pnpm/linuxstatic-x64/arm64) that run on both glibc and +# musl, so this works on Alpine — unlike pnpm 10.x releases that shipped no +# musl build (the failure mode cdxgen documented in its binary-builds.yml). +RUN npm install --global "pnpm@$(node -p "require('./package.json').packageManager.split('pnpm@')[1].split('+')[0]")" \ + && pnpm install --frozen-lockfile --ignore-scripts RUN if case $VARIANT in "alpine"*) true;; *) false;; esac; then \ - npm run prebuild -- --tag-libc; \ + pnpm run prebuild --tag-libc; \ else \ CFLAGS="${CFLAGS:-} -include ../src/gcc-preinclude.h" \ CXXFLAGS="${CXXFLAGS:-} -include ../src/gcc-preinclude.h" \ - npm run prebuild -- --tag-libc; \ + pnpm run prebuild --tag-libc; \ fi RUN if case $VARIANT in "alpine"*) false;; *) true;; esac; then ldd prebuilds/*/*.node; nm prebuilds/*/*.node | grep \"GLIBC_\" | c++filt || true ; fi -RUN npm run test && ls -l prebuilds +RUN pnpm run test && ls -l prebuilds -CMD ["sh"] \ No newline at end of file +CMD ["sh"] diff --git a/tools/benchmark/insert.js b/tools/benchmark/insert.js index e5f479d..0ca3018 100644 --- a/tools/benchmark/insert.js +++ b/tools/benchmark/insert.js @@ -1,73 +1,77 @@ +import fs from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + import sqlite3 from '../../lib/sqlite3.js'; -import fs from 'fs'; -import { fileURLToPath } from 'url'; -import { dirname, join } from 'path'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); -let iterations = 10000; +const iterations = 10000; export const compare = { - 'insert literal file': function(finished) { - let db = new sqlite3.Database(''); - let file = fs.readFileSync(join(__dirname, 'insert-transaction.sql'), 'utf8'); - db.exec(file); - db.close(finished); - }, + 'insert literal file': function (finished) { + const db = new sqlite3.Database(''); + const file = fs.readFileSync( + join(__dirname, 'insert-transaction.sql'), + 'utf8', + ); + db.exec(file); + db.close(finished); + }, - 'insert with transaction and two statements': function(finished) { - let db = new sqlite3.Database(''); + 'insert with transaction and two statements': function (finished) { + const db = new sqlite3.Database(''); - db.serialize(function() { - db.run("CREATE TABLE foo (id INT, txt TEXT)"); - db.run("BEGIN"); + db.serialize(function () { + db.run('CREATE TABLE foo (id INT, txt TEXT)'); + db.run('BEGIN'); - db.parallelize(function() { - let stmt1 = db.prepare("INSERT INTO foo VALUES (?, ?)"); - let stmt2 = db.prepare("INSERT INTO foo VALUES (?, ?)"); - for (let i = 0; i < iterations; i++) { - stmt1.run(i, 'Row ' + i); - i++; - stmt2.run(i, 'Row ' + i); - } - stmt1.finalize(); - stmt2.finalize(); - }); + db.parallelize(function () { + const stmt1 = db.prepare('INSERT INTO foo VALUES (?, ?)'); + const stmt2 = db.prepare('INSERT INTO foo VALUES (?, ?)'); + for (let i = 0; i < iterations; i++) { + stmt1.run(i, `Row ${i}`); + i++; + stmt2.run(i, `Row ${i}`); + } + stmt1.finalize(); + stmt2.finalize(); + }); - db.run("COMMIT"); - }); + db.run('COMMIT'); + }); - db.close(finished); - }, - 'insert with transaction': function(finished) { - let db = new sqlite3.Database(''); + db.close(finished); + }, + 'insert with transaction': function (finished) { + const db = new sqlite3.Database(''); - db.serialize(function() { - db.run("CREATE TABLE foo (id INT, txt TEXT)"); - db.run("BEGIN"); - let stmt = db.prepare("INSERT INTO foo VALUES (?, ?)"); - for (let i = 0; i < iterations; i++) { - stmt.run(i, 'Row ' + i); - } - stmt.finalize(); - db.run("COMMIT"); - }); + db.serialize(function () { + db.run('CREATE TABLE foo (id INT, txt TEXT)'); + db.run('BEGIN'); + const stmt = db.prepare('INSERT INTO foo VALUES (?, ?)'); + for (let i = 0; i < iterations; i++) { + stmt.run(i, `Row ${i}`); + } + stmt.finalize(); + db.run('COMMIT'); + }); - db.close(finished); - }, - 'insert without transaction': function(finished) { - let db = new sqlite3.Database(''); + db.close(finished); + }, + 'insert without transaction': function (finished) { + const db = new sqlite3.Database(''); - db.serialize(function() { - db.run("CREATE TABLE foo (id INT, txt TEXT)"); - let stmt = db.prepare("INSERT INTO foo VALUES (?, ?)"); - for (let i = 0; i < iterations; i++) { - stmt.run(i, 'Row ' + i); - } - stmt.finalize(); - }); + db.serialize(function () { + db.run('CREATE TABLE foo (id INT, txt TEXT)'); + const stmt = db.prepare('INSERT INTO foo VALUES (?, ?)'); + for (let i = 0; i < iterations; i++) { + stmt.run(i, `Row ${i}`); + } + stmt.finalize(); + }); - db.close(finished); - } -}; \ No newline at end of file + db.close(finished); + }, +}; diff --git a/tools/benchmark/select.js b/tools/benchmark/select.js index 96bbadc..6014331 100644 --- a/tools/benchmark/select.js +++ b/tools/benchmark/select.js @@ -1,7 +1,8 @@ +import { readFileSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + import sqlite3 from '../../lib/sqlite3.js'; -import { readFileSync } from 'fs'; -import { fileURLToPath } from 'url'; -import { dirname } from 'path'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -9,26 +10,30 @@ const __dirname = dirname(__filename); const db = new sqlite3.Database(':memory:'); db.serialize(() => { - db.exec(readFileSync(`${__dirname}/select-data.sql`, 'utf8'), (err) => { - if (err) throw err; - console.time('db.each'); - }); - - { - const results = []; - db.each('SELECT * FROM foo', (err, row) => { - if (err) throw err; - results.push(row); - }, () => { - console.timeEnd('db.each'); - console.time('db.all'); + db.exec(readFileSync(`${__dirname}/select-data.sql`, 'utf8'), (err) => { + if (err) throw err; + console.time('db.each'); }); - } - db.all('SELECT * FROM foo', (err, rows) => { - console.timeEnd('db.all'); - if (err) throw err; - }); + { + const results = []; + db.each( + 'SELECT * FROM foo', + (err, row) => { + if (err) throw err; + results.push(row); + }, + () => { + console.timeEnd('db.each'); + console.time('db.all'); + }, + ); + } + + db.all('SELECT * FROM foo', (err, _rows) => { + console.timeEnd('db.all'); + if (err) throw err; + }); - db.close(); -}); \ No newline at end of file + db.close(); +}); diff --git a/tools/check-no-only.js b/tools/check-no-only.js new file mode 100644 index 0000000..192b9c9 --- /dev/null +++ b/tools/check-no-only.js @@ -0,0 +1,28 @@ +// Fails the run if any test file still carries an .only marker. +// node:test ignores `only` without --test-only (and --test-only would +// silently skip every unmarked test), so the only safe posture is to +// reject the marker outright. +import { readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +// fileURLToPath, not URL.pathname: on Windows the latter yields +// "/C:/...", which join() turns into an unusable path. +const dir = fileURLToPath(new URL('../test/', import.meta.url)); +let found = 0; +for (const f of readdirSync(dir).filter((f) => f.endsWith('.test.js'))) { + const src = readFileSync(join(dir, f), 'utf8'); + for (const [i, line] of src.split('\n').entries()) { + if (/\b(?:it|describe|test)\.only\s*[.(]/.test(line)) { + console.error(`${f}:${i + 1}: stray .only marker: ${line.trim()}`); + found++; + } + } +} +if (found > 0) { + console.error( + `check-no-only: ${found} stray .only marker(s) — remove them before running the suite`, + ); + process.exit(1); +} +console.log('check-no-only: no stray .only markers.'); diff --git a/tools/gen-types.js b/tools/gen-types.js new file mode 100644 index 0000000..ca01d44 --- /dev/null +++ b/tools/gen-types.js @@ -0,0 +1,126 @@ +// Generates the shipped type declarations. +// +// pnpm run gen-types +// +// Runs `tsc -p tsconfig.types.json`, which typechecks lib/*.js (checkJs) +// against the hand-written native declarations in lib/native.d.ts and +// emits declarations into the gitignored types-gen/ scratch directory. +// This script then copies them into lib/, post-processing the package's +// `types` entry (lib/sqlite3.d.ts) in three deterministic steps: +// +// 1. prepend the GENERATED header, +// 2. append `import './augment.js'` so consumers of the package load +// the JS-layer member augmentation in lib/augment.d.ts, +// 3. append re-exports of the public type declarations of +// lib/native.d.ts and lib/promises.d.ts, so the type surface +// (`Row`, `BindValue`, `SignalOptions`, …) is importable from the +// package root, as it was when the .d.ts was hand-written. +// +// The previously generated lib/sqlite3.d.ts, lib/promises.d.ts and +// lib/trace.d.ts are deleted up front: sitting next to their .js +// sources they would win module resolution, and tsc would check the JS +// against the stale declarations instead of emitting them. +// +// The result is committed; CI regenerates and fails on any diff, so a +// declaration can neither drift from the JSDoc nor be silently dropped. +import { execFileSync } from 'node:child_process'; +import { copyFileSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.dirname(path.dirname(fileURLToPath(import.meta.url))); +const tsc = path.join(root, 'node_modules', '.bin', 'tsc'); +const generated = ['sqlite3.d.ts', 'promises.d.ts', 'trace.d.ts', 'pool.d.ts']; + +// Stale outputs first, so resolution during the run sees the sources. +for (const file of generated) { + rmSync(path.join(root, 'lib', file), { force: true }); +} + +execFileSync(tsc, ['-p', path.join(root, 'tsconfig.types.json')], { + stdio: 'inherit', + cwd: root, +}); + +const emitDir = path.join(root, 'types-gen'); +const entry = path.join(root, 'lib', 'sqlite3.d.ts'); + +// tsc's JS emit attaches the original JSDoc verbatim, so a rendered +// `export type X` can be followed by a second, raw copy of its @typedef +// block. Drop those: the rendered declaration carries the same docs. +function stripRawTypedefComments(text) { + return text.replace(/\/\*\*(?:[^*]|\*(?!\/))*?@typedef[\s\S]*?\*\/\n/g, ''); +} + +const emitted = stripRawTypedefComments( + readFileSync(path.join(emitDir, 'sqlite3.d.ts'), 'utf8'), +); + +// Every `export type X` / `export interface X` in the hand-written +// island, keys sorted for a stable diff. Classes are re-exported by the +// emitted module re-export already; functions stay in their module. +const nativeTypes = [ + ...readFileSync(path.join(root, 'lib', 'native.d.ts'), 'utf8').matchAll( + /^export (?:type|interface) ([A-Za-z_$][\w$]*)/gm, + ), +].map((m) => m[1]); + +// The public types authored in lib/promises.js' JSDoc. Internal helpers +// (Installed, IteratorOptions, …) stay inside lib/promises.d.ts. +const promisesPublicTypes = [ + 'FetchCallback', + 'OpenFunction', + 'PromiseRunResult', + 'SignalOptions', + 'TransactionOptions', +]; + +// The pool's public types, authored in lib/pool.js' JSDoc. +const poolPublicTypes = [ + 'PoolOptions', + 'PoolQueryOptions', + 'PoolTransaction', + 'SqlitePool', +]; + +const augmentImport = "import './augment.js';"; +const block = (from, names) => + `export type {\n${[...names] + .sort() + .map((n) => ` ${n},`) + .join('\n')}\n} from './${from}';\n`; + +const header = `// GENERATED FILE — DO NOT EDIT. +// Regenerate with \`pnpm run gen-types\`. Sources, in order of truth: +// 1. the native layer's shape, hand-written in lib/native.d.ts, +// 2. the JS layer's members in lib/augment.d.ts, +// 3. the JSDoc of lib/*.js, from which tsc emits this file plus +// lib/promises.d.ts and lib/trace.d.ts. +// The three shipped .d.ts files together form the public types. + +`; + +writeFileSync( + entry, + `${header}${emitted}${augmentImport}\n` + + block('promises.js', promisesPublicTypes) + + block('pool.js', poolPublicTypes) + + block('native.js', nativeTypes), +); + +copyFileSync( + path.join(emitDir, 'promises.d.ts'), + path.join(root, 'lib', 'promises.d.ts'), +); +copyFileSync( + path.join(emitDir, 'trace.d.ts'), + path.join(root, 'lib', 'trace.d.ts'), +); +copyFileSync( + path.join(emitDir, 'pool.d.ts'), + path.join(root, 'lib', 'pool.d.ts'), +); + +console.log( + 'gen-types: lib/sqlite3.d.ts, lib/promises.d.ts, lib/trace.d.ts, lib/pool.d.ts regenerated.', +); diff --git a/tools/run-tests.mjs b/tools/run-tests.mjs new file mode 100644 index 0000000..2049edb --- /dev/null +++ b/tools/run-tests.mjs @@ -0,0 +1,58 @@ +// Resolves the test files in Node and runs them, instead of handing a +// glob to the shell. +// +// `node --test 'test/*.test.js'` looks portable and is not: POSIX sh +// strips the single quotes and Node expands the glob, but cmd.exe treats +// them as ordinary characters, so Node receives a pattern with quotes in +// it, matches nothing, reports "tests 0" and **exits 0**. Every Windows +// CI job was green on zero tests for months. A silently empty test run +// is worse than a failing one, so this script also refuses to pass when +// it finds implausibly few files. +// +// node tools/run-tests.mjs # the whole suite +// node tools/run-tests.mjs test/pool.test.js … # explicit files +import { spawnSync } from 'node:child_process'; +import { globSync } from 'node:fs'; +import { dirname, join, relative } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = join(dirname(fileURLToPath(import.meta.url)), '..'); + +// A floor, not an exact count, so adding tests never edits this. It only +// has to be high enough that "the glob broke" cannot slip through. +const MINIMUM_TEST_FILES = 30; + +const explicit = process.argv.slice(2).filter((a) => !a.startsWith('-')); +const passthrough = process.argv.slice(2).filter((a) => a.startsWith('-')); + +const files = explicit.length + ? explicit + : globSync('test/*.test.js', { cwd: root }) + .map((f) => relative(root, join(root, f))) + .sort(); + +if (!explicit.length && files.length < MINIMUM_TEST_FILES) { + console.error( + `run-tests: found only ${files.length} test file(s) under test/, expected at least ` + + `${MINIMUM_TEST_FILES}. Refusing to report success on an empty or truncated run — ` + + 'this is the failure mode where a broken glob makes CI green on zero tests.', + ); + process.exit(1); +} + +const args = [ + '--test', + '--test-reporter=spec', + '--test-timeout=20000', + ...passthrough, + ...files, +]; + +// process.execPath, so this follows whichever runtime invoked it — plain +// Node, or Electron's Node build under ELECTRON_RUN_AS_NODE. +const res = spawnSync(process.execPath, args, { cwd: root, stdio: 'inherit' }); +if (res.error) { + console.error(`run-tests: failed to start: ${res.error.message}`); + process.exit(1); +} +process.exit(res.status ?? 1); diff --git a/tools/semver-check.js b/tools/semver-check.js index 1b7e22a..001ee8a 100644 --- a/tools/semver-check.js +++ b/tools/semver-check.js @@ -1,8 +1,8 @@ -import fs from 'fs'; -import path from 'path'; +import fs from 'node:fs'; +import path, { dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + import semver from 'semver'; -import { fileURLToPath } from 'url'; -import { dirname } from 'path'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -10,28 +10,34 @@ const __dirname = dirname(__filename); const supportedVersions = '24.0.0'; function checkEngines(modulePath) { - const packageJsonPath = path.join(modulePath, 'package.json'); + const packageJsonPath = path.join(modulePath, 'package.json'); - if (!fs.existsSync(packageJsonPath)) return; + if (!fs.existsSync(packageJsonPath)) return; - const packageJson = JSON.parse(fs.readFileSync(packageJsonPath)); - const engines = packageJson.engines; + const packageJson = JSON.parse(fs.readFileSync(packageJsonPath)); + const engines = packageJson.engines; - if (engines && engines.node) { - const minVersion = semver.minVersion(engines.node); + if (engines?.node) { + const minVersion = semver.minVersion(engines.node); - if (semver.gt(minVersion, supportedVersions)) { - console.log(`${packageJson.name}@${packageJson.version} requires ${engines.node}`); - process.exit(1); + if (semver.gt(minVersion, supportedVersions)) { + console.log( + `${packageJson.name}@${packageJson.version} requires ${engines.node}`, + ); + process.exit(1); + } } - } } -const packageJson = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'package.json'))); +const packageJson = JSON.parse( + fs.readFileSync(path.join(__dirname, '..', 'package.json')), +); -const allDependencies = Object.keys(packageJson.dependencies || {}).concat(Object.keys(packageJson.optionalDependencies || {})); +const allDependencies = Object.keys(packageJson.dependencies || {}).concat( + Object.keys(packageJson.optionalDependencies || {}), +); for (const dependency of allDependencies) { - const modulePath = path.join(__dirname, '..', 'node_modules', dependency); - checkEngines(modulePath); -} \ No newline at end of file + const modulePath = path.join(__dirname, '..', 'node_modules', dependency); + checkEngines(modulePath); +} diff --git a/tools/test-matrix.mjs b/tools/test-matrix.mjs new file mode 100644 index 0000000..5d643c8 --- /dev/null +++ b/tools/test-matrix.mjs @@ -0,0 +1,247 @@ +// Runs the test suite across a matrix of container targets locally, so a +// platform-specific or load-sensitive failure can be reproduced without +// pushing and waiting for CI. +// +// This exists because CI failures have repeatedly not reproduced on a +// developer machine: the D08 segfault was musl-only, and an +// ubuntu-22.04 flake in D10 needed glibc *and* Node 26 *and* a starved +// CPU before it showed up at all. +// +// node tools/test-matrix.mjs # every target, full suite +// node tools/test-matrix.mjs --only=ubuntu22-node26 +// node tools/test-matrix.mjs --cpus=1 --load=6 # reproduce a slow runner +// node tools/test-matrix.mjs --repeat=20 --cmd='node test/support/foo.mjs' +// node tools/test-matrix.mjs --list +// +// Notes on fidelity: +// * The working tree is copied in, but node_modules/, build/, +// prebuilds/ and test/tmp/ are left behind and the addon is rebuilt +// inside the container. Copying those in silently changes results — +// a stale test/tmp made a backup fixture fail fast and hid a race +// that only appeared once the directory existed. +// * Fixtures are generated inside the container (test/support/createdb.js), +// not copied, for the same reason. +import { execFileSync, spawnSync } from 'node:child_process'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = join(dirname(fileURLToPath(import.meta.url)), '..'); + +// `node: null` means the base image already ships Node. +const TARGETS = { + // Mirrors the CI ubuntu-22.04 job, which builds against Node 26. + 'ubuntu22-node26': { + base: 'ubuntu:22.04', + node: '26.0.0', + pkg: 'apt', + note: 'glibc 2.35, matches the CI ubuntu-22.04 job', + }, + 'ubuntu22-node24': { + base: 'ubuntu:22.04', + node: '24.19.0', + pkg: 'apt', + note: 'glibc 2.35, the engines floor', + }, + 'alpine-node24': { + base: 'node:24-alpine', + node: null, + pkg: 'apk', + note: 'musl — the D08 segfault was musl-only', + }, + 'debian-node24': { + base: 'node:24', + node: null, + pkg: 'apt', + note: 'glibc 2.36+', + }, +}; + +function parseArgs(argv) { + const opts = { + only: null, + cpus: null, + load: 0, + repeat: 1, + cmd: 'pnpm run test', + platform: 'linux/amd64', + list: false, + keepGoing: true, + }; + for (const arg of argv) { + const [key, ...rest] = arg.replace(/^--/, '').split('='); + const value = rest.join('='); + if (key === 'list') opts.list = true; + else if (key === 'only') opts.only = value.split(',').filter(Boolean); + else if (key === 'cpus') opts.cpus = value; + else if (key === 'load') opts.load = Number(value); + else if (key === 'repeat') opts.repeat = Number(value); + else if (key === 'cmd') opts.cmd = value; + else if (key === 'platform') opts.platform = value; + else { + console.error(`unknown option: ${arg}`); + process.exit(2); + } + } + return opts; +} + +const opts = parseArgs(process.argv.slice(2)); + +if (opts.list) { + for (const [name, t] of Object.entries(TARGETS)) { + console.log(`${name.padEnd(18)} ${t.base.padEnd(16)} ${t.note}`); + } + process.exit(0); +} + +const selected = opts.only ?? Object.keys(TARGETS); +for (const name of selected) { + if (!TARGETS[name]) { + console.error( + `unknown target: ${name}\nknown: ${Object.keys(TARGETS).join(', ')}`, + ); + process.exit(2); + } +} + +// The pinned pnpm, so the container matches the repo rather than +// whatever npm's dist-tag happens to be today. +const packageManager = + JSON.parse( + execFileSync( + 'node', + ['-p', 'JSON.stringify(require("./package.json"))'], + { + cwd: root, + encoding: 'utf8', + }, + ), + ).packageManager ?? 'pnpm@11'; + +function dockerfileFor(target) { + const lines = [`FROM ${target.base}`]; + if (target.pkg === 'apt') { + lines.push( + 'ENV DEBIAN_FRONTEND=noninteractive', + 'RUN apt-get update && apt-get install -y --no-install-recommends ' + + 'curl python3 make g++ xz-utils ca-certificates ' + + '>/dev/null 2>&1 && rm -rf /var/lib/apt/lists/*', + ); + } else { + lines.push('RUN apk add --no-cache python3 make g++ >/dev/null 2>&1'); + } + if (target.node) { + lines.push( + `RUN curl -fsSL https://nodejs.org/dist/v${target.node}/node-v${target.node}-linux-x64.tar.xz -o /n.tar.xz \\ + && tar -xJf /n.tar.xz -C /usr/local --strip-components=1 && rm /n.tar.xz`, + ); + } + // Node 26 no longer bundles corepack, so install pnpm outright. + lines.push(`RUN npm i -g ${packageManager} >/dev/null 2>&1`); + return lines.join('\n'); +} + +function buildImage(name, target) { + const tag = `sq3-matrix-${name}`; + const dir = mkdtempSync(join(tmpdir(), 'sq3-matrix-')); + writeFileSync(join(dir, 'Dockerfile'), dockerfileFor(target)); + const res = spawnSync( + 'docker', + ['build', '--platform', opts.platform, '-t', tag, dir], + { stdio: ['ignore', 'ignore', 'pipe'], encoding: 'utf8' }, + ); + if (res.status !== 0) { + throw new Error( + `docker build failed for ${name}:\n${res.stderr?.slice(-1500)}`, + ); + } + return tag; +} + +// Everything volatile is rebuilt inside the container; see the header. +const SETUP = [ + 'cp -R /src /work', + 'cd /work', + 'rm -rf node_modules prebuilds build test/tmp', + 'pnpm install --ignore-scripts >/dev/null 2>&1', + 'pnpm run rebuild >/dev/null 2>&1 || { echo "REBUILD FAILED"; exit 90; }', + 'mkdir -p test/tmp', + 'node test/support/createdb.js >/dev/null 2>&1', +].join('; '); + +function runTarget(name, target) { + const tag = buildImage(name, target); + const spinners = + opts.load > 0 + ? `for i in $(seq 1 ${opts.load}); do (while :; do :; done) & done; ` + : ''; + const body = + opts.repeat > 1 + ? `F=0; for i in $(seq 1 ${opts.repeat}); do ${opts.cmd} >/dev/null 2>&1 || F=$((F+1)); done; ` + + `echo "REPEAT_FAILURES=$F/${opts.repeat}"; [ "$F" = "0" ]` + : opts.cmd; + const args = ['run', '--rm', '--platform', opts.platform]; + if (opts.cpus) args.push(`--cpus=${opts.cpus}`); + args.push( + '-v', + `${root}:/src:ro`, + tag, + 'sh', + '-c', + `${SETUP}; ${spinners}${body}`, + ); + + const started = Date.now(); + const res = spawnSync('docker', args, { encoding: 'utf8' }); + const out = `${res.stdout ?? ''}${res.stderr ?? ''}`; + process.stdout.write(out); + return { + name, + status: res.status, + seconds: Math.round((Date.now() - started) / 1000), + summary: summarise(out, res.status), + }; +} + +function summarise(out, status) { + const repeat = out.match(/REPEAT_FAILURES=(\S+)/)?.[1]; + if (repeat) return `failures ${repeat}`; + if (out.includes('REBUILD FAILED')) return 'native build failed'; + const pass = out.match(/^ℹ pass (\d+)$/m)?.[1]; + const fail = out.match(/^ℹ fail (\d+)$/m)?.[1]; + if (pass !== undefined) return `pass ${pass}, fail ${fail ?? '?'}`; + return status === 0 ? 'ok' : `exit ${status}`; +} + +const results = []; +for (const name of selected) { + console.log(`\n=== ${name} (${TARGETS[name].base}) ===`); + try { + results.push(runTarget(name, TARGETS[name])); + } catch (err) { + console.error(err.message); + results.push({ + name, + status: 1, + seconds: 0, + summary: 'image build failed', + }); + } +} + +console.log('\n──────── matrix summary ────────'); +for (const r of results) { + const mark = r.status === 0 ? 'PASS' : 'FAIL'; + console.log( + `${mark} ${r.name.padEnd(18)} ${r.summary.padEnd(22)} ${r.seconds}s`, + ); +} +const failed = results.filter((r) => r.status !== 0); +console.log( + failed.length + ? `\n${failed.length} of ${results.length} targets failed` + : `\nall ${results.length} targets passed`, +); +process.exit(failed.length ? 1 : 0); diff --git a/tsconfig.check.json b/tsconfig.check.json new file mode 100644 index 0000000..fbc94f0 --- /dev/null +++ b/tsconfig.check.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "strict": true, + "module": "node16", + "moduleResolution": "node16", + "target": "esnext", + "lib": ["esnext"], + "types": ["node"], + "noEmit": true, + "skipLibCheck": false + }, + "include": ["types/**/*.check.ts"] +} diff --git a/tsconfig.types.json b/tsconfig.types.json new file mode 100644 index 0000000..6980671 --- /dev/null +++ b/tsconfig.types.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "allowJs": true, + "checkJs": true, + "strict": true, + "alwaysStrict": true, + "module": "nodenext", + "moduleResolution": "nodenext", + "target": "esnext", + "lib": ["esnext"], + "types": ["node"], + "declaration": true, + "emitDeclarationOnly": true, + "skipLibCheck": false, + "outDir": "types-gen", + "rootDir": "lib" + }, + "include": [ + "lib/sqlite3.js", + "lib/promises.js", + "lib/trace.js", + "lib/augment.d.ts", + "lib/native.d.ts", + "lib/pool.js", + "lib/worker.js" + ] +} diff --git a/types/consumer.check.ts b/types/consumer.check.ts new file mode 100644 index 0000000..05d8b49 --- /dev/null +++ b/types/consumer.check.ts @@ -0,0 +1,101 @@ +// Consumer-side compile checks under strict + node16 resolution, +// complementing the tsd assertions: `await using` disposal, async +// iteration, generic propagation, and the negatives the marshalling +// rules promise (unsupported bind types, invalid configure literals). +// Any line marked @ts-expect-error MUST fail; the file fails to compile +// if one of them starts passing. + +import type { Database, Row } from '../lib/sqlite3.js'; +import sqlite3 from '../lib/sqlite3.js'; + +async function consumer(): Promise { + // open + await using: the promise-native lifecycle + await using db: Database = await sqlite3.open('file.db'); + + // Generic propagation: db.all<{a: number}> gives {a: number}[] + const rows: { a: number }[] = await db.all<{ a: number }>('SELECT a'); + void rows; + + // untyped all: Row[] + const untyped: Row[] = await db.all('SELECT a'); + void untyped; + + // iterate: AsyncIterableIterator + const iterator = db.iterate('SELECT a'); + for await (const row of iterator) { + const value: unknown = row.a; + void value; + } + + // using: sync prepare + sync dispose + using stmt = db.prepareSync('SELECT a'); + const first: Row | undefined = stmt.getSync(); + void first; + + // statement transaction shape + const count = await db.transaction(async (tx) => { + await tx.run('INSERT INTO t VALUES (?)', 1); + return 42; + }); + const check: number = count; + void check; + + // --- Deliverable 11: open options, extension policy, attach paths --- + + // open() accepts flags or a v9 options object. + const untrusted = await sqlite3.open('downloaded.db', { + mode: sqlite3.OPEN_READONLY, + untrusted: true, + }); + await untrusted.close(); + const flagged = await sqlite3.open('file.db', sqlite3.OPEN_READWRITE); + await flagged.close(); + + // The constructor takes the same options object (namespace or named + // export — both are the wrapper). + const constructed = new sqlite3.Database(':memory:', { + untrusted: true, + }); + // @ts-expect-error untrusted is boolean + const constructed2 = new sqlite3.Database(':memory:', { + untrusted: 'yes', + }); + void constructed; + void constructed2; + + // The security configure options typecheck with their shapes. + db.configure('extensionPolicy', { allow: ['/abs/ext.so'] }); + db.configure('extensionPolicy', { deny: true }); + db.configure('attachPaths', ['/abs/aux.db']); + db.configure('attachPaths', null); + // @ts-expect-error unknown policy key + db.configure('extensionPolicy', { maybe: true }); + // @ts-expect-error attachPaths wants paths or null + db.configure('attachPaths', 'nope'); + + // --- Negatives: these must NOT compile ------------------------------- + + // Symbol is not a BindValue (strict binding, Deliverable 02). + // @ts-expect-error + await db.run('SELECT ?', Symbol('nope')); + + // configure takes the documented literals only. + // @ts-expect-error + db.configure('integerMode', 'float'); + + // @ts-expect-error + db.on('nonexistent-event', () => undefined); + + // the constants are literal-typed; a wrong value is a type error. + const flag: 1 = sqlite3.OPEN_READONLY; + void flag; + // @ts-expect-error + const wrong: 2 = sqlite3.OPEN_READONLY; + void wrong; + + // open() needs a filename. + // @ts-expect-error + await sqlite3.open(); +} + +void consumer; diff --git a/types/sqlite3.test-d.ts b/types/sqlite3.test-d.ts new file mode 100644 index 0000000..ea4228b --- /dev/null +++ b/types/sqlite3.test-d.ts @@ -0,0 +1,450 @@ +// Type tests for the generated declarations, run by tsd via +// `pnpm run test:types`. Callbacks are written contextually: their +// parameter types come from the declarations, so a signature change +// anywhere in the pipeline fails here. + +import type { Readable } from 'node:stream'; + +import { expectType } from 'tsd'; + +import type { + Backup, + Database, + FetchCallback, + IntegerMode, + PromiseRunResult, + Row, + SignalOptions, + sqlite3 as Sqlite3Namespace, + SqliteError, + Statement, + StatementRunSyncResult, + TransactionOptions, +} from '../lib/sqlite3.js'; +import sqlite3 from '../lib/sqlite3.js'; + +declare const db: Database; +declare const stmt: Statement; +declare const err: SqliteError; +declare const signal: AbortSignal; + +// --- Namespace object ----------------------------------------------------- + +// Constants carry their literal values, so flag arithmetic is checkable. +expectType<1>(sqlite3.OPEN_READONLY); +expectType<2>(sqlite3.OPEN_READWRITE); +expectType<4>(sqlite3.OPEN_CREATE); +expectType<1555>(sqlite3.CONSTRAINT_PRIMARYKEY); +expectType<'3.53.4'>(sqlite3.VERSION); +expectType<3053004>(sqlite3.VERSION_NUMBER); +expectType<11>(sqlite3.LIMIT_WORKER_THREADS); + +// verbose() returns the same namespace shape. +expectType(sqlite3.verbose()); + +// cached.Database reuses or opens connections; the registry is public. +expectType(sqlite3.cached.Database('file.db')); +expectType>(sqlite3.cached.objects); + +// open is the promise-native constructor form. +expectType>(sqlite3.open('file.db')); +expectType>(sqlite3.open('file.db', sqlite3.OPEN_READONLY)); + +// --- Database: callback mode --------------------------------------------- + +expectType( + db.run('INSERT INTO t VALUES (?)', 1, (e) => { + expectType(e); + }), +); +expectType( + db.run('INSERT INTO t VALUES (?)', [1], (e) => { + expectType(e); + }), +); +expectType( + db.get('SELECT 1', (e, row) => { + expectType(e); + expectType(row); + }), +); +expectType( + db.all('SELECT 1', (e, rows) => { + expectType(e); + expectType(rows); + }), +); +expectType( + db.each( + 'SELECT 1', + (e, row) => { + expectType(e); + expectType(row); + }, + (e, count) => { + expectType(e); + expectType(count); + }, + ), +); +expectType( + db.exec('CREATE TABLE t (a)', (e) => { + expectType(e); + }), +); +expectType(db.prepare('SELECT 1')); +expectType(db.prepare('SELECT ?', 1, () => undefined)); + +// --- Database: promise mode with bound parameters (D03 follow-up) -------- + +expectType>(db.run('INSERT INTO t VALUES (?)')); +expectType>(db.run('INSERT INTO t VALUES (?)', 1)); +expectType>(db.run('INSERT INTO t VALUES (?)', [1])); +expectType>( + db.run('INSERT INTO t VALUES (?)', 1, { signal }), +); +expectType>( + db.run('INSERT INTO t VALUES (?)', [1], { signal }), +); + +expectType>(db.get('SELECT 1')); +expectType>(db.get('SELECT 1', 1)); +expectType>(db.get('SELECT 1', [1])); +expectType>(db.get('SELECT 1', 1, { signal })); + +expectType>(db.all('SELECT 1')); +expectType>(db.all('SELECT 1', 1)); +expectType>(db.all('SELECT 1', [1], { signal })); + +expectType>>(db.map('SELECT 1')); +expectType>>(db.map('SELECT 1', 1, { signal })); + +expectType>(db.exec('CREATE TABLE t (a)')); +expectType>(db.exec('CREATE TABLE t (a)', { signal })); +expectType>(db.close()); +expectType>(db.wait()); +expectType>(db.loadExtension('ext.so')); + +// Generic propagation: the type argument flows through every promise form. +expectType>( + db.get<{ a: number }>('SELECT a'), +); +expectType>( + db.get<{ a: number }>('SELECT a', 1), +); +expectType>(db.all<{ a: number }>('SELECT a')); +expectType>( + db.all<{ a: number }>('SELECT a', 1, { signal }), +); +// and through the callback forms. +expectType( + db.get<{ a: number }>('SELECT a', (e, row) => { + expectType(e); + expectType<{ a: number } | undefined>(row); + }), +); + +// --- Database: sync fast path -------------------------------------------- + +expectType(db.getSync('SELECT 1')); +expectType<{ a: number } | undefined>(db.getSync<{ a: number }>('SELECT a')); +// lastID is number | bigint: it applies the connection's integer mode. +expectType(db.runSync('SELECT 1')); +expectType(db.allSync('SELECT 1')); +expectType<{ a: number }[]>(db.allSync<{ a: number }>('SELECT a')); +expectType(db.prepareSync('SELECT 1')); +// rowMode: 'array' opts into the bulk-reader row shape (arrays). +expectType(db.getSync('SELECT 1', { rowMode: 'array' })); +expectType(db.allSync('SELECT 1', { rowMode: 'array' })); +expectType( + db.allSync('SELECT a FROM t WHERE b = ?', 5, { rowMode: 'array' }), +); + +// --- Database: statement cache, backup, transactions, iteration ---------- + +expectType(db.cacheStatements()); +expectType(db.cacheStatements(128)); +expectType(db.backup('copy.db')); +expectType(db.backup('copy.db', 'main', 'main', true, () => undefined)); + +expectType>(db.iterate('SELECT 1')); +expectType>(db.iterate('SELECT 1', 1)); +expectType>(db.iterate('SELECT 1', [1])); +expectType>(db.iterate('SELECT 1', 1, { signal })); + +expectType(db.stream('SELECT 1')); +expectType(db.stream('SELECT 1', 1, { signal })); + +expectType>( + db.transaction(async (tx) => { + expectType(tx); + return { a: 1 }; + }), +); +expectType>(db.transaction(async () => undefined)); +const txOptions: TransactionOptions = { + mode: 'immediate', + savepoint: true, + serialize: false, + signal, +}; +expectType>( + db.transaction(async () => undefined, txOptions), +); +const signalOptions: SignalOptions = { signal }; +expectType(signalOptions); + +// --- Database: events, accessors, disposal ------------------------------- + +expectType( + db.on('trace', (sql) => { + expectType(sql); + }), +); +expectType( + db.on('profile', (sql, time) => { + expectType(sql); + expectType(time); + }), +); +expectType( + db.on('change', (type, database, table, rowid) => { + expectType(type); + expectType(database); + expectType(table); + expectType(rowid); + }), +); +expectType( + db.on('error', (e) => { + expectType(e); + }), +); +expectType(db.on('open', () => undefined)); + +expectType(db.open); +expectType(db.filename); +expectType(db.mode); +expectType(db.integerMode); +expectType(db.configure('integerMode', 'mixed')); +expectType(db.configure('busyTimeout', 1000)); +expectType(db.configure('limit', sqlite3.LIMIT_LENGTH, 1 << 30)); +expectType(db.interrupt()); +expectType(db.serialize()); +expectType(db.parallelize()); +expectType>(db[Symbol.asyncDispose]()); + +// --- Statement: callback mode (contextual) -------------------------------- + +expectType( + stmt.bind(1, (e) => { + expectType(e); + }), +); +expectType( + stmt.run(1, (e) => { + expectType(e); + }), +); +expectType( + stmt.get(1, (e, row) => { + expectType(e); + expectType(row); + }), +); +expectType( + stmt.all(1, (e, rows) => { + expectType(e); + expectType(rows); + }), +); +expectType( + // biome-ignore lint/suspicious/useIterableCallbackReturn: Statement#map is the row-mapping SQL method, not Array#map + stmt.map((e, map) => { + expectType(e); + expectType(map); + }), +); +expectType( + stmt.reset((e) => { + expectType(e); + }), +); +expectType(stmt.finalize(() => undefined)); + +// --- Statement: promise mode with bound parameters ----------------------- + +expectType>(stmt.bind()); +expectType>(stmt.bind(1)); +expectType>(stmt.bind([1])); +expectType>(stmt.run()); +expectType>(stmt.run(1)); +expectType>(stmt.run([1])); +expectType>(stmt.run(1, { signal })); +expectType>(stmt.get()); +expectType>(stmt.get(1)); +expectType>(stmt.get(1, { signal })); +expectType>(stmt.all()); +expectType>(stmt.all(1, { signal })); +expectType>>(stmt.map(1, { signal })); +expectType>(stmt.reset()); +expectType>(stmt.finalize()); + +// fetch, iteration, sync, accessors +expectType( + stmt.fetch(100, (e, rows, done) => { + expectType(e); + expectType(rows); + expectType(done); + }), +); +const fetchCallback: FetchCallback = (e, rows, done) => { + expectType(e); + expectType(rows); + expectType(done); +}; +void fetchCallback; +expectType( + stmt.fetch(100, 1, (e, rows, done) => { + expectType(e); + expectType(rows); + expectType(done); + }), +); +expectType>(stmt.iterate()); +expectType>(stmt.iterate(1)); +expectType>(stmt.iterate(1, { signal })); +expectType(stmt.getSync(1)); +expectType(stmt.allSync(1)); +expectType(stmt.getSync(1, { rowMode: 'array' })); +expectType(stmt.allSync({ rowMode: 'array' })); +expectType(stmt.runSync(1)); +expectType(stmt.sql); +expectType(stmt.lastID); +expectType(stmt.lastIDBigInt); +expectType(stmt.changes); +expectType>(stmt[Symbol.asyncDispose]()); +expectType(stmt[Symbol.dispose]()); + +// --- Backup --------------------------------------------------------------- + +declare const backup: Backup; +expectType( + backup.step(1, (e, completed) => { + expectType(e); + expectType(completed); + }), +); +expectType(backup.finish(() => undefined)); +expectType>(backup.step(1)); +expectType>(backup.finish()); +expectType(backup.idle); +expectType(backup.completed); +expectType(backup.failed); +expectType(backup.remaining); +expectType(backup.pageCount); +expectType(backup.retryErrors); +expectType(backup.filename); +expectType(backup.sourceName); +expectType(backup.destName); +expectType(backup.filenameIsDest); +expectType>(backup[Symbol.asyncDispose]()); + +// --- User-defined functions, aggregates, collations ------------------------ + +import type { AggregateDefinition, FunctionOptions } from '../lib/sqlite3.js'; + +expectType(db.function('regexp', (_pattern, _value) => 1)); +expectType(db.function('regexp', { deterministic: true }, (_p) => 1)); +expectType( + db.function('seventh', { varargs: true }, (..._args: unknown[]) => 1), +); +expectType( + db.aggregate('median', { + start: () => [], + step: (acc, _v) => acc, + result: (acc) => acc, + inverse: (acc, _v) => acc, + }), +); +expectType( + db.collation('german', (a, b) => a.localeCompare(b, 'de')), +); +expectType(db.removeFunction('regexp')); +expectType(db.removeCollation('german')); + +// The option and aggregate types are exported from the package root. +declare const opts: FunctionOptions; +expectType(opts.deterministic); +expectType(opts.directOnly); +expectType(opts.innocuous); +expectType(opts.varargs); +declare const spec: AggregateDefinition; +// Assignability (not invocation: the implementations are `this: undefined`). +const aggStart: (this: undefined) => unknown = spec.start; +const aggStep: (this: undefined, acc: unknown, ...args: unknown[]) => unknown = + spec.step; +const aggResult: (this: undefined, acc: unknown) => unknown = spec.result; +type InverseFn = (this: undefined, acc: unknown, ...args: unknown[]) => unknown; +const aggInverse: InverseFn | undefined = spec.inverse; +void aggStart; +void aggStep; +void aggResult; +void aggInverse; +// The aggregate definition carries the function options. +declare const asOptions: FunctionOptions; +const withFlags: AggregateDefinition = { + deterministic: true, + ...asOptions, + start: () => 0, + step: (acc) => acc, + result: (acc) => acc, +}; +void withFlags; + +// --- Errors ---------------------------------------------------------------- + +expectType(err.code); +expectType(err.primaryCode); +expectType(err.errno); +expectType(err.message); + +// --- Worker pool (Deliverable 09) ------------------------------------------- + +import type { + PoolOptions, + PoolQueryOptions, + SqlitePool, +} from '../lib/sqlite3.js'; + +// The factory resolves a fully-typed pool. +declare const poolPromise: Promise; +expectType>(sqlite3.pool('app.db')); +expectType>(sqlite3.pool('app.db', { readers: 2 })); +expectType>(poolPromise); + +// The option surface is closed over the declared keys. +declare const poolOpts: PoolOptions; +expectType(poolOpts.readers); +expectType(poolOpts.walMode); +expectType(poolOpts.busyTimeout); +expectType<'number' | 'bigint' | 'mixed' | undefined>(poolOpts.integerMode); + +// Query options carry the signal. +declare const queryOpts: PoolQueryOptions; +expectType(queryOpts.signal); + +// The query surface. +declare const p: SqlitePool; +expectType>(p.read('SELECT a')); +expectType>(p.get('SELECT a')); +expectType>( + p.write('INSERT'), +); +expectType>(p.exec('VACUUM')); +expectType>(p.transaction(async () => 42)); +expectType(p.filename); +expectType(p.readers); +expectType(p.closed); +expectType>(p.close()); +expectType>(p[Symbol.asyncDispose]());