From da912a33ced78c98181f7235de09682541eae488 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 29 May 2026 17:19:17 +0000 Subject: [PATCH 01/40] Add consumer projection persistence support --- docs/api.md | 21 +++++++++++ docs/consumers.md | 26 +++++++++++++ src/Consumer.js | 86 ++++++++++++++++++++++++++++++++++++++++++- test/Consumer.spec.js | 42 +++++++++++++++++++++ 4 files changed, 173 insertions(+), 2 deletions(-) diff --git a/docs/api.md b/docs/api.md index 7791dc1a..26c80309 100644 --- a/docs/api.md +++ b/docs/api.md @@ -260,6 +260,27 @@ Asynchronously scan all consumer state files and return their identifiers. --- +#### `consumer.createProjection(projectionFn, [options])` + +```javascript +consumer.createProjection(projectionFn [, options]) +``` + +Register a reducer-style projection as the consumer's `'data'` handler and persist it so it is auto-restored when reopening the same consumer. + +`projectionFn` can be either: + +- a reducer function: `(state, event) => state` +- an object map: `{ [eventType]: (state, event) => state }` + +Options: + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `options.hmac` | `function(string): string` | storage HMAC | HMAC function used to sign/verify serialized function projections. Required for trusted function projection restore. | + +--- + ### Events emitted | Event | Payload | Description | diff --git a/docs/consumers.md b/docs/consumers.md index 9b24b4b7..e27ea5d1 100644 --- a/docs/consumers.md +++ b/docs/consumers.md @@ -69,6 +69,32 @@ consumer.setState({ count: 0 }); consumer.setState((state) => ({ ...state, count: state.count + 1 })); ``` +## Projections + +Instead of registering `'data'` manually, you can register a projection reducer via `createProjection`. +The reducer is persisted and automatically reloaded when the same consumer is reopened. + +```javascript +import crypto from 'crypto'; + +const hmac = (code) => crypto.createHmac('sha256', 'your-private-secret').update(code).digest('hex'); + +const consumer = eventstore.getConsumer('orders', 'orders-projection', { total: 0 }); +consumer.createProjection( + (state, event) => ({ ...state, total: state.total + (event.amount || 0) }), + { hmac } +); +``` + +You can also pass per-event reducers: + +```javascript +consumer.createProjection({ + OrderCreated: (state, event) => ({ ...state, orders: [...state.orders, event] }), + OrderCancelled: (state, event) => ({ ...state, cancelled: state.cancelled + 1 }) +}, { hmac }); +``` + ## Resetting a Consumer Force the consumer to reprocess events from a given position: diff --git a/src/Consumer.js b/src/Consumer.js index 9b2e813d..faefe7b8 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -3,6 +3,7 @@ import fs from 'fs'; import path from 'path'; import { assert } from './utils/util.js'; import { ensureDirectory } from './utils/fsUtil.js'; +import { buildMetadataForMatcher, buildMatcherFromMetadata } from './utils/metadataUtil.js'; import Storage from './Storage/ReadableStorage.js'; const MAX_CATCHUP_BATCH = 10; @@ -33,7 +34,7 @@ class Consumer extends stream.Readable { * @param {object} [initialState={}] The initial state of the consumer. * @param {number} [startFrom=0] The revision to start from within the index to consume. */ - constructor(storage, indexName, identifier, initialState = {}, startFrom = 0) { + constructor(storage, indexName, identifier, initialState = {}, startFrom = 0, options = {}) { super({ objectMode: true }); assert(storage instanceof Storage, 'Must provide a storage for the consumer.'); @@ -42,6 +43,7 @@ class Consumer extends stream.Readable { this.initializeStorage(storage, indexName, identifier); this.restoreState(initialState, startFrom); + this.restoreProjection(options); this.handler = this.handleNewDocument.bind(this); this.on('error', () => (this.handleDocument = false)); } @@ -58,6 +60,7 @@ class Consumer extends stream.Readable { this.indexName = indexName; const consumerDirectory = path.join(this.storage.indexDirectory, 'consumers'); this.fileName = path.join(consumerDirectory, this.storage.storageFile + '.' + indexName + '.' + identifier); + this.projectionFileName = this.fileName + '.projection'; if (ensureDirectory(consumerDirectory)) { this.cleanUpFailedWrites(); } @@ -72,7 +75,8 @@ class Consumer extends stream.Readable { const consumerDirectory = path.dirname(this.fileName); const files = fs.readdirSync(consumerDirectory); for (let file of files) { - if (file.startsWith(consumerNamePrefix)) { + const suffix = file.slice(consumerNamePrefix.length); + if (file.startsWith(consumerNamePrefix) && /^\d+$/.test(suffix)) { safeUnlink(path.join(consumerDirectory, file)); } } @@ -106,6 +110,84 @@ class Consumer extends stream.Readable { this.consuming = false; } + /** + * Restore a persisted projection and register it as data handler. + * @private + * @param {object} [options={}] + * @param {function(string): string} [options.hmac] + */ + restoreProjection(options = {}) { + this.projection = null; + this.projectionHandler = null; + if (!this.projectionFileName || !fs.existsSync(this.projectionFileName)) { + return; + } + const projectionMetadata = JSON.parse(fs.readFileSync(this.projectionFileName, 'utf8')); + const hmac = options.hmac || this.storage.hmac; + if (typeof projectionMetadata.matcher === 'string') { + assert(typeof hmac === 'function', 'Must provide options.hmac to restore a function projection.'); + } + const projection = buildMatcherFromMetadata(projectionMetadata, hmac); + this.registerProjection(projection); + } + + /** + * Register a projection function as `data` event handler. + * @private + * @param {function|object} projection + */ + registerProjection(projection) { + if (this.projectionHandler) { + this.removeListener('data', this.projectionHandler); + } + this.projection = projection; + this.projectionHandler = (event) => { + let reducer = projection; + if (typeof projection === 'object') { + const type = event?.type || event?.payload?.type; + reducer = projection[type]; + if (typeof reducer !== 'function') { + return; + } + } + this.setState(reducer(this.state, event)); + }; + this.on('data', this.projectionHandler); + } + + /** + * Create and persist a projection reducer and register it as data handler. + * + * @api + * @param {function(object, object): object|object} projectionFn + * @param {object} [options] + * @param {function(string): string} [options.hmac] Required for function projections. + */ + createProjection(projectionFn, options = {}) { + assert((typeof projectionFn === 'function') || (projectionFn && typeof projectionFn === 'object' && !Array.isArray(projectionFn)), 'Projection must be a reducer function or an object map of reducer functions.'); + if (typeof projectionFn === 'object') { + for (const reducer of Object.values(projectionFn)) { + assert(typeof reducer === 'function', 'Projection object values must be reducer functions.'); + } + } + const hmac = options.hmac || this.storage.hmac; + if (typeof projectionFn === 'function') { + assert(typeof hmac === 'function', 'Must provide options.hmac for function projections.'); + } + + const projectionMetadata = buildMetadataForMatcher(projectionFn, hmac); + const tmpProjectionFileName = this.projectionFileName + '.tmp'; + try { + fs.writeFileSync(tmpProjectionFileName, JSON.stringify(projectionMetadata), 'utf8'); + fs.renameSync(tmpProjectionFileName, this.projectionFileName); + } catch (e) { + safeUnlink(tmpProjectionFileName); + throw e; + } + + this.registerProjection(projectionFn); + } + /** * Update the state of this consumer transactionally with the position. * May only be called from within the document handling callback. diff --git a/test/Consumer.spec.js b/test/Consumer.spec.js index a5de02b7..b3f10e6e 100644 --- a/test/Consumer.spec.js +++ b/test/Consumer.spec.js @@ -3,6 +3,7 @@ import fs from 'fs-extra'; import fsNative from 'fs'; import Storage from '../src/Storage.js'; import Consumer from '../src/Consumer.js'; +import { createHmac } from '../src/utils/metadataUtil.js'; import { fileURLToPath } from 'url'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -519,6 +520,47 @@ describe('Consumer', function() { }); }); + it('can create projections from a reducer function', function(done) { + consumer = new Consumer(storage, 'foobar', 'consumer-projection', { count: 0 }); + consumer.createProjection((state, event) => ({ ...state, count: state.count + event.id }), { hmac: createHmac('test-secret') }); + consumer.on('caught-up', () => { + expect(consumer.state.count).to.be(6); + done(); + }); + + storage.write({ type: 'Foobar', id: 1 }); + storage.write({ type: 'Foobar', id: 2 }); + storage.write({ type: 'Foobar', id: 3 }); + }); + + it('can create projections from event-type reducer maps and restore them on reopen', function(done) { + consumer = new Consumer(storage, 'foobar', 'consumer-projection-map', { count: 0 }); + consumer.createProjection({ + Foobar: (state, event) => ({ ...state, count: state.count + event.id }), + Bazinga: (state) => state + }, { hmac: createHmac('test-secret') }); + + consumer.on('caught-up', () => { + consumer.stop(); + consumer = new Consumer(storage, 'foobar', 'consumer-projection-map'); + consumer.on('caught-up', () => { + expect(consumer.state.count).to.be(10); + done(); + }); + storage.write({ type: 'Foobar', id: 4 }); + }); + + storage.write({ type: 'Foobar', id: 1 }); + storage.write({ type: 'Foobar', id: 2 }); + storage.write({ type: 'Foobar', id: 3 }); + }); + + it('throws if function projection is restored without trusted hmac', function() { + consumer = new Consumer(storage, 'foobar', 'consumer-projection-hmac'); + consumer.createProjection((state, event) => ({ ...state, lastId: event.id }), { hmac: createHmac('test-secret') }); + expect(() => new Consumer(storage, 'foobar', 'consumer-projection-hmac', {}, 0, { hmac: createHmac('wrong-secret') })).to.throwError(/Invalid HMAC/); + }); + it('can build consistency guards (aggregates)', function(done) { const guard = new Consumer(storage, 'foobar', 'unique-bar-guard'); guard.apply = function(event) { From e8095680483bc743b365b3e0a4f3ba084c0c393a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 29 May 2026 17:25:45 +0000 Subject: [PATCH 02/40] Implement durable consumer projections with HMAC validation --- docs/api.md | 2 +- src/Consumer.js | 54 +++++++++++++++++++++++++++++++++---------- test/Consumer.spec.js | 9 ++++---- 3 files changed, 48 insertions(+), 17 deletions(-) diff --git a/docs/api.md b/docs/api.md index 26c80309..9e12183d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -277,7 +277,7 @@ Options: | Parameter | Type | Default | Description | |-----------|------|---------|-------------| -| `options.hmac` | `function(string): string` | storage HMAC | HMAC function used to sign/verify serialized function projections. Required for trusted function projection restore. | +| `options.hmac` | `function(string): string` | storage HMAC | HMAC function used to sign/verify serialized projections. Required when no storage-level HMAC is configured. | --- diff --git a/src/Consumer.js b/src/Consumer.js index faefe7b8..51092eb3 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -71,12 +71,17 @@ class Consumer extends stream.Readable { * @private */ cleanUpFailedWrites() { - const consumerNamePrefix = path.basename(this.fileName) + '.'; + const consumerBaseName = path.basename(this.fileName); + const projectionBaseName = consumerBaseName + '.projection'; + const escapedConsumerBaseName = consumerBaseName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const failedStateFilePattern = new RegExp(`^${escapedConsumerBaseName}\\.\\d+$`); const consumerDirectory = path.dirname(this.fileName); const files = fs.readdirSync(consumerDirectory); for (let file of files) { - const suffix = file.slice(consumerNamePrefix.length); - if (file.startsWith(consumerNamePrefix) && /^\d+$/.test(suffix)) { + if (file === projectionBaseName) { + continue; + } + if (file === projectionBaseName + '.tmp' || failedStateFilePattern.test(file)) { safeUnlink(path.join(consumerDirectory, file)); } } @@ -124,13 +129,33 @@ class Consumer extends stream.Readable { } const projectionMetadata = JSON.parse(fs.readFileSync(this.projectionFileName, 'utf8')); const hmac = options.hmac || this.storage.hmac; - if (typeof projectionMetadata.matcher === 'string') { - assert(typeof hmac === 'function', 'Must provide options.hmac to restore a function projection.'); - } - const projection = buildMatcherFromMetadata(projectionMetadata, hmac); + const projection = this.deserializeProjectionMetadata(projectionMetadata, hmac); this.registerProjection(projection); } + /** + * @private + * @param {{kind?: string, projection?: object, matcher?: string|object, hmac?: string}} metadata + * @param {function(string): string} hmac + * @returns {function|object} + */ + deserializeProjectionMetadata(metadata, hmac) { + assert(metadata && typeof metadata === 'object', 'Invalid projection metadata.'); + if (metadata.kind === 'map') { + assert(typeof hmac === 'function', 'Must provide options.hmac to restore mapped function projections.'); + const projectionMap = {}; + for (const [eventType, reducerMetadata] of Object.entries(metadata.projection || {})) { + projectionMap[eventType] = buildMatcherFromMetadata(reducerMetadata, hmac); + } + return projectionMap; + } + assert(metadata.kind === 'function', 'Invalid projection metadata kind.'); + if (typeof metadata.projection?.matcher === 'string') { + assert(typeof hmac === 'function', 'Must provide options.hmac to restore function projections.'); + } + return buildMatcherFromMetadata(metadata.projection, hmac); + } + /** * Register a projection function as `data` event handler. * @private @@ -144,6 +169,7 @@ class Consumer extends stream.Readable { this.projectionHandler = (event) => { let reducer = projection; if (typeof projection === 'object') { + // Direct Storage consumers emit `{ type }`, EventStore consumers emit `{ payload: { type } }`. const type = event?.type || event?.payload?.type; reducer = projection[type]; if (typeof reducer !== 'function') { @@ -171,11 +197,15 @@ class Consumer extends stream.Readable { } } const hmac = options.hmac || this.storage.hmac; - if (typeof projectionFn === 'function') { - assert(typeof hmac === 'function', 'Must provide options.hmac for function projections.'); - } - - const projectionMetadata = buildMetadataForMatcher(projectionFn, hmac); + assert(typeof hmac === 'function', 'Must provide options.hmac for projections.'); + const projectionMetadata = (typeof projectionFn === 'function') + ? { kind: 'function', projection: buildMetadataForMatcher(projectionFn, hmac) } + : { + kind: 'map', + projection: Object.fromEntries( + Object.entries(projectionFn).map(([eventType, reducer]) => [eventType, buildMetadataForMatcher(reducer, hmac)]) + ) + }; const tmpProjectionFileName = this.projectionFileName + '.tmp'; try { fs.writeFileSync(tmpProjectionFileName, JSON.stringify(projectionMetadata), 'utf8'); diff --git a/test/Consumer.spec.js b/test/Consumer.spec.js index b3f10e6e..db669117 100644 --- a/test/Consumer.spec.js +++ b/test/Consumer.spec.js @@ -542,10 +542,11 @@ describe('Consumer', function() { consumer.on('caught-up', () => { consumer.stop(); - consumer = new Consumer(storage, 'foobar', 'consumer-projection-map'); - consumer.on('caught-up', () => { - expect(consumer.state.count).to.be(10); - done(); + consumer = new Consumer(storage, 'foobar', 'consumer-projection-map', {}, 0, { hmac: createHmac('test-secret') }); + consumer.on('progress', () => { + if (consumer.state.count === 10) { + done(); + } }); storage.write({ type: 'Foobar', id: 4 }); }); From 2e6f8498752454fc47cf64ecb1608ac765eff617 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 30 May 2026 20:01:39 +0000 Subject: [PATCH 03/40] Refactor projections into Projection class and integrate Consumer/EventStore APIs --- docs/api.md | 40 ++++++ docs/consumers.md | 30 +++-- index.js | 1 + src/Consumer.js | 99 +++++--------- src/EventStore.js | 35 ++++- src/Projection.js | 276 ++++++++++++++++++++++++++++++++++++++++ test/Consumer.spec.js | 43 +++++++ test/EventStore.spec.js | 71 +++++++++++ 8 files changed, 520 insertions(+), 75 deletions(-) create mode 100644 src/Projection.js diff --git a/docs/api.md b/docs/api.md index 9e12183d..4db01f3d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -281,6 +281,46 @@ Options: --- +#### `eventstore.getProjection(name, [definition], [options])` + +```javascript +eventstore.getProjection(name [, definition [, options]]) → Projection +``` + +Create a `Projection` with EventStore defaults (`typeAccessor`, storage HMAC), or restore a previously persisted one when `definition` is omitted. + +`definition` shape: + +```javascript +{ + initialState: any, + handlers: (state, event) => state | { [eventType]: (state, event) => state }, + matcher: object|function // optional +} +``` + +--- + +#### `consumer.project(projection, [options])` + +```javascript +consumer.project(projection [, options]) +``` + +Attach a `Projection` instance to a durable consumer and persist its definition for automatic restore when reopening the same consumer. + +--- + +#### `projection.subscribe(consumer, [options])` + +```javascript +projection.subscribe(consumer [, options]) +``` + +Alias for `consumer.project(projection, options)`. + +--- + ### Events emitted | Event | Payload | Description | diff --git a/docs/consumers.md b/docs/consumers.md index e27ea5d1..a6ced869 100644 --- a/docs/consumers.md +++ b/docs/consumers.md @@ -71,22 +71,25 @@ consumer.setState((state) => ({ ...state, count: state.count + 1 })); ## Projections -Instead of registering `'data'` manually, you can register a projection reducer via `createProjection`. -The reducer is persisted and automatically reloaded when the same consumer is reopened. +Use a `Projection` to define *how* events are projected into state, then connect it to a `Consumer` for durable continuous updates. ```javascript import crypto from 'crypto'; const hmac = (code) => crypto.createHmac('sha256', 'your-private-secret').update(code).digest('hex'); -const consumer = eventstore.getConsumer('orders', 'orders-projection', { total: 0 }); -consumer.createProjection( - (state, event) => ({ ...state, total: state.total + (event.amount || 0) }), - { hmac } -); +const projection = eventstore.getProjection('orders-total', { + initialState: { total: 0 }, + handlers: { + OrderCreated: (state, event) => ({ ...state, total: state.total + (event.payload.amount || 0) }) + } +}, { hmac }); + +const consumer = eventstore.getConsumer('orders', 'orders-projection', projection.initialState); +consumer.project(projection); ``` -You can also pass per-event reducers: +You can still use `consumer.createProjection(...)` for the shorter in-place API: ```javascript consumer.createProjection({ @@ -95,6 +98,17 @@ consumer.createProjection({ }, { hmac }); ``` +Projections are composable via `CompositeProjection`: + +```javascript +import { CompositeProjection } from 'event-storage'; + +const overview = new CompositeProjection('overview', { + count: { initialState: 0, handlers: { OrderCreated: (state) => state + 1 } }, + last: { initialState: null, handlers: { OrderCreated: (state, event) => event.payload } } +}); +``` + ## Resetting a Consumer Force the consumer to reprocess events from a given position: diff --git a/index.js b/index.js index 4cfbb2de..0253e0ab 100644 --- a/index.js +++ b/index.js @@ -3,3 +3,4 @@ export { default as EventStream } from './src/EventStream.js'; export { default as Storage, StorageLockedError } from './src/Storage.js'; export { default as Index } from './src/Index.js'; export { default as Consumer } from './src/Consumer.js'; +export { default as Projection, CompositeProjection } from './src/Projection.js'; diff --git a/src/Consumer.js b/src/Consumer.js index 51092eb3..948f0bad 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -3,7 +3,7 @@ import fs from 'fs'; import path from 'path'; import { assert } from './utils/util.js'; import { ensureDirectory } from './utils/fsUtil.js'; -import { buildMetadataForMatcher, buildMatcherFromMetadata } from './utils/metadataUtil.js'; +import Projection from './Projection.js'; import Storage from './Storage/ReadableStorage.js'; const MAX_CATCHUP_BATCH = 10; @@ -58,6 +58,7 @@ class Consumer extends stream.Readable { this.storage = storage; this.index = this.storage.openIndex(indexName); this.indexName = indexName; + this.identifier = identifier; const consumerDirectory = path.join(this.storage.indexDirectory, 'consumers'); this.fileName = path.join(consumerDirectory, this.storage.storageFile + '.' + indexName + '.' + identifier); this.projectionFileName = this.fileName + '.projection'; @@ -127,58 +128,39 @@ class Consumer extends stream.Readable { if (!this.projectionFileName || !fs.existsSync(this.projectionFileName)) { return; } - const projectionMetadata = JSON.parse(fs.readFileSync(this.projectionFileName, 'utf8')); - const hmac = options.hmac || this.storage.hmac; - const projection = this.deserializeProjectionMetadata(projectionMetadata, hmac); - this.registerProjection(projection); - } - - /** - * @private - * @param {{kind?: string, projection?: object, matcher?: string|object, hmac?: string}} metadata - * @param {function(string): string} hmac - * @returns {function|object} - */ - deserializeProjectionMetadata(metadata, hmac) { - assert(metadata && typeof metadata === 'object', 'Invalid projection metadata.'); - if (metadata.kind === 'map') { - assert(typeof hmac === 'function', 'Must provide options.hmac to restore mapped function projections.'); - const projectionMap = {}; - for (const [eventType, reducerMetadata] of Object.entries(metadata.projection || {})) { - projectionMap[eventType] = buildMatcherFromMetadata(reducerMetadata, hmac); - } - return projectionMap; - } - assert(metadata.kind === 'function', 'Invalid projection metadata kind.'); - if (typeof metadata.projection?.matcher === 'string') { - assert(typeof hmac === 'function', 'Must provide options.hmac to restore function projections.'); - } - return buildMatcherFromMetadata(metadata.projection, hmac); + const projection = Projection.restoreFromFile(this.projectionFileName, { + hmac: options.hmac || this.storage.hmac, + typeAccessor: options.typeAccessor + }); + this.project(projection, { persist: false }); } /** * Register a projection function as `data` event handler. * @private - * @param {function|object} projection + * @param {Projection} projection + * @param {object} [options] + * @param {boolean} [options.persist=true] + * @param {function(string): string} [options.hmac] */ - registerProjection(projection) { + project(projection, options = {}) { + assert(projection instanceof Projection, 'Projection must be an instance of Projection.'); + const { persist = true, hmac } = options; if (this.projectionHandler) { this.removeListener('data', this.projectionHandler); } this.projection = projection; this.projectionHandler = (event) => { - let reducer = projection; - if (typeof projection === 'object') { - // Direct Storage consumers emit `{ type }`, EventStore consumers emit `{ payload: { type } }`. - const type = event?.type || event?.payload?.type; - reducer = projection[type]; - if (typeof reducer !== 'function') { - return; - } - } - this.setState(reducer(this.state, event)); + this.setState(projection.apply(this.state, event)); }; this.on('data', this.projectionHandler); + if (persist) { + projection.persist({ + fileName: this.projectionFileName, + hmac: hmac || projection.hmac || this.storage.hmac + }); + } + return projection; } /** @@ -190,32 +172,17 @@ class Consumer extends stream.Readable { * @param {function(string): string} [options.hmac] Required for function projections. */ createProjection(projectionFn, options = {}) { - assert((typeof projectionFn === 'function') || (projectionFn && typeof projectionFn === 'object' && !Array.isArray(projectionFn)), 'Projection must be a reducer function or an object map of reducer functions.'); - if (typeof projectionFn === 'object') { - for (const reducer of Object.values(projectionFn)) { - assert(typeof reducer === 'function', 'Projection object values must be reducer functions.'); - } - } - const hmac = options.hmac || this.storage.hmac; - assert(typeof hmac === 'function', 'Must provide options.hmac for projections.'); - const projectionMetadata = (typeof projectionFn === 'function') - ? { kind: 'function', projection: buildMetadataForMatcher(projectionFn, hmac) } - : { - kind: 'map', - projection: Object.fromEntries( - Object.entries(projectionFn).map(([eventType, reducer]) => [eventType, buildMetadataForMatcher(reducer, hmac)]) - ) - }; - const tmpProjectionFileName = this.projectionFileName + '.tmp'; - try { - fs.writeFileSync(tmpProjectionFileName, JSON.stringify(projectionMetadata), 'utf8'); - fs.renameSync(tmpProjectionFileName, this.projectionFileName); - } catch (e) { - safeUnlink(tmpProjectionFileName); - throw e; - } - - this.registerProjection(projectionFn); + const projection = projectionFn instanceof Projection + ? projectionFn + : new Projection(options.name || this.identifier, { + initialState: this.state, + matcher: options.matcher, + handlers: projectionFn + }, { + hmac: options.hmac || this.storage.hmac, + typeAccessor: options.typeAccessor + }); + return this.project(projection, options); } /** diff --git a/src/EventStore.js b/src/EventStore.js index 80ae74ec..035feb25 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -6,6 +6,7 @@ import events from 'events'; import Storage, { ReadOnly as ReadOnlyStorage, LOCK_THROW, LOCK_RECLAIM } from './Storage.js'; import Index from './Index.js'; import Consumer from './Consumer.js'; +import Projection from './Projection.js'; import { assert, getPropertyAtPath } from './utils/util.js'; import { ensureDirectory, scanForFiles } from './utils/fsUtil.js'; import { buildTypeMatcherFn } from './utils/metadataUtil.js'; @@ -784,12 +785,44 @@ class EventStore extends events.EventEmitter { if (this.consumers.has(identifier)) { return this.consumers.get(identifier); } - const consumer = new Consumer(this.storage, streamName === '_all' ? '_all' : 'stream-' + streamName, identifier, initialState, since); + const projectionTypeAccessor = this.typeAccessor + ? (event) => this.typeAccessor(event?.payload || event) + : undefined; + const consumer = new Consumer(this.storage, streamName === '_all' ? '_all' : 'stream-' + streamName, identifier, initialState, since, { + hmac: this.storage.hmac, + typeAccessor: projectionTypeAccessor + }); consumer.streamName = streamName; this.consumers.set(identifier, consumer); return consumer; } + /** + * Get or create a projection with EventStore defaults. + * + * @param {string} name Projection name. + * @param {object} [definition] Projection definition. + * @param {object} [options] Projection options. + * @returns {Projection} + */ + getProjection(name, definition, options = {}) { + assert(typeof name === 'string' && name !== '', 'Must provide a projection name.'); + const projectionTypeAccessor = this.typeAccessor + ? (event) => this.typeAccessor(event?.payload || event) + : options.typeAccessor; + const projectionFileName = path.join(this.storage.indexDirectory, 'projections', this.storage.storageFile + '.' + name + '.projection'); + const projectionOptions = { + ...options, + fileName: options.fileName || projectionFileName, + hmac: options.hmac || this.storage.hmac, + typeAccessor: projectionTypeAccessor + }; + if (definition) { + return new Projection(name, definition, projectionOptions); + } + return Projection.restore(name, projectionOptions); + } + /** * Scan the existing consumers on this EventStore and asynchronously invoke a callback with the parsed list. * diff --git a/src/Projection.js b/src/Projection.js new file mode 100644 index 00000000..7b580b65 --- /dev/null +++ b/src/Projection.js @@ -0,0 +1,276 @@ +import fs from 'fs'; +import path from 'path'; +import { assert } from './utils/util.js'; +import { ensureDirectory } from './utils/fsUtil.js'; +import { buildMatcherFromMetadata, buildMetadataForMatcher, matches } from './utils/metadataUtil.js'; + +const DEFAULT_TYPE_ACCESSOR = (event) => event?.type || event?.payload?.type; + +const safeUnlink = (filename) => { + try { + fs.unlinkSync(filename); + } catch (e) { + if (e.code !== "ENOENT") { + throw e; + } + } +}; + +class Projection { + + constructor(name, definition = {}, options = {}) { + assert(typeof name === 'string' && name !== '', 'Projection must have a name.'); + const { initialState = {}, handlers, matcher } = definition; + assert((typeof handlers === 'function') || (handlers && typeof handlers === 'object' && !Array.isArray(handlers)), 'Projection handlers must be a function or an object map of functions.'); + if (typeof handlers === 'object') { + for (const reducer of Object.values(handlers)) { + assert(typeof reducer === 'function', 'Projection handler maps must contain reducer functions.'); + } + } + this.name = name; + this.initialState = Object.freeze(initialState); + this.handlers = handlers; + this.matcher = matcher; + this.hmac = options.hmac || null; + this.typeAccessor = options.typeAccessor || DEFAULT_TYPE_ACCESSOR; + this.fileName = options.fileName || null; + this.state = this.initialState; + } + + get types() { + if (typeof this.handlers === 'function') { + return []; + } + return Object.keys(this.handlers); + } + + apply(state, event) { + if (!this.matches(event)) { + this.state = state; + return state; + } + let reducer = this.handlers; + if (typeof this.handlers === 'object') { + reducer = this.handlers[this.typeAccessor(event)]; + if (typeof reducer !== 'function') { + this.state = state; + return state; + } + } + const nextState = reducer(state, event); + this.state = nextState; + return nextState; + } + + handle(stream) { + this.reset(); + for (const event of stream) { + this.state = this.apply(this.state, event); + } + return this.state; + } + + reset() { + this.state = this.initialState; + return this.state; + } + + matches(event) { + if (!this.matcher) { + return true; + } + if (typeof this.matcher === 'function') { + return this.matcher(event); + } + return matches(event, this.matcher); + } + + subscribe(consumer, options = {}) { + assert(consumer && typeof consumer.project === 'function', 'Projection.subscribe expects a Consumer instance.'); + consumer.project(this, options); + return this; + } + + persist(options = {}) { + const hmac = options.hmac || this.hmac; + const fileName = options.fileName || this.fileName || `${this.name}.projection`; + const metadata = this.toMetadata(hmac); + const tmpFile = fileName + '.tmp'; + ensureDirectory(path.dirname(fileName)); + try { + fs.writeFileSync(tmpFile, JSON.stringify(metadata), 'utf8'); + fs.renameSync(tmpFile, fileName); + } catch (e) { + safeUnlink(tmpFile); + throw e; + } + this.fileName = fileName; + this.hmac = hmac; + return fileName; + } + + toMetadata(hmac = this.hmac) { + const serializeFn = (fn) => { + assert(typeof hmac === 'function', 'Must provide options.hmac for function projections.'); + return buildMetadataForMatcher(fn, hmac); + }; + const matcherMetadata = this.matcher ? ( + typeof this.matcher === 'function' ? serializeFn(this.matcher) : buildMetadataForMatcher(this.matcher, hmac) + ) : null; + const handlersMetadata = (typeof this.handlers === 'function') + ? serializeFn(this.handlers) + : Object.fromEntries( + Object.entries(this.handlers).map(([eventType, reducer]) => [eventType, serializeFn(reducer)]) + ); + return { + kind: 'projection', + name: this.name, + initialState: this.initialState, + matcher: matcherMetadata, + handlersKind: typeof this.handlers === 'function' ? 'function' : 'map', + handlers: handlersMetadata + }; + } + + static restore(name, options = {}) { + assert(typeof name === 'string' && name !== '', 'Projection.restore requires a projection name.'); + const fileName = options.fileName || `${name}.projection`; + return Projection.restoreFromFile(fileName, options); + } + + static restoreFromFile(fileName, options = {}) { + assert(fs.existsSync(fileName), `Projection file does not exist: ${fileName}`); + const metadata = JSON.parse(fs.readFileSync(fileName, 'utf8')); + return Projection.fromMetadata(metadata, { ...options, fileName }); + } + + static fromMetadata(metadata, options = {}) { + assert(metadata && typeof metadata === 'object', 'Invalid projection metadata.'); + if (metadata.kind === 'composite-projection') { + return CompositeProjection.fromMetadata(metadata, options); + } + assert(metadata.kind === 'projection', 'Invalid projection metadata kind.'); + const hmac = options.hmac; + const deserialize = (matcherMetadata) => { + if (!matcherMetadata) { + return undefined; + } + if (typeof matcherMetadata.matcher === 'string') { + assert(typeof hmac === 'function', 'Must provide options.hmac to restore function projections.'); + } + return buildMatcherFromMetadata(matcherMetadata, hmac); + }; + const handlers = metadata.handlersKind === 'function' + ? deserialize(metadata.handlers) + : Object.fromEntries( + Object.entries(metadata.handlers || {}).map(([eventType, reducerMetadata]) => [eventType, deserialize(reducerMetadata)]) + ); + const projection = new Projection(metadata.name, { + initialState: metadata.initialState, + matcher: deserialize(metadata.matcher), + handlers + }, { + ...options, + fileName: options.fileName || null + }); + projection.reset(); + return projection; + } + + static compose(name, projections, options = {}) { + return new CompositeProjection(name, projections, options); + } +} + +class CompositeProjection extends Projection { + + constructor(name, projections, options = {}) { + assert(projections && typeof projections === 'object' && !Array.isArray(projections), 'CompositeProjection requires an object map of projections.'); + const normalized = {}; + for (const [projectionName, projection] of Object.entries(projections)) { + normalized[projectionName] = projection instanceof Projection + ? projection + : new Projection(projectionName, projection, options); + } + super(name, { + initialState: Object.fromEntries( + Object.entries(normalized).map(([projectionName, projection]) => [projectionName, projection.initialState]) + ), + handlers: (state) => state, + matcher: options.matcher + }, options); + this.projections = normalized; + this.reset(); + } + + get types() { + const types = new Set(); + for (const projection of Object.values(this.projections)) { + for (const type of projection.types) { + types.add(type); + } + } + return [...types]; + } + + apply(state, event) { + if (!this.matches(event)) { + this.state = state; + return state; + } + const currentState = state || this.initialState; + const nextState = {}; + for (const [name, projection] of Object.entries(this.projections)) { + nextState[name] = projection.apply(currentState[name], event); + } + this.state = nextState; + return nextState; + } + + reset() { + for (const projection of Object.values(this.projections)) { + projection.reset(); + } + this.state = Object.fromEntries( + Object.entries(this.projections).map(([projectionName, projection]) => [projectionName, projection.state]) + ); + return this.state; + } + + toMetadata(hmac = this.hmac) { + return { + kind: 'composite-projection', + name: this.name, + matcher: this.matcher ? buildMetadataForMatcher(this.matcher, hmac) : null, + projections: Object.fromEntries( + Object.entries(this.projections).map(([name, projection]) => [name, projection.toMetadata(hmac)]) + ) + }; + } + + static fromMetadata(metadata, options = {}) { + const hmac = options.hmac; + const deserializeMatcher = (matcherMetadata) => { + if (!matcherMetadata) { + return undefined; + } + if (typeof matcherMetadata.matcher === 'string') { + assert(typeof hmac === 'function', 'Must provide options.hmac to restore function projections.'); + } + return buildMatcherFromMetadata(matcherMetadata, hmac); + }; + const projections = Object.fromEntries( + Object.entries(metadata.projections || {}).map(([name, projectionMetadata]) => [ + name, + Projection.fromMetadata(projectionMetadata, options) + ]) + ); + return new CompositeProjection(metadata.name, projections, { + ...options, + matcher: deserializeMatcher(metadata.matcher) + }); + } +} + +export default Projection; +export { CompositeProjection }; diff --git a/test/Consumer.spec.js b/test/Consumer.spec.js index db669117..47cf53b0 100644 --- a/test/Consumer.spec.js +++ b/test/Consumer.spec.js @@ -3,6 +3,7 @@ import fs from 'fs-extra'; import fsNative from 'fs'; import Storage from '../src/Storage.js'; import Consumer from '../src/Consumer.js'; +import Projection, { CompositeProjection } from '../src/Projection.js'; import { createHmac } from '../src/utils/metadataUtil.js'; import { fileURLToPath } from 'url'; @@ -562,6 +563,48 @@ describe('Consumer', function() { expect(() => new Consumer(storage, 'foobar', 'consumer-projection-hmac', {}, 0, { hmac: createHmac('wrong-secret') })).to.throwError(/Invalid HMAC/); }); + it('can attach a projection instance and restore it on reopen', function(done) { + consumer = new Consumer(storage, 'foobar', 'consumer-projection-instance', { count: 0 }); + const projection = new Projection('consumer-projection-instance', { + initialState: { count: 0 }, + handlers: { + Foobar: (state, event) => ({ ...state, count: state.count + event.id }) + } + }, { + hmac: createHmac('test-secret') + }); + consumer.project(projection); + consumer.on('caught-up', () => { + expect(consumer.state.count).to.be(6); + consumer.stop(); + consumer = new Consumer(storage, 'foobar', 'consumer-projection-instance', {}, 0, { hmac: createHmac('test-secret') }); + consumer.on('progress', () => { + if (consumer.state.count === 10) { + done(); + } + }); + storage.write({ type: 'Foobar', id: 4 }); + }); + storage.write({ type: 'Foobar', id: 1 }); + storage.write({ type: 'Foobar', id: 2 }); + storage.write({ type: 'Foobar', id: 3 }); + }); + + it('supports composite projections', function() { + const projection = new CompositeProjection('overview', { + count: { + initialState: 0, + handlers: { Foobar: (state) => state + 1 } + }, + last: { + initialState: null, + handlers: { Foobar: (state, event) => event.id || state } + } + }); + projection.handle([{ type: 'Foobar', id: 1 }, { type: 'Foobar', id: 2 }]); + expect(projection.state).to.eql({ count: 2, last: 2 }); + }); + it('can build consistency guards (aggregates)', function(done) { const guard = new Consumer(storage, 'foobar', 'unique-bar-guard'); guard.apply = function(event) { diff --git a/test/EventStore.spec.js b/test/EventStore.spec.js index 8e56f02d..5ef7e282 100644 --- a/test/EventStore.spec.js +++ b/test/EventStore.spec.js @@ -4,6 +4,8 @@ import fsSync from 'fs'; import path from 'path'; import EventStore, { ExpectedVersion, OptimisticConcurrencyError, CommitCondition, LOCK_RECLAIM } from '../src/EventStore.js'; import Consumer from '../src/Consumer.js'; +import Projection from '../src/Projection.js'; +import { createHmac } from '../src/utils/metadataUtil.js'; import { fileURLToPath } from 'url'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -1377,6 +1379,75 @@ describe('EventStore', function() { eventstore.commit('foo', { foo: 'bar', id: 1 }); eventstore.commit('bar', { foo: 'baz', id: 2 }); }); + + it('restores a projected consumer with eventStore typeAccessor defaults', function(done) { + eventstore = new EventStore({ + storageDirectory, + typeAccessor: 'type', + storageConfig: { + hmac: createHmac('test-secret') + } + }); + eventstore.createEventStream('user-stream', (event) => event.stream === 'user-stream'); + + const consumer = eventstore.getConsumer('user-stream', 'user-counter', { count: 0 }); + consumer.project(new Projection('user-counter', { + initialState: { count: 0 }, + handlers: { + UserCreated: (state) => ({ ...state, count: state.count + 1 }) + } + })); + eventstore.commit('user-stream', [{ type: 'UserCreated', id: 1 }]); + eventstore.commit('user-stream', [{ type: 'UserCreated', id: 2 }]); + + consumer.on('progress', () => { + if (consumer.state.count !== 2) { + return; + } + eventstore.close(); + eventstore = new EventStore({ + storageDirectory, + typeAccessor: 'type', + storageConfig: { + hmac: createHmac('test-secret') + } + }); + const reopened = eventstore.getConsumer('user-stream', 'user-counter', { count: 0 }); + reopened.on('progress', () => { + if (reopened.state.count === 3) { + done(); + } + }); + eventstore.commit('user-stream', [{ type: 'UserCreated', id: 3 }]); + }); + }); + }); + + describe('getProjection', function() { + + it('creates and restores persisted projections with EventStore defaults', function() { + eventstore = new EventStore({ + storageDirectory, + typeAccessor: 'type', + storageConfig: { + hmac: createHmac('test-secret') + } + }); + const projection = eventstore.getProjection('user-count', { + initialState: 0, + handlers: { + UserCreated: (state) => state + 1 + } + }); + projection.persist(); + + const restored = eventstore.getProjection('user-count'); + const state = restored.handle([ + { payload: { type: 'UserCreated' } }, + { payload: { type: 'UserCreated' } } + ]); + expect(state).to.be(2); + }); }); describe('scanConsumers', function() { From 2f46addf2ee5c9a25ec8efad991f23d844b37c6b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 31 May 2026 11:06:30 +0000 Subject: [PATCH 04/40] Address projection API review feedback --- docs/api.md | 37 +++++---------------- docs/consumers.md | 17 ++-------- src/Consumer.js | 74 +++++------------------------------------ src/EventStore.js | 25 ++++++++------ src/Projection.js | 12 +++++-- test/Consumer.spec.js | 44 ++++++++++++++++++------ test/EventStore.spec.js | 6 ++-- 7 files changed, 80 insertions(+), 135 deletions(-) diff --git a/docs/api.md b/docs/api.md index 4db01f3d..fa6ac740 100644 --- a/docs/api.md +++ b/docs/api.md @@ -260,31 +260,10 @@ Asynchronously scan all consumer state files and return their identifiers. --- -#### `consumer.createProjection(projectionFn, [options])` +#### `eventstore.getProjection(name, [definition])` ```javascript -consumer.createProjection(projectionFn [, options]) -``` - -Register a reducer-style projection as the consumer's `'data'` handler and persist it so it is auto-restored when reopening the same consumer. - -`projectionFn` can be either: - -- a reducer function: `(state, event) => state` -- an object map: `{ [eventType]: (state, event) => state }` - -Options: - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `options.hmac` | `function(string): string` | storage HMAC | HMAC function used to sign/verify serialized projections. Required when no storage-level HMAC is configured. | - ---- - -#### `eventstore.getProjection(name, [definition], [options])` - -```javascript -eventstore.getProjection(name [, definition [, options]]) → Projection +eventstore.getProjection(name [, definition]) → Projection ``` Create a `Projection` with EventStore defaults (`typeAccessor`, storage HMAC), or restore a previously persisted one when `definition` is omitted. @@ -301,23 +280,23 @@ Create a `Projection` with EventStore defaults (`typeAccessor`, storage HMAC), o --- -#### `consumer.project(projection, [options])` +#### `consumer.project(projection)` ```javascript -consumer.project(projection [, options]) +consumer.project(projection) ``` -Attach a `Projection` instance to a durable consumer and persist its definition for automatic restore when reopening the same consumer. +Attach a projection-like object (`apply(state, event)`) as the consumer `'data'` handler. --- -#### `projection.subscribe(consumer, [options])` +#### `projection.subscribe(consumer)` ```javascript -projection.subscribe(consumer [, options]) +projection.subscribe(consumer) ``` -Alias for `consumer.project(projection, options)`. +Attach this projection to the consumer and persist its definition next to the consumer state file so `eventstore.getConsumer(...)` can restore and reconnect it automatically. --- diff --git a/docs/consumers.md b/docs/consumers.md index a6ced869..9e34348c 100644 --- a/docs/consumers.md +++ b/docs/consumers.md @@ -74,28 +74,15 @@ consumer.setState((state) => ({ ...state, count: state.count + 1 })); Use a `Projection` to define *how* events are projected into state, then connect it to a `Consumer` for durable continuous updates. ```javascript -import crypto from 'crypto'; - -const hmac = (code) => crypto.createHmac('sha256', 'your-private-secret').update(code).digest('hex'); - const projection = eventstore.getProjection('orders-total', { initialState: { total: 0 }, handlers: { OrderCreated: (state, event) => ({ ...state, total: state.total + (event.payload.amount || 0) }) } -}, { hmac }); +}); const consumer = eventstore.getConsumer('orders', 'orders-projection', projection.initialState); -consumer.project(projection); -``` - -You can still use `consumer.createProjection(...)` for the shorter in-place API: - -```javascript -consumer.createProjection({ - OrderCreated: (state, event) => ({ ...state, orders: [...state.orders, event] }), - OrderCancelled: (state, event) => ({ ...state, cancelled: state.cancelled + 1 }) -}, { hmac }); +projection.subscribe(consumer); ``` Projections are composable via `CompositeProjection`: diff --git a/src/Consumer.js b/src/Consumer.js index 948f0bad..2609651e 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -3,7 +3,6 @@ import fs from 'fs'; import path from 'path'; import { assert } from './utils/util.js'; import { ensureDirectory } from './utils/fsUtil.js'; -import Projection from './Projection.js'; import Storage from './Storage/ReadableStorage.js'; const MAX_CATCHUP_BATCH = 10; @@ -34,7 +33,7 @@ class Consumer extends stream.Readable { * @param {object} [initialState={}] The initial state of the consumer. * @param {number} [startFrom=0] The revision to start from within the index to consume. */ - constructor(storage, indexName, identifier, initialState = {}, startFrom = 0, options = {}) { + constructor(storage, indexName, identifier, initialState = {}, startFrom = 0) { super({ objectMode: true }); assert(storage instanceof Storage, 'Must provide a storage for the consumer.'); @@ -43,7 +42,6 @@ class Consumer extends stream.Readable { this.initializeStorage(storage, indexName, identifier); this.restoreState(initialState, startFrom); - this.restoreProjection(options); this.handler = this.handleNewDocument.bind(this); this.on('error', () => (this.handleDocument = false)); } @@ -61,7 +59,6 @@ class Consumer extends stream.Readable { this.identifier = identifier; const consumerDirectory = path.join(this.storage.indexDirectory, 'consumers'); this.fileName = path.join(consumerDirectory, this.storage.storageFile + '.' + indexName + '.' + identifier); - this.projectionFileName = this.fileName + '.projection'; if (ensureDirectory(consumerDirectory)) { this.cleanUpFailedWrites(); } @@ -73,16 +70,12 @@ class Consumer extends stream.Readable { */ cleanUpFailedWrites() { const consumerBaseName = path.basename(this.fileName); - const projectionBaseName = consumerBaseName + '.projection'; const escapedConsumerBaseName = consumerBaseName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); const failedStateFilePattern = new RegExp(`^${escapedConsumerBaseName}\\.\\d+$`); const consumerDirectory = path.dirname(this.fileName); const files = fs.readdirSync(consumerDirectory); for (let file of files) { - if (file === projectionBaseName) { - continue; - } - if (file === projectionBaseName + '.tmp' || failedStateFilePattern.test(file)) { + if (failedStateFilePattern.test(file)) { safeUnlink(path.join(consumerDirectory, file)); } } @@ -117,35 +110,12 @@ class Consumer extends stream.Readable { } /** - * Restore a persisted projection and register it as data handler. - * @private - * @param {object} [options={}] - * @param {function(string): string} [options.hmac] - */ - restoreProjection(options = {}) { - this.projection = null; - this.projectionHandler = null; - if (!this.projectionFileName || !fs.existsSync(this.projectionFileName)) { - return; - } - const projection = Projection.restoreFromFile(this.projectionFileName, { - hmac: options.hmac || this.storage.hmac, - typeAccessor: options.typeAccessor - }); - this.project(projection, { persist: false }); - } - - /** - * Register a projection function as `data` event handler. - * @private - * @param {Projection} projection - * @param {object} [options] - * @param {boolean} [options.persist=true] - * @param {function(string): string} [options.hmac] + * Register a projection as `data` event handler. + * @api + * @param {{ apply: function(object, object): object }} projection */ - project(projection, options = {}) { - assert(projection instanceof Projection, 'Projection must be an instance of Projection.'); - const { persist = true, hmac } = options; + project(projection) { + assert(projection && typeof projection.apply === 'function', 'Projection must implement apply(state, event).'); if (this.projectionHandler) { this.removeListener('data', this.projectionHandler); } @@ -154,35 +124,7 @@ class Consumer extends stream.Readable { this.setState(projection.apply(this.state, event)); }; this.on('data', this.projectionHandler); - if (persist) { - projection.persist({ - fileName: this.projectionFileName, - hmac: hmac || projection.hmac || this.storage.hmac - }); - } - return projection; - } - - /** - * Create and persist a projection reducer and register it as data handler. - * - * @api - * @param {function(object, object): object|object} projectionFn - * @param {object} [options] - * @param {function(string): string} [options.hmac] Required for function projections. - */ - createProjection(projectionFn, options = {}) { - const projection = projectionFn instanceof Projection - ? projectionFn - : new Projection(options.name || this.identifier, { - initialState: this.state, - matcher: options.matcher, - handlers: projectionFn - }, { - hmac: options.hmac || this.storage.hmac, - typeAccessor: options.typeAccessor - }); - return this.project(projection, options); + return this; } /** diff --git a/src/EventStore.js b/src/EventStore.js index 035feb25..e82d4212 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -121,6 +121,9 @@ class EventStore extends events.EventEmitter { } this.initialize(storeName, storageConfig); + this.projectionHmac = typeof storageConfig.hmac === 'function' + ? storageConfig.hmac + : this.storage.hmac; } /** @@ -788,10 +791,14 @@ class EventStore extends events.EventEmitter { const projectionTypeAccessor = this.typeAccessor ? (event) => this.typeAccessor(event?.payload || event) : undefined; - const consumer = new Consumer(this.storage, streamName === '_all' ? '_all' : 'stream-' + streamName, identifier, initialState, since, { - hmac: this.storage.hmac, - typeAccessor: projectionTypeAccessor - }); + const consumer = new Consumer(this.storage, streamName === '_all' ? '_all' : 'stream-' + streamName, identifier, initialState, since); + const consumerProjectionFileName = `${consumer.fileName}.projection`; + if (fs.existsSync(consumerProjectionFileName)) { + Projection.restoreFromFile(consumerProjectionFileName, { + hmac: this.projectionHmac, + typeAccessor: projectionTypeAccessor + }).subscribe(consumer); + } consumer.streamName = streamName; this.consumers.set(identifier, consumer); return consumer; @@ -802,19 +809,17 @@ class EventStore extends events.EventEmitter { * * @param {string} name Projection name. * @param {object} [definition] Projection definition. - * @param {object} [options] Projection options. * @returns {Projection} */ - getProjection(name, definition, options = {}) { + getProjection(name, definition) { assert(typeof name === 'string' && name !== '', 'Must provide a projection name.'); const projectionTypeAccessor = this.typeAccessor ? (event) => this.typeAccessor(event?.payload || event) - : options.typeAccessor; + : undefined; const projectionFileName = path.join(this.storage.indexDirectory, 'projections', this.storage.storageFile + '.' + name + '.projection'); const projectionOptions = { - ...options, - fileName: options.fileName || projectionFileName, - hmac: options.hmac || this.storage.hmac, + fileName: projectionFileName, + hmac: this.projectionHmac, typeAccessor: projectionTypeAccessor }; if (definition) { diff --git a/src/Projection.js b/src/Projection.js index 7b580b65..09c7e5b2 100644 --- a/src/Projection.js +++ b/src/Projection.js @@ -85,9 +85,17 @@ class Projection { return matches(event, this.matcher); } - subscribe(consumer, options = {}) { + subscribe(consumer) { assert(consumer && typeof consumer.project === 'function', 'Projection.subscribe expects a Consumer instance.'); - consumer.project(this, options); + const projectionFileName = consumer.fileName ? `${consumer.fileName}.projection` : null; + const isAlreadySubscribed = consumer.projection === this; + const isAlreadyPersisted = projectionFileName && this.fileName === projectionFileName && fs.existsSync(projectionFileName); + consumer.project(this); + if (!isAlreadySubscribed && !isAlreadyPersisted) { + this.persist({ + fileName: projectionFileName || this.fileName + }); + } return this; } diff --git a/test/Consumer.spec.js b/test/Consumer.spec.js index 47cf53b0..760315f3 100644 --- a/test/Consumer.spec.js +++ b/test/Consumer.spec.js @@ -521,9 +521,14 @@ describe('Consumer', function() { }); }); - it('can create projections from a reducer function', function(done) { + it('can attach projections from a reducer function', function(done) { consumer = new Consumer(storage, 'foobar', 'consumer-projection', { count: 0 }); - consumer.createProjection((state, event) => ({ ...state, count: state.count + event.id }), { hmac: createHmac('test-secret') }); + new Projection('consumer-projection', { + initialState: { count: 0 }, + handlers: (state, event) => ({ ...state, count: state.count + event.id }) + }, { + hmac: createHmac('test-secret') + }).subscribe(consumer); consumer.on('caught-up', () => { expect(consumer.state.count).to.be(6); done(); @@ -534,16 +539,23 @@ describe('Consumer', function() { storage.write({ type: 'Foobar', id: 3 }); }); - it('can create projections from event-type reducer maps and restore them on reopen', function(done) { + it('can attach and restore projections from event-type reducer maps', function(done) { consumer = new Consumer(storage, 'foobar', 'consumer-projection-map', { count: 0 }); - consumer.createProjection({ - Foobar: (state, event) => ({ ...state, count: state.count + event.id }), - Bazinga: (state) => state + const projection = new Projection('consumer-projection-map', { + initialState: { count: 0 }, + handlers: { + Foobar: (state, event) => ({ ...state, count: state.count + event.id }), + Bazinga: (state) => state + } }, { hmac: createHmac('test-secret') }); + projection.subscribe(consumer); consumer.on('caught-up', () => { consumer.stop(); - consumer = new Consumer(storage, 'foobar', 'consumer-projection-map', {}, 0, { hmac: createHmac('test-secret') }); + consumer = new Consumer(storage, 'foobar', 'consumer-projection-map', {}); + Projection.restoreFromFile(`${consumer.fileName}.projection`, { + hmac: createHmac('test-secret') + }).subscribe(consumer); consumer.on('progress', () => { if (consumer.state.count === 10) { done(); @@ -559,8 +571,15 @@ describe('Consumer', function() { it('throws if function projection is restored without trusted hmac', function() { consumer = new Consumer(storage, 'foobar', 'consumer-projection-hmac'); - consumer.createProjection((state, event) => ({ ...state, lastId: event.id }), { hmac: createHmac('test-secret') }); - expect(() => new Consumer(storage, 'foobar', 'consumer-projection-hmac', {}, 0, { hmac: createHmac('wrong-secret') })).to.throwError(/Invalid HMAC/); + new Projection('consumer-projection-hmac', { + initialState: {}, + handlers: (state, event) => ({ ...state, lastId: event.id }) + }, { + hmac: createHmac('test-secret') + }).subscribe(consumer); + expect(() => Projection.restoreFromFile(`${consumer.fileName}.projection`, { + hmac: createHmac('wrong-secret') + })).to.throwError(/Invalid HMAC/); }); it('can attach a projection instance and restore it on reopen', function(done) { @@ -573,11 +592,14 @@ describe('Consumer', function() { }, { hmac: createHmac('test-secret') }); - consumer.project(projection); + projection.subscribe(consumer); consumer.on('caught-up', () => { expect(consumer.state.count).to.be(6); consumer.stop(); - consumer = new Consumer(storage, 'foobar', 'consumer-projection-instance', {}, 0, { hmac: createHmac('test-secret') }); + consumer = new Consumer(storage, 'foobar', 'consumer-projection-instance', {}); + Projection.restoreFromFile(`${consumer.fileName}.projection`, { + hmac: createHmac('test-secret') + }).subscribe(consumer); consumer.on('progress', () => { if (consumer.state.count === 10) { done(); diff --git a/test/EventStore.spec.js b/test/EventStore.spec.js index 5ef7e282..58a12eda 100644 --- a/test/EventStore.spec.js +++ b/test/EventStore.spec.js @@ -1391,12 +1391,14 @@ describe('EventStore', function() { eventstore.createEventStream('user-stream', (event) => event.stream === 'user-stream'); const consumer = eventstore.getConsumer('user-stream', 'user-counter', { count: 0 }); - consumer.project(new Projection('user-counter', { + new Projection('user-counter', { initialState: { count: 0 }, handlers: { UserCreated: (state) => ({ ...state, count: state.count + 1 }) } - })); + }, { + hmac: createHmac('test-secret') + }).subscribe(consumer); eventstore.commit('user-stream', [{ type: 'UserCreated', id: 1 }]); eventstore.commit('user-stream', [{ type: 'UserCreated', id: 2 }]); From a222f4ce4ac8d9d79af686d7dcc54a2c07855926 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Jun 2026 17:06:07 +0000 Subject: [PATCH 05/40] Address projection API and utility review feedback --- docs/api.md | 19 +++++++------------ docs/consumers.md | 16 +++++++++------- src/Consumer.js | 19 ++----------------- src/EventStore.js | 31 +++++++++++++++---------------- src/Projection.js | 22 +++++----------------- src/utils/fsUtil.js | 39 +++++++++++++++++++++++++++++++++++++++ test/EventStore.spec.js | 15 ++++++--------- 7 files changed, 83 insertions(+), 78 deletions(-) diff --git a/docs/api.md b/docs/api.md index fa6ac740..e4885bf1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -260,23 +260,18 @@ Asynchronously scan all consumer state files and return their identifiers. --- -#### `eventstore.getProjection(name, [definition])` +#### `eventstore.getProjection(name, [handlers], [initialState], [matcher])` ```javascript -eventstore.getProjection(name [, definition]) → Projection +eventstore.getProjection(name [, handlers] [, initialState] [, matcher]) → Projection ``` -Create a `Projection` with EventStore defaults (`typeAccessor`, storage HMAC), or restore a previously persisted one when `definition` is omitted. +Create a `Projection` with EventStore defaults (`typeAccessor`, storage HMAC), or restore a previously persisted one when `handlers` is omitted. -`definition` shape: +- `handlers`: reducer function `(state, event) => state` or map `{ [eventType]: reducer }` +- `initialState`: initial projection state (default `{}`) +- `matcher`: optional object/function matcher (same shape as stream/query matchers) -```javascript -{ - initialState: any, - handlers: (state, event) => state | { [eventType]: (state, event) => state }, - matcher: object|function // optional -} -``` --- @@ -296,7 +291,7 @@ Attach a projection-like object (`apply(state, event)`) as the consumer `'data'` projection.subscribe(consumer) ``` -Attach this projection to the consumer and persist its definition next to the consumer state file so `eventstore.getConsumer(...)` can restore and reconnect it automatically. +Attach this projection to the consumer (same wiring behavior as `consumer.project(projection)`) and persist its definition next to the consumer state file so `eventstore.getConsumer(...)` can restore and reconnect it automatically. --- diff --git a/docs/consumers.md b/docs/consumers.md index 9e34348c..75436a92 100644 --- a/docs/consumers.md +++ b/docs/consumers.md @@ -75,11 +75,8 @@ Use a `Projection` to define *how* events are projected into state, then connect ```javascript const projection = eventstore.getProjection('orders-total', { - initialState: { total: 0 }, - handlers: { - OrderCreated: (state, event) => ({ ...state, total: state.total + (event.payload.amount || 0) }) - } -}); + OrderCreated: (state, event) => ({ ...state, total: state.total + (event.payload.amount || 0) }) +}, { total: 0 }); const consumer = eventstore.getConsumer('orders', 'orders-projection', projection.initialState); projection.subscribe(consumer); @@ -88,10 +85,15 @@ projection.subscribe(consumer); Projections are composable via `CompositeProjection`: ```javascript -import { CompositeProjection } from 'event-storage'; +import { CompositeProjection, Projection } from 'event-storage'; + +const count = new Projection('count', { + initialState: 0, + handlers: { OrderCreated: (state) => state + 1 } +}); const overview = new CompositeProjection('overview', { - count: { initialState: 0, handlers: { OrderCreated: (state) => state + 1 } }, + count, last: { initialState: null, handlers: { OrderCreated: (state, event) => event.payload } } }); ``` diff --git a/src/Consumer.js b/src/Consumer.js index 2609651e..4dc90110 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -2,24 +2,10 @@ import stream from 'stream'; import fs from 'fs'; import path from 'path'; import { assert } from './utils/util.js'; -import { ensureDirectory } from './utils/fsUtil.js'; +import { ensureDirectory, safeUnlink, writeFileAtomic } from './utils/fsUtil.js'; import Storage from './Storage/ReadableStorage.js'; const MAX_CATCHUP_BATCH = 10; -/** - * Safely unlink a file and ignore if it doesn't exist. - * @param {string} filename - */ -const safeUnlink = (filename) => { - /* istanbul ignore next */ - try { - fs.unlinkSync(filename); - } catch (e) { - if (e.code !== "ENOENT") { - throw e; - } - } -}; /** * Implements an event-driven durable Consumer that provides at-least-once delivery semantics or exactly-once processing semantics if only using setState(). @@ -197,9 +183,8 @@ class Consumer extends stream.Readable { throw new Error(`Trying to update consumer ${this.name} concurrently. Keep each single consumer within a single process.`); } try { - fs.writeFileSync(tmpFile, consumerData); // If the write fails (half-way), the consumer state file will not be corrupted - fs.renameSync(tmpFile, this.fileName); + writeFileAtomic(this.fileName, consumerData, { tmpFileName: tmpFile }); this.emit('persisted', consumerState); } catch (e) { /* istanbul ignore next */ diff --git a/src/EventStore.js b/src/EventStore.js index e82d4212..ce6d7e77 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -120,10 +120,10 @@ class EventStore extends events.EventEmitter { } } + this.projectionTypeAccessor = this.typeAccessor + ? (event) => this.typeAccessor(event?.payload || event) + : undefined; this.initialize(storeName, storageConfig); - this.projectionHmac = typeof storageConfig.hmac === 'function' - ? storageConfig.hmac - : this.storage.hmac; } /** @@ -788,15 +788,12 @@ class EventStore extends events.EventEmitter { if (this.consumers.has(identifier)) { return this.consumers.get(identifier); } - const projectionTypeAccessor = this.typeAccessor - ? (event) => this.typeAccessor(event?.payload || event) - : undefined; const consumer = new Consumer(this.storage, streamName === '_all' ? '_all' : 'stream-' + streamName, identifier, initialState, since); const consumerProjectionFileName = `${consumer.fileName}.projection`; if (fs.existsSync(consumerProjectionFileName)) { Projection.restoreFromFile(consumerProjectionFileName, { - hmac: this.projectionHmac, - typeAccessor: projectionTypeAccessor + hmac: this.storage.hmac, + typeAccessor: this.projectionTypeAccessor }).subscribe(consumer); } consumer.streamName = streamName; @@ -808,21 +805,23 @@ class EventStore extends events.EventEmitter { * Get or create a projection with EventStore defaults. * * @param {string} name Projection name. - * @param {object} [definition] Projection definition. + * @param {function(object, object): object|object} [handlers] Projection handlers (reducer fn or reducer map). + * @param {object} [initialState={}] Projection initial state. + * @param {object|function(object): boolean} [matcher] Optional projection matcher. * @returns {Projection} */ - getProjection(name, definition) { + getProjection(name, handlers, initialState = {}, matcher) { assert(typeof name === 'string' && name !== '', 'Must provide a projection name.'); - const projectionTypeAccessor = this.typeAccessor - ? (event) => this.typeAccessor(event?.payload || event) - : undefined; const projectionFileName = path.join(this.storage.indexDirectory, 'projections', this.storage.storageFile + '.' + name + '.projection'); const projectionOptions = { fileName: projectionFileName, - hmac: this.projectionHmac, - typeAccessor: projectionTypeAccessor + hmac: this.storage.hmac, + typeAccessor: this.projectionTypeAccessor }; - if (definition) { + if (handlers !== undefined) { + const definition = (handlers && typeof handlers === 'object' && !Array.isArray(handlers) && Object.prototype.hasOwnProperty.call(handlers, 'handlers')) + ? handlers + : { handlers, initialState, matcher }; return new Projection(name, definition, projectionOptions); } return Projection.restore(name, projectionOptions); diff --git a/src/Projection.js b/src/Projection.js index 09c7e5b2..742c2efc 100644 --- a/src/Projection.js +++ b/src/Projection.js @@ -1,20 +1,11 @@ import fs from 'fs'; import path from 'path'; import { assert } from './utils/util.js'; -import { ensureDirectory } from './utils/fsUtil.js'; +import { ensureDirectory, writeFileAtomic } from './utils/fsUtil.js'; import { buildMatcherFromMetadata, buildMetadataForMatcher, matches } from './utils/metadataUtil.js'; const DEFAULT_TYPE_ACCESSOR = (event) => event?.type || event?.payload?.type; -const safeUnlink = (filename) => { - try { - fs.unlinkSync(filename); - } catch (e) { - if (e.code !== "ENOENT") { - throw e; - } - } -}; class Projection { @@ -105,13 +96,10 @@ class Projection { const metadata = this.toMetadata(hmac); const tmpFile = fileName + '.tmp'; ensureDirectory(path.dirname(fileName)); - try { - fs.writeFileSync(tmpFile, JSON.stringify(metadata), 'utf8'); - fs.renameSync(tmpFile, fileName); - } catch (e) { - safeUnlink(tmpFile); - throw e; - } + writeFileAtomic(fileName, JSON.stringify(metadata), { + tmpFileName: tmpFile, + encoding: 'utf8' + }); this.fileName = fileName; this.hmac = hmac; return fileName; diff --git a/src/utils/fsUtil.js b/src/utils/fsUtil.js index 1dd2086f..37cafbe2 100644 --- a/src/utils/fsUtil.js +++ b/src/utils/fsUtil.js @@ -2,6 +2,43 @@ import fs from 'fs'; import path from 'path'; import { mkdirpSync } from 'mkdirp'; +/** + * Safely unlink a file and ignore ENOENT. + * @param {string} fileName + */ +function safeUnlink(fileName) { + try { + fs.unlinkSync(fileName); + } catch (e) { + if (e.code !== 'ENOENT') { + throw e; + } + } +} + +/** + * Atomically write a file by writing to a temporary file first and then renaming it. + * @param {string} fileName + * @param {string|Buffer} data + * @param {{ tmpFileName?: string, encoding?: BufferEncoding }} [options] + * @returns {string} + */ +function writeFileAtomic(fileName, data, options = {}) { + const tmpFileName = options.tmpFileName || `${fileName}.tmp`; + try { + if (options.encoding) { + fs.writeFileSync(tmpFileName, data, options.encoding); + } else { + fs.writeFileSync(tmpFileName, data); + } + fs.renameSync(tmpFileName, fileName); + } catch (e) { + safeUnlink(tmpFileName); + throw e; + } + return fileName; +} + /** * Ensure that the given directory exists. * @param {string} dirName @@ -119,5 +156,7 @@ function scanForFiles(directory, regexPattern, onEach, onDone) { export { ensureDirectory, + safeUnlink, + writeFileAtomic, scanForFiles, }; diff --git a/test/EventStore.spec.js b/test/EventStore.spec.js index 58a12eda..1a7ac4a5 100644 --- a/test/EventStore.spec.js +++ b/test/EventStore.spec.js @@ -1385,7 +1385,7 @@ describe('EventStore', function() { storageDirectory, typeAccessor: 'type', storageConfig: { - hmac: createHmac('test-secret') + hmacSecret: 'test-secret' } }); eventstore.createEventStream('user-stream', (event) => event.stream === 'user-stream'); @@ -1397,7 +1397,7 @@ describe('EventStore', function() { UserCreated: (state) => ({ ...state, count: state.count + 1 }) } }, { - hmac: createHmac('test-secret') + hmac: eventstore.storage.hmac }).subscribe(consumer); eventstore.commit('user-stream', [{ type: 'UserCreated', id: 1 }]); eventstore.commit('user-stream', [{ type: 'UserCreated', id: 2 }]); @@ -1411,7 +1411,7 @@ describe('EventStore', function() { storageDirectory, typeAccessor: 'type', storageConfig: { - hmac: createHmac('test-secret') + hmacSecret: 'test-secret' } }); const reopened = eventstore.getConsumer('user-stream', 'user-counter', { count: 0 }); @@ -1432,15 +1432,12 @@ describe('EventStore', function() { storageDirectory, typeAccessor: 'type', storageConfig: { - hmac: createHmac('test-secret') + hmacSecret: 'test-secret' } }); const projection = eventstore.getProjection('user-count', { - initialState: 0, - handlers: { - UserCreated: (state) => state + 1 - } - }); + UserCreated: (state) => state + 1 + }, 0); projection.persist(); const restored = eventstore.getProjection('user-count'); From f9e7fdd711878afe8ccb420c72c17d74ec0878e1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Jun 2026 17:09:08 +0000 Subject: [PATCH 06/40] Refine projection API docs and atomic fs helpers --- src/EventStore.js | 9 ++++++++- src/utils/fsUtil.js | 20 ++++---------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/src/EventStore.js b/src/EventStore.js index ce6d7e77..a56d5e37 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -819,7 +819,7 @@ class EventStore extends events.EventEmitter { typeAccessor: this.projectionTypeAccessor }; if (handlers !== undefined) { - const definition = (handlers && typeof handlers === 'object' && !Array.isArray(handlers) && Object.prototype.hasOwnProperty.call(handlers, 'handlers')) + const definition = isProjectionDefinitionObject(handlers) ? handlers : { handlers, initialState, matcher }; return new Projection(name, definition, projectionOptions); @@ -883,6 +883,13 @@ function normalizePredicateRaw(predicate, raw) { return { predicate, raw }; } +function isProjectionDefinitionObject(value) { + return value + && typeof value === 'object' + && !Array.isArray(value) + && Object.hasOwn(value, 'handlers'); +} + EventStore.Storage = Storage; EventStore.Index = Index; diff --git a/src/utils/fsUtil.js b/src/utils/fsUtil.js index 37cafbe2..08ea7f86 100644 --- a/src/utils/fsUtil.js +++ b/src/utils/fsUtil.js @@ -2,10 +2,7 @@ import fs from 'fs'; import path from 'path'; import { mkdirpSync } from 'mkdirp'; -/** - * Safely unlink a file and ignore ENOENT. - * @param {string} fileName - */ +// Best-effort cleanup for temporary files after interrupted/failed writes. function safeUnlink(fileName) { try { fs.unlinkSync(fileName); @@ -16,21 +13,12 @@ function safeUnlink(fileName) { } } -/** - * Atomically write a file by writing to a temporary file first and then renaming it. - * @param {string} fileName - * @param {string|Buffer} data - * @param {{ tmpFileName?: string, encoding?: BufferEncoding }} [options] - * @returns {string} - */ +// Prevent partially written persistence files from replacing the last valid state. function writeFileAtomic(fileName, data, options = {}) { const tmpFileName = options.tmpFileName || `${fileName}.tmp`; + const writeOptions = options.encoding ? { encoding: options.encoding } : undefined; try { - if (options.encoding) { - fs.writeFileSync(tmpFileName, data, options.encoding); - } else { - fs.writeFileSync(tmpFileName, data); - } + fs.writeFileSync(tmpFileName, data, writeOptions); fs.renameSync(tmpFileName, fileName); } catch (e) { safeUnlink(tmpFileName); From 30a73dbc5b14ef4af5b34a132d379ad91aa56820 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Jun 2026 20:16:01 +0000 Subject: [PATCH 07/40] Address remaining projection and consumer review feedback --- docs/consumers.md | 1 + src/Consumer.js | 30 ++++++++---- src/EventStore.js | 12 ++--- src/Projection.js | 99 +++++++++++++++++++++++++++++++++++---- src/utils/fsUtil.js | 25 +++++++++- src/utils/metadataUtil.js | 1 + test/Consumer.spec.js | 28 +++++++++++ test/util.spec.js | 22 ++++++++- 8 files changed, 192 insertions(+), 26 deletions(-) diff --git a/docs/consumers.md b/docs/consumers.md index 75436a92..411c6694 100644 --- a/docs/consumers.md +++ b/docs/consumers.md @@ -96,6 +96,7 @@ const overview = new CompositeProjection('overview', { count, last: { initialState: null, handlers: { OrderCreated: (state, event) => event.payload } } }); +// overview.state -> { count: number, last: object|null } ``` ## Resetting a Consumer diff --git a/src/Consumer.js b/src/Consumer.js index 4dc90110..778980c7 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -2,7 +2,7 @@ import stream from 'stream'; import fs from 'fs'; import path from 'path'; import { assert } from './utils/util.js'; -import { ensureDirectory, safeUnlink, writeFileAtomic } from './utils/fsUtil.js'; +import { ensureDirectory, isSafeRelativeName, resolvePathWithinRoot, safeUnlink, writeFileAtomic } from './utils/fsUtil.js'; import Storage from './Storage/ReadableStorage.js'; const MAX_CATCHUP_BATCH = 10; @@ -39,12 +39,15 @@ class Consumer extends stream.Readable { * @param {string} identifier The unique name to identify this consumer. */ initializeStorage(storage, indexName, identifier) { + assert(indexName === '_all' || isSafeRelativeName(indexName), `Invalid index name "${indexName}" for consumer.`); + assert(isSafeRelativeName(identifier), `Invalid identifier "${identifier}" for consumer.`); this.storage = storage; this.index = this.storage.openIndex(indexName); this.indexName = indexName; this.identifier = identifier; const consumerDirectory = path.join(this.storage.indexDirectory, 'consumers'); - this.fileName = path.join(consumerDirectory, this.storage.storageFile + '.' + indexName + '.' + identifier); + this.fileName = resolvePathWithinRoot(consumerDirectory, `${this.storage.storageFile}.${indexName}.${identifier}`); + ensureDirectory(path.dirname(this.fileName)); if (ensureDirectory(consumerDirectory)) { this.cleanUpFailedWrites(); } @@ -56,12 +59,14 @@ class Consumer extends stream.Readable { */ cleanUpFailedWrites() { const consumerBaseName = path.basename(this.fileName); - const escapedConsumerBaseName = consumerBaseName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - const failedStateFilePattern = new RegExp(`^${escapedConsumerBaseName}\\.\\d+$`); const consumerDirectory = path.dirname(this.fileName); const files = fs.readdirSync(consumerDirectory); for (let file of files) { - if (failedStateFilePattern.test(file)) { + if (!file.startsWith(consumerBaseName + '.')) { + continue; + } + const suffix = file.slice(consumerBaseName.length + 1); + if (/^\d+$/.test(suffix)) { safeUnlink(path.join(consumerDirectory, file)); } } @@ -102,6 +107,11 @@ class Consumer extends stream.Readable { */ project(projection) { assert(projection && typeof projection.apply === 'function', 'Projection must implement apply(state, event).'); + const projectionFileName = this.fileName ? `${this.fileName}.projection` : null; + const isAlreadySubscribed = this.projection === projection; + const isAlreadyPersisted = projectionFileName + && projection.fileName === projectionFileName + && fs.existsSync(projectionFileName); if (this.projectionHandler) { this.removeListener('data', this.projectionHandler); } @@ -110,6 +120,9 @@ class Consumer extends stream.Readable { this.setState(projection.apply(this.state, event)); }; this.on('data', this.projectionHandler); + if (!isAlreadySubscribed && !isAlreadyPersisted && typeof projection.persist === 'function') { + projection.persist({ fileName: projectionFileName || projection.fileName }); + } return this; } @@ -127,6 +140,9 @@ class Consumer extends stream.Readable { if (typeof newState === 'function') { newState = newState(this.state); } + if (this.state === newState) { + return; + } this.state = Object.freeze(newState); this.doPersist = persist; } @@ -183,9 +199,7 @@ class Consumer extends stream.Readable { throw new Error(`Trying to update consumer ${this.name} concurrently. Keep each single consumer within a single process.`); } try { - // If the write fails (half-way), the consumer state file will not be corrupted - writeFileAtomic(this.fileName, consumerData, { tmpFileName: tmpFile }); - this.emit('persisted', consumerState); + writeFileAtomic(this.fileName, consumerData, { tmpFileName: tmpFile }, () => this.emit('persisted', consumerState)); } catch (e) { /* istanbul ignore next */ safeUnlink(tmpFile); diff --git a/src/EventStore.js b/src/EventStore.js index a56d5e37..28462339 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -8,8 +8,8 @@ import Index from './Index.js'; import Consumer from './Consumer.js'; import Projection from './Projection.js'; import { assert, getPropertyAtPath } from './utils/util.js'; -import { ensureDirectory, scanForFiles } from './utils/fsUtil.js'; -import { buildTypeMatcherFn } from './utils/metadataUtil.js'; +import { ensureDirectory, isSafeRelativeName, scanForFiles } from './utils/fsUtil.js'; +import { buildTypeMatcherFn, isPlainObject } from './utils/metadataUtil.js'; const ExpectedVersion = { Any: -1, @@ -20,7 +20,6 @@ const ExpectedVersion = { * Default matcher property paths mirroring the Storage default, used for index optimization. */ const DEFAULT_MATCHER_PROPERTIES = ['stream', 'payload.type']; -const STREAM_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_]*(?:[\/:@~+=\-#.][A-Za-z0-9_]+)*$/; const STORAGE_HOOK_EVENTS = new Set(['preCommit', 'preRead']); class OptimisticConcurrencyError extends Error {} @@ -435,7 +434,7 @@ class EventStore extends events.EventEmitter { return null; } assert(typeof type === 'string', 'typeAccessor must return a string.'); - assert(STREAM_NAME_PATTERN.test(type), `typeAccessor must return a valid stream name. Got: "${type}"`); + assert(isSafeRelativeName(type), `typeAccessor must return a valid stream name. Got: "${type}"`); return type; } @@ -884,10 +883,7 @@ function normalizePredicateRaw(predicate, raw) { } function isProjectionDefinitionObject(value) { - return value - && typeof value === 'object' - && !Array.isArray(value) - && Object.hasOwn(value, 'handlers'); + return isPlainObject(value) && Object.hasOwn(value, 'handlers'); } EventStore.Storage = Storage; diff --git a/src/Projection.js b/src/Projection.js index 742c2efc..5425efe7 100644 --- a/src/Projection.js +++ b/src/Projection.js @@ -9,6 +9,11 @@ const DEFAULT_TYPE_ACCESSOR = (event) => event?.type || event?.payload?.type; class Projection { + /** + * @param {string} name Projection name. + * @param {{ initialState?: object, handlers: function(object, object): object|object, matcher?: object|function(object): boolean }} [definition] + * @param {{ hmac?: function(string): string, typeAccessor?: function(object): string, fileName?: string }} [options] + */ constructor(name, definition = {}, options = {}) { assert(typeof name === 'string' && name !== '', 'Projection must have a name.'); const { initialState = {}, handlers, matcher } = definition; @@ -35,6 +40,12 @@ class Projection { return Object.keys(this.handlers); } + /** + * Apply one event to the provided state and return the next state. + * @param {*} state + * @param {object} event + * @returns {*} + */ apply(state, event) { if (!this.matches(event)) { this.state = state; @@ -53,6 +64,11 @@ class Projection { return nextState; } + /** + * Reset to initialState and project all events from the given iterable stream. + * @param {Iterable} stream + * @returns {*} + */ handle(stream) { this.reset(); for (const event of stream) { @@ -61,11 +77,20 @@ class Projection { return this.state; } + /** + * Reset current projection state to its initial state. + * @returns {*} + */ reset() { this.state = this.initialState; return this.state; } + /** + * Check whether an event matches this projection's matcher definition. + * @param {object} event + * @returns {boolean} + */ matches(event) { if (!this.matcher) { return true; @@ -76,20 +101,22 @@ class Projection { return matches(event, this.matcher); } + /** + * Subscribe this projection to a consumer and persist when needed. + * @param {{ project: function(Projection): object }} consumer + * @returns {Projection} + */ subscribe(consumer) { assert(consumer && typeof consumer.project === 'function', 'Projection.subscribe expects a Consumer instance.'); - const projectionFileName = consumer.fileName ? `${consumer.fileName}.projection` : null; - const isAlreadySubscribed = consumer.projection === this; - const isAlreadyPersisted = projectionFileName && this.fileName === projectionFileName && fs.existsSync(projectionFileName); consumer.project(this); - if (!isAlreadySubscribed && !isAlreadyPersisted) { - this.persist({ - fileName: projectionFileName || this.fileName - }); - } return this; } + /** + * Persist projection definition metadata to disk. + * @param {{ hmac?: function(string): string, fileName?: string }} [options] + * @returns {string} Persisted file name. + */ persist(options = {}) { const hmac = options.hmac || this.hmac; const fileName = options.fileName || this.fileName || `${this.name}.projection`; @@ -105,6 +132,11 @@ class Projection { return fileName; } + /** + * Serialize this projection definition into trusted metadata. + * @param {function(string): string} [hmac] + * @returns {object} + */ toMetadata(hmac = this.hmac) { const serializeFn = (fn) => { assert(typeof hmac === 'function', 'Must provide options.hmac for function projections.'); @@ -128,18 +160,36 @@ class Projection { }; } + /** + * Restore a projection by name from default or configured file location. + * @param {string} name + * @param {{ fileName?: string, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {Projection} + */ static restore(name, options = {}) { assert(typeof name === 'string' && name !== '', 'Projection.restore requires a projection name.'); const fileName = options.fileName || `${name}.projection`; return Projection.restoreFromFile(fileName, options); } + /** + * Restore a projection from an explicit metadata file path. + * @param {string} fileName + * @param {{ hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {Projection} + */ static restoreFromFile(fileName, options = {}) { assert(fs.existsSync(fileName), `Projection file does not exist: ${fileName}`); const metadata = JSON.parse(fs.readFileSync(fileName, 'utf8')); return Projection.fromMetadata(metadata, { ...options, fileName }); } + /** + * Recreate a projection instance from serialized metadata. + * @param {object} metadata + * @param {{ fileName?: string, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {Projection} + */ static fromMetadata(metadata, options = {}) { assert(metadata && typeof metadata === 'object', 'Invalid projection metadata.'); if (metadata.kind === 'composite-projection') { @@ -173,6 +223,13 @@ class Projection { return projection; } + /** + * Compose multiple projections into one composite projection. + * @param {string} name + * @param {object} projections + * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {CompositeProjection} + */ static compose(name, projections, options = {}) { return new CompositeProjection(name, projections, options); } @@ -180,6 +237,11 @@ class Projection { class CompositeProjection extends Projection { + /** + * @param {string} name + * @param {object} projections + * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + */ constructor(name, projections, options = {}) { assert(projections && typeof projections === 'object' && !Array.isArray(projections), 'CompositeProjection requires an object map of projections.'); const normalized = {}; @@ -209,6 +271,12 @@ class CompositeProjection extends Projection { return [...types]; } + /** + * Apply one event across all child projections and return composed state. + * @param {object} state + * @param {object} event + * @returns {object} + */ apply(state, event) { if (!this.matches(event)) { this.state = state; @@ -223,6 +291,10 @@ class CompositeProjection extends Projection { return nextState; } + /** + * Reset all child projections and rebuild composed state. + * @returns {object} + */ reset() { for (const projection of Object.values(this.projections)) { projection.reset(); @@ -233,6 +305,11 @@ class CompositeProjection extends Projection { return this.state; } + /** + * Serialize composed projection metadata recursively. + * @param {function(string): string} [hmac] + * @returns {object} + */ toMetadata(hmac = this.hmac) { return { kind: 'composite-projection', @@ -244,6 +321,12 @@ class CompositeProjection extends Projection { }; } + /** + * Restore a composed projection from serialized metadata. + * @param {object} metadata + * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {CompositeProjection} + */ static fromMetadata(metadata, options = {}) { const hmac = options.hmac; const deserializeMatcher = (matcherMetadata) => { diff --git a/src/utils/fsUtil.js b/src/utils/fsUtil.js index 08ea7f86..17e38fce 100644 --- a/src/utils/fsUtil.js +++ b/src/utils/fsUtil.js @@ -2,6 +2,8 @@ import fs from 'fs'; import path from 'path'; import { mkdirpSync } from 'mkdirp'; +const SAFE_RELATIVE_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_]*(?:[\/:@~+=\-#.][A-Za-z0-9_]+)*$/; + // Best-effort cleanup for temporary files after interrupted/failed writes. function safeUnlink(fileName) { try { @@ -14,12 +16,15 @@ function safeUnlink(fileName) { } // Prevent partially written persistence files from replacing the last valid state. -function writeFileAtomic(fileName, data, options = {}) { +function writeFileAtomic(fileName, data, options = {}, onSuccess = null) { const tmpFileName = options.tmpFileName || `${fileName}.tmp`; const writeOptions = options.encoding ? { encoding: options.encoding } : undefined; try { fs.writeFileSync(tmpFileName, data, writeOptions); fs.renameSync(tmpFileName, fileName); + if (typeof onSuccess === 'function') { + onSuccess(); + } } catch (e) { safeUnlink(tmpFileName); throw e; @@ -27,6 +32,22 @@ function writeFileAtomic(fileName, data, options = {}) { return fileName; } +function isSafeRelativeName(name) { + return typeof name === 'string' + && name !== '' + && SAFE_RELATIVE_NAME_PATTERN.test(name); +} + +function resolvePathWithinRoot(rootDirectory, relativePath) { + const root = path.resolve(rootDirectory); + const resolvedPath = path.resolve(root, relativePath); + const rootRelativePath = path.relative(root, resolvedPath); + if (rootRelativePath.startsWith('..') || path.isAbsolute(rootRelativePath)) { + throw new Error(`Invalid relative path "${relativePath}".`); + } + return resolvedPath; +} + /** * Ensure that the given directory exists. * @param {string} dirName @@ -147,4 +168,6 @@ export { safeUnlink, writeFileAtomic, scanForFiles, + isSafeRelativeName, + resolvePathWithinRoot, }; diff --git a/src/utils/metadataUtil.js b/src/utils/metadataUtil.js index fa089fca..19764532 100644 --- a/src/utils/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -238,6 +238,7 @@ function matchesNode(buffer, startOffset, node) { export { createHmac, + isPlainObject, matches, buildMetadataHeader, buildMetadataForMatcher, diff --git a/test/Consumer.spec.js b/test/Consumer.spec.js index 760315f3..e53d8ddd 100644 --- a/test/Consumer.spec.js +++ b/test/Consumer.spec.js @@ -41,6 +41,14 @@ describe('Consumer', function() { expect(() => new Consumer(storage, 'foobar')).to.throwError(/identifier/); }); + it('throws for invalid consumer index names', function() { + expect(() => new Consumer(storage, '../foobar', 'consumer1')).to.throwError(/Invalid index name/); + }); + + it('throws for invalid consumer identifiers', function() { + expect(() => new Consumer(storage, 'foobar', '../consumer1')).to.throwError(/Invalid identifier/); + }); + it('creates consumer directory if not existing', function() { consumer = new Consumer(storage, 'foobar', 'consumer1'); expect(fs.existsSync(dataDirectory + '/consumers')).to.be(true); @@ -539,6 +547,26 @@ describe('Consumer', function() { storage.write({ type: 'Foobar', id: 3 }); }); + it('consumer.project persists a projection when attaching', function(done) { + consumer = new Consumer(storage, 'foobar', 'consumer-project-method', { count: 0 }); + const projection = new Projection('consumer-project-method', { + initialState: { count: 0 }, + handlers: { + Foobar: (state, event) => ({ ...state, count: state.count + event.id }) + } + }, { hmac: createHmac('test-secret') }); + + consumer.project(projection); + expect(fs.existsSync(`${consumer.fileName}.projection`)).to.be(true); + consumer.on('caught-up', () => { + expect(consumer.state.count).to.be(6); + done(); + }); + storage.write({ type: 'Foobar', id: 1 }); + storage.write({ type: 'Foobar', id: 2 }); + storage.write({ type: 'Foobar', id: 3 }); + }); + it('can attach and restore projections from event-type reducer maps', function(done) { consumer = new Consumer(storage, 'foobar', 'consumer-projection-map', { count: 0 }); const projection = new Projection('consumer-projection-map', { diff --git a/test/util.spec.js b/test/util.spec.js index b9adc01e..6124af08 100644 --- a/test/util.spec.js +++ b/test/util.spec.js @@ -2,7 +2,7 @@ import expect from 'expect.js'; import fs from 'fs-extra'; import path from 'path'; import { iterate } from '../src/utils/util.js'; -import { scanForFiles } from '../src/utils/fsUtil.js'; +import { isSafeRelativeName, resolvePathWithinRoot, scanForFiles } from '../src/utils/fsUtil.js'; import { fileURLToPath } from 'url'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -90,6 +90,26 @@ describe('util', function() { }); }); + describe('file path helpers', function() { + + it('detects safe relative names', function() { + expect(isSafeRelativeName('stream-orders')).to.be(true); + expect(isSafeRelativeName('../orders')).to.be(false); + }); + + it('resolves paths within a root directory', function() { + const root = path.join(testDir, 'root'); + const resolved = resolvePathWithinRoot(root, 'a/b.file'); + expect(resolved.startsWith(path.resolve(root))).to.be(true); + }); + + it('rejects traversal paths outside of root directory', function() { + const root = path.join(testDir, 'root'); + expect(() => resolvePathWithinRoot(root, '../outside.file')).to.throwError(/Invalid relative path/); + }); + + }); + }); }); From ecc828265c9db48979faacf770e32788b211c5d1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 19:23:07 +0000 Subject: [PATCH 08/40] refactor projection composition and simplify consumer persist --- src/CompositeProjection.js | 129 +++++++++++++++++++++++++++++++++++++ src/Consumer.js | 7 +- src/Projection.js | 117 +-------------------------------- test/Consumer.spec.js | 18 +++--- 4 files changed, 142 insertions(+), 129 deletions(-) create mode 100644 src/CompositeProjection.js diff --git a/src/CompositeProjection.js b/src/CompositeProjection.js new file mode 100644 index 00000000..91ff9b9f --- /dev/null +++ b/src/CompositeProjection.js @@ -0,0 +1,129 @@ +import { assert } from './utils/util.js'; +import { buildMatcherFromMetadata, buildMetadataForMatcher } from './utils/metadataUtil.js'; + +/** + * Build the CompositeProjection class on top of the provided Projection base class. + * This avoids cyclic imports between Projection and CompositeProjection modules. + * + * @param {typeof import('./Projection.js').default} Projection + * @returns {typeof import('./Projection.js').CompositeProjection} + */ +function createCompositeProjectionClass(Projection) { + return class CompositeProjection extends Projection { + + /** + * @param {string} name + * @param {object} projections + * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + */ + constructor(name, projections, options = {}) { + assert(projections && typeof projections === 'object' && !Array.isArray(projections), 'CompositeProjection requires an object map of projections.'); + const normalized = {}; + for (const [projectionName, projection] of Object.entries(projections)) { + normalized[projectionName] = projection instanceof Projection + ? projection + : new Projection(projectionName, projection, options); + } + super(name, { + initialState: Object.fromEntries( + Object.entries(normalized).map(([projectionName, projection]) => [projectionName, projection.initialState]) + ), + handlers: (state) => state, + matcher: options.matcher + }, options); + this.projections = normalized; + this.reset(); + } + + get types() { + const types = new Set(); + for (const projection of Object.values(this.projections)) { + for (const type of projection.types) { + types.add(type); + } + } + return [...types]; + } + + /** + * Apply one event across all child projections and return composed state. + * @param {object} state + * @param {object} event + * @returns {object} + */ + apply(state, event) { + if (!this.matches(event)) { + this.state = state; + return state; + } + const currentState = state || this.initialState; + const nextState = {}; + for (const [name, projection] of Object.entries(this.projections)) { + nextState[name] = projection.apply(currentState[name], event); + } + this.state = nextState; + return nextState; + } + + /** + * Reset all child projections and rebuild composed state. + * @returns {object} + */ + reset() { + for (const projection of Object.values(this.projections)) { + projection.reset(); + } + this.state = Object.fromEntries( + Object.entries(this.projections).map(([projectionName, projection]) => [projectionName, projection.state]) + ); + return this.state; + } + + /** + * Serialize composed projection metadata recursively. + * @param {function(string): string} [hmac] + * @returns {object} + */ + toMetadata(hmac = this.hmac) { + return { + kind: 'composite-projection', + name: this.name, + matcher: this.matcher ? buildMetadataForMatcher(this.matcher, hmac) : null, + projections: Object.fromEntries( + Object.entries(this.projections).map(([name, projection]) => [name, projection.toMetadata(hmac)]) + ) + }; + } + + /** + * Restore a composed projection from serialized metadata. + * @param {object} metadata + * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {CompositeProjection} + */ + static fromMetadata(metadata, options = {}) { + const hmac = options.hmac; + const deserializeMatcher = (matcherMetadata) => { + if (!matcherMetadata) { + return undefined; + } + if (typeof matcherMetadata.matcher === 'string') { + assert(typeof hmac === 'function', 'Must provide options.hmac to restore function projections.'); + } + return buildMatcherFromMetadata(matcherMetadata, hmac); + }; + const projections = Object.fromEntries( + Object.entries(metadata.projections || {}).map(([name, projectionMetadata]) => [ + name, + Projection.fromMetadata(projectionMetadata, options) + ]) + ); + return new this(metadata.name, projections, { + ...options, + matcher: deserializeMatcher(metadata.matcher) + }); + } + }; +} + +export { createCompositeProjectionClass }; diff --git a/src/Consumer.js b/src/Consumer.js index 7d379942..18c15550 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -194,12 +194,7 @@ class Consumer extends stream.Readable { if (fs.existsSync(tmpFile)) { throw new Error(`Trying to update consumer ${this.name} concurrently. Keep each single consumer within a single process.`); } - try { - writeFileAtomic(this.fileName, consumerData, { tmpFileName: tmpFile }, () => this.emit('persisted', consumerState)); - } catch (e) { - /* c8 ignore next */ - safeUnlink(tmpFile); - } + writeFileAtomic(this.fileName, consumerData, { tmpFileName: tmpFile }, () => this.emit('persisted', consumerState)); }); } diff --git a/src/Projection.js b/src/Projection.js index 5425efe7..a61b5569 100644 --- a/src/Projection.js +++ b/src/Projection.js @@ -3,6 +3,7 @@ import path from 'path'; import { assert } from './utils/util.js'; import { ensureDirectory, writeFileAtomic } from './utils/fsUtil.js'; import { buildMatcherFromMetadata, buildMetadataForMatcher, matches } from './utils/metadataUtil.js'; +import { createCompositeProjectionClass } from './CompositeProjection.js'; const DEFAULT_TYPE_ACCESSOR = (event) => event?.type || event?.payload?.type; @@ -235,121 +236,7 @@ class Projection { } } -class CompositeProjection extends Projection { - - /** - * @param {string} name - * @param {object} projections - * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] - */ - constructor(name, projections, options = {}) { - assert(projections && typeof projections === 'object' && !Array.isArray(projections), 'CompositeProjection requires an object map of projections.'); - const normalized = {}; - for (const [projectionName, projection] of Object.entries(projections)) { - normalized[projectionName] = projection instanceof Projection - ? projection - : new Projection(projectionName, projection, options); - } - super(name, { - initialState: Object.fromEntries( - Object.entries(normalized).map(([projectionName, projection]) => [projectionName, projection.initialState]) - ), - handlers: (state) => state, - matcher: options.matcher - }, options); - this.projections = normalized; - this.reset(); - } - - get types() { - const types = new Set(); - for (const projection of Object.values(this.projections)) { - for (const type of projection.types) { - types.add(type); - } - } - return [...types]; - } - - /** - * Apply one event across all child projections and return composed state. - * @param {object} state - * @param {object} event - * @returns {object} - */ - apply(state, event) { - if (!this.matches(event)) { - this.state = state; - return state; - } - const currentState = state || this.initialState; - const nextState = {}; - for (const [name, projection] of Object.entries(this.projections)) { - nextState[name] = projection.apply(currentState[name], event); - } - this.state = nextState; - return nextState; - } - - /** - * Reset all child projections and rebuild composed state. - * @returns {object} - */ - reset() { - for (const projection of Object.values(this.projections)) { - projection.reset(); - } - this.state = Object.fromEntries( - Object.entries(this.projections).map(([projectionName, projection]) => [projectionName, projection.state]) - ); - return this.state; - } - - /** - * Serialize composed projection metadata recursively. - * @param {function(string): string} [hmac] - * @returns {object} - */ - toMetadata(hmac = this.hmac) { - return { - kind: 'composite-projection', - name: this.name, - matcher: this.matcher ? buildMetadataForMatcher(this.matcher, hmac) : null, - projections: Object.fromEntries( - Object.entries(this.projections).map(([name, projection]) => [name, projection.toMetadata(hmac)]) - ) - }; - } - - /** - * Restore a composed projection from serialized metadata. - * @param {object} metadata - * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] - * @returns {CompositeProjection} - */ - static fromMetadata(metadata, options = {}) { - const hmac = options.hmac; - const deserializeMatcher = (matcherMetadata) => { - if (!matcherMetadata) { - return undefined; - } - if (typeof matcherMetadata.matcher === 'string') { - assert(typeof hmac === 'function', 'Must provide options.hmac to restore function projections.'); - } - return buildMatcherFromMetadata(matcherMetadata, hmac); - }; - const projections = Object.fromEntries( - Object.entries(metadata.projections || {}).map(([name, projectionMetadata]) => [ - name, - Projection.fromMetadata(projectionMetadata, options) - ]) - ); - return new CompositeProjection(metadata.name, projections, { - ...options, - matcher: deserializeMatcher(metadata.matcher) - }); - } -} +const CompositeProjection = createCompositeProjectionClass(Projection); export default Projection; export { CompositeProjection }; diff --git a/test/Consumer.spec.js b/test/Consumer.spec.js index e53d8ddd..f8d36528 100644 --- a/test/Consumer.spec.js +++ b/test/Consumer.spec.js @@ -355,26 +355,28 @@ describe('Consumer', function() { }, 15); }); - it('swallows persistence write errors and removes temp files', function(done) { + it('rethrows persistence write errors and removes temp files', function() { consumer = new Consumer(storage, 'foobar', 'consumer-1'); consumer.position = 1; consumer.state = Object.freeze({ foo: 1 }); const tmpFile = consumer.fileName + '.1'; const originalWriteFileSync = fsNative.writeFileSync; + const originalSetImmediate = global.setImmediate; + global.setImmediate = (fn) => { + fn(); + return null; + }; fsNative.writeFileSync = () => { const error = new Error('disk full'); error.code = 'ENOSPC'; throw error; }; - consumer.persist(); - - setTimeout(() => { - fsNative.writeFileSync = originalWriteFileSync; - expect(fs.existsSync(tmpFile)).to.be(false); - done(); - }, 15); + expect(() => consumer.persist()).to.throwError(/disk full/); + fsNative.writeFileSync = originalWriteFileSync; + global.setImmediate = originalSetImmediate; + expect(fs.existsSync(tmpFile)).to.be(false); }); it('restores state after reopening', function(done) { From ace421894e769e36d338df4027312f1c658ea9c3 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sat, 6 Jun 2026 11:58:10 +0200 Subject: [PATCH 09/40] Code cleanup for the matcher logic --- src/utils/jsonUtil.js | 5 +- src/utils/metadataUtil.js | 182 ++++++++++++++++---------------------- 2 files changed, 78 insertions(+), 109 deletions(-) diff --git a/src/utils/jsonUtil.js b/src/utils/jsonUtil.js index 29372970..6dbb949b 100644 --- a/src/utils/jsonUtil.js +++ b/src/utils/jsonUtil.js @@ -149,11 +149,8 @@ function findJsonValueEnd(buffer, offset) { * Supports strings, numbers, booleans, and null. */ function parseJsonValue(buffer, startOffset, endOffset) { - const valueStr = buffer.toString('utf8', startOffset, endOffset).trim(); - if (!valueStr) { - return undefined; - } try { + const valueStr = buffer.toString('utf8', startOffset, endOffset); return JSON.parse(valueStr); } catch { return undefined; diff --git a/src/utils/metadataUtil.js b/src/utils/metadataUtil.js index 3316bf31..a036423d 100644 --- a/src/utils/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -4,14 +4,15 @@ import { BYTE_OPEN_OBJECT, indexOfSameLevel, findJsonValueEnd, parseJsonValue } const compiledOperatorMatcherCache = new WeakMap(); +function isObject(value) { + return value !== null && typeof value === 'object'; +} + function isPlainObject(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); } function isOperatorObject(obj) { - if (!isPlainObject(obj)) { - return false; - } const keys = Object.keys(obj); return keys.length > 0 && keys.every(key => key.startsWith('$')); } @@ -21,11 +22,11 @@ function isOperatorObject(obj) { * so callers don't need to know the shape of `matcherValue`. */ function propertyMatchesValue(documentValue, matcherValue) { - if (Array.isArray(matcherValue)) { - return matcherValue.includes(documentValue); - } else if (matcherValue && typeof matcherValue === 'object') { - const operatorChecks = getCompiledOperatorChecks(matcherValue); - if (operatorChecks) { + if (isObject(matcherValue)) { + if (Array.isArray(matcherValue)) { + return matcherValue.includes(documentValue); + } else if (isOperatorObject(matcherValue)) { + const operatorChecks = getCompiledOperatorChecks(matcherValue); return matchesCompiledOperators(documentValue, operatorChecks); } return matches(documentValue, matcherValue); @@ -75,10 +76,6 @@ function getCompiledOperatorChecks(operatorObj) { if (cachedChecks) { return cachedChecks; } - /* c8 ignore next */ - if (!isOperatorObject(operatorObj)) { - return null; - } const checks = buildOperatorChecks(operatorObj); compiledOperatorMatcherCache.set(operatorObj, checks); return checks; @@ -232,39 +229,6 @@ function buildRawBufferMatcher(matcher = {}) { }; } -/** - * Optimization pass: check that every required byte pattern is present anywhere in the buffer - * before spending the more expensive per-depth scan in `matchesNode`. - */ -function preCheck(buffer, startOffset, node) { - for (const child of node.children) { - if (child.valuePatterns && !child.valuePatterns.some((pattern, i) => { - child.valueMatches[i] = buffer.indexOf(pattern, startOffset); - return child.valueMatches[i] !== -1; - })) { - return false; - } - if (child.operators) { - const keyPos = buffer.indexOf(child.operatorKeyPrefix, startOffset); - if (keyPos === -1) { - return false; - } - child.operatorMatch = keyPos; - } - if (child.objectPattern) { - const objectMatch = buffer.indexOf(child.objectPattern, startOffset); - if (objectMatch === -1) { - return false; - } - child.objMatch = objectMatch; - if (!preCheck(buffer, objectMatch, child.node)) { - return false; - } - } - } - return true; -} - /** * Pre-compile a plain object matcher into a tree of byte patterns so `matchesNode` can scan * raw JSON buffers without deserializing them. @@ -279,50 +243,50 @@ function buildMatcherTree(matcher) { return node; } +/** + * Build a matcher over a predicate for an array of patterns that checks if any one pattern matches + */ +function matchAny(patterns) { + return (predicate) => patterns.some(predicate); +} +function matchOne(pattern) { + return (predicate) => predicate(pattern, 0); +} + /** * Decide which matching strategy to use for a key/value pair and build the corresponding * child node with pre-computed byte patterns or compiled operator checks. */ function buildMatcherTreeChild(key, value) { const keyPrefix = Buffer.from(`${JSON.stringify(key)}:`, 'utf8'); - const operatorChecks = isPlainObject(value) - ? getCompiledOperatorChecks(value) - : null; const child = { - objectPattern: null, - valuePatterns: null, - operators: null, + match: () => false, + isKeyPattern: false, operatorChecks: null, - operatorKeyPrefix: null, - operatorMatch: null, node: null, - objMatch: null, - valueMatches: [] + lastMatches: [] // Cached positions of indexOf patterns }; - if (Array.isArray(value)) { - if (value.some(item => item && typeof item === 'object')) { - throw new TypeError('Array matcher values must be scalars.'); + if (isObject(value)) { + if (Array.isArray(value)) { + assert(!value.some(isObject), 'Array matcher values must be scalars.', TypeError); + + child.match = matchAny(value.map(item => buildValuePattern(keyPrefix, item))); + } else if ('$eq' in value && Object.keys(value).length === 1) { + // A lone $eq is semantically identical to a scalar equality check — fold it into a + // value pattern at compile time so the buffer scan takes the same fast path as { key: value }. + child.match = matchOne(buildValuePattern(keyPrefix, value['$eq'])); + } else if (isOperatorObject(value)) { + child.operatorChecks = getCompiledOperatorChecks(value); + child.isKeyPattern = true; + child.match = matchOne(keyPrefix); + } else { + child.match = matchOne(Buffer.concat([keyPrefix, Buffer.from('{', 'utf8')])); + child.node = buildMatcherTree(value); } - child.valuePatterns = value.map(item => buildValuePattern(keyPrefix, item)); - return child; - } else if (operatorChecks) { - // A lone $eq is semantically identical to a scalar equality check — fold it into a - // value pattern at compile time so the buffer scan takes the same fast path as { key: value }. - if ('$eq' in value && Object.keys(value).length === 1) { - child.valuePatterns = [buildValuePattern(keyPrefix, value['$eq'])]; - return child; - } - child.operators = value; - child.operatorChecks = operatorChecks; - child.operatorKeyPrefix = keyPrefix; - return child; - } else if (value && typeof value === 'object') { - child.objectPattern = Buffer.concat([keyPrefix, Buffer.from('{', 'utf8')]); - child.node = buildMatcherTree(value); - return child; + } else { + child.match = matchOne(buildValuePattern(keyPrefix, value)); } - child.valuePatterns = [buildValuePattern(keyPrefix, value)]; return child; } @@ -330,30 +294,47 @@ function buildValuePattern(keyPrefix, value) { return Buffer.concat([keyPrefix, Buffer.from(JSON.stringify(value), 'utf8')]); } +function childPatternsExist(buffer, pattern, i, startOffset, child) { + child.lastMatches[i] = buffer.indexOf(pattern, startOffset); + if (child.lastMatches[i] === -1) { + return false; + } + return !(child.node && !preCheck(buffer, child.lastMatches[0], child.node)); +} + /** - * Verify that each required byte pattern in the tree is present at the correct JSON nesting - * depth so values inside nested objects don't satisfy a top-level match requirement. + * Optimization pass: check that every required byte pattern is present anywhere in the buffer + * before spending the more expensive per-depth scan in `matchesNode`. */ -function matchesNode(buffer, startOffset, node) { +function preCheck(buffer, startOffset, node) { for (const child of node.children) { - if (child.valuePatterns && !child.valuePatterns.some((pattern, i) => { - return indexOfSameLevel(buffer, pattern, startOffset, child.valueMatches[i]) !== -1; - })) { + //child.lastMatches.fill(-1); // this would be the correct thing to do, but would do an array fill on every level for every check + if (!child.match((pattern, i) => childPatternsExist(buffer, pattern, i, startOffset, child))) { return false; } + } + return true; +} - if (child.operators && !matchesOperatorInBuffer(buffer, startOffset, child)) { - return false; - } +function matchesChild(buffer, pattern, startOffset, lastMatchPosition, child) { + const matchPosition = indexOfSameLevel(buffer, pattern, startOffset, lastMatchPosition, child.isKeyPattern); + if (matchPosition === -1) { + return false; + } + if (child.operatorChecks && !matchesOperatorInBuffer(buffer, matchPosition + pattern.length, child.operatorChecks)) { + return false; + } + return !child.node || matchesNode(buffer, matchPosition + pattern.length, child.node); +} - if (child.node) { - const objectIndex = indexOfSameLevel(buffer, child.objectPattern, startOffset, child.objMatch); - if (objectIndex === -1) { - return false; - } - if (!matchesNode(buffer, objectIndex + child.objectPattern.length, child.node)) { - return false; - } +/** + * Verify that each required byte pattern in the tree is present at the correct JSON nesting + * depth so values inside nested objects don't satisfy a top-level match requirement. + */ +function matchesNode(buffer, startOffset, node) { + for (const child of node.children) { + if (!child.match((pattern, i) => matchesChild(buffer, pattern, startOffset, child.lastMatches[i], child))) { + return false; } } @@ -364,17 +345,8 @@ function matchesNode(buffer, startOffset, node) { * Check if a value at the given key-offset matches the specified operators. * Finds the key pattern, extracts the value, parses it, and applies operators. */ -function matchesOperatorInBuffer(buffer, startOffset, child) { - const keyPos = indexOfSameLevel(buffer, child.operatorKeyPrefix, startOffset, child.operatorMatch, true); - if (keyPos === -1) { - return false; - } - - const valueStart = keyPos + child.operatorKeyPrefix.length; - if (valueStart >= buffer.length) { - return false; - } - +function matchesOperatorInBuffer(buffer, startOffset, operatorChecks) { + const valueStart = startOffset; const valueEnd = findJsonValueEnd(buffer, valueStart); if (valueEnd === -1 || valueEnd <= valueStart) { return false; @@ -382,7 +354,7 @@ function matchesOperatorInBuffer(buffer, startOffset, child) { const parsedValue = parseJsonValue(buffer, valueStart, valueEnd); - return matchesCompiledOperators(parsedValue, child.operatorChecks); + return matchesCompiledOperators(parsedValue, operatorChecks); } export { From 8b102ff45a4dc04c4fdcc91395fbfc6956128787 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sat, 6 Jun 2026 14:22:07 +0200 Subject: [PATCH 10/40] Optimize multi value matcher Matching against any of an array of values is now faster by a factor of ~2 than previous implementation, scales better for more match items and stays close to the normal deep exact match case. --- AGENTS.md | 1 + bench/bench-matcher.js | 7 +- src/utils/jsonUtil.js | 72 +++++++++++++-- src/utils/metadataUtil.js | 184 ++++++++++++++++++++++++-------------- test/jsonUtil.spec.js | 30 ++++++- 5 files changed, 217 insertions(+), 77 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 26f3969e..7db81a99 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,7 @@ Do not repeat yourself. Be concise and precise in your answers. No paraphrasing - **Pass cheap prefilter results as hints to expensive scans**: when a cheap pass (e.g. `Buffer.indexOf`) already locates a candidate position, carry that result forward as a hint to the costlier depth- or context-aware pass rather than letting it re-scan from the start. Eliminates a full O(n) scan on every call. - **Custom implementations only when benchmarks justify the complexity**: a hand-rolled solution may edge out a standard library call by 10–15 %, but the maintenance cost is rarely worth it unless the gain is substantial and measurable at realistic data sizes (e.g. a custom numeric parser vs. `JSON.parse` with try/catch). - **Prefer a flag on a shared helper over near-duplicate functions**: when two functions differ only in a single behavioral detail, add a boolean parameter to the shared function rather than maintaining two near-identical copies. Keep the flag's semantics explicit and limited to one axis of variation. +- **Inline single-use hot-path helpers once the surrounding flow becomes simpler**: if a helper is only called from one place and its logic can be embedded while keeping the caller below the local complexity/return-count budget, prefer the inline version. Removes call indirection and often makes the hot path easier to read as straight-line pseudo-code. ## Architecture diff --git a/bench/bench-matcher.js b/bench/bench-matcher.js index 7b91256c..83be7ac0 100644 --- a/bench/bench-matcher.js +++ b/bench/bench-matcher.js @@ -11,7 +11,8 @@ const MEASURED_RUNS = 5; const matcherObjects = [ { name: 'flat', matcher: { type: 'Foo' } }, { name: 'scoped', matcher: { payload: { type: 'Foo' } } }, - { name: 'multi', matcher: { payload: { type: ['Foo', 'Baz'] } } }, + { name: 'multi', matcher: { payload: { type: ['BazingaHappened', 'Foo'] } } }, + { name: 'multi4', matcher: { payload: { type: ['BazingaHappened', 'BarxingaHappened', 'QuuxingaHappened', 'Foo'] } } }, { name: '$gt', matcher: { payload: { value: { $gt: 100 } } } }, { name: '$eq', matcher: { payload: { type: { $eq: 'Foo' } } } } ]; @@ -19,7 +20,9 @@ const matcherObjects = [ const implementations = item => [ { name: `${item.name}-obj`, impl: 'obj', fn: (doc) => matches(doc, item.matcher) }, { name: `${item.name}-raw`, impl: 'raw', fn: buildRawBufferMatcher(item.matcher) }, - { name: `${item.name}-+obj`, impl: '+obj', fn: (buffer) => matches(JSON.parse(buffer.toString('utf8')), item.matcher) } + // The following is a reference benchmark against realistic usage, where an obj matcher requires a deserialization, which is costly. + // Do NOT remove the commented implementation case. + //{ name: `${item.name}-+obj`, impl: '+obj', fn: (buffer) => matches(JSON.parse(buffer.toString('utf8')), item.matcher) } ]; const implementationsCount = implementations(matcherObjects[0]).length; // assuming all have same count diff --git a/src/utils/jsonUtil.js b/src/utils/jsonUtil.js index 6dbb949b..82fe84a0 100644 --- a/src/utils/jsonUtil.js +++ b/src/utils/jsonUtil.js @@ -9,6 +9,10 @@ const BYTE_COMMA = 0x2c; /** * Advance past a JSON string whose opening `"` is at `i`. * Returns the position after the closing `"`, or -1 if the string is unterminated. + * + * @param {Buffer} buffer + * @param {number} i + * @returns {number} */ function skipString(buffer, i) { let j = i + 1; @@ -51,10 +55,21 @@ function isClosingBracket(char) { return char === BYTE_CLOSE_OBJECT || char === BYTE_CLOSE_ARRAY; } +/** + * @param {number} char + * @returns {boolean} + */ function isOpeningObject(char) { return char === BYTE_OPEN_OBJECT; } +/** + * @param {Buffer} buffer + * @param {Buffer} pattern + * @param {number} startOffset + * @param {number|undefined} lastMatchPosition + * @returns {number} + */ function nextIndexOf(buffer, pattern, startOffset, lastMatchPosition) { if (lastMatchPosition === undefined || lastMatchPosition < startOffset) { return buffer.indexOf(pattern, startOffset); @@ -72,6 +87,13 @@ function nextIndexOf(buffer, pattern, startOffset, lastMatchPosition) { * For key patterns (`"key":`) pass `isKeyPattern=true` to skip that trailing delimiter check. * Returns -1 when no such position exists before the end of the buffer or when a closing brace * reduces depth below zero (the top-level object has ended). + * + * @param {Buffer} buffer + * @param {Buffer} pattern + * @param {number} [startOffset=0] + * @param {number|undefined} [matchPosition=undefined] + * @param {boolean} [isKeyPattern=false] + * @returns {number} */ function indexOfSameLevel(buffer, pattern, startOffset = 0, matchPosition = undefined, isKeyPattern = false) { let depth = 0; @@ -123,10 +145,11 @@ function indexOfSameLevel(buffer, pattern, startOffset = 0, matchPosition = unde } /** - * Extract the end position (exclusive) of a JSON scalar value starting at `offset`. - * `offset` should point to the first character of the value ('"' for strings, digit/-/true/false/null for others). - * Returns the position after the value ends (past the closing quote for strings, past the last digit/char for others). - * Returns -1 if the buffer is malformed. + * Find the end of a scalar JSON value so operator matching can parse only the relevant slice. + * + * @param {Buffer} buffer + * @param {number} offset + * @returns {number} */ function findJsonValueEnd(buffer, offset) { /* c8 ignore next 3 */ @@ -145,8 +168,12 @@ function findJsonValueEnd(buffer, offset) { } /** - * Convert a JSON scalar value buffer to a comparable JavaScript value for range operators. - * Supports strings, numbers, booleans, and null. + * Parse one scalar JSON slice only after a byte-level match has already narrowed the candidate. + * + * @param {Buffer} buffer + * @param {number} startOffset + * @param {number} endOffset + * @returns {string|number|boolean|null|undefined} */ function parseJsonValue(buffer, startOffset, endOffset) { try { @@ -157,4 +184,35 @@ function parseJsonValue(buffer, startOffset, endOffset) { } } -export { BYTE_OPEN_OBJECT, BYTE_CLOSE_OBJECT, isOpeningObject, indexOfSameLevel, findJsonValueEnd, parseJsonValue }; +/** + * Compare a matched key's scalar value against pre-serialized candidates without reparsing JSON. + * + * @param {Buffer} buffer + * @param {number} valueStart + * @param {Buffer[]} patterns + * @returns {boolean} + */ +function matchesAnyValuePattern(buffer, valueStart, patterns) { + for (const pattern of patterns) { + const valueEnd = valueStart + pattern.length; + if (valueEnd > buffer.length) { + continue; + } + let matches = true; + for (let i = 0; i < pattern.length; i++) { + if (buffer[valueStart + i] !== pattern[i]) { + matches = false; + break; + } + } + if (!matches) { + continue; + } + if (isDelimiter(buffer[valueEnd])) { + return true; + } + } + return false; +} + +export { isOpeningObject, indexOfSameLevel, findJsonValueEnd, parseJsonValue, matchesAnyValuePattern }; diff --git a/src/utils/metadataUtil.js b/src/utils/metadataUtil.js index a036423d..b7140c81 100644 --- a/src/utils/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -1,17 +1,35 @@ import crypto from 'crypto'; -import { assert, assertEqual } from './util.js'; -import { BYTE_OPEN_OBJECT, indexOfSameLevel, findJsonValueEnd, parseJsonValue } from './jsonUtil.js'; +import {assert, assertEqual} from './util.js'; +import { + indexOfSameLevel, + findJsonValueEnd, + parseJsonValue, + matchesAnyValuePattern, + isOpeningObject +} from './jsonUtil.js'; const compiledOperatorMatcherCache = new WeakMap(); +/** + * @param {any} value + * @returns {boolean} + */ function isObject(value) { return value !== null && typeof value === 'object'; } +/** + * @param {any} value + * @returns {boolean} + */ function isPlainObject(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); } +/** + * @param {object} obj + * @returns {boolean} + */ function isOperatorObject(obj) { const keys = Object.keys(obj); return keys.length > 0 && keys.every(key => key.startsWith('$')); @@ -20,6 +38,10 @@ function isOperatorObject(obj) { /** * Dispatch between array (OR), operator object, nested object, and scalar equality matching * so callers don't need to know the shape of `matcherValue`. + * + * @param {any} documentValue + * @param {any} matcherValue + * @returns {boolean} */ function propertyMatchesValue(documentValue, matcherValue) { if (isObject(matcherValue)) { @@ -37,6 +59,9 @@ function propertyMatchesValue(documentValue, matcherValue) { /** * Pre-compile an operator object into an array of comparison closures so the hot path avoids * repeated `Object.entries` + switch dispatch per matched document. + * + * @param {object} operatorObj + * @returns {Array} */ function buildOperatorChecks(operatorObj) { const checks = []; @@ -70,6 +95,9 @@ function buildOperatorChecks(operatorObj) { /** * Return cached compiled checks for `operatorObj`, compiling and caching on first access. * Using WeakMap keeps operator objects GC-eligible and avoids mutating user-supplied objects. + * + * @param {object} operatorObj + * @returns {Array} */ function getCompiledOperatorChecks(operatorObj) { const cachedChecks = compiledOperatorMatcherCache.get(operatorObj); @@ -81,6 +109,11 @@ function getCompiledOperatorChecks(operatorObj) { return checks; } +/** + * @param {any} documentValue + * @param {Array} checks + * @returns {boolean} + */ function matchesCompiledOperators(documentValue, checks) { if (documentValue === undefined) { return false; @@ -120,10 +153,10 @@ function buildMetadataHeader(magic, metadata) { * @returns {function(string)} A function that calculates the HMAC for a given string */ const createHmac = secret => string => { - const hmac = crypto.createHmac('sha256', secret); - hmac.update(string); - return hmac.digest('hex'); - }; + const hmac = crypto.createHmac('sha256', secret); + hmac.update(string); + return hmac.digest('hex'); +}; /** * @typedef {object|function(object):boolean} Matcher @@ -159,10 +192,10 @@ function buildMetadataForMatcher(matcher, hmac) { return undefined; } if (typeof matcher === 'object') { - return { matcher }; + return {matcher}; } const matcherString = matcher.toString(); - return { matcher: matcherString, hmac: hmac(matcherString) }; + return {matcher: matcherString, hmac: hmac(matcherString)}; } /** @@ -192,19 +225,18 @@ function buildMatcherFromMetadata(matcherMetadata, hmac) { */ function buildTypeMatcherFn(payloadPath) { const parts = payloadPath.split('.'); - return function(typeValue) { + return function (typeValue) { let obj = typeValue; for (let i = parts.length - 1; i >= 0; i--) { - obj = { [parts[i]]: obj }; + obj = {[parts[i]]: obj}; } - return { payload: obj }; + return {payload: obj}; }; } /** - * Builds a raw-buffer matcher. - * It expects the Buffer to contain compact stringified JSON - * and supports matcher objects with sub properties and multi-value matches (OR/any of). + * Compile an object matcher into a raw-buffer predicate so raw-mode reads can filter compact + * JSON without parsing every document first. * * @param {object} matcher Object matcher. * @returns {function(Buffer): boolean} @@ -219,7 +251,7 @@ function buildRawBufferMatcher(matcher = {}) { } return function matchesRawBuffer(buffer) { - if (buffer[0] !== BYTE_OPEN_OBJECT) { + if (!isOpeningObject(buffer[0])) { return false; } if (!preCheck(buffer, 1, root)) { @@ -230,11 +262,14 @@ function buildRawBufferMatcher(matcher = {}) { } /** - * Pre-compile a plain object matcher into a tree of byte patterns so `matchesNode` can scan - * raw JSON buffers without deserializing them. + * Compile a matcher object into a tree whose children each carry one primary byte pattern plus + * optional follow-up checks for nested objects, operators, or multi-value scalars. + * + * @param {object} matcher + * @returns {{children: Array}} */ function buildMatcherTree(matcher) { - const node = { children: [] }; + const node = {children: []}; for (const [key, value] of Object.entries(matcher)) { node.children.push(buildMatcherTreeChild(key, value)); @@ -244,96 +279,106 @@ function buildMatcherTree(matcher) { } /** - * Build a matcher over a predicate for an array of patterns that checks if any one pattern matches - */ -function matchAny(patterns) { - return (predicate) => patterns.some(predicate); -} -function matchOne(pattern) { - return (predicate) => predicate(pattern, 0); -} - -/** - * Decide which matching strategy to use for a key/value pair and build the corresponding - * child node with pre-computed byte patterns or compiled operator checks. + * Normalize one matcher property into the cheapest raw-buffer strategy for that value shape. + * + * @param {string} key + * @param {any} value + * @returns {{pattern: Buffer, isKeyPattern: boolean, operatorChecks: (Array|null), valuePatterns: (Buffer[]|null), node: ({children: Array}|null), lastMatch: number}} */ function buildMatcherTreeChild(key, value) { const keyPrefix = Buffer.from(`${JSON.stringify(key)}:`, 'utf8'); const child = { - match: () => false, + pattern: null, isKeyPattern: false, operatorChecks: null, + valuePatterns: null, node: null, - lastMatches: [] // Cached positions of indexOf patterns + lastMatch: -1 }; if (isObject(value)) { if (Array.isArray(value)) { assert(!value.some(isObject), 'Array matcher values must be scalars.', TypeError); - - child.match = matchAny(value.map(item => buildValuePattern(keyPrefix, item))); + if (value.length === 1) { + child.pattern = buildKeyValuePattern(keyPrefix, value[0]); + } else { + child.isKeyPattern = true; + child.pattern = keyPrefix; + child.valuePatterns = value.map(item => Buffer.from(JSON.stringify(item), 'utf8')); + } } else if ('$eq' in value && Object.keys(value).length === 1) { // A lone $eq is semantically identical to a scalar equality check — fold it into a // value pattern at compile time so the buffer scan takes the same fast path as { key: value }. - child.match = matchOne(buildValuePattern(keyPrefix, value['$eq'])); + child.pattern = buildKeyValuePattern(keyPrefix, value['$eq']); } else if (isOperatorObject(value)) { child.operatorChecks = getCompiledOperatorChecks(value); child.isKeyPattern = true; - child.match = matchOne(keyPrefix); + child.pattern = keyPrefix; } else { - child.match = matchOne(Buffer.concat([keyPrefix, Buffer.from('{', 'utf8')])); + child.pattern = Buffer.concat([keyPrefix, Buffer.from('{', 'utf8')]); child.node = buildMatcherTree(value); } } else { - child.match = matchOne(buildValuePattern(keyPrefix, value)); + child.pattern = buildKeyValuePattern(keyPrefix, value); } return child; } -function buildValuePattern(keyPrefix, value) { +/** + * @param {Buffer} keyPrefix + * @param {any} value + * @returns {Buffer} + */ +function buildKeyValuePattern(keyPrefix, value) { return Buffer.concat([keyPrefix, Buffer.from(JSON.stringify(value), 'utf8')]); } -function childPatternsExist(buffer, pattern, i, startOffset, child) { - child.lastMatches[i] = buffer.indexOf(pattern, startOffset); - if (child.lastMatches[i] === -1) { - return false; - } - return !(child.node && !preCheck(buffer, child.lastMatches[0], child.node)); -} - /** - * Optimization pass: check that every required byte pattern is present anywhere in the buffer - * before spending the more expensive per-depth scan in `matchesNode`. + * Cheap pass: confirm each child's primary pattern exists somewhere and cache that position as a + * hint for the depth-aware pass that follows. + * + * @param {Buffer} buffer + * @param {number} startOffset + * @param {{children: Array}} node + * @returns {boolean} */ function preCheck(buffer, startOffset, node) { for (const child of node.children) { - //child.lastMatches.fill(-1); // this would be the correct thing to do, but would do an array fill on every level for every check - if (!child.match((pattern, i) => childPatternsExist(buffer, pattern, i, startOffset, child))) { + child.lastMatch = buffer.indexOf(child.pattern, startOffset); + if (child.lastMatch === -1) { + return false; + } + if (child.node && !preCheck(buffer, child.lastMatch, child.node)) { return false; } } return true; } -function matchesChild(buffer, pattern, startOffset, lastMatchPosition, child) { - const matchPosition = indexOfSameLevel(buffer, pattern, startOffset, lastMatchPosition, child.isKeyPattern); - if (matchPosition === -1) { - return false; - } - if (child.operatorChecks && !matchesOperatorInBuffer(buffer, matchPosition + pattern.length, child.operatorChecks)) { - return false; - } - return !child.node || matchesNode(buffer, matchPosition + pattern.length, child.node); -} - /** - * Verify that each required byte pattern in the tree is present at the correct JSON nesting - * depth so values inside nested objects don't satisfy a top-level match requirement. + * Confirm that each prechecked child really matches at the requested JSON level and then run the + * optional value-specific follow-up checks from the compiled tree. + * + * @param {Buffer} buffer + * @param {number} startOffset + * @param {{children: Array}} node + * @returns {boolean} */ function matchesNode(buffer, startOffset, node) { for (const child of node.children) { - if (!child.match((pattern, i) => matchesChild(buffer, pattern, startOffset, child.lastMatches[i], child))) { + const matchPosition = indexOfSameLevel(buffer, child.pattern, startOffset, child.lastMatch, child.isKeyPattern); + if (matchPosition === -1) { + return false; + } + + const valueStart = matchPosition + child.pattern.length; + if (child.valuePatterns && !matchesAnyValuePattern(buffer, valueStart, child.valuePatterns)) { + return false; + } + if (child.operatorChecks && !matchesOperatorInBuffer(buffer, valueStart, child.operatorChecks)) { + return false; + } + if (child.node && !matchesNode(buffer, valueStart, child.node)) { return false; } } @@ -342,8 +387,13 @@ function matchesNode(buffer, startOffset, node) { } /** - * Check if a value at the given key-offset matches the specified operators. - * Finds the key pattern, extracts the value, parses it, and applies operators. + * Parse the scalar at a matched key position only when operator objects require a real JavaScript + * comparison instead of a pure byte-pattern check. + * + * @param {Buffer} buffer + * @param {number} startOffset + * @param {Array} operatorChecks + * @returns {boolean} */ function matchesOperatorInBuffer(buffer, startOffset, operatorChecks) { const valueStart = startOffset; diff --git a/test/jsonUtil.spec.js b/test/jsonUtil.spec.js index bcff6b26..4fcd8c99 100644 --- a/test/jsonUtil.spec.js +++ b/test/jsonUtil.spec.js @@ -1,5 +1,5 @@ import expect from 'expect.js'; -import { indexOfSameLevel, findJsonValueEnd, parseJsonValue } from '../src/utils/jsonUtil.js'; +import { indexOfSameLevel, findJsonValueEnd, parseJsonValue, matchesAnyValuePattern } from '../src/utils/jsonUtil.js'; describe('jsonUtil', function() { @@ -84,6 +84,34 @@ describe('jsonUtil', function() { }); + describe('matchesAnyValuePattern', function() { + + it('matches any exact scalar candidate at the given offset', function() { + const buffer = Buffer.from('{"type":"Foo","id":1}', 'utf8'); + const valueStart = buffer.indexOf(Buffer.from('"Foo"', 'utf8')); + const patterns = [Buffer.from('"Bar"', 'utf8'), Buffer.from('"Foo"', 'utf8')]; + + expect(matchesAnyValuePattern(buffer, valueStart, patterns)).to.be(true); + }); + + it('rejects prefix matches like 5 for an actual value 50', function() { + const buffer = Buffer.from('{"amount":50}', 'utf8'); + const valueStart = buffer.indexOf(Buffer.from('50', 'utf8')); + const patterns = [Buffer.from('5', 'utf8')]; + + expect(matchesAnyValuePattern(buffer, valueStart, patterns)).to.be(false); + }); + + it('returns false when none of the candidates match at the exact offset', function() { + const buffer = Buffer.from('{"type":"Foo"}', 'utf8'); + const valueStart = buffer.indexOf(Buffer.from('"Foo"', 'utf8')); + const patterns = [Buffer.from('"Bar"', 'utf8'), Buffer.from('"Baz"', 'utf8')]; + + expect(matchesAnyValuePattern(buffer, valueStart, patterns)).to.be(false); + }); + + }); + }); From 46f248a9940821b287e978eab33daa2e586fec85 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sat, 6 Jun 2026 14:36:34 +0200 Subject: [PATCH 11/40] Update docblocks --- AGENTS.md | 1 + src/utils/apiHelpers.js | 53 +++++++++++++++++++++++++++ src/utils/fsUtil.js | 48 +++++++++++++------------ src/utils/jsonUtil.js | 66 +++++++++++++++++----------------- src/utils/metadataUtil.js | 76 +++++++++++++++++++-------------------- src/utils/util.js | 35 +++++++++--------- 6 files changed, 170 insertions(+), 109 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7db81a99..e70738bb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,6 +3,7 @@ ## General guidelines for interaction Do not repeat yourself. Be concise and precise in your answers. No paraphrasing of previous information unless specifically asked for. +The code and documentation language is English, even if the user is communicating in another language. **Keep this file up-to-date**: after any code review that surfaces new architectural insights or a new principle, compact this file and integrate what was learned. ## Principles (in priority order) diff --git a/src/utils/apiHelpers.js b/src/utils/apiHelpers.js index 93a40980..d6f18aa3 100644 --- a/src/utils/apiHelpers.js +++ b/src/utils/apiHelpers.js @@ -1,3 +1,14 @@ +/** + * Normalize commit overloads into a single argument object. + * + * @param {object|object[]} events Event or event list. + * @param {number|object|function} expectedVersion Expected version, CommitCondition, or already metadata/callback. + * @param {object|function} metadata Commit metadata or callback. + * @param {function} callback Completion callback. + * @param {number} ExpectedVersionAny Fallback value for "any" expectedVersion. + * @param {Function} CommitConditionClass Class used for CommitCondition checks. + * @returns {{events: object[], expectedVersion: number|object, metadata: object, callback: function}} Normalized commit arguments. + */ function fixCommitArgumentTypes(events, expectedVersion, metadata, callback, ExpectedVersionAny, CommitConditionClass) { if (!(events instanceof Array)) { events = [events]; @@ -17,6 +28,12 @@ function fixCommitArgumentTypes(events, expectedVersion, metadata, callback, Exp return { events, expectedVersion, metadata, callback }; } +/** + * Derive the stream name from an index name. + * + * @param {string} indexName Index file/index name. + * @returns {string} Corresponding stream name. + */ function parseStreamFromIndexName(indexName) { if (indexName === '_all') { return '_all'; @@ -27,6 +44,13 @@ function parseStreamFromIndexName(indexName) { return indexName; } +/** + * Support predicate/raw shorthand (`predicate=true`). + * + * @param {object|function|boolean|null} predicate Filter predicate or raw shorthand. + * @param {boolean} raw Raw flag from the call signature. + * @returns {{predicate: object|function|null, raw: boolean}} Normalized predicate/raw pair. + */ function normalizePredicateRaw(predicate, raw) { if (typeof predicate === 'boolean' && raw === false) { return { predicate: null, raw: predicate }; @@ -34,6 +58,14 @@ function normalizePredicateRaw(predicate, raw) { return { predicate, raw }; } +/** + * Support constructor overloads with optional `name`. + * + * @param {string|object} name Name or already the options object. + * @param {object} options Options object when `name` is provided. + * @param {string|undefined} [fallbackName=undefined] Fallback name when no explicit `name` is passed. + * @returns {{name: string|undefined, options: object}} Normalized name/options pair. + */ function normalizeNamedCtorArgs(name, options, fallbackName = undefined) { if (typeof name !== 'string') { return { name: fallbackName, options: name }; @@ -41,14 +73,35 @@ function normalizeNamedCtorArgs(name, options, fallbackName = undefined) { return { name, options }; } +/** + * Normalize negative revisions relative to stream length. + * + * @param {number} version Requested revision. + * @param {number} length Current stream length. + * @returns {number} Resolved revision. + */ function normalizeRevision(version, length) { return version < 0 ? version + length + 1 : version; } +/** + * Clamp and normalize maxRevision, including negative values. + * + * @param {number} length Current stream length. + * @param {number} maxRevision Requested max revision. + * @returns {number} Effective max revision in valid range. + */ function normalizeMaxRevision(length, maxRevision) { return Math.min(length, maxRevision < 0 ? length + maxRevision + 1 : maxRevision); } +/** + * Support the consumer overload where the first argument is a numeric start offset. + * + * @param {object|number} initialState Initial state or numeric start offset. + * @param {number} startFrom Start offset from the call signature. + * @returns {{initialState: object, startFrom: number}} Normalized consumer initialization values. + */ function normalizeConsumerStateArgs(initialState, startFrom) { if (typeof initialState === 'number') { return { initialState: {}, startFrom: initialState }; diff --git a/src/utils/fsUtil.js b/src/utils/fsUtil.js index f84bde57..7c0df566 100644 --- a/src/utils/fsUtil.js +++ b/src/utils/fsUtil.js @@ -4,8 +4,8 @@ import { mkdirpSync } from 'mkdirp'; /** * Ensure that the given directory exists. - * @param {string} dirName - * @return {boolean} true if the directory existed already + * @param {string} dirName Target directory. + * @returns {boolean} True when the directory already existed. */ function ensureDirectory(dirName) { if (!fs.existsSync(dirName)) { @@ -21,9 +21,10 @@ function ensureDirectory(dirName) { /** * Invoke `onEach` if `relativePath` matches `regexPattern`, passing the first capture group or the full match. * - * @param {string} relativePath - * @param {RegExp} regexPattern - * @param {function(string)} onEach + * @param {string} relativePath Relative file path. + * @param {RegExp} regexPattern Regex used for path matching. + * @param {function(string): void} onEach Callback invoked per match. + * @returns {void} */ function visitMatchingPath(relativePath, regexPattern, onEach) { const match = relativePath.match(regexPattern); @@ -35,11 +36,11 @@ function visitMatchingPath(relativePath, regexPattern, onEach) { /** * Classify `entries` into matching files (visited via `onEach`) and subdirectory names (returned). * - * @param {fs.Dirent[]} entries - * @param {string} relativePrefix - * @param {RegExp} regexPattern - * @param {function(string)} onEach - * @returns {string[]} names of subdirectory entries + * @param {fs.Dirent[]} entries Directory entries from one level. + * @param {string} relativePrefix Relative prefix for child entries. + * @param {RegExp} regexPattern Regex for file paths. + * @param {function(string): void} onEach Callback for matching files. + * @returns {string[]} Names of subdirectory entries. */ function classifyEntries(entries, relativePrefix, regexPattern, onEach) { const subdirs = []; @@ -56,12 +57,13 @@ function classifyEntries(entries, relativePrefix, regexPattern, onEach) { /** * Sequentially scan each name in `subdirs`, calling `done` when all are complete or on first error. * - * @param {string[]} subdirs - * @param {string} dir - * @param {string} relativePrefix - * @param {RegExp} regexPattern - * @param {function(string)} onEach - * @param {function(Error?)} done + * @param {string[]} subdirs Subdirectory names. + * @param {string} dir Absolute parent path. + * @param {string} relativePrefix Relative prefix used during recursion. + * @param {RegExp} regexPattern Regex for file paths. + * @param {function(string): void} onEach Callback for matching files. + * @param {function(Error=): void} done Completion callback. + * @returns {void} */ function scanSubdirs(subdirs, dir, relativePrefix, regexPattern, onEach, done) { let i = 0; @@ -79,12 +81,13 @@ function scanSubdirs(subdirs, dir, relativePrefix, regexPattern, onEach, done) { /** * Asynchronously scan one directory level, then recurse into subdirectories sequentially. * - * @param {string} dir - * @param {string} relativePrefix - * @param {boolean} isRoot - * @param {RegExp} regexPattern - * @param {function(string)} onEach - * @param {function(Error?)} done + * @param {string} dir Absolute directory path. + * @param {string} relativePrefix Relative prefix for match paths. + * @param {boolean} isRoot True for the initial call. + * @param {RegExp} regexPattern Regex for file paths. + * @param {function(string): void} onEach Callback for matching files. + * @param {function(Error=): void} done Completion callback. + * @returns {void} */ function scanDir(dir, relativePrefix, isRoot, regexPattern, onEach, done) { fs.readdir(dir, { withFileTypes: true }, (err, entries) => { @@ -112,6 +115,7 @@ function scanDir(dir, relativePrefix, isRoot, regexPattern, onEach, done) { * @param {RegExp} regexPattern The pattern to match relative file paths against. * @param {function(string)} onEach Called with the first capturing group (or full match) for each matching path. * @param {function(Error?)} onDone Called when the scan is complete, or with an error if one occurred. + * @returns {void} */ function scanForFiles(directory, regexPattern, onEach, onDone) { scanDir(directory, '', true, regexPattern, onEach, onDone); diff --git a/src/utils/jsonUtil.js b/src/utils/jsonUtil.js index 82fe84a0..86cd1f4b 100644 --- a/src/utils/jsonUtil.js +++ b/src/utils/jsonUtil.js @@ -10,9 +10,9 @@ const BYTE_COMMA = 0x2c; * Advance past a JSON string whose opening `"` is at `i`. * Returns the position after the closing `"`, or -1 if the string is unterminated. * - * @param {Buffer} buffer - * @param {number} i - * @returns {number} + * @param {Buffer} buffer Source JSON buffer. + * @param {number} i Offset of the opening quote. + * @returns {number} Offset after the closing quote, or -1 if unterminated. */ function skipString(buffer, i) { let j = i + 1; @@ -32,43 +32,43 @@ function skipString(buffer, i) { /** * Check if a character byte is a valid JSON value delimiter (comma, closing brace, or closing bracket). - * @param {number} char - * @returns {boolean} + * @param {number} char Byte value to test. + * @returns {boolean} True when `char` is `,`, `}` or `]`. */ function isDelimiter(char) { return (char === BYTE_COMMA || char === BYTE_CLOSE_OBJECT || char === BYTE_CLOSE_ARRAY); } /** - * @param {number} char - * @returns {boolean} + * @param {number} char Byte value to test. + * @returns {boolean} True when `char` opens an object or array. */ function isOpeningBracket(char) { return char === BYTE_OPEN_OBJECT || char === BYTE_OPEN_ARRAY; } /** - * @param {number} char - * @returns {boolean} + * @param {number} char Byte value to test. + * @returns {boolean} True when `char` closes an object or array. */ function isClosingBracket(char) { return char === BYTE_CLOSE_OBJECT || char === BYTE_CLOSE_ARRAY; } /** - * @param {number} char - * @returns {boolean} + * @param {number} char Byte value to test. + * @returns {boolean} True when `char` is `{`. */ function isOpeningObject(char) { return char === BYTE_OPEN_OBJECT; } /** - * @param {Buffer} buffer - * @param {Buffer} pattern - * @param {number} startOffset - * @param {number|undefined} lastMatchPosition - * @returns {number} + * @param {Buffer} buffer Source JSON buffer. + * @param {Buffer} pattern Pattern to find. + * @param {number} startOffset Search start offset. + * @param {number|undefined} lastMatchPosition Optional cached candidate position. + * @returns {number} Match position or -1. */ function nextIndexOf(buffer, pattern, startOffset, lastMatchPosition) { if (lastMatchPosition === undefined || lastMatchPosition < startOffset) { @@ -88,12 +88,12 @@ function nextIndexOf(buffer, pattern, startOffset, lastMatchPosition) { * Returns -1 when no such position exists before the end of the buffer or when a closing brace * reduces depth below zero (the top-level object has ended). * - * @param {Buffer} buffer - * @param {Buffer} pattern - * @param {number} [startOffset=0] - * @param {number|undefined} [matchPosition=undefined] - * @param {boolean} [isKeyPattern=false] - * @returns {number} + * @param {Buffer} buffer Source JSON buffer. + * @param {Buffer} pattern Serialized key/value pattern. + * @param {number} [startOffset=0] Offset where scanning begins. + * @param {number|undefined} [matchPosition=undefined] Optional cached candidate position. + * @param {boolean} [isKeyPattern=false] Skip value-delimiter validation for key-only patterns. + * @returns {number} Match position at the same JSON depth, or -1. */ function indexOfSameLevel(buffer, pattern, startOffset = 0, matchPosition = undefined, isKeyPattern = false) { let depth = 0; @@ -147,9 +147,9 @@ function indexOfSameLevel(buffer, pattern, startOffset = 0, matchPosition = unde /** * Find the end of a scalar JSON value so operator matching can parse only the relevant slice. * - * @param {Buffer} buffer - * @param {number} offset - * @returns {number} + * @param {Buffer} buffer Source JSON buffer. + * @param {number} offset Offset where the scalar starts. + * @returns {number} End offset (exclusive), or -1 for invalid start. */ function findJsonValueEnd(buffer, offset) { /* c8 ignore next 3 */ @@ -170,10 +170,10 @@ function findJsonValueEnd(buffer, offset) { /** * Parse one scalar JSON slice only after a byte-level match has already narrowed the candidate. * - * @param {Buffer} buffer - * @param {number} startOffset - * @param {number} endOffset - * @returns {string|number|boolean|null|undefined} + * @param {Buffer} buffer Source JSON buffer. + * @param {number} startOffset Start offset (inclusive). + * @param {number} endOffset End offset (exclusive). + * @returns {string|number|boolean|null|undefined} Parsed scalar, or `undefined` on parse failure. */ function parseJsonValue(buffer, startOffset, endOffset) { try { @@ -187,10 +187,10 @@ function parseJsonValue(buffer, startOffset, endOffset) { /** * Compare a matched key's scalar value against pre-serialized candidates without reparsing JSON. * - * @param {Buffer} buffer - * @param {number} valueStart - * @param {Buffer[]} patterns - * @returns {boolean} + * @param {Buffer} buffer Source JSON buffer. + * @param {number} valueStart Offset where the candidate scalar begins. + * @param {Buffer[]} patterns Pre-serialized scalar candidates. + * @returns {boolean} True when any candidate matches exactly and is delimiter-terminated. */ function matchesAnyValuePattern(buffer, valueStart, patterns) { for (const pattern of patterns) { diff --git a/src/utils/metadataUtil.js b/src/utils/metadataUtil.js index b7140c81..0b41c923 100644 --- a/src/utils/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -11,24 +11,24 @@ import { const compiledOperatorMatcherCache = new WeakMap(); /** - * @param {any} value - * @returns {boolean} + * @param {any} value Value to classify. + * @returns {boolean} True when `value` is a non-null object. */ function isObject(value) { return value !== null && typeof value === 'object'; } /** - * @param {any} value - * @returns {boolean} + * @param {any} value Value to classify. + * @returns {boolean} True when `value` is a non-array object. */ function isPlainObject(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); } /** - * @param {object} obj - * @returns {boolean} + * @param {object} obj Candidate matcher object. + * @returns {boolean} True when all keys are operator keys (`$...`). */ function isOperatorObject(obj) { const keys = Object.keys(obj); @@ -39,9 +39,9 @@ function isOperatorObject(obj) { * Dispatch between array (OR), operator object, nested object, and scalar equality matching * so callers don't need to know the shape of `matcherValue`. * - * @param {any} documentValue - * @param {any} matcherValue - * @returns {boolean} + * @param {any} documentValue Value from the document. + * @param {any} matcherValue Value from the matcher definition. + * @returns {boolean} True when both values match under matcher semantics. */ function propertyMatchesValue(documentValue, matcherValue) { if (isObject(matcherValue)) { @@ -60,8 +60,8 @@ function propertyMatchesValue(documentValue, matcherValue) { * Pre-compile an operator object into an array of comparison closures so the hot path avoids * repeated `Object.entries` + switch dispatch per matched document. * - * @param {object} operatorObj - * @returns {Array} + * @param {object} operatorObj Object containing operator/value pairs. + * @returns {Array} Compiled predicate checks in evaluation order. */ function buildOperatorChecks(operatorObj) { const checks = []; @@ -96,8 +96,8 @@ function buildOperatorChecks(operatorObj) { * Return cached compiled checks for `operatorObj`, compiling and caching on first access. * Using WeakMap keeps operator objects GC-eligible and avoids mutating user-supplied objects. * - * @param {object} operatorObj - * @returns {Array} + * @param {object} operatorObj Object containing operator/value pairs. + * @returns {Array} Cached or newly compiled operator checks. */ function getCompiledOperatorChecks(operatorObj) { const cachedChecks = compiledOperatorMatcherCache.get(operatorObj); @@ -110,9 +110,9 @@ function getCompiledOperatorChecks(operatorObj) { } /** - * @param {any} documentValue - * @param {Array} checks - * @returns {boolean} + * @param {any} documentValue Parsed scalar value from the document. + * @param {Array} checks Compiled operator checks. + * @returns {boolean} True when all checks pass. */ function matchesCompiledOperators(documentValue, checks) { if (documentValue === undefined) { @@ -238,8 +238,8 @@ function buildTypeMatcherFn(payloadPath) { * Compile an object matcher into a raw-buffer predicate so raw-mode reads can filter compact * JSON without parsing every document first. * - * @param {object} matcher Object matcher. - * @returns {function(Buffer): boolean} + * @param {object} matcher Object matcher to compile. + * @returns {function(Buffer): boolean} Predicate over compact JSON buffers. */ function buildRawBufferMatcher(matcher = {}) { assert(isPlainObject(matcher), 'Matcher must be an object.', TypeError); @@ -265,8 +265,8 @@ function buildRawBufferMatcher(matcher = {}) { * Compile a matcher object into a tree whose children each carry one primary byte pattern plus * optional follow-up checks for nested objects, operators, or multi-value scalars. * - * @param {object} matcher - * @returns {{children: Array}} + * @param {object} matcher Matcher object for this tree level. + * @returns {{children: Array}} Compiled child descriptors for this level. */ function buildMatcherTree(matcher) { const node = {children: []}; @@ -281,9 +281,9 @@ function buildMatcherTree(matcher) { /** * Normalize one matcher property into the cheapest raw-buffer strategy for that value shape. * - * @param {string} key - * @param {any} value - * @returns {{pattern: Buffer, isKeyPattern: boolean, operatorChecks: (Array|null), valuePatterns: (Buffer[]|null), node: ({children: Array}|null), lastMatch: number}} + * @param {string} key Property name at this matcher level. + * @param {any} value Matcher value for `key`. + * @returns {{pattern: Buffer, isKeyPattern: boolean, operatorChecks: (Array|null), valuePatterns: (Buffer[]|null), node: ({children: Array}|null), lastMatch: number}} Compiled descriptor consumed by preCheck/matchesNode. */ function buildMatcherTreeChild(key, value) { const keyPrefix = Buffer.from(`${JSON.stringify(key)}:`, 'utf8'); @@ -325,9 +325,9 @@ function buildMatcherTreeChild(key, value) { } /** - * @param {Buffer} keyPrefix - * @param {any} value - * @returns {Buffer} + * @param {Buffer} keyPrefix Serialized key prefix (`"key":`). + * @param {any} value Scalar value to append. + * @returns {Buffer} Full serialized `"key":value` pattern. */ function buildKeyValuePattern(keyPrefix, value) { return Buffer.concat([keyPrefix, Buffer.from(JSON.stringify(value), 'utf8')]); @@ -337,10 +337,10 @@ function buildKeyValuePattern(keyPrefix, value) { * Cheap pass: confirm each child's primary pattern exists somewhere and cache that position as a * hint for the depth-aware pass that follows. * - * @param {Buffer} buffer - * @param {number} startOffset - * @param {{children: Array}} node - * @returns {boolean} + * @param {Buffer} buffer Compact JSON document buffer. + * @param {number} startOffset Start offset within `buffer`. + * @param {{children: Array}} node Compiled matcher node for this level. + * @returns {boolean} True when every child pattern exists somewhere from `startOffset`. */ function preCheck(buffer, startOffset, node) { for (const child of node.children) { @@ -359,10 +359,10 @@ function preCheck(buffer, startOffset, node) { * Confirm that each prechecked child really matches at the requested JSON level and then run the * optional value-specific follow-up checks from the compiled tree. * - * @param {Buffer} buffer - * @param {number} startOffset - * @param {{children: Array}} node - * @returns {boolean} + * @param {Buffer} buffer Compact JSON document buffer. + * @param {number} startOffset Start offset within `buffer`. + * @param {{children: Array}} node Compiled matcher node for this level. + * @returns {boolean} True when all children match at the requested JSON level. */ function matchesNode(buffer, startOffset, node) { for (const child of node.children) { @@ -390,10 +390,10 @@ function matchesNode(buffer, startOffset, node) { * Parse the scalar at a matched key position only when operator objects require a real JavaScript * comparison instead of a pure byte-pattern check. * - * @param {Buffer} buffer - * @param {number} startOffset - * @param {Array} operatorChecks - * @returns {boolean} + * @param {Buffer} buffer Compact JSON document buffer. + * @param {number} startOffset Offset of the scalar value to parse. + * @param {Array} operatorChecks Compiled operator checks. + * @returns {boolean} True when the parsed scalar satisfies all operators. */ function matchesOperatorInBuffer(buffer, startOffset, operatorChecks) { const valueStart = startOffset; diff --git a/src/utils/util.js b/src/utils/util.js index c39a79a1..1b787374 100644 --- a/src/utils/util.js +++ b/src/utils/util.js @@ -1,9 +1,10 @@ /** * Assert that actual and expected match or throw an Error with the given message appended by information about expected and actual value. * - * @param {*} actual - * @param {*} expected - * @param {string} message + * @param {*} actual Actual value. + * @param {*} expected Expected value. + * @param {string} message Error message prefix. + * @returns {void} */ function assertEqual(actual, expected, message) { if (actual !== expected) { @@ -14,9 +15,10 @@ function assertEqual(actual, expected, message) { /** * Assert that the condition holds and if not, throw an error with the given message. * - * @param {boolean} condition - * @param {string} message - * @param {typeof Error} ErrorType + * @param {boolean} condition Condition to verify. + * @param {string} message Error message when the condition fails. + * @param {typeof Error} ErrorType Error class to throw. + * @returns {void} */ function assert(condition, message, ErrorType = Error) { if (!condition) { @@ -27,9 +29,9 @@ function assert(condition, message, ErrorType = Error) { /** * Return the amount required to align value to the given alignment. * It calculates the difference of the alignment and the modulo of value by alignment. - * @param {number} value - * @param {number} alignment - * @returns {number} + * @param {number} value Source value. + * @param {number} alignment Target alignment. + * @returns {number} Additional offset needed to reach the next aligned value. */ function alignTo(value, alignment) { return (alignment - (value % alignment)) % alignment; @@ -38,8 +40,8 @@ function alignTo(value, alignment) { /** * Method for hashing a string (e.g. a partition name) to a 32-bit unsigned integer. * - * @param {string} str - * @returns {number} + * @param {string} str Input string. + * @returns {number} 32-bit unsigned hash value. */ function hash(str) { /* c8 ignore next 3 */ @@ -115,8 +117,9 @@ function wrapAndCheck(index, length) { /** * Iterate an array-like list in forward or reverse order. * - * @param {Iterable} entries - * @param {boolean} forwards + * @param {Iterable|ArrayLike} entries Entries to iterate. + * @param {boolean} forwards Iteration direction. + * @returns {Generator} */ function* iterate(entries, forwards) { if (forwards) { @@ -141,7 +144,7 @@ function* iterate(entries, forwards) { * @param {boolean} [ascending=true] When true, yields items in ascending key order (min-merge). * When false, yields in descending key order (max-merge). * @param {function(*): *} [visit] Optional extractor for the yielded value. Defaults to identity. - * @returns {Generator<*>} + * @returns {Generator<*>} Merged sequence in key order. */ function *kWayMerge(iterables, getSortKey, ascending = true, visit = v => v) { const states = []; @@ -175,9 +178,9 @@ function *kWayMerge(iterables, getSortKey, ascending = true, visit = v => v) { * Read a scalar value at a dot-notation path from an object. * Returns `undefined` if any path segment is absent or an intermediate value is not an object. * - * @param {object} obj + * @param {object} obj Source object. * @param {string} dotPath Dot-separated property path, e.g. `'payload.type'`. - * @returns {*} + * @returns {*} Value at the path or `undefined`. */ function getPropertyAtPath(obj, dotPath) { let current = obj; From d9f23ddfa18f50f3ec806f91e813a81dfe037ad2 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sat, 6 Jun 2026 14:39:24 +0200 Subject: [PATCH 12/40] Improve AGENTS.md --- AGENTS.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index e70738bb..fbd1442e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,7 @@ The code and documentation language is English, even if the user is communicatin ## Principles (in priority order) 1. **Clean API surface** — usage for common cases should be straightforward and well-documented. Avoid breaking changes unless they significantly improve usability. -2. **Understandable code** — cyclomatic complexity per method should stay around or below current levels. Higher-level methods should delegate to clearly-named helpers so the logic reads like pseudo-code. Use utility functions (e.g. `kWayMerge`, binary search) for generic algorithms. +2. **Understandable code** — cyclomatic complexity per method should stay around or below current levels (~15). Higher-level methods should delegate to clearly-named helpers so the logic reads like pseudo-code. Use utility functions (e.g. `kWayMerge`, binary search) for generic algorithms. 3. **Performance** — maintain good performance across all code paths. The Index and Partition layers are most performance-sensitive and may prefer performance over readability at the margin. Elsewhere, prefer simplicity; simpler code is often faster. ### Performance trade-off lessons @@ -22,6 +22,14 @@ The code and documentation language is English, even if the user is communicatin - **Inline single-use hot-path helpers once the surrounding flow becomes simpler**: if a helper is only called from one place and its logic can be embedded while keeping the caller below the local complexity/return-count budget, prefer the inline version. Removes call indirection and often makes the hot path easier to read as straight-line pseudo-code. +### Code Quality standards + +- Keep cyclomatic complexity per method around 15 or below. If a method exceeds that, look for ways to break it down into smaller, clearly-named helper methods. Exceptions can be made for very performance-sensitive code paths, but prefer readability in general. +- Keep number of return statements per method below 6. Multiple return points can be acceptable if they significantly improve readability, but watch for methods that become hard to follow due to too many exit points. +- Avoid deep nesting of conditionals. If you find yourself nesting more than 3 levels deep, consider refactoring to flatten the structure, such as by using guard clauses or extracting nested logic into separate methods. +- Use descriptive method and variable names to make the code self-documenting. Higher level methods should read like pseudo-code, delegating to well-named helpers that encapsulate specific logic or steps in the process. +- In general, optimize the code for readability first, and only introduce complexity when there is a clear performance benefit that has been measured and justified. + ## Architecture ``` From abe922e0d53f5538041a7e9a3b5cc33f5b4cee5c Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sat, 6 Jun 2026 18:44:49 +0200 Subject: [PATCH 13/40] Optimize numeric operator matching for single operators by ~100% --- src/utils/jsonUtil.js | 86 +++++- src/utils/metadataUtil.js | 149 +++++++--- test/metadataUtil.spec.js | 564 +++++++++++++++++++++++--------------- 3 files changed, 544 insertions(+), 255 deletions(-) diff --git a/src/utils/jsonUtil.js b/src/utils/jsonUtil.js index 86cd1f4b..77688cd5 100644 --- a/src/utils/jsonUtil.js +++ b/src/utils/jsonUtil.js @@ -5,6 +5,8 @@ const BYTE_CLOSE_OBJECT = 0x7d; const BYTE_OPEN_ARRAY = 0x5b; const BYTE_CLOSE_ARRAY = 0x5d; const BYTE_COMMA = 0x2c; +const BYTE_SIGN_MINUS = 0x2d; +const BYTE_DECIMAL_SEP = 0x2e; /** * Advance past a JSON string whose opening `"` is at `i`. @@ -184,6 +186,88 @@ function parseJsonValue(buffer, startOffset, endOffset) { } } +/** + * @param {number} byte + * @returns {boolean} True when `byte` is an ASCII digit. + */ +function isAsciiDigit(byte) { + return byte >= 0x30 && byte <= 0x39; +} + +/** + * Compare a contiguous ASCII digit sequence in `buffer` against expected digits. + * For JSON.stringify()-normalized numbers this is enough for both integer and fraction parts. + * Returns only the ordering; when ordering === 0, callers can compute the consumed length as startOffset + expectedDigits.length. + * + * @param {Buffer} buffer + * @param {number} startOffset + * @param {string} expectedDigits + * @returns {-1|0|1} + */ +function compareDigits(buffer, startOffset, expectedDigits) { + let index = startOffset; + let position = 0; + let ordering = 0; + const expectedLength = expectedDigits.length; + + while (index < buffer.length && isAsciiDigit(buffer[index])) { + if (ordering === 0 && position < expectedLength) { + const expectedByte = expectedDigits.charCodeAt(position); + if (buffer[index] !== expectedByte) { + ordering = buffer[index] < expectedByte ? -1 : 1; + } + } + position++; + index++; + } + + if (position !== expectedLength) { + ordering = position > expectedLength ? 1 : -1; + } + + return ordering; +} + +/** + * Compare a compact JSON numeric token in `buffer` against a precompiled expected number, + * using one linear pass over the buffer slice and no `parseJsonValue` call. + * + * @param {Buffer} buffer + * @param {number} startOffset + * @param {{isNegative: boolean, integerPart: string, fractionPart: string}} expected + * @returns {-1|0|1|null} Ordering (`actual` vs `expected`) or null for invalid/non-numeric token. + */ +function compareNumeric(buffer, startOffset, expected) { + let index = startOffset; + const firstByte = buffer[index]; + if (firstByte !== BYTE_SIGN_MINUS && !isAsciiDigit(firstByte)) { + return null; + } + + const isNegative = firstByte === BYTE_SIGN_MINUS; + if (isNegative !== expected.isNegative) { + return isNegative ? -1 : 1; + } + + if (isNegative) { + index++; + } + + let result = compareDigits(buffer, index, expected.integerPart); + if (result === 0) { + index += expected.integerPart.length; + + const hasFraction = index < buffer.length && buffer[index] === BYTE_DECIMAL_SEP; + const expectedHasFraction = expected.fractionPart.length > 0; + if (hasFraction !== expectedHasFraction) { + result = hasFraction ? 1 : -1; + } else if (hasFraction) { + result = compareDigits(buffer, index + 1, expected.fractionPart); + } + } + return isNegative ? -result : result; +} + /** * Compare a matched key's scalar value against pre-serialized candidates without reparsing JSON. * @@ -215,4 +299,4 @@ function matchesAnyValuePattern(buffer, valueStart, patterns) { return false; } -export { isOpeningObject, indexOfSameLevel, findJsonValueEnd, parseJsonValue, matchesAnyValuePattern }; +export { isOpeningObject, indexOfSameLevel, findJsonValueEnd, parseJsonValue, compareNumeric, matchesAnyValuePattern }; diff --git a/src/utils/metadataUtil.js b/src/utils/metadataUtil.js index 0b41c923..75ea0fa1 100644 --- a/src/utils/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -5,7 +5,8 @@ import { findJsonValueEnd, parseJsonValue, matchesAnyValuePattern, - isOpeningObject + isOpeningObject, + compareNumeric } from './jsonUtil.js'; const compiledOperatorMatcherCache = new WeakMap(); @@ -187,10 +188,10 @@ function matches(document, matcher) { * @returns {{matcher: string|object, hmac?: string}} */ function buildMetadataForMatcher(matcher, hmac) { - /* c8 ignore next */ - if (!matcher) { - return undefined; - } + /* c8 ignore next 2 */ + if (!matcher) { + return undefined; + } if (typeof matcher === 'object') { return {matcher}; } @@ -204,12 +205,12 @@ function buildMetadataForMatcher(matcher, hmac) { * @returns {Matcher} The matcher object or function. */ function buildMatcherFromMetadata(matcherMetadata, hmac) { - let matcher; - if (typeof matcherMetadata.matcher === 'object') { - matcher = matcherMetadata.matcher; - } else { - /* c8 ignore next */ - assert(matcherMetadata.hmac === hmac(matcherMetadata.matcher), 'Invalid HMAC for matcher.'); + let matcher; + if (typeof matcherMetadata.matcher === 'object') { + matcher = matcherMetadata.matcher; + } else { + /* c8 ignore next 1 */ + assert(matcherMetadata.hmac === hmac(matcherMetadata.matcher), 'Invalid HMAC for matcher.'); matcher = eval('(' + matcherMetadata.matcher + ')').bind({}); // jshint ignore:line } @@ -239,12 +240,14 @@ function buildTypeMatcherFn(payloadPath) { * JSON without parsing every document first. * * @param {object} matcher Object matcher to compile. + * @param {{enableOperatorBufferMatcher?: boolean}} [options] Raw matcher build options. * @returns {function(Buffer): boolean} Predicate over compact JSON buffers. */ -function buildRawBufferMatcher(matcher = {}) { +function buildRawBufferMatcher(matcher = {}, options = {}) { assert(isPlainObject(matcher), 'Matcher must be an object.', TypeError); + const enableOperatorBufferMatcher = options.enableOperatorBufferMatcher !== false; - const root = buildMatcherTree(matcher); + const root = buildMatcherTree(matcher, enableOperatorBufferMatcher); /* c8 ignore next 3 */ if (root.children.length === 0) { return () => true; @@ -266,13 +269,14 @@ function buildRawBufferMatcher(matcher = {}) { * optional follow-up checks for nested objects, operators, or multi-value scalars. * * @param {object} matcher Matcher object for this tree level. + * @param {boolean} enableOperatorBufferMatcher Enables specialized byte-level operator shortcuts. * @returns {{children: Array}} Compiled child descriptors for this level. */ -function buildMatcherTree(matcher) { +function buildMatcherTree(matcher, enableOperatorBufferMatcher) { const node = {children: []}; for (const [key, value] of Object.entries(matcher)) { - node.children.push(buildMatcherTreeChild(key, value)); + node.children.push(buildMatcherTreeChild(key, value, enableOperatorBufferMatcher)); } return node; @@ -283,15 +287,15 @@ function buildMatcherTree(matcher) { * * @param {string} key Property name at this matcher level. * @param {any} value Matcher value for `key`. - * @returns {{pattern: Buffer, isKeyPattern: boolean, operatorChecks: (Array|null), valuePatterns: (Buffer[]|null), node: ({children: Array}|null), lastMatch: number}} Compiled descriptor consumed by preCheck/matchesNode. + * @param {boolean} enableOperatorBufferMatcher Enables specialized byte-level operator shortcuts. + * @returns {{pattern: Buffer, isKeyPattern: boolean, matches: ((function(Buffer, number): boolean)|null), node: ({children: Array}|null), lastMatch: number}} Compiled descriptor consumed by preCheck/matchesNode. */ -function buildMatcherTreeChild(key, value) { +function buildMatcherTreeChild(key, value, enableOperatorBufferMatcher) { const keyPrefix = Buffer.from(`${JSON.stringify(key)}:`, 'utf8'); const child = { pattern: null, isKeyPattern: false, - operatorChecks: null, - valuePatterns: null, + matches: null, node: null, lastMatch: -1 }; @@ -304,19 +308,26 @@ function buildMatcherTreeChild(key, value) { } else { child.isKeyPattern = true; child.pattern = keyPrefix; - child.valuePatterns = value.map(item => Buffer.from(JSON.stringify(item), 'utf8')); + const valuePatterns = value.map(item => Buffer.from(JSON.stringify(item), 'utf8')); + child.matches = (buffer, startOffset) => matchesAnyValuePattern(buffer, startOffset, valuePatterns); } } else if ('$eq' in value && Object.keys(value).length === 1) { // A lone $eq is semantically identical to a scalar equality check — fold it into a // value pattern at compile time so the buffer scan takes the same fast path as { key: value }. child.pattern = buildKeyValuePattern(keyPrefix, value['$eq']); } else if (isOperatorObject(value)) { - child.operatorChecks = getCompiledOperatorChecks(value); child.isKeyPattern = true; child.pattern = keyPrefix; + if (enableOperatorBufferMatcher) { + child.matches = buildOperatorBufferMatcher(value); + } else { + const operatorChecks = getCompiledOperatorChecks(value); + child.matches = (buffer, startOffset) => matchesOperatorInBuffer(buffer, startOffset, operatorChecks); + } } else { child.pattern = Buffer.concat([keyPrefix, Buffer.from('{', 'utf8')]); - child.node = buildMatcherTree(value); + child.node = buildMatcherTree(value, enableOperatorBufferMatcher); + child.matches = (buffer, startOffset) => matchesNode(buffer, startOffset, child.node); } } else { child.pattern = buildKeyValuePattern(keyPrefix, value); @@ -372,13 +383,7 @@ function matchesNode(buffer, startOffset, node) { } const valueStart = matchPosition + child.pattern.length; - if (child.valuePatterns && !matchesAnyValuePattern(buffer, valueStart, child.valuePatterns)) { - return false; - } - if (child.operatorChecks && !matchesOperatorInBuffer(buffer, valueStart, child.operatorChecks)) { - return false; - } - if (child.node && !matchesNode(buffer, valueStart, child.node)) { + if (child.matches && !child.matches(buffer, valueStart)) { return false; } } @@ -396,17 +401,93 @@ function matchesNode(buffer, startOffset, node) { * @returns {boolean} True when the parsed scalar satisfies all operators. */ function matchesOperatorInBuffer(buffer, startOffset, operatorChecks) { - const valueStart = startOffset; - const valueEnd = findJsonValueEnd(buffer, valueStart); - if (valueEnd === -1 || valueEnd <= valueStart) { - return false; - } + const valueStart = startOffset; + const valueEnd = findJsonValueEnd(buffer, valueStart); + /* c8 ignore next 2 */ + if (valueEnd === -1 || valueEnd <= valueStart) { + return false; + } const parsedValue = parseJsonValue(buffer, valueStart, valueEnd); return matchesCompiledOperators(parsedValue, operatorChecks); } +/** + * Map pre-computed ordering information to operator semantics. + * ordering: -1 (actual < expected), 0 (equal), 1 (actual > expected) + * + * @param {string} operator + * @param {-1|0|1} ordering + * @returns {boolean} + */ +function matchesOrdering(operator, ordering) { + switch (operator) { + case '$gt': + return ordering === 1; + case '$gte': + return ordering >= 0; + case '$lt': + return ordering === -1; + case '$lte': + return ordering <= 0; + case '$eq': + return ordering === 0; + case '$ne': + return ordering !== 0; + /* c8 ignore next 1 */ + default: + return false; + } + } + +/** + * Build a specialized buffer-based operator comparator by pre-compiling operator-specific + * byte shortcuts at matcher build time. This avoids runtime dispatch and enables aggressive + * short-circuit evaluation for sign mismatches and digit-count differences. + * + * Assumes no scientific notation in the JSON buffer. + * + * @param {object} operatorObj Operator object (e.g., { $gt: 100 }). + * @returns {function(Buffer, number): boolean} Predicate that reads the buffer at given offset. + */ +function buildOperatorBufferMatcher(operatorObj) { + const entries = Object.entries(operatorObj); + const operatorChecks = getCompiledOperatorChecks(operatorObj); + + if (entries.length !== 1) { + // Multi-operator case: fall back to generic path. + return (buffer, startOffset) => matchesOperatorInBuffer(buffer, startOffset, operatorChecks); + } + + const [operator, expectedValue] = entries[0]; + + // Single comparison operator on a number: generate a single-pass comparator without parsing. + if (typeof expectedValue === 'number' && (operator === '$gt' || operator === '$gte' || operator === '$lt' || operator === '$lte')) { + const expectedStr = JSON.stringify(expectedValue); + const expectedIsNegative = expectedStr[0] === '-'; + const intStart = expectedIsNegative ? 1 : 0; + const [expectedIntegerPart, expectedFractionPart = ''] = expectedStr.substring(intStart).split('.'); + const expectedNumeric = { + isNegative: expectedIsNegative, + integerPart: expectedIntegerPart, + fractionPart: expectedFractionPart + }; + + return (buffer, startOffset) => { + const ordering = compareNumeric(buffer, startOffset, expectedNumeric); + /* c8 ignore next 2 */ + if (ordering === null) { + return false; + } + return matchesOrdering(operator, ordering); + }; + } + + // Non-numeric expected value: use generic operator checks. + return (buffer, startOffset) => matchesOperatorInBuffer(buffer, startOffset, operatorChecks); +} + export { createHmac, matches, diff --git a/test/metadataUtil.spec.js b/test/metadataUtil.spec.js index 99fe91f4..e8320ac4 100644 --- a/test/metadataUtil.spec.js +++ b/test/metadataUtil.spec.js @@ -9,58 +9,58 @@ import { createHmac } from '../src/utils/metadataUtil.js'; -describe('metadataUtil', function() { +describe('metadataUtil', function () { - describe('buildRawBufferMatcher', function() { + describe('buildRawBufferMatcher', function () { - it('matches top-level scalar properties in raw JSON buffers', function() { - const matchesTypeFoo = buildRawBufferMatcher({ type: 'Foo' }); + it('matches top-level scalar properties in raw JSON buffers', function () { + const matchesTypeFoo = buildRawBufferMatcher({type: 'Foo'}); const buffer = Buffer.from('{"type":"Foo","id":1}', 'utf8'); expect(matchesTypeFoo(buffer)).to.be(true); }); - it('does not match nested values with the same key/value', function() { - const matchesTypeFoo = buildRawBufferMatcher({ type: 'Foo' }); + it('does not match nested values with the same key/value', function () { + const matchesTypeFoo = buildRawBufferMatcher({type: 'Foo'}); const buffer = Buffer.from('{"payload":{"type":"Foo"},"type":"Bar"}', 'utf8'); expect(matchesTypeFoo(buffer)).to.be(false); }); - it('supports matchers with multiple top-level properties', function() { - const matcher = buildRawBufferMatcher({ type: 'Foo', version: 3 }); + it('supports matchers with multiple top-level properties', function () { + const matcher = buildRawBufferMatcher({type: 'Foo', version: 3}); const buffer = Buffer.from('{"type":"Foo","version":3,"payload":"ok"}', 'utf8'); expect(matcher(buffer)).to.be(true); }); - it('requires compact key-value separators without whitespace', function() { - const matcher = buildRawBufferMatcher({ type: 'Foo', enabled: true }); + it('requires compact key-value separators without whitespace', function () { + const matcher = buildRawBufferMatcher({type: 'Foo', enabled: true}); const buffer = Buffer.from('{ "type" : "Foo", "enabled" : true }', 'utf8'); expect(matcher(buffer)).to.be(false); }); - it('supports nested matcher objects via the scoped fallback', function() { - const matcher = buildRawBufferMatcher({ payload: { type: 'Foo' } }); + it('supports nested matcher objects via the scoped fallback', function () { + const matcher = buildRawBufferMatcher({payload: {type: 'Foo'}}); const buffer = Buffer.from('{"payload":{"type":"Foo"},"id":1}', 'utf8'); expect(matcher(buffer)).to.be(true); }); }); - describe('buildRawBufferMatcher (nested/object-array mode)', function() { + describe('buildRawBufferMatcher (nested/object-array mode)', function () { - it('matches one-level nested object properties', function() { - const matcher = buildRawBufferMatcher({ payload: { type: 'Foo' } }); + it('matches one-level nested object properties', function () { + const matcher = buildRawBufferMatcher({payload: {type: 'Foo'}}); const buffer = Buffer.from('{"payload":{"type":"Foo"},"id":1}', 'utf8'); expect(matcher(buffer)).to.be(true); }); - it('does not match when nested value differs', function() { - const matcher = buildRawBufferMatcher({ payload: { type: 'Foo' } }); + it('does not match when nested value differs', function () { + const matcher = buildRawBufferMatcher({payload: {type: 'Foo'}}); const buffer = Buffer.from('{"payload":{"type":"Bar"},"id":1}', 'utf8'); expect(matcher(buffer)).to.be(false); }); - it('matches one of two allowed nested scalar values', function() { - const matcher = buildRawBufferMatcher({ payload: { type: ['Foo', 'Bar'] } }); + it('matches one of two allowed nested scalar values', function () { + const matcher = buildRawBufferMatcher({payload: {type: ['Foo', 'Bar']}}); const fooBuffer = Buffer.from('{"payload":{"type":"Foo"},"id":1}', 'utf8'); const barBuffer = Buffer.from('{"payload":{"type":"Bar"},"id":2}', 'utf8'); const bazBuffer = Buffer.from('{"payload":{"type":"Baz"},"id":3}', 'utf8'); @@ -70,35 +70,47 @@ describe('metadataUtil', function() { expect(matcher(bazBuffer)).to.be(false); }); - it('throws when matcher is not an object', function() { + it('matches with array containing single scalar value', function () { + const matcher = buildRawBufferMatcher({status: ['active']}); + const buffer = Buffer.from('{"status":"active"}', 'utf8'); + expect(matcher(buffer)).to.be(true); + }); + + it('matches nested array with single value', function () { + const matcher = buildRawBufferMatcher({payload: {type: ['Foo']}}); + const buffer = Buffer.from('{"payload":{"type":"Foo"}}', 'utf8'); + expect(matcher(buffer)).to.be(true); + }); + + it('throws when matcher is not an object', function () { expect(() => buildRawBufferMatcher(null)).to.throwError(TypeError); expect(() => buildRawBufferMatcher(['Foo'])).to.throwError(TypeError); expect(() => buildRawBufferMatcher('Foo')).to.throwError(TypeError); }); - it('throws when an array matcher contains object values', function() { - expect(() => buildRawBufferMatcher({ type: [{ value: 'Foo' }] })).to.throwError(TypeError); + it('throws when an array matcher contains object values', function () { + expect(() => buildRawBufferMatcher({type: [{value: 'Foo'}]})).to.throwError(TypeError); }); - it('matches every object when matcher is empty', function() { + it('matches every object when matcher is empty', function () { const matcher = buildRawBufferMatcher({}); expect(matcher(Buffer.from('{"type":"Foo"}', 'utf8'))).to.be(true); expect(matcher(Buffer.from('{"nested":{"value":1}}', 'utf8'))).to.be(true); }); - it('does not match non-object JSON buffers', function() { - const matcher = buildRawBufferMatcher({ type: 'Foo' }); + it('does not match non-object JSON buffers', function () { + const matcher = buildRawBufferMatcher({type: 'Foo'}); expect(matcher(Buffer.from('[{"type":"Foo"}]', 'utf8'))).to.be(false); }); - it('does not match when the required nested object key is missing', function() { - const matcher = buildRawBufferMatcher({ payload: { type: 'Foo' } }); + it('does not match when the required nested object key is missing', function () { + const matcher = buildRawBufferMatcher({payload: {type: 'Foo'}}); const buffer = Buffer.from('{"meta":{"type":"Foo"},"id":1}', 'utf8'); expect(matcher(buffer)).to.be(false); }); - it('matches string values containing escaped quotes', function() { - const matcher = buildRawBufferMatcher({ payload: { text: 'a"b' } }); + it('matches string values containing escaped quotes', function () { + const matcher = buildRawBufferMatcher({payload: {text: 'a"b'}}); const matchingBuffer = Buffer.from('{"payload":{"text":"a\\\"b"}}', 'utf8'); const nonMatchingBuffer = Buffer.from('{"payload":{"text":"ab"}}', 'utf8'); @@ -106,217 +118,329 @@ describe('metadataUtil', function() { expect(matcher(nonMatchingBuffer)).to.be(false); }); - it('matches when escaped characters appear before the matched key on the same level', function() { - const matcher = buildRawBufferMatcher({ type: 'Foo' }); + it('matches when escaped characters appear before the matched key on the same level', function () { + const matcher = buildRawBufferMatcher({type: 'Foo'}); const buffer = Buffer.from('{"note":"a\\\\b and \\\"quoted\\\"","type":"Foo"}', 'utf8'); expect(matcher(buffer)).to.be(true); }); - it('does not match malformed objects with a premature closing brace before the key', function() { - const matcher = buildRawBufferMatcher({ type: 'Foo' }); + it('does not match malformed objects with a premature closing brace before the key', function () { + const matcher = buildRawBufferMatcher({type: 'Foo'}); const buffer = Buffer.from('{"payload":{"id":1}},"type":"Foo"}', 'utf8'); expect(matcher(buffer)).to.be(false); }); - it('does not match nested object keys that are only present deeper than the requested level', function() { - const matcher = buildRawBufferMatcher({ payload: { source: { kind: 'A' } } }); + it('does not match nested object keys that are only present deeper than the requested level', function () { + const matcher = buildRawBufferMatcher({payload: {source: {kind: 'A'}}}); const buffer = Buffer.from('{"payload":{"wrapper":{"source":{"kind":"A"}}}}', 'utf8'); expect(matcher(buffer)).to.be(false); }); - it('does not match when a nested object exists but its nested value differs', function() { - const matcher = buildRawBufferMatcher({ payload: { source: { kind: 'A' } } }); + it('does not match when a nested object exists but its nested value differs', function () { + const matcher = buildRawBufferMatcher({payload: {source: {kind: 'A'}}}); const buffer = Buffer.from('{"payload":{"source":{"kind":"B"}}}', 'utf8'); expect(matcher(buffer)).to.be(false); }); - it('does not match scalar prefixes like 30 for an expected value of 3', function() { - const matcher = buildRawBufferMatcher({ version: 3 }); + it('does not match scalar prefixes like 30 for an expected value of 3', function () { + const matcher = buildRawBufferMatcher({version: 3}); const buffer = Buffer.from('{"version":30}', 'utf8'); expect(matcher(buffer)).to.be(false); }); - }); - - describe('operators: $gt, $gte, $lt, $lte', function() { - - it('matches values greater than threshold with $gt', function() { - const matcher = buildRawBufferMatcher({ amount: { $gt: 100 } }); - const buffer = Buffer.from('{"amount":150}', 'utf8'); - expect(matcher(buffer)).to.be(true); - }); - - it('does not match values equal or less than threshold with $gt', function() { - const matcher = buildRawBufferMatcher({ amount: { $gt: 100 } }); - expect(matcher(Buffer.from('{"amount":100}', 'utf8'))).to.be(false); - expect(matcher(Buffer.from('{"amount":50}', 'utf8'))).to.be(false); - }); - - it('matches values greater or equal to threshold with $gte', function() { - const matcher = buildRawBufferMatcher({ amount: { $gte: 100 } }); - expect(matcher(Buffer.from('{"amount":150}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"amount":100}', 'utf8'))).to.be(true); - }); - - it('does not match values less than threshold with $gte', function() { - const matcher = buildRawBufferMatcher({ amount: { $gte: 100 } }); - expect(matcher(Buffer.from('{"amount":99}', 'utf8'))).to.be(false); - }); - - it('matches string values with $gte', function() { - const matcher = buildRawBufferMatcher({ status: { $gte: 'pending' } }); - expect(matcher(Buffer.from('{"status":"pending"}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"status":"submitted"}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"status":"draft"}', 'utf8'))).to.be(false); - }); - - it('combines multiple operators in one matcher', function() { - const matcher = buildRawBufferMatcher({ version: { $gte: 2, $lt: 5 } }); - expect(matcher(Buffer.from('{"version":2}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"version":3}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"version":4}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"version":5}', 'utf8'))).to.be(false); - expect(matcher(Buffer.from('{"version":1}', 'utf8'))).to.be(false); - }); - - it('works with nested objects', function() { - const matcher = buildRawBufferMatcher({ payload: { amount: { $gte: 50 } } }); - expect(matcher(Buffer.from('{"payload":{"amount":50}}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"payload":{"amount":100}}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"payload":{"amount":25}}', 'utf8'))).to.be(false); - }); - - it('works with floating point numbers', function() { - const matcher = buildRawBufferMatcher({ price: { $gt: 19.99 } }); - expect(matcher(Buffer.from('{"price":20.0}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"price":19.99}', 'utf8'))).to.be(false); - expect(matcher(Buffer.from('{"price":19.98}', 'utf8'))).to.be(false); - }); - - it('works with negative numbers', function() { - const matcher = buildRawBufferMatcher({ temperature: { $gte: -10 } }); - expect(matcher(Buffer.from('{"temperature":-5}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"temperature":-10}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"temperature":-15}', 'utf8'))).to.be(false); - }); - - it('works with scientific notation numbers', function() { - const matcher = buildRawBufferMatcher({ amount: { $gt: 1000 } }); - expect(matcher(Buffer.from('{"amount":1.1e3}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"amount":1e3}', 'utf8'))).to.be(false); - }); - - it('does not match malformed numeric values for numeric operators', function() { - const matcher = buildRawBufferMatcher({ amount: { $gte: 10 } }); - expect(matcher(Buffer.from('{"amount":10abc}', 'utf8'))).to.be(false); - }); - - }); - - describe('matches with operators ($gt, $gte, $lt, $lte, $eq, $ne)', function() { - - it('matches with $gt', function() { - expect(matches({ amount: 150 }, { amount: { $gt: 100 } })).to.be(true); - expect(matches({ amount: 100 }, { amount: { $gt: 100 } })).to.be(false); - expect(matches({ amount: 50 }, { amount: { $gt: 100 } })).to.be(false); - }); - - it('matches with $gte', function() { - expect(matches({ amount: 150 }, { amount: { $gte: 100 } })).to.be(true); - expect(matches({ amount: 100 }, { amount: { $gte: 100 } })).to.be(true); - expect(matches({ amount: 99 }, { amount: { $gte: 100 } })).to.be(false); - }); - - it('matches with $lt', function() { - expect(matches({ amount: 50 }, { amount: { $lt: 100 } })).to.be(true); - expect(matches({ amount: 100 }, { amount: { $lt: 100 } })).to.be(false); - expect(matches({ amount: 150 }, { amount: { $lt: 100 } })).to.be(false); - }); - - it('matches with $lte', function() { - expect(matches({ amount: 50 }, { amount: { $lte: 100 } })).to.be(true); - expect(matches({ amount: 100 }, { amount: { $lte: 100 } })).to.be(true); - expect(matches({ amount: 101 }, { amount: { $lte: 100 } })).to.be(false); - }); - - it('matches with $eq', function() { - expect(matches({ status: 'active' }, { status: { $eq: 'active' } })).to.be(true); - expect(matches({ status: 'inactive' }, { status: { $eq: 'active' } })).to.be(false); - }); - - it('matches with $ne', function() { - expect(matches({ status: 'inactive' }, { status: { $ne: 'active' } })).to.be(true); - expect(matches({ status: 'active' }, { status: { $ne: 'active' } })).to.be(false); - }); - - it('combines multiple operators', function() { - expect(matches({ version: 3 }, { version: { $gte: 2, $lt: 5 } })).to.be(true); - expect(matches({ version: 1 }, { version: { $gte: 2, $lt: 5 } })).to.be(false); - expect(matches({ version: 5 }, { version: { $gte: 2, $lt: 5 } })).to.be(false); - }); - - it('throws on unknown operator', function() { - expect(() => matches({ x: 1 }, { x: { $unknown: 1 } })).to.throwError(TypeError); - }); - - it('does not confuse operator objects with plain nested matchers', function() { - // Plain nested object (no $ keys) → deep match - expect(matches({ meta: { kind: 'A' } }, { meta: { kind: 'A' } })).to.be(true); - // Operator object → range comparison - expect(matches({ amount: 50 }, { amount: { $gt: 10 } })).to.be(true); - }); - - it('reuses compiled operator checks on repeated matcher objects', function() { - const matcher = { amount: { $gte: 10 } }; - expect(matches({ amount: 12 }, matcher)).to.be(true); - expect(matches({ amount: 9 }, matcher)).to.be(false); - expect(matches({ amount: 11 }, matcher)).to.be(true); - }); - - }); - - describe('matcher metadata helpers', function() { - - it('returns undefined when no matcher is provided', function() { - expect(buildMetadataForMatcher(undefined, createHmac('secret'))).to.be(undefined); - }); - - it('serializes and restores function matchers with hmac', function() { - const hmac = createHmac('secret'); - const matcher = event => event.type === 'Foo'; - const metadata = buildMetadataForMatcher(matcher, hmac); - - expect(metadata.matcher).to.be.a('string'); - expect(metadata.hmac).to.be(hmac(metadata.matcher)); - - const restored = buildMatcherFromMetadata(metadata, hmac); - expect(restored({ type: 'Foo' })).to.be(true); - expect(restored({ type: 'Bar' })).to.be(false); - }); - - it('passes through object matchers unchanged in metadata', function() { - const matcher = { payload: { type: 'Foo' } }; - const metadata = buildMetadataForMatcher(matcher, createHmac('secret')); - const restored = buildMatcherFromMetadata(metadata, createHmac('secret')); - - expect(metadata).to.eql({ matcher }); - expect(restored).to.eql(matcher); - }); - - it('builds type matcher functions for nested paths', function() { - const typeMatcher = buildTypeMatcherFn('meta.kind'); - expect(typeMatcher('OrderPlaced')).to.eql({ payload: { meta: { kind: 'OrderPlaced' } } }); - }); - - it('builds a padded metadata header', function() { - const header = buildMetadataHeader('MAGICHDR', { version: 1 }); - - expect(header.slice(0, 8).toString('utf8')).to.be('MAGICHDR'); - expect(header.length % 16).to.be(0); - expect(header.readUInt32BE(8)).to.be.greaterThan(0); - }); - - }); + }); + + describe('operators: $gt, $gte, $lt, $lte', function () { + + it('matches values greater than threshold with $gt', function () { + const matcher = buildRawBufferMatcher({amount: {$gt: 100}}); + const buffer = Buffer.from('{"amount":150}', 'utf8'); + expect(matcher(buffer)).to.be(true); + }); + + it('does not match values equal or less than threshold with $gt', function () { + const matcher = buildRawBufferMatcher({amount: {$gt: 100}}); + expect(matcher(Buffer.from('{"amount":100}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"amount":50}', 'utf8'))).to.be(false); + }); + + it('matches values greater or equal to threshold with $gte', function () { + const matcher = buildRawBufferMatcher({amount: {$gte: 100}}); + expect(matcher(Buffer.from('{"amount":150}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"amount":100}', 'utf8'))).to.be(true); + }); + + it('does not match values less than threshold with $gte', function () { + const matcher = buildRawBufferMatcher({amount: {$gte: 100}}); + expect(matcher(Buffer.from('{"amount":99}', 'utf8'))).to.be(false); + }); + + it('matches string values with $gte', function () { + const matcher = buildRawBufferMatcher({status: {$gte: 'pending'}}); + expect(matcher(Buffer.from('{"status":"pending"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"submitted"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"draft"}', 'utf8'))).to.be(false); + }); + + it('combines multiple operators in one matcher', function () { + const matcher = buildRawBufferMatcher({version: {$gte: 2, $lt: 5}}); + expect(matcher(Buffer.from('{"version":2}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"version":3}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"version":4}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"version":5}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"version":1}', 'utf8'))).to.be(false); + }); + + it('works with nested objects', function () { + const matcher = buildRawBufferMatcher({payload: {amount: {$gte: 50}}}); + expect(matcher(Buffer.from('{"payload":{"amount":50}}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"payload":{"amount":100}}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"payload":{"amount":25}}', 'utf8'))).to.be(false); + }); + + it('works with floating point numbers', function () { + const matcher = buildRawBufferMatcher({price: {$gt: 19.99}}); + expect(matcher(Buffer.from('{"price":20.0}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"price":19.99}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"price":19.98}', 'utf8'))).to.be(false); + }); + + it('works with negative numbers', function () { + const matcher = buildRawBufferMatcher({temperature: {$gte: -10}}); + expect(matcher(Buffer.from('{"temperature":-5}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"temperature":-10}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"temperature":-15}', 'utf8'))).to.be(false); + }); + + it('matches values less than threshold with $lt', function () { + const matcher = buildRawBufferMatcher({amount: {$lt: 100}}); + expect(matcher(Buffer.from('{"amount":50}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"amount":100}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"amount":150}', 'utf8'))).to.be(false); + }); + + it('matches values less or equal to threshold with $lte', function () { + const matcher = buildRawBufferMatcher({amount: {$lte: 100}}); + expect(matcher(Buffer.from('{"amount":50}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"amount":100}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"amount":101}', 'utf8'))).to.be(false); + }); + + it('matches with standalone $eq operator', function () { + const matcher = buildRawBufferMatcher({status: {$eq: 'active'}}); + expect(matcher(Buffer.from('{"status":"active"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"inactive"}', 'utf8'))).to.be(false); + }); + + it('matches with $ne operator', function () { + const matcher = buildRawBufferMatcher({status: {$ne: 'active'}}); + expect(matcher(Buffer.from('{"status":"inactive"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"active"}', 'utf8'))).to.be(false); + }); + + it('handles fractional numbers with operators', function () { + const matcher = buildRawBufferMatcher({price: {$lte: 19.99}}); + expect(matcher(Buffer.from('{"price":19.99}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"price":19.98}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"price":20.0}', 'utf8'))).to.be(false); + }); + + it('matches with $eq and $ne on string values', function () { + const eqMatcher = buildRawBufferMatcher({type: {$eq: 'EventType'}}); + expect(eqMatcher(Buffer.from('{"type":"EventType"}', 'utf8'))).to.be(true); + expect(eqMatcher(Buffer.from('{"type":"OtherType"}', 'utf8'))).to.be(false); + }); + + it('matches when multiple operators and properties are combined', function () { + const matcher = buildRawBufferMatcher({ + status: {$ne: 'draft'}, + priority: {$gte: 5} + }); + expect(matcher(Buffer.from('{"status":"published","priority":7}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"draft","priority":7}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"status":"published","priority":3}', 'utf8'))).to.be(false); + }); + + it('uses generic operator checks when operator buffer matcher is disabled', function () { + const matcher = buildRawBufferMatcher( + {amount: {$gt: 100}}, + {enableOperatorBufferMatcher: false} + ); + expect(matcher(Buffer.from('{"amount":150}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"amount":100}', 'utf8'))).to.be(false); + }); + + }); + + describe('matches with operators ($gt, $gte, $lt, $lte, $eq, $ne)', function () { + + it('matches with $gt', function () { + expect(matches({amount: 150}, {amount: {$gt: 100}})).to.be(true); + expect(matches({amount: 100}, {amount: {$gt: 100}})).to.be(false); + expect(matches({amount: 50}, {amount: {$gt: 100}})).to.be(false); + }); + + it('matches with $gte', function () { + expect(matches({amount: 150}, {amount: {$gte: 100}})).to.be(true); + expect(matches({amount: 100}, {amount: {$gte: 100}})).to.be(true); + expect(matches({amount: 99}, {amount: {$gte: 100}})).to.be(false); + }); + + it('matches with $lt', function () { + expect(matches({amount: 50}, {amount: {$lt: 100}})).to.be(true); + expect(matches({amount: 100}, {amount: {$lt: 100}})).to.be(false); + expect(matches({amount: 150}, {amount: {$lt: 100}})).to.be(false); + }); + + it('matches with $lte', function () { + expect(matches({amount: 50}, {amount: {$lte: 100}})).to.be(true); + expect(matches({amount: 100}, {amount: {$lte: 100}})).to.be(true); + expect(matches({amount: 101}, {amount: {$lte: 100}})).to.be(false); + }); + + it('matches with $eq', function () { + expect(matches({status: 'active'}, {status: {$eq: 'active'}})).to.be(true); + expect(matches({status: 'inactive'}, {status: {$eq: 'active'}})).to.be(false); + }); + + it('matches with $ne', function () { + expect(matches({status: 'inactive'}, {status: {$ne: 'active'}})).to.be(true); + expect(matches({status: 'active'}, {status: {$ne: 'active'}})).to.be(false); + }); + + it('combines multiple operators', function () { + expect(matches({version: 3}, {version: {$gte: 2, $lt: 5}})).to.be(true); + expect(matches({version: 1}, {version: {$gte: 2, $lt: 5}})).to.be(false); + expect(matches({version: 5}, {version: {$gte: 2, $lt: 5}})).to.be(false); + }); + + it('throws on unknown operator', function () { + expect(() => matches({x: 1}, {x: {$unknown: 1}})).to.throwError(TypeError); + }); + + it('does not confuse operator objects with plain nested matchers', function () { + // Plain nested object (no $ keys) → deep match + expect(matches({meta: {kind: 'A'}}, {meta: {kind: 'A'}})).to.be(true); + // Operator object → range comparison + expect(matches({amount: 50}, {amount: {$gt: 10}})).to.be(true); + }); + + it('reuses compiled operator checks on repeated matcher objects', function () { + const matcher = {amount: {$gte: 10}}; + expect(matches({amount: 12}, matcher)).to.be(true); + expect(matches({amount: 9}, matcher)).to.be(false); + expect(matches({amount: 11}, matcher)).to.be(true); + }); + + it('matches with $lt', function () { + expect(matches({amount: 50}, {amount: {$lt: 100}})).to.be(true); + expect(matches({amount: 100}, {amount: {$lt: 100}})).to.be(false); + expect(matches({amount: 150}, {amount: {$lt: 100}})).to.be(false); + }); + + it('matches with $lte', function () { + expect(matches({amount: 50}, {amount: {$lte: 100}})).to.be(true); + expect(matches({amount: 100}, {amount: {$lte: 100}})).to.be(true); + expect(matches({amount: 101}, {amount: {$lte: 100}})).to.be(false); + }); + + it('matches with $ne', function () { + expect(matches({status: 'inactive'}, {status: {$ne: 'active'}})).to.be(true); + expect(matches({status: 'active'}, {status: {$ne: 'active'}})).to.be(false); + }); + + it('matches when operator value is undefined (matches all non-undefined values)', function () { + expect(matches({amount: 50}, {amount: undefined})).to.be(true); + expect(matches({amount: undefined}, {amount: undefined})).to.be(true); + }); + + it('matches when document property is undefined', function () { + expect(matches({}, {missing: undefined})).to.be(true); + expect(matches({other: 1}, {missing: {$gt: 10}})).to.be(false); + }); + + it('handles array equality matching', function () { + expect(matches({status: 'active'}, {status: ['active', 'pending']})).to.be(true); + expect(matches({status: 'completed'}, {status: ['active', 'pending']})).to.be(false); + }); + + it('handles nested object matching without operators', function () { + expect(matches({meta: {kind: 'A', version: 1}}, {meta: {kind: 'A'}})).to.be(true); + expect(matches({meta: {kind: 'B'}}, {meta: {kind: 'A'}})).to.be(false); + }); + + }); + + describe('matcher metadata helpers', function () { + + it('returns undefined when no matcher is provided', function () { + expect(buildMetadataForMatcher(undefined, createHmac('secret'))).to.be(undefined); + }); + + it('serializes and restores function matchers with hmac', function () { + const hmac = createHmac('secret'); + const matcher = event => event.type === 'Foo'; + const metadata = buildMetadataForMatcher(matcher, hmac); + + expect(metadata.matcher).to.be.a('string'); + expect(metadata.hmac).to.be(hmac(metadata.matcher)); + + const restored = buildMatcherFromMetadata(metadata, hmac); + expect(restored({type: 'Foo'})).to.be(true); + expect(restored({type: 'Bar'})).to.be(false); + }); + + it('passes through object matchers unchanged in metadata', function () { + const matcher = {payload: {type: 'Foo'}}; + const metadata = buildMetadataForMatcher(matcher, createHmac('secret')); + const restored = buildMatcherFromMetadata(metadata, createHmac('secret')); + + expect(metadata).to.eql({matcher}); + expect(restored).to.eql(matcher); + }); + + it('builds type matcher functions for single-level paths', function () { + const typeMatcher = buildTypeMatcherFn('type'); + expect(typeMatcher('OrderPlaced')).to.eql({payload: {type: 'OrderPlaced'}}); + }); + + it('builds type matcher functions for nested paths', function () { + const typeMatcher = buildTypeMatcherFn('meta.kind'); + expect(typeMatcher('OrderPlaced')).to.eql({payload: {meta: {kind: 'OrderPlaced'}}}); + }); + + it('builds type matcher functions for deeply nested paths', function () { + const typeMatcher = buildTypeMatcherFn('deeply.nested.type'); + expect(typeMatcher('MyEvent')).to.eql({ + payload: {deeply: {nested: {type: 'MyEvent'}}} + }); + }); + + it('builds a padded metadata header', function () { + const header = buildMetadataHeader('MAGICHDR', {version: 1}); + + expect(header.slice(0, 8).toString('utf8')).to.be('MAGICHDR'); + expect(header.length % 16).to.be(0); + expect(header.readUInt32BE(8)).to.be.greaterThan(0); + }); + + it('builds metadata header with large metadata object', function () { + const largeMetadata = {key: 'value'.repeat(50)}; + const header = buildMetadataHeader('MAGICHDR', largeMetadata); + + expect(header.slice(0, 8).toString('utf8')).to.be('MAGICHDR'); + expect(header.length % 16).to.be(0); + }); + + it('creates different HMACs for different secrets', function () { + const hmac1 = createHmac('secret1'); + const hmac2 = createHmac('secret2'); + const testString = 'test'; + + expect(hmac1(testString)).not.to.be(hmac2(testString)); + }); + + }); }); From 5cd481e11ec39e62fa6b6ddd5a4182a11db56dbd Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sat, 6 Jun 2026 20:45:55 +0200 Subject: [PATCH 14/40] Remove toggle for new operator matcher --- src/utils/metadataUtil.js | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/utils/metadataUtil.js b/src/utils/metadataUtil.js index 75ea0fa1..760fdcca 100644 --- a/src/utils/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -267,30 +267,33 @@ function buildRawBufferMatcher(matcher = {}, options = {}) { /** * Compile a matcher object into a tree whose children each carry one primary byte pattern plus * optional follow-up checks for nested objects, operators, or multi-value scalars. + * Fast scalar-equality children are placed first so preCheck and matchesNode short-circuit early. * * @param {object} matcher Matcher object for this tree level. - * @param {boolean} enableOperatorBufferMatcher Enables specialized byte-level operator shortcuts. * @returns {{children: Array}} Compiled child descriptors for this level. */ -function buildMatcherTree(matcher, enableOperatorBufferMatcher) { - const node = {children: []}; +function buildMatcherTree(matcher) { + const fast = []; + const slow = []; for (const [key, value] of Object.entries(matcher)) { - node.children.push(buildMatcherTreeChild(key, value, enableOperatorBufferMatcher)); + const child = buildMatcherTreeChild(key, value); + // A child with only one byte pattern and no follow-up matcher cannot be outperformed by any extra matcher logic. + (!child.matches ? fast : slow).push(child); } - return node; + return {children: [...fast, ...slow]}; } /** * Normalize one matcher property into the cheapest raw-buffer strategy for that value shape. + * Children that compile to a plain byte-equality pattern leave `matches` as null. * * @param {string} key Property name at this matcher level. * @param {any} value Matcher value for `key`. - * @param {boolean} enableOperatorBufferMatcher Enables specialized byte-level operator shortcuts. * @returns {{pattern: Buffer, isKeyPattern: boolean, matches: ((function(Buffer, number): boolean)|null), node: ({children: Array}|null), lastMatch: number}} Compiled descriptor consumed by preCheck/matchesNode. */ -function buildMatcherTreeChild(key, value, enableOperatorBufferMatcher) { +function buildMatcherTreeChild(key, value) { const keyPrefix = Buffer.from(`${JSON.stringify(key)}:`, 'utf8'); const child = { pattern: null, @@ -318,15 +321,10 @@ function buildMatcherTreeChild(key, value, enableOperatorBufferMatcher) { } else if (isOperatorObject(value)) { child.isKeyPattern = true; child.pattern = keyPrefix; - if (enableOperatorBufferMatcher) { - child.matches = buildOperatorBufferMatcher(value); - } else { - const operatorChecks = getCompiledOperatorChecks(value); - child.matches = (buffer, startOffset) => matchesOperatorInBuffer(buffer, startOffset, operatorChecks); - } + child.matches = buildOperatorBufferMatcher(value); } else { child.pattern = Buffer.concat([keyPrefix, Buffer.from('{', 'utf8')]); - child.node = buildMatcherTree(value, enableOperatorBufferMatcher); + child.node = buildMatcherTree(value); child.matches = (buffer, startOffset) => matchesNode(buffer, startOffset, child.node); } } else { From 2ba937c8764f08dbfc7e922fe0f98258f40aee68 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sat, 6 Jun 2026 20:51:22 +0200 Subject: [PATCH 15/40] Optimize multi-operator numeric checks { $gt(e): X, $lt(e): Y } operator matchers now gain from the same speed improvement as single numeric operator cases. --- src/utils/metadataUtil.js | 69 +++++++++++++++++++++++---------------- test/metadataUtil.spec.js | 53 ++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 28 deletions(-) diff --git a/src/utils/metadataUtil.js b/src/utils/metadataUtil.js index 760fdcca..81e62f60 100644 --- a/src/utils/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -439,6 +439,32 @@ function matchesOrdering(operator, ordering) { } } +/** + * @param {Array<[string, any]>} entries + * @returns {Array<{operator: string, expectedNumeric: {isNegative: boolean, integerPart: string, fractionPart: string}}>|null} + */ +function buildNumericOperatorComparisons(entries) { + const comparisons = []; + for (const [operator, expectedValue] of entries) { + if (typeof expectedValue !== 'number') { + return null; + } + const expectedStr = JSON.stringify(expectedValue); + const expectedIsNegative = expectedStr[0] === '-'; + const intStart = expectedIsNegative ? 1 : 0; + const [expectedIntegerPart, expectedFractionPart = ''] = expectedStr.substring(intStart).split('.'); + comparisons.push({ + operator, + expectedNumeric: { + isNegative: expectedIsNegative, + integerPart: expectedIntegerPart, + fractionPart: expectedFractionPart + } + }); + } + return comparisons; +} + /** * Build a specialized buffer-based operator comparator by pre-compiling operator-specific * byte shortcuts at matcher build time. This avoids runtime dispatch and enables aggressive @@ -451,38 +477,25 @@ function matchesOrdering(operator, ordering) { */ function buildOperatorBufferMatcher(operatorObj) { const entries = Object.entries(operatorObj); - const operatorChecks = getCompiledOperatorChecks(operatorObj); - - if (entries.length !== 1) { - // Multi-operator case: fall back to generic path. - return (buffer, startOffset) => matchesOperatorInBuffer(buffer, startOffset, operatorChecks); - } - - const [operator, expectedValue] = entries[0]; - - // Single comparison operator on a number: generate a single-pass comparator without parsing. - if (typeof expectedValue === 'number' && (operator === '$gt' || operator === '$gte' || operator === '$lt' || operator === '$lte')) { - const expectedStr = JSON.stringify(expectedValue); - const expectedIsNegative = expectedStr[0] === '-'; - const intStart = expectedIsNegative ? 1 : 0; - const [expectedIntegerPart, expectedFractionPart = ''] = expectedStr.substring(intStart).split('.'); - const expectedNumeric = { - isNegative: expectedIsNegative, - integerPart: expectedIntegerPart, - fractionPart: expectedFractionPart - }; - + const numericComparisons = buildNumericOperatorComparisons(entries); + if (numericComparisons) { return (buffer, startOffset) => { - const ordering = compareNumeric(buffer, startOffset, expectedNumeric); - /* c8 ignore next 2 */ - if (ordering === null) { - return false; - } - return matchesOrdering(operator, ordering); - }; + for (const comparison of numericComparisons) { + const ordering = compareNumeric(buffer, startOffset, comparison.expectedNumeric); + /* c8 ignore next 2 */ + if (ordering === null) { + return false; + } + if (!matchesOrdering(comparison.operator, ordering)) { + return false; + } + } + return true; + }; } // Non-numeric expected value: use generic operator checks. + const operatorChecks = getCompiledOperatorChecks(operatorObj); return (buffer, startOffset) => matchesOperatorInBuffer(buffer, startOffset, operatorChecks); } diff --git a/test/metadataUtil.spec.js b/test/metadataUtil.spec.js index e8320ac4..b0a0dbcf 100644 --- a/test/metadataUtil.spec.js +++ b/test/metadataUtil.spec.js @@ -191,6 +191,22 @@ describe('metadataUtil', function () { expect(matcher(Buffer.from('{"version":1}', 'utf8'))).to.be(false); }); + it('combines multiple numeric operators including $ne', function () { + const matcher = buildRawBufferMatcher({version: {$gt: 1, $lt: 5, $ne: 3}}); + expect(matcher(Buffer.from('{"version":2}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"version":3}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"version":4}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"version":5}', 'utf8'))).to.be(false); + }); + + it('supports multiple string operators via generic fallback', function () { + const matcher = buildRawBufferMatcher({status: {$gte: 'b', $lt: 'd'}}); + expect(matcher(Buffer.from('{"status":"b"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"c"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"d"}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"status":"a"}', 'utf8'))).to.be(false); + }); + it('works with nested objects', function () { const matcher = buildRawBufferMatcher({payload: {amount: {$gte: 50}}}); expect(matcher(Buffer.from('{"payload":{"amount":50}}', 'utf8'))).to.be(true); @@ -238,6 +254,24 @@ describe('metadataUtil', function () { expect(matcher(Buffer.from('{"status":"active"}', 'utf8'))).to.be(false); }); + it('lone $ne does not match when the key is absent', function () { + const matcher = buildRawBufferMatcher({status: {$ne: 'active'}}); + expect(matcher(Buffer.from('{"other":"active"}', 'utf8'))).to.be(false); + }); + + it('lone $ne ignores matching value at the wrong nesting level', function () { + // "status":"active" appears only inside payload — must not match the top-level $ne check + const matcher = buildRawBufferMatcher({status: {$ne: 'active'}}); + expect(matcher(Buffer.from('{"payload":{"status":"active"},"status":"inactive"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"payload":{"status":"inactive"},"status":"active"}', 'utf8'))).to.be(false); + }); + + it('lone $ne matches a numeric value that differs', function () { + const matcher = buildRawBufferMatcher({version: {$ne: 3}}); + expect(matcher(Buffer.from('{"version":4}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"version":3}', 'utf8'))).to.be(false); + }); + it('handles fractional numbers with operators', function () { const matcher = buildRawBufferMatcher({price: {$lte: 19.99}}); expect(matcher(Buffer.from('{"price":19.99}', 'utf8'))).to.be(true); @@ -270,6 +304,25 @@ describe('metadataUtil', function () { expect(matcher(Buffer.from('{"amount":100}', 'utf8'))).to.be(false); }); + it('rejects early via fast scalar pattern when string equals precedes operator', function () { + // status:'active' is a fast scalar pattern; amount:{$gte:50} is slow. + // With reordering, the status check runs first in both preCheck and matchesNode, + // so documents with wrong status never reach the numeric operator. + const matcher = buildRawBufferMatcher({status: 'active', amount: {$gte: 50}}); + expect(matcher(Buffer.from('{"status":"active","amount":100}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"inactive","amount":100}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"status":"active","amount":10}', 'utf8'))).to.be(false); + }); + + it('rejects early when type equality is listed after a multi-operator in source order', function () { + // Matcher keys: amount first (slow), type second (fast). + // After reordering, type:"Foo" moves to front and acts as a prefilter. + const matcher = buildRawBufferMatcher({amount: {$gt: 10, $lt: 100}, type: 'Foo'}); + expect(matcher(Buffer.from('{"amount":50,"type":"Foo"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"amount":50,"type":"Bar"}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"amount":150,"type":"Foo"}', 'utf8'))).to.be(false); + }); + }); describe('matches with operators ($gt, $gte, $lt, $lte, $eq, $ne)', function () { From f2f6033728208ce888136cb7e40addaa6a1708b6 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sat, 6 Jun 2026 21:04:42 +0200 Subject: [PATCH 16/40] Add more matcher benchmarks --- bench/bench-matcher.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/bench/bench-matcher.js b/bench/bench-matcher.js index 83be7ac0..a146603f 100644 --- a/bench/bench-matcher.js +++ b/bench/bench-matcher.js @@ -9,12 +9,16 @@ const WARMUP_RUNS = 1; const MEASURED_RUNS = 5; const matcherObjects = [ - { name: 'flat', matcher: { type: 'Foo' } }, - { name: 'scoped', matcher: { payload: { type: 'Foo' } } }, - { name: 'multi', matcher: { payload: { type: ['BazingaHappened', 'Foo'] } } }, - { name: 'multi4', matcher: { payload: { type: ['BazingaHappened', 'BarxingaHappened', 'QuuxingaHappened', 'Foo'] } } }, - { name: '$gt', matcher: { payload: { value: { $gt: 100 } } } }, - { name: '$eq', matcher: { payload: { type: { $eq: 'Foo' } } } } + { name: 'flat', matcher: { type: 'Foo' } }, + { name: 'scoped', matcher: { payload: { type: 'Foo' } } }, + { name: 'anyOf2', matcher: { payload: { type: ['BazingaHappened', 'Foo'] } } }, + { name: 'anyOf4', matcher: { payload: { type: ['BazingaHappened', 'BarxingaHappened', 'QuuxingaHappened', 'Foo'] } } }, + { name: '$gt', matcher: { payload: { value: { $gt: 100 } } } }, + { name: '$eq', matcher: { payload: { type: { $eq: 'Foo' } } } }, + // ── multi-operator numeric: two numeric ops → fast path (no JSON.parse) ── + { name: '$gte+$lt', matcher: { payload: { value: { $gte: 100, $lt: 2000 } } } }, + // ── two numeric + string: numeric ops on value field + string op on type field ── + { name: '$gte+$lt+str', matcher: { payload: { value: { $gte: 100, $lt: 2000 }, type: 'Foo' } } } ]; const implementations = item => [ From bffeffd4fc387d219d1dad1abdac361a90abfea5 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sat, 6 Jun 2026 21:11:36 +0200 Subject: [PATCH 17/40] Add performance section for matchers --- docs/performance.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/performance.md b/docs/performance.md index 462d7078..d544fc47 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -128,3 +128,34 @@ These benchmarks measure pure **read operations** (no writes). The range scan re | range scan | stable | 453,288 | ±0.28% | 86 | | range scan | latest | 423,291 | ±0.45% | 81 | | range scan | latest(raw) | 886,578 | ±0.59% | 90 | + +### Raw Buffer Operator Matcher Benchmark + +This benchmark compares the same matcher definitions in two execution modes: + +- **`obj` (baseline):** run `matches(document, matcher)` on already deserialized events +- **`raw`:** run `buildRawBufferMatcher(matcher)` directly on NDJSON buffers + +Matcher objects used in `bench/bench-matcher.js`: + +```js +{ type: 'Foo' } +{ payload: { type: 'Foo' } } +{ payload: { type: ['BazingaHappened', 'Foo'] } } +{ payload: { type: ['BazingaHappened', 'BarxingaHappened', 'QuuxingaHappened', 'Foo'] } } +{ payload: { value: { $gt: 100 } } } +{ payload: { type: { $eq: 'Foo' } } } +{ payload: { value: { $gte: 100, $lt: 2000 } } } +{ payload: { value: { $gte: 100, $lt: 2000 }, type: 'Foo' } } +``` + +Observed median delta (`raw` vs `obj`) from the benchmark run: + +At **ANY match ratio**, raw is roughly **40–55% slower** than object mode. +At **ANY match ratio**, raw is roughly **50–60% faster** than object mode when deserializing the event first. + +Interpretation: + +- When events are **already deserialized**, object matchers are the correct baseline and typically faster. +- In workloads where data is still in **raw buffer form** (streaming), raw matchers avoid `JSON.parse` and can outperform a deserialize+object-match pipeline by roughly **~50%** in practice. + From 0a26f7f65ff9120f65cb7eae2836d57d06c649a6 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sat, 6 Jun 2026 21:19:36 +0200 Subject: [PATCH 18/40] Optimize $ne operator by ~20% --- src/utils/metadataUtil.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/utils/metadataUtil.js b/src/utils/metadataUtil.js index 81e62f60..2c89b5fa 100644 --- a/src/utils/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -318,6 +318,13 @@ function buildMatcherTreeChild(key, value) { // A lone $eq is semantically identical to a scalar equality check — fold it into a // value pattern at compile time so the buffer scan takes the same fast path as { key: value }. child.pattern = buildKeyValuePattern(keyPrefix, value['$eq']); + } else if ('$ne' in value && Object.keys(value).length === 1) { + // A lone $ne is the logical negation of $eq: confirm the key exists at this level, + // then reject only when the value byte-matches the excluded pattern. + child.isKeyPattern = true; + child.pattern = keyPrefix; + const nePattern = [Buffer.from(JSON.stringify(value['$ne']), 'utf8')]; + child.matches = (buffer, valueStart) => !matchesAnyValuePattern(buffer, valueStart, nePattern); } else if (isOperatorObject(value)) { child.isKeyPattern = true; child.pattern = keyPrefix; From 08059f8345836ec9bdf6aa2eed93e92c00bf8572 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sat, 6 Jun 2026 23:22:04 +0200 Subject: [PATCH 19/40] Add ISO DateTime matching tests --- test/metadataUtil.spec.js | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/metadataUtil.spec.js b/test/metadataUtil.spec.js index b0a0dbcf..0db4a631 100644 --- a/test/metadataUtil.spec.js +++ b/test/metadataUtil.spec.js @@ -207,6 +207,32 @@ describe('metadataUtil', function () { expect(matcher(Buffer.from('{"status":"a"}', 'utf8'))).to.be(false); }); + it('matches ISO datetime strings in a nested range', function () { + const matcher = buildRawBufferMatcher({ + payload: { + at: { + $gte: '2021-06-09T21:03:28.297Z', + $lt: '2021-06-09T21:41:53.775Z' + } + } + }); + + const events = [ + {payload: {at: '2021-06-09T21:03:28.297Z'}}, + {payload: {at: '2021-06-09T21:20:00.000Z'}}, + {payload: {at: '2021-06-09T21:41:53.774Z'}}, + {payload: {at: '2021-06-09T21:03:28.296Z'}}, + {payload: {at: '2021-06-09T21:41:53.775Z'}} + ]; + + const matching = events.filter(event => matcher(Buffer.from(JSON.stringify(event), 'utf8'))); + expect(matching).to.eql([ + {payload: {at: '2021-06-09T21:03:28.297Z'}}, + {payload: {at: '2021-06-09T21:20:00.000Z'}}, + {payload: {at: '2021-06-09T21:41:53.774Z'}} + ]); + }); + it('works with nested objects', function () { const matcher = buildRawBufferMatcher({payload: {amount: {$gte: 50}}}); expect(matcher(Buffer.from('{"payload":{"amount":50}}', 'utf8'))).to.be(true); From 95f3ad0541051d383d77f5f7af2c9ad9c56a0d9c Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sat, 6 Jun 2026 23:23:38 +0200 Subject: [PATCH 20/40] Add platform hint for Agents --- .gitignore | 3 ++- AGENTS.md | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e877ba4e..9c16bbea 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ node_modules /data *.tgz /event-storage-ui -/event-storage-http \ No newline at end of file +/event-storage-http +/Platform.md diff --git a/AGENTS.md b/AGENTS.md index fbd1442e..9672f6a1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,6 +4,7 @@ Do not repeat yourself. Be concise and precise in your answers. No paraphrasing of previous information unless specifically asked for. The code and documentation language is English, even if the user is communicating in another language. +Read the Platform.md file if it exists to know which platform you are running on and what capabilities you have, instead of trying to run linux tools on a Windows machine. **Keep this file up-to-date**: after any code review that surfaces new architectural insights or a new principle, compact this file and integrate what was learned. ## Principles (in priority order) From 9b302d61ff559688ec78791d14a69e74c15d1862 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sun, 7 Jun 2026 00:26:04 +0200 Subject: [PATCH 21/40] Fix type of CommitCondition matcher --- src/EventStore.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/EventStore.js b/src/EventStore.js index 848fc9e3..0131333d 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -40,7 +40,7 @@ class OptimisticConcurrencyError extends Error {} class CommitCondition { /** * @param {string[]} types - * @param {function(object, object): boolean|object|null} [matcher] + * @param {(function(object, object): boolean)|object|null} [matcher] * @param {number} noneMatchAfter * @param {boolean} [raw=false] */ From 999f4acdc5bd717818b23cc06c1043900559b8ba Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sun, 7 Jun 2026 19:39:34 +0200 Subject: [PATCH 22/40] Add type information --- docs/changelog.md | 7 ++ index.d.ts | 142 +++++++++++++++++++++++++++++++++++++++ package.json | 2 + test/EventStream.spec.js | 21 ++++++ 4 files changed, 172 insertions(+) create mode 100644 index.d.ts diff --git a/docs/changelog.md b/docs/changelog.md index f5df7a77..fdf5bfdd 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,12 @@ # Changelog +## 1.3.1 + +- Added published TypeScript declarations (`index.d.ts`) and package-level `types` metadata so consumers can reference `EventStore` and related public APIs from `event-storage`. +- `EventStream` now exposes `where()` for matcher-based filtering. +- `EventStream.filter()` now supports `Readable.filter(callback, options)` delegation when options are provided. +- Matcher-style `EventStream.filter(matcher)` remains supported for backward compatibility but now emits a deprecation warning and should be migrated to `where()`. + ## 1.3.0 New in 1.3: diff --git a/index.d.ts b/index.d.ts new file mode 100644 index 00000000..87552ce1 --- /dev/null +++ b/index.d.ts @@ -0,0 +1,142 @@ +import { EventEmitter } from 'node:events'; +import { Readable } from 'node:stream'; + +export type EventMatcher = Record | ((payload: any, metadata?: any) => boolean); +export type EventPredicate = EventMatcher | ((buffer: Buffer) => boolean) | null; + +export interface StreamIndexInfo { + length: number; + metadata?: Record | null; +} + +export interface StreamInfo { + index: StreamIndexInfo; + matcher?: EventMatcher; + closed?: boolean; +} + +export class CommitCondition { + constructor(types: string[], matcher: EventMatcher | null | undefined, noneMatchAfter: number, raw?: boolean); + types: string[]; + matcher: EventMatcher | null; + raw: boolean; + noneMatchAfter: number; +} + +export class OptimisticConcurrencyError extends Error {} + +export const ExpectedVersion: { + Any: -1; + EmptyStream: 0; +}; + +export class EventStream extends Readable { + constructor(name: string, eventStore: EventStore, minRevision?: number, maxRevision?: number, predicate?: EventPredicate, raw?: boolean); + name: string; + raw: boolean; + version: number; + minRevision: number; + maxRevision: number; + streamIndex: StreamIndexInfo; + predicate: EventPredicate; + + from(revision: number): this; + until(revision: number): this; + fromStart(): this; + fromEnd(): this; + previous(amount: number): this; + following(amount: number): this; + backwards(): this; + forwards(): this; + where(predicate?: EventPredicate): this; + filter(predicate?: EventPredicate): this; + filter( + predicate: (chunk: any, options?: { signal?: AbortSignal }) => boolean | Promise, + options: { concurrency?: number; signal?: AbortSignal } + ): Readable; + next(): any | false; +} + +export class Consumer extends Readable { + constructor(storage: Storage, indexName: string, identifier: string, initialState?: Record, startFrom?: number); + streamName: string; + position: number; + state: Record; + start(): this; + stop(): void; + reset(initialState?: Record, startFrom?: number): this; + setState(state: Record): this; +} + +export class Storage extends EventEmitter { + static ReadOnly: typeof Storage; + static StorageLockedError: typeof StorageLockedError; + static LOCK_THROW: number; + static LOCK_RECLAIM: number; + + initialized: boolean | null; + length: number; + indexDirectory: string; + storageFile: string; +} + +export class StorageLockedError extends Error {} + +export class Index { + static ReadOnly: typeof Index; + static Entry: unknown; + + length: number; + metadata?: Record | null; +} + +export class EventStore extends EventEmitter { + static Storage: typeof Storage; + static Index: typeof Index; + + storage: Storage; + streams: Record; + consumers: Map; + length: number; + + constructor(storeName?: string, config?: Record); + + open(callback?: (err?: Error | null) => void): void; + close(callback?: (err?: Error | null) => void): void; + + getStreamVersion(streamName: string): number; + getEventStream(streamName: string, minRevision?: number, maxRevision?: number, predicate?: EventPredicate, raw?: boolean): EventStream | false; + getAllEvents(minRevision?: number, maxRevision?: number, predicate?: EventPredicate, raw?: boolean): EventStream; + fromStreams(streamName: string, streamNames: string[], minRevision?: number, maxRevision?: number, predicate?: EventPredicate, raw?: boolean): EventStream; + getEventStreamForCategory(categoryName: string, minRevision?: number, maxRevision?: number, predicate?: EventPredicate, raw?: boolean): EventStream; + + query(types: string[], matcher?: EventMatcher | null, minRevision?: number, raw?: boolean): { stream: EventStream; condition: CommitCondition }; + createEventStream(streamName: string, matcher: EventMatcher, reindex?: boolean): EventStream; + + commit( + streamName: string, + events: any[], + expectedVersion: number | CommitCondition, + metadata: Record, + callback: (error: Error | null, commit?: Record) => void + ): void; + + getConsumer(streamName: string, identifier: string, initialState?: Record, since?: number): Consumer; + getConsumer(identifier: string): Consumer | null; + scanConsumers( + callback: (error: Error | null, consumers: Array<{ name: string; stream: string; identifier: string }>) => void, + autoStart?: boolean + ): void; +} + +export const LOCK_THROW: number; +export const LOCK_RECLAIM: number; + +export function matches(document: Record, matcher: Record): boolean; +export function buildRawBufferMatcher(matcher: Record): (buffer: Buffer, startOffset?: number) => boolean; + +export default EventStore; + + + + diff --git a/package.json b/package.json index 1db79c7c..f798e4cf 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "exports": { ".": "./index.js" }, + "types": "./index.d.ts", "scripts": { "test": "c8 --reporter=lcov --reporter=text mocha test/*.spec.js", "test:grep": "c8 --reporter=lcov --reporter=text mocha test/*.spec.js --grep", @@ -45,6 +46,7 @@ "src/Storage/*.js", "src/WatchesFile.js", "index.js", + "index.d.ts", "src/utils/*.js" ], "license": "MIT", diff --git a/test/EventStream.spec.js b/test/EventStream.spec.js index cb263d08..e58effb8 100644 --- a/test/EventStream.spec.js +++ b/test/EventStream.spec.js @@ -248,6 +248,27 @@ describe('EventStream', function() { expect(stream.events).to.eql(events); }); + it('delegates to Readable.filter() when called with options', async function() { + const filteredReadable = stream.filter((payload) => payload === 'foo', { concurrency: 1 }); + expect(filteredReadable).not.to.be(stream); + + const values = []; + for await (const value of filteredReadable) { + values.push(value); + } + expect(values).to.eql(['foo']); + }); + + }); + + describe('where()', function() { + + it('applies matcher semantics and resets the iterator', function() { + const filtered = stream.where({ payload: 'foo' }); + expect(filtered).to.be(stream); + expect(stream.events).to.eql(['foo']); + }); + }); describe('object matcher (non-raw mode)', function() { From c8de46080a971fb38bef154c458c7595f4495835 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sun, 7 Jun 2026 19:39:58 +0200 Subject: [PATCH 23/40] Deprecate EventStream.filter(Matcher) in favor of EventStream.where() --- src/EventStream.js | 35 +++++++++++++++++++++++++++++------ src/utils/deprecations.js | 19 +++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 src/utils/deprecations.js diff --git a/src/EventStream.js b/src/EventStream.js index 6865eb36..97a5f2b0 100644 --- a/src/EventStream.js +++ b/src/EventStream.js @@ -2,8 +2,10 @@ import stream from 'stream'; import { assert } from './utils/util.js'; import { buildRawBufferMatcher, matches } from './utils/metadataUtil.js'; import { normalizeRevision, normalizeMaxRevision } from './utils/apiHelpers.js'; +import { emitDeprecationWarningOnce } from './utils/deprecations.js'; const NDJSON_NEWLINE = Buffer.from('\n'); +const FILTER_MATCHER_DEPRECATION_CODE = 'EVENTSTREAM_FILTER_MATCHER_DEPRECATED'; /** * An event stream is a simple wrapper around an iterator over storage documents. @@ -245,16 +247,15 @@ class EventStream extends stream.Readable { } /** - * Apply a filter predicate to this stream. Only events for which `predicate(payload, metadata)` - * returns a truthy value will be yielded. The predicate is stored as a first-class property - * of the stream and applied in {@link EventStream#next}. + * Apply matcher semantics to this EventStream instance (legacy `filter()` behavior). * * @api - * @param {function(object, object): boolean} predicate A function receiving `(payload, metadata)`. - * Events for which the predicate returns falsy are skipped. + * @param {(function(object, object): boolean)|(function(Buffer): boolean)|object|null} [predicate] + * Matcher function/object for object mode, or buffer matcher function/object for raw mode. + * Omit to clear the current predicate. * @returns {EventStream} `this` */ - filter(predicate) { + where(predicate) { this.predicate = predicate || null; this.rawMatcher = null; this._iterator = null; @@ -262,6 +263,28 @@ class EventStream extends stream.Readable { return this; } + /** + * Readable-compatible filter entry point. + * + * - `filter(callback, options)` delegates to Node's `Readable.filter(...)`. + * - `filter(matcher)` keeps legacy EventStream matcher behavior and is deprecated. + * + * @api + * @param {function|object|null} [predicate] + * @param {{concurrency?: number, signal?: AbortSignal}|undefined} [options] + * @returns {stream.Readable|EventStream} + */ + filter(predicate, options) { + if (options !== undefined) { + return super.filter(predicate, options); + } + emitDeprecationWarningOnce( + FILTER_MATCHER_DEPRECATION_CODE, + 'EventStream.filter() with matcher semantics is deprecated. Use EventStream.where() for matcher/object predicates or pass options to filter() to use Readable.filter().' + ); + return this.where(predicate); + } + matchesPredicate(entry) { if (!this.predicate) { return true; diff --git a/src/utils/deprecations.js b/src/utils/deprecations.js new file mode 100644 index 00000000..faed9704 --- /dev/null +++ b/src/utils/deprecations.js @@ -0,0 +1,19 @@ +const emittedDeprecationWarnings = new Set(); + +/** + * Emit a deprecation warning only once per warning code. + * + * @param {string} code Unique warning code. + * @param {string} message Warning message. + * @returns {void} + */ +function emitDeprecationWarningOnce(code, message) { + if (emittedDeprecationWarnings.has(code)) { + return; + } + emittedDeprecationWarnings.add(code); + process.emitWarning(message, 'DeprecationWarning', code); +} + +export { emitDeprecationWarningOnce }; + From c56c5420c143376489d2fec576cc93cbe4e0243d Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sun, 7 Jun 2026 19:40:27 +0200 Subject: [PATCH 24/40] Update AGENTS.md --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index 9672f6a1..ed5b0958 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,6 +3,7 @@ ## General guidelines for interaction Do not repeat yourself. Be concise and precise in your answers. No paraphrasing of previous information unless specifically asked for. +If you run into a decision point, maybe because the instructions given were contradictory or incomplete, try to identify the underlying principle or intent and unless that is absolutely clear, ask for clarification and/or a decision. Explain the issue you stumbled upon and the options you see, and ask for guidance on how to proceed. The code and documentation language is English, even if the user is communicating in another language. Read the Platform.md file if it exists to know which platform you are running on and what capabilities you have, instead of trying to run linux tools on a Windows machine. **Keep this file up-to-date**: after any code review that surfaces new architectural insights or a new principle, compact this file and integrate what was learned. From c9960ef040af87926501ebe742480604428d5599 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Sun, 7 Jun 2026 19:40:50 +0200 Subject: [PATCH 25/40] Bump version 1.3.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f798e4cf..15961284 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "event-storage", - "version": "1.3.0", + "version": "1.3.1", "type": "module", "description": "An optimized embedded event store for node.js", "keywords": [ From f3a5746918e7f89e5c93b2e1bf57331a23fe3b59 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Mon, 8 Jun 2026 18:36:04 +0200 Subject: [PATCH 26/40] Fix bug in ReadOnlyStorage open with callback --- src/Storage/ReadOnlyStorage.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Storage/ReadOnlyStorage.js b/src/Storage/ReadOnlyStorage.js index 87d7cf4d..560abf65 100644 --- a/src/Storage/ReadOnlyStorage.js +++ b/src/Storage/ReadOnlyStorage.js @@ -31,14 +31,16 @@ class ReadOnlyStorage extends ReadableStorage { * Will emit an 'opened' event if finished. * * @api + * @param {function(): void} [callback] Called after indexes open, before `'opened'` is emitted. + * Can be used as a synchronous alternative to listening to the `'opened'` event. * @returns {boolean} */ - open() { + open(callback) { if (!this.watcher) { this.watcher = new Watcher([this.dataDirectory, this.indexDirectory], this.storageFilesFilter); this.watcher.on('rename', this.onStorageFileChanged); } - return super.open(); + return super.open(callback); } /** From 90e87adcfe2aaa1eedddb149493774562c713901 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Mon, 8 Jun 2026 18:40:22 +0200 Subject: [PATCH 27/40] Add EventStore makeReadOnly() API method This allows to step back from writing without fully stopping handling any operation, which is useful for a graceful handover of write ownership. --- src/EventStore.js | 57 +++++++++++++++++++++++++++------ test/EventStore.spec.js | 71 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 9 deletions(-) diff --git a/src/EventStore.js b/src/EventStore.js index 0131333d..8b0d41a4 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -129,24 +129,34 @@ class EventStore extends events.EventEmitter { * @param {object} storageConfig */ initialize(storeName, storageConfig) { + this.storageConfig = storageConfig; this.streamsDirectory = path.resolve(storageConfig.indexDirectory); - this.storeName = storeName; - this.storage = (storageConfig.readOnly === true) ? - new ReadOnlyStorage(storeName, storageConfig) - : new Storage(storeName, storageConfig); - this.streams = Object.create(null); - this.streams._all = { index: this.storage.index }; this.consumers = new Map(); - this.storage.on('index-created', this.registerStream.bind(this)); + const storage = storageConfig.readOnly === true + ? new ReadOnlyStorage(storeName, storageConfig) + : new Storage(storeName, storageConfig); - this.storage.on('opened', () => { + this.mountStorage(storage, () => { this.checkUnfinishedCommits(); this.emit('ready'); }); + } - this.storage.open(); + /** + * Wire up a storage instance: reset the streams map, attach the index-created listener, + * register a one-shot opened handler, then open the storage. + * @private + * @param {ReadableStorage} storage + * @param {function} [onOpened] Called once when the storage is opened. + */ + mountStorage(storage, onOpened) { + this.storage = storage; + this.streams = Object.create(null); + this.streams._all = { index: this.storage.index }; + this.storage.on('index-created', this.registerStream.bind(this)); + this.storage.open(onOpened); } /** @@ -231,6 +241,35 @@ class EventStore extends events.EventEmitter { this.storage.close(); } + /** + * Flush all pending writes, then re-open the store in read-only mode. + * Any registered consumers are stopped. The `callback` is invoked once the + * new read-only storage has finished opening. + * + * Does not re-emit `'ready'` — use the callback to react to the transition. + * + * @api + * @param {function} [callback] Called when the store is ready in read-only mode. + */ + makeReadOnly(callback) { + if (this.storage instanceof ReadOnlyStorage) { + callback?.(); + return + } + for (const consumer of this.consumers.values()) { + consumer.stop(); + } + this.consumers.clear(); + + this.storage.flush(); + this.storage.close(); + + const readOnlyConfig = Object.assign({}, this.storageConfig, { readOnly: true }); + this.storageConfig = readOnlyConfig; + + this.mountStorage(new ReadOnlyStorage(this.storeName, readOnlyConfig), callback); + } + /** * Override EventEmitter.on() to delegate 'preCommit' and 'preRead' event registrations * to the underlying storage, so that `eventstore.on('preCommit', handler)` works naturally. diff --git a/test/EventStore.spec.js b/test/EventStore.spec.js index 970f47ee..684aad50 100644 --- a/test/EventStore.spec.js +++ b/test/EventStore.spec.js @@ -2210,4 +2210,75 @@ describe('EventStore', function() { }); + describe('makeReadOnly()', function() { + + it('switches to read-only mode and calls the callback', function(done) { + eventstore = new EventStore({ storageDirectory }); + eventstore.on('ready', () => { + eventstore.commit('foo', [{ x: 1 }], () => { + eventstore.makeReadOnly(() => { + expect(eventstore.storage.constructor.name).to.be('ReadOnlyStorage'); + expect(() => eventstore.commit('foo', [{ x: 2 }])).to.throwError(/read-only/); + done(); + }); + }); + }); + }); + + it('can switch to read-only outside a commit callback and keeps already written events', function(done) { + eventstore = new EventStore({ storageDirectory }); + eventstore.on('ready', () => { + eventstore.commit('deployment-stream', [{ step: 'before-switch' }]); + eventstore.makeReadOnly(() => { + expect(eventstore.storage.constructor.name).to.be('ReadOnlyStorage'); + + const stream = eventstore.getEventStream('deployment-stream'); + expect(stream).to.not.be(false); + expect(stream.events.length).to.be(1); + expect(stream.events[0].step).to.be('before-switch'); + + expect(() => eventstore.commit('deployment-stream', [{ step: 'after-switch' }])).to.throwError(/read-only/); + done(); + }); + }); + }); + + it('preserves existing events after switching to read-only', function(done) { + eventstore = new EventStore({ storageDirectory }); + eventstore.on('ready', () => { + eventstore.commit('bar', [{ y: 1 }, { y: 2 }], () => { + eventstore.makeReadOnly(() => { + const stream = eventstore.getEventStream('bar'); + const events = stream.events; + expect(events.length).to.be(2); + expect(events[0].y).to.be(1); + expect(events[1].y).to.be(2); + done(); + }); + }); + }); + }); + + it('prevents new commits after switching to read-only', function(done) { + eventstore = new EventStore({ storageDirectory }); + eventstore.on('ready', () => { + eventstore.makeReadOnly(() => { + expect(() => eventstore.commit('baz', [{ z: 1 }])).to.throwError(/read-only/); + done(); + }); + }); + }); + + it('prevents createEventStream after switching to read-only', function(done) { + eventstore = new EventStore({ storageDirectory }); + eventstore.on('ready', () => { + eventstore.makeReadOnly(() => { + expect(() => eventstore.createEventStream('new-stream', { stream: 'new-stream' })).to.throwError(/read-only/); + done(); + }); + }); + }); + + }); + }); From df7132af46bb363c66fd3aab9c97f17ef2dcb703 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Mon, 8 Jun 2026 19:03:58 +0200 Subject: [PATCH 28/40] Add documentation for makeReadOnly() --- docs/api.md | 13 +++++++++++++ docs/changelog.md | 5 +++++ docs/consumers.md | 2 ++ 3 files changed, 20 insertions(+) diff --git a/docs/api.md b/docs/api.md index 33a70413..221e057a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -47,6 +47,19 @@ Close the event store and free all resources. --- +#### `makeReadOnly([callback])` ✅ Stable + +```javascript +eventstore.makeReadOnly([callback]) +``` + +Flush pending writes, close the writable storage, and reopen the store in read-only mode. +The optional `callback` is called after the new read-only storage has emitted `'opened'`. + +This is mainly useful for deployment handover workflows where one process stops writing and the same process switches into read-only mode; for most applications, `eventstore.close(); new EventStore(..., { readOnly: true })` is the simpler option. + +--- + #### `commit(streamName, events, [expectedVersion], [metadata], [callback])` ✅ Stable ```javascript diff --git a/docs/changelog.md b/docs/changelog.md index fdf5bfdd..38c65ff6 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,10 @@ # Changelog +## 1.4.0 + +- Added `EventStore.makeReadOnly([callback])` as a niche handover API for switching a running writer process into read-only mode without restarting it. For the common case, creating a fresh read-only `EventStore` instance remains the recommended approach. +- Fixed `ReadOnlyStorage.open()` to keep the same opened-event semantics as the readable and writable storage variants, so read-only reopen paths and the `makeReadOnly()` callback now work consistently. + ## 1.3.1 - Added published TypeScript declarations (`index.d.ts`) and package-level `types` metadata so consumers can reference `EventStore` and related public APIs from `event-storage`. diff --git a/docs/consumers.md b/docs/consumers.md index 9b24b4b7..f83e6801 100644 --- a/docs/consumers.md +++ b/docs/consumers.md @@ -144,4 +144,6 @@ eventstore.on('ready', () => { - Multiple read-only instances can run simultaneously. - This enables a multi-process projection pattern: one writer process + N reader processes building different projections. +If you need to hand the *current* process over from writing to reading during a deployment, use `EventStore.makeReadOnly([callback])` instead of starting a new read-only instance. + > In principle this also works across machines sharing a common filesystem (e.g. NFS), as long as the Node.js file watcher functions correctly on that filesystem. From d280d81df3fbab5de0c8ebb7141f65d10d34d0a0 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Mon, 8 Jun 2026 20:30:09 +0200 Subject: [PATCH 29/40] Add VERSION export --- index.d.ts | 3 +++ index.js | 34 ++++++++++++++++++++++++++++------ test/Exports.spec.js | 11 +++++++++++ 3 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 test/Exports.spec.js diff --git a/index.d.ts b/index.d.ts index 87552ce1..6ef54039 100644 --- a/index.d.ts +++ b/index.d.ts @@ -30,6 +30,8 @@ export const ExpectedVersion: { EmptyStream: 0; }; +export const VERSION: string; + export class EventStream extends Readable { constructor(name: string, eventStore: EventStore, minRevision?: number, maxRevision?: number, predicate?: EventPredicate, raw?: boolean); name: string; @@ -93,6 +95,7 @@ export class Index { export class EventStore extends EventEmitter { static Storage: typeof Storage; static Index: typeof Index; + static VERSION: string; storage: Storage; streams: Record; diff --git a/index.js b/index.js index b6d72de1..fc9780be 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,28 @@ -export { default as EventStore, default, ExpectedVersion, OptimisticConcurrencyError, CommitCondition, LOCK_THROW, LOCK_RECLAIM } from './src/EventStore.js'; -export { default as EventStream } from './src/EventStream.js'; -export { default as Storage, StorageLockedError } from './src/Storage.js'; -export { default as Index } from './src/Index.js'; -export { default as Consumer } from './src/Consumer.js'; -export { matches, buildRawBufferMatcher } from './src/utils/metadataUtil.js'; +import packageJson from './package.json' with { type: 'json' }; +import EventStore, { ExpectedVersion, OptimisticConcurrencyError, CommitCondition, LOCK_THROW, LOCK_RECLAIM } from './src/EventStore.js'; +import EventStream from './src/EventStream.js'; +import Storage, { StorageLockedError } from './src/Storage.js'; +import Index from './src/Index.js'; +import Consumer from './src/Consumer.js'; +import { matches, buildRawBufferMatcher } from './src/utils/metadataUtil.js'; + +const VERSION = packageJson.version; +EventStore.VERSION = VERSION; + +export { + EventStore, + EventStore as default, + ExpectedVersion, + OptimisticConcurrencyError, + CommitCondition, + LOCK_THROW, + LOCK_RECLAIM, + VERSION, + EventStream, + Storage, + StorageLockedError, + Index, + Consumer, + matches, + buildRawBufferMatcher +}; diff --git a/test/Exports.spec.js b/test/Exports.spec.js new file mode 100644 index 00000000..890e1277 --- /dev/null +++ b/test/Exports.spec.js @@ -0,0 +1,11 @@ +import expect from 'expect.js'; +import EventStore, { VERSION } from '../index.js'; + +describe('Public exports', function() { + it('exports VERSION and mirrors it on EventStore.VERSION', function() { + expect(VERSION).to.be.a('string'); + expect(VERSION.length > 0).to.be(true); + expect(EventStore.VERSION).to.be(VERSION); + }); +}); + From 5499008b6e3069292b79b55281dd53e0b3adcd8b Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Mon, 8 Jun 2026 20:40:26 +0200 Subject: [PATCH 30/40] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9c16bbea..913f6d62 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ node_modules /event-storage-ui /event-storage-http /Platform.md +/*/node_modules From 388cb5d1f96fcabd9ad2f797e1c43b2586f9df44 Mon Sep 17 00:00:00 2001 From: Alexander Berl Date: Mon, 8 Jun 2026 20:43:22 +0200 Subject: [PATCH 31/40] Bump version --- docs/changelog.md | 3 ++- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 38c65ff6..01ea047f 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,8 +1,9 @@ # Changelog -## 1.4.0 +## 1.3.2 - Added `EventStore.makeReadOnly([callback])` as a niche handover API for switching a running writer process into read-only mode without restarting it. For the common case, creating a fresh read-only `EventStore` instance remains the recommended approach. +- Added public `VERSION` export at the package root and mirrored it on `EventStore.VERSION` so dependent layers can read the `event-storage` version directly without resolving/parsing `package.json` at runtime. - Fixed `ReadOnlyStorage.open()` to keep the same opened-event semantics as the readable and writable storage variants, so read-only reopen paths and the `makeReadOnly()` callback now work consistently. ## 1.3.1 diff --git a/package-lock.json b/package-lock.json index 181a112b..390a8a91 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "event-storage", - "version": "1.2.0", + "version": "1.3.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "event-storage", - "version": "1.2.0", + "version": "1.3.2", "license": "MIT", "dependencies": { "mkdirp": "^3.0.1" diff --git a/package.json b/package.json index 15961284..3af33b3b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "event-storage", - "version": "1.3.1", + "version": "1.3.2", "type": "module", "description": "An optimized embedded event store for node.js", "keywords": [ From f053257ab55cbfc6d2d17b0563ac23559f5e3593 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 29 May 2026 17:19:17 +0000 Subject: [PATCH 32/40] Add consumer projection persistence support --- docs/api.md | 21 +++++++++++ docs/consumers.md | 26 +++++++++++++ src/Consumer.js | 86 ++++++++++++++++++++++++++++++++++++++++++- test/Consumer.spec.js | 42 +++++++++++++++++++++ 4 files changed, 173 insertions(+), 2 deletions(-) diff --git a/docs/api.md b/docs/api.md index 221e057a..03664e79 100644 --- a/docs/api.md +++ b/docs/api.md @@ -280,6 +280,27 @@ Asynchronously scan all consumer state files and return their identifiers. --- +#### `consumer.createProjection(projectionFn, [options])` + +```javascript +consumer.createProjection(projectionFn [, options]) +``` + +Register a reducer-style projection as the consumer's `'data'` handler and persist it so it is auto-restored when reopening the same consumer. + +`projectionFn` can be either: + +- a reducer function: `(state, event) => state` +- an object map: `{ [eventType]: (state, event) => state }` + +Options: + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `options.hmac` | `function(string): string` | storage HMAC | HMAC function used to sign/verify serialized function projections. Required for trusted function projection restore. | + +--- + ### Events emitted | Event | Payload | Description | diff --git a/docs/consumers.md b/docs/consumers.md index f83e6801..ffac2e84 100644 --- a/docs/consumers.md +++ b/docs/consumers.md @@ -69,6 +69,32 @@ consumer.setState({ count: 0 }); consumer.setState((state) => ({ ...state, count: state.count + 1 })); ``` +## Projections + +Instead of registering `'data'` manually, you can register a projection reducer via `createProjection`. +The reducer is persisted and automatically reloaded when the same consumer is reopened. + +```javascript +import crypto from 'crypto'; + +const hmac = (code) => crypto.createHmac('sha256', 'your-private-secret').update(code).digest('hex'); + +const consumer = eventstore.getConsumer('orders', 'orders-projection', { total: 0 }); +consumer.createProjection( + (state, event) => ({ ...state, total: state.total + (event.amount || 0) }), + { hmac } +); +``` + +You can also pass per-event reducers: + +```javascript +consumer.createProjection({ + OrderCreated: (state, event) => ({ ...state, orders: [...state.orders, event] }), + OrderCancelled: (state, event) => ({ ...state, cancelled: state.cancelled + 1 }) +}, { hmac }); +``` + ## Resetting a Consumer Force the consumer to reprocess events from a given position: diff --git a/src/Consumer.js b/src/Consumer.js index 58411b57..4b44024e 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -4,6 +4,7 @@ import path from 'path'; import { assert } from './utils/util.js'; import { ensureDirectory } from './utils/fsUtil.js'; import { normalizeConsumerStateArgs } from './utils/apiHelpers.js'; +import { buildMetadataForMatcher, buildMatcherFromMetadata } from './utils/metadataUtil.js'; import Storage from './Storage/ReadableStorage.js'; const MAX_CATCHUP_BATCH = 10; @@ -34,7 +35,7 @@ class Consumer extends stream.Readable { * @param {object} [initialState={}] The initial state of the consumer. * @param {number} [startFrom=0] The revision to start from within the index to consume. */ - constructor(storage, indexName, identifier, initialState = {}, startFrom = 0) { + constructor(storage, indexName, identifier, initialState = {}, startFrom = 0, options = {}) { super({ objectMode: true }); assert(storage instanceof Storage, 'Must provide a storage for the consumer.'); @@ -43,6 +44,7 @@ class Consumer extends stream.Readable { this.initializeStorage(storage, indexName, identifier); this.restoreState(initialState, startFrom); + this.restoreProjection(options); this.handler = this.handleNewDocument.bind(this); this.on('error', () => (this.handleDocument = false)); } @@ -59,6 +61,7 @@ class Consumer extends stream.Readable { this.indexName = indexName; const consumerDirectory = path.join(this.storage.indexDirectory, 'consumers'); this.fileName = path.join(consumerDirectory, this.storage.storageFile + '.' + indexName + '.' + identifier); + this.projectionFileName = this.fileName + '.projection'; if (ensureDirectory(consumerDirectory)) { this.cleanUpFailedWrites(); } @@ -73,7 +76,8 @@ class Consumer extends stream.Readable { const consumerDirectory = path.dirname(this.fileName); const files = fs.readdirSync(consumerDirectory); for (let file of files) { - if (file.startsWith(consumerNamePrefix)) { + const suffix = file.slice(consumerNamePrefix.length); + if (file.startsWith(consumerNamePrefix) && /^\d+$/.test(suffix)) { safeUnlink(path.join(consumerDirectory, file)); } } @@ -104,6 +108,84 @@ class Consumer extends stream.Readable { this.consuming = false; } + /** + * Restore a persisted projection and register it as data handler. + * @private + * @param {object} [options={}] + * @param {function(string): string} [options.hmac] + */ + restoreProjection(options = {}) { + this.projection = null; + this.projectionHandler = null; + if (!this.projectionFileName || !fs.existsSync(this.projectionFileName)) { + return; + } + const projectionMetadata = JSON.parse(fs.readFileSync(this.projectionFileName, 'utf8')); + const hmac = options.hmac || this.storage.hmac; + if (typeof projectionMetadata.matcher === 'string') { + assert(typeof hmac === 'function', 'Must provide options.hmac to restore a function projection.'); + } + const projection = buildMatcherFromMetadata(projectionMetadata, hmac); + this.registerProjection(projection); + } + + /** + * Register a projection function as `data` event handler. + * @private + * @param {function|object} projection + */ + registerProjection(projection) { + if (this.projectionHandler) { + this.removeListener('data', this.projectionHandler); + } + this.projection = projection; + this.projectionHandler = (event) => { + let reducer = projection; + if (typeof projection === 'object') { + const type = event?.type || event?.payload?.type; + reducer = projection[type]; + if (typeof reducer !== 'function') { + return; + } + } + this.setState(reducer(this.state, event)); + }; + this.on('data', this.projectionHandler); + } + + /** + * Create and persist a projection reducer and register it as data handler. + * + * @api + * @param {function(object, object): object|object} projectionFn + * @param {object} [options] + * @param {function(string): string} [options.hmac] Required for function projections. + */ + createProjection(projectionFn, options = {}) { + assert((typeof projectionFn === 'function') || (projectionFn && typeof projectionFn === 'object' && !Array.isArray(projectionFn)), 'Projection must be a reducer function or an object map of reducer functions.'); + if (typeof projectionFn === 'object') { + for (const reducer of Object.values(projectionFn)) { + assert(typeof reducer === 'function', 'Projection object values must be reducer functions.'); + } + } + const hmac = options.hmac || this.storage.hmac; + if (typeof projectionFn === 'function') { + assert(typeof hmac === 'function', 'Must provide options.hmac for function projections.'); + } + + const projectionMetadata = buildMetadataForMatcher(projectionFn, hmac); + const tmpProjectionFileName = this.projectionFileName + '.tmp'; + try { + fs.writeFileSync(tmpProjectionFileName, JSON.stringify(projectionMetadata), 'utf8'); + fs.renameSync(tmpProjectionFileName, this.projectionFileName); + } catch (e) { + safeUnlink(tmpProjectionFileName); + throw e; + } + + this.registerProjection(projectionFn); + } + /** * Update the state of this consumer transactionally with the position. * May only be called from within the document handling callback. diff --git a/test/Consumer.spec.js b/test/Consumer.spec.js index a5de02b7..b3f10e6e 100644 --- a/test/Consumer.spec.js +++ b/test/Consumer.spec.js @@ -3,6 +3,7 @@ import fs from 'fs-extra'; import fsNative from 'fs'; import Storage from '../src/Storage.js'; import Consumer from '../src/Consumer.js'; +import { createHmac } from '../src/utils/metadataUtil.js'; import { fileURLToPath } from 'url'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -519,6 +520,47 @@ describe('Consumer', function() { }); }); + it('can create projections from a reducer function', function(done) { + consumer = new Consumer(storage, 'foobar', 'consumer-projection', { count: 0 }); + consumer.createProjection((state, event) => ({ ...state, count: state.count + event.id }), { hmac: createHmac('test-secret') }); + consumer.on('caught-up', () => { + expect(consumer.state.count).to.be(6); + done(); + }); + + storage.write({ type: 'Foobar', id: 1 }); + storage.write({ type: 'Foobar', id: 2 }); + storage.write({ type: 'Foobar', id: 3 }); + }); + + it('can create projections from event-type reducer maps and restore them on reopen', function(done) { + consumer = new Consumer(storage, 'foobar', 'consumer-projection-map', { count: 0 }); + consumer.createProjection({ + Foobar: (state, event) => ({ ...state, count: state.count + event.id }), + Bazinga: (state) => state + }, { hmac: createHmac('test-secret') }); + + consumer.on('caught-up', () => { + consumer.stop(); + consumer = new Consumer(storage, 'foobar', 'consumer-projection-map'); + consumer.on('caught-up', () => { + expect(consumer.state.count).to.be(10); + done(); + }); + storage.write({ type: 'Foobar', id: 4 }); + }); + + storage.write({ type: 'Foobar', id: 1 }); + storage.write({ type: 'Foobar', id: 2 }); + storage.write({ type: 'Foobar', id: 3 }); + }); + + it('throws if function projection is restored without trusted hmac', function() { + consumer = new Consumer(storage, 'foobar', 'consumer-projection-hmac'); + consumer.createProjection((state, event) => ({ ...state, lastId: event.id }), { hmac: createHmac('test-secret') }); + expect(() => new Consumer(storage, 'foobar', 'consumer-projection-hmac', {}, 0, { hmac: createHmac('wrong-secret') })).to.throwError(/Invalid HMAC/); + }); + it('can build consistency guards (aggregates)', function(done) { const guard = new Consumer(storage, 'foobar', 'unique-bar-guard'); guard.apply = function(event) { From cf21ab9a6a81d228d0b7c75e51cd3881ec60b966 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 29 May 2026 17:25:45 +0000 Subject: [PATCH 33/40] Implement durable consumer projections with HMAC validation --- docs/api.md | 2 +- src/Consumer.js | 54 +++++++++++++++++++++++++++++++++---------- test/Consumer.spec.js | 9 ++++---- 3 files changed, 48 insertions(+), 17 deletions(-) diff --git a/docs/api.md b/docs/api.md index 03664e79..b1370803 100644 --- a/docs/api.md +++ b/docs/api.md @@ -297,7 +297,7 @@ Options: | Parameter | Type | Default | Description | |-----------|------|---------|-------------| -| `options.hmac` | `function(string): string` | storage HMAC | HMAC function used to sign/verify serialized function projections. Required for trusted function projection restore. | +| `options.hmac` | `function(string): string` | storage HMAC | HMAC function used to sign/verify serialized projections. Required when no storage-level HMAC is configured. | --- diff --git a/src/Consumer.js b/src/Consumer.js index 4b44024e..022eb1b8 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -72,12 +72,17 @@ class Consumer extends stream.Readable { * @private */ cleanUpFailedWrites() { - const consumerNamePrefix = path.basename(this.fileName) + '.'; + const consumerBaseName = path.basename(this.fileName); + const projectionBaseName = consumerBaseName + '.projection'; + const escapedConsumerBaseName = consumerBaseName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const failedStateFilePattern = new RegExp(`^${escapedConsumerBaseName}\\.\\d+$`); const consumerDirectory = path.dirname(this.fileName); const files = fs.readdirSync(consumerDirectory); for (let file of files) { - const suffix = file.slice(consumerNamePrefix.length); - if (file.startsWith(consumerNamePrefix) && /^\d+$/.test(suffix)) { + if (file === projectionBaseName) { + continue; + } + if (file === projectionBaseName + '.tmp' || failedStateFilePattern.test(file)) { safeUnlink(path.join(consumerDirectory, file)); } } @@ -122,13 +127,33 @@ class Consumer extends stream.Readable { } const projectionMetadata = JSON.parse(fs.readFileSync(this.projectionFileName, 'utf8')); const hmac = options.hmac || this.storage.hmac; - if (typeof projectionMetadata.matcher === 'string') { - assert(typeof hmac === 'function', 'Must provide options.hmac to restore a function projection.'); - } - const projection = buildMatcherFromMetadata(projectionMetadata, hmac); + const projection = this.deserializeProjectionMetadata(projectionMetadata, hmac); this.registerProjection(projection); } + /** + * @private + * @param {{kind?: string, projection?: object, matcher?: string|object, hmac?: string}} metadata + * @param {function(string): string} hmac + * @returns {function|object} + */ + deserializeProjectionMetadata(metadata, hmac) { + assert(metadata && typeof metadata === 'object', 'Invalid projection metadata.'); + if (metadata.kind === 'map') { + assert(typeof hmac === 'function', 'Must provide options.hmac to restore mapped function projections.'); + const projectionMap = {}; + for (const [eventType, reducerMetadata] of Object.entries(metadata.projection || {})) { + projectionMap[eventType] = buildMatcherFromMetadata(reducerMetadata, hmac); + } + return projectionMap; + } + assert(metadata.kind === 'function', 'Invalid projection metadata kind.'); + if (typeof metadata.projection?.matcher === 'string') { + assert(typeof hmac === 'function', 'Must provide options.hmac to restore function projections.'); + } + return buildMatcherFromMetadata(metadata.projection, hmac); + } + /** * Register a projection function as `data` event handler. * @private @@ -142,6 +167,7 @@ class Consumer extends stream.Readable { this.projectionHandler = (event) => { let reducer = projection; if (typeof projection === 'object') { + // Direct Storage consumers emit `{ type }`, EventStore consumers emit `{ payload: { type } }`. const type = event?.type || event?.payload?.type; reducer = projection[type]; if (typeof reducer !== 'function') { @@ -169,11 +195,15 @@ class Consumer extends stream.Readable { } } const hmac = options.hmac || this.storage.hmac; - if (typeof projectionFn === 'function') { - assert(typeof hmac === 'function', 'Must provide options.hmac for function projections.'); - } - - const projectionMetadata = buildMetadataForMatcher(projectionFn, hmac); + assert(typeof hmac === 'function', 'Must provide options.hmac for projections.'); + const projectionMetadata = (typeof projectionFn === 'function') + ? { kind: 'function', projection: buildMetadataForMatcher(projectionFn, hmac) } + : { + kind: 'map', + projection: Object.fromEntries( + Object.entries(projectionFn).map(([eventType, reducer]) => [eventType, buildMetadataForMatcher(reducer, hmac)]) + ) + }; const tmpProjectionFileName = this.projectionFileName + '.tmp'; try { fs.writeFileSync(tmpProjectionFileName, JSON.stringify(projectionMetadata), 'utf8'); diff --git a/test/Consumer.spec.js b/test/Consumer.spec.js index b3f10e6e..db669117 100644 --- a/test/Consumer.spec.js +++ b/test/Consumer.spec.js @@ -542,10 +542,11 @@ describe('Consumer', function() { consumer.on('caught-up', () => { consumer.stop(); - consumer = new Consumer(storage, 'foobar', 'consumer-projection-map'); - consumer.on('caught-up', () => { - expect(consumer.state.count).to.be(10); - done(); + consumer = new Consumer(storage, 'foobar', 'consumer-projection-map', {}, 0, { hmac: createHmac('test-secret') }); + consumer.on('progress', () => { + if (consumer.state.count === 10) { + done(); + } }); storage.write({ type: 'Foobar', id: 4 }); }); From 2204aa88eb027fe05ee2269dedc4ae763003a5df Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 30 May 2026 20:01:39 +0000 Subject: [PATCH 34/40] Refactor projections into Projection class and integrate Consumer/EventStore APIs --- docs/api.md | 40 ++++++ docs/consumers.md | 30 +++-- index.js | 3 + src/Consumer.js | 99 +++++--------- src/EventStore.js | 35 ++++- src/Projection.js | 276 ++++++++++++++++++++++++++++++++++++++++ test/Consumer.spec.js | 43 +++++++ test/EventStore.spec.js | 71 +++++++++++ 8 files changed, 522 insertions(+), 75 deletions(-) create mode 100644 src/Projection.js diff --git a/docs/api.md b/docs/api.md index b1370803..d64c59b9 100644 --- a/docs/api.md +++ b/docs/api.md @@ -301,6 +301,46 @@ Options: --- +#### `eventstore.getProjection(name, [definition], [options])` + +```javascript +eventstore.getProjection(name [, definition [, options]]) → Projection +``` + +Create a `Projection` with EventStore defaults (`typeAccessor`, storage HMAC), or restore a previously persisted one when `definition` is omitted. + +`definition` shape: + +```javascript +{ + initialState: any, + handlers: (state, event) => state | { [eventType]: (state, event) => state }, + matcher: object|function // optional +} +``` + +--- + +#### `consumer.project(projection, [options])` + +```javascript +consumer.project(projection [, options]) +``` + +Attach a `Projection` instance to a durable consumer and persist its definition for automatic restore when reopening the same consumer. + +--- + +#### `projection.subscribe(consumer, [options])` + +```javascript +projection.subscribe(consumer [, options]) +``` + +Alias for `consumer.project(projection, options)`. + +--- + ### Events emitted | Event | Payload | Description | diff --git a/docs/consumers.md b/docs/consumers.md index ffac2e84..51b27835 100644 --- a/docs/consumers.md +++ b/docs/consumers.md @@ -71,22 +71,25 @@ consumer.setState((state) => ({ ...state, count: state.count + 1 })); ## Projections -Instead of registering `'data'` manually, you can register a projection reducer via `createProjection`. -The reducer is persisted and automatically reloaded when the same consumer is reopened. +Use a `Projection` to define *how* events are projected into state, then connect it to a `Consumer` for durable continuous updates. ```javascript import crypto from 'crypto'; const hmac = (code) => crypto.createHmac('sha256', 'your-private-secret').update(code).digest('hex'); -const consumer = eventstore.getConsumer('orders', 'orders-projection', { total: 0 }); -consumer.createProjection( - (state, event) => ({ ...state, total: state.total + (event.amount || 0) }), - { hmac } -); +const projection = eventstore.getProjection('orders-total', { + initialState: { total: 0 }, + handlers: { + OrderCreated: (state, event) => ({ ...state, total: state.total + (event.payload.amount || 0) }) + } +}, { hmac }); + +const consumer = eventstore.getConsumer('orders', 'orders-projection', projection.initialState); +consumer.project(projection); ``` -You can also pass per-event reducers: +You can still use `consumer.createProjection(...)` for the shorter in-place API: ```javascript consumer.createProjection({ @@ -95,6 +98,17 @@ consumer.createProjection({ }, { hmac }); ``` +Projections are composable via `CompositeProjection`: + +```javascript +import { CompositeProjection } from 'event-storage'; + +const overview = new CompositeProjection('overview', { + count: { initialState: 0, handlers: { OrderCreated: (state) => state + 1 } }, + last: { initialState: null, handlers: { OrderCreated: (state, event) => event.payload } } +}); +``` + ## Resetting a Consumer Force the consumer to reprocess events from a given position: diff --git a/index.js b/index.js index fc9780be..2f8e0df4 100644 --- a/index.js +++ b/index.js @@ -4,6 +4,7 @@ import EventStream from './src/EventStream.js'; import Storage, { StorageLockedError } from './src/Storage.js'; import Index from './src/Index.js'; import Consumer from './src/Consumer.js'; +import Projection, { CompositeProjection } from './src/Projection.js'; import { matches, buildRawBufferMatcher } from './src/utils/metadataUtil.js'; const VERSION = packageJson.version; @@ -23,6 +24,8 @@ export { StorageLockedError, Index, Consumer, + Projection, + CompositeProjection, matches, buildRawBufferMatcher }; diff --git a/src/Consumer.js b/src/Consumer.js index 022eb1b8..0ebcd6c9 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -4,7 +4,7 @@ import path from 'path'; import { assert } from './utils/util.js'; import { ensureDirectory } from './utils/fsUtil.js'; import { normalizeConsumerStateArgs } from './utils/apiHelpers.js'; -import { buildMetadataForMatcher, buildMatcherFromMetadata } from './utils/metadataUtil.js'; +import Projection from './Projection.js'; import Storage from './Storage/ReadableStorage.js'; const MAX_CATCHUP_BATCH = 10; @@ -59,6 +59,7 @@ class Consumer extends stream.Readable { this.storage = storage; this.index = this.storage.openIndex(indexName); this.indexName = indexName; + this.identifier = identifier; const consumerDirectory = path.join(this.storage.indexDirectory, 'consumers'); this.fileName = path.join(consumerDirectory, this.storage.storageFile + '.' + indexName + '.' + identifier); this.projectionFileName = this.fileName + '.projection'; @@ -125,58 +126,39 @@ class Consumer extends stream.Readable { if (!this.projectionFileName || !fs.existsSync(this.projectionFileName)) { return; } - const projectionMetadata = JSON.parse(fs.readFileSync(this.projectionFileName, 'utf8')); - const hmac = options.hmac || this.storage.hmac; - const projection = this.deserializeProjectionMetadata(projectionMetadata, hmac); - this.registerProjection(projection); - } - - /** - * @private - * @param {{kind?: string, projection?: object, matcher?: string|object, hmac?: string}} metadata - * @param {function(string): string} hmac - * @returns {function|object} - */ - deserializeProjectionMetadata(metadata, hmac) { - assert(metadata && typeof metadata === 'object', 'Invalid projection metadata.'); - if (metadata.kind === 'map') { - assert(typeof hmac === 'function', 'Must provide options.hmac to restore mapped function projections.'); - const projectionMap = {}; - for (const [eventType, reducerMetadata] of Object.entries(metadata.projection || {})) { - projectionMap[eventType] = buildMatcherFromMetadata(reducerMetadata, hmac); - } - return projectionMap; - } - assert(metadata.kind === 'function', 'Invalid projection metadata kind.'); - if (typeof metadata.projection?.matcher === 'string') { - assert(typeof hmac === 'function', 'Must provide options.hmac to restore function projections.'); - } - return buildMatcherFromMetadata(metadata.projection, hmac); + const projection = Projection.restoreFromFile(this.projectionFileName, { + hmac: options.hmac || this.storage.hmac, + typeAccessor: options.typeAccessor + }); + this.project(projection, { persist: false }); } /** * Register a projection function as `data` event handler. * @private - * @param {function|object} projection + * @param {Projection} projection + * @param {object} [options] + * @param {boolean} [options.persist=true] + * @param {function(string): string} [options.hmac] */ - registerProjection(projection) { + project(projection, options = {}) { + assert(projection instanceof Projection, 'Projection must be an instance of Projection.'); + const { persist = true, hmac } = options; if (this.projectionHandler) { this.removeListener('data', this.projectionHandler); } this.projection = projection; this.projectionHandler = (event) => { - let reducer = projection; - if (typeof projection === 'object') { - // Direct Storage consumers emit `{ type }`, EventStore consumers emit `{ payload: { type } }`. - const type = event?.type || event?.payload?.type; - reducer = projection[type]; - if (typeof reducer !== 'function') { - return; - } - } - this.setState(reducer(this.state, event)); + this.setState(projection.apply(this.state, event)); }; this.on('data', this.projectionHandler); + if (persist) { + projection.persist({ + fileName: this.projectionFileName, + hmac: hmac || projection.hmac || this.storage.hmac + }); + } + return projection; } /** @@ -188,32 +170,17 @@ class Consumer extends stream.Readable { * @param {function(string): string} [options.hmac] Required for function projections. */ createProjection(projectionFn, options = {}) { - assert((typeof projectionFn === 'function') || (projectionFn && typeof projectionFn === 'object' && !Array.isArray(projectionFn)), 'Projection must be a reducer function or an object map of reducer functions.'); - if (typeof projectionFn === 'object') { - for (const reducer of Object.values(projectionFn)) { - assert(typeof reducer === 'function', 'Projection object values must be reducer functions.'); - } - } - const hmac = options.hmac || this.storage.hmac; - assert(typeof hmac === 'function', 'Must provide options.hmac for projections.'); - const projectionMetadata = (typeof projectionFn === 'function') - ? { kind: 'function', projection: buildMetadataForMatcher(projectionFn, hmac) } - : { - kind: 'map', - projection: Object.fromEntries( - Object.entries(projectionFn).map(([eventType, reducer]) => [eventType, buildMetadataForMatcher(reducer, hmac)]) - ) - }; - const tmpProjectionFileName = this.projectionFileName + '.tmp'; - try { - fs.writeFileSync(tmpProjectionFileName, JSON.stringify(projectionMetadata), 'utf8'); - fs.renameSync(tmpProjectionFileName, this.projectionFileName); - } catch (e) { - safeUnlink(tmpProjectionFileName); - throw e; - } - - this.registerProjection(projectionFn); + const projection = projectionFn instanceof Projection + ? projectionFn + : new Projection(options.name || this.identifier, { + initialState: this.state, + matcher: options.matcher, + handlers: projectionFn + }, { + hmac: options.hmac || this.storage.hmac, + typeAccessor: options.typeAccessor + }); + return this.project(projection, options); } /** diff --git a/src/EventStore.js b/src/EventStore.js index 8b0d41a4..f91b7655 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -6,6 +6,7 @@ import events from 'events'; import Storage, { ReadOnly as ReadOnlyStorage, LOCK_THROW, LOCK_RECLAIM } from './Storage.js'; import Index from './Index.js'; import Consumer from './Consumer.js'; +import Projection from './Projection.js'; import { assert, getPropertyAtPath } from './utils/util.js'; import { ensureDirectory, scanForFiles } from './utils/fsUtil.js'; import { buildTypeMatcherFn } from './utils/metadataUtil.js'; @@ -815,12 +816,44 @@ class EventStore extends events.EventEmitter { // identifier already exists for another stream. existingConsumer.stop(); } - const consumer = new Consumer(this.storage, streamName === '_all' ? '_all' : 'stream-' + streamName, identifier, initialState, since); + const projectionTypeAccessor = this.typeAccessor + ? (event) => this.typeAccessor(event?.payload || event) + : undefined; + const consumer = new Consumer(this.storage, streamName === '_all' ? '_all' : 'stream-' + streamName, identifier, initialState, since, { + hmac: this.storage.hmac, + typeAccessor: projectionTypeAccessor + }); consumer.streamName = streamName; this.consumers.set(identifier, consumer); return consumer; } + /** + * Get or create a projection with EventStore defaults. + * + * @param {string} name Projection name. + * @param {object} [definition] Projection definition. + * @param {object} [options] Projection options. + * @returns {Projection} + */ + getProjection(name, definition, options = {}) { + assert(typeof name === 'string' && name !== '', 'Must provide a projection name.'); + const projectionTypeAccessor = this.typeAccessor + ? (event) => this.typeAccessor(event?.payload || event) + : options.typeAccessor; + const projectionFileName = path.join(this.storage.indexDirectory, 'projections', this.storage.storageFile + '.' + name + '.projection'); + const projectionOptions = { + ...options, + fileName: options.fileName || projectionFileName, + hmac: options.hmac || this.storage.hmac, + typeAccessor: projectionTypeAccessor + }; + if (definition) { + return new Projection(name, definition, projectionOptions); + } + return Projection.restore(name, projectionOptions); + } + /** * Scan the existing consumers on this EventStore and asynchronously invoke a callback with the parsed list. * diff --git a/src/Projection.js b/src/Projection.js new file mode 100644 index 00000000..7b580b65 --- /dev/null +++ b/src/Projection.js @@ -0,0 +1,276 @@ +import fs from 'fs'; +import path from 'path'; +import { assert } from './utils/util.js'; +import { ensureDirectory } from './utils/fsUtil.js'; +import { buildMatcherFromMetadata, buildMetadataForMatcher, matches } from './utils/metadataUtil.js'; + +const DEFAULT_TYPE_ACCESSOR = (event) => event?.type || event?.payload?.type; + +const safeUnlink = (filename) => { + try { + fs.unlinkSync(filename); + } catch (e) { + if (e.code !== "ENOENT") { + throw e; + } + } +}; + +class Projection { + + constructor(name, definition = {}, options = {}) { + assert(typeof name === 'string' && name !== '', 'Projection must have a name.'); + const { initialState = {}, handlers, matcher } = definition; + assert((typeof handlers === 'function') || (handlers && typeof handlers === 'object' && !Array.isArray(handlers)), 'Projection handlers must be a function or an object map of functions.'); + if (typeof handlers === 'object') { + for (const reducer of Object.values(handlers)) { + assert(typeof reducer === 'function', 'Projection handler maps must contain reducer functions.'); + } + } + this.name = name; + this.initialState = Object.freeze(initialState); + this.handlers = handlers; + this.matcher = matcher; + this.hmac = options.hmac || null; + this.typeAccessor = options.typeAccessor || DEFAULT_TYPE_ACCESSOR; + this.fileName = options.fileName || null; + this.state = this.initialState; + } + + get types() { + if (typeof this.handlers === 'function') { + return []; + } + return Object.keys(this.handlers); + } + + apply(state, event) { + if (!this.matches(event)) { + this.state = state; + return state; + } + let reducer = this.handlers; + if (typeof this.handlers === 'object') { + reducer = this.handlers[this.typeAccessor(event)]; + if (typeof reducer !== 'function') { + this.state = state; + return state; + } + } + const nextState = reducer(state, event); + this.state = nextState; + return nextState; + } + + handle(stream) { + this.reset(); + for (const event of stream) { + this.state = this.apply(this.state, event); + } + return this.state; + } + + reset() { + this.state = this.initialState; + return this.state; + } + + matches(event) { + if (!this.matcher) { + return true; + } + if (typeof this.matcher === 'function') { + return this.matcher(event); + } + return matches(event, this.matcher); + } + + subscribe(consumer, options = {}) { + assert(consumer && typeof consumer.project === 'function', 'Projection.subscribe expects a Consumer instance.'); + consumer.project(this, options); + return this; + } + + persist(options = {}) { + const hmac = options.hmac || this.hmac; + const fileName = options.fileName || this.fileName || `${this.name}.projection`; + const metadata = this.toMetadata(hmac); + const tmpFile = fileName + '.tmp'; + ensureDirectory(path.dirname(fileName)); + try { + fs.writeFileSync(tmpFile, JSON.stringify(metadata), 'utf8'); + fs.renameSync(tmpFile, fileName); + } catch (e) { + safeUnlink(tmpFile); + throw e; + } + this.fileName = fileName; + this.hmac = hmac; + return fileName; + } + + toMetadata(hmac = this.hmac) { + const serializeFn = (fn) => { + assert(typeof hmac === 'function', 'Must provide options.hmac for function projections.'); + return buildMetadataForMatcher(fn, hmac); + }; + const matcherMetadata = this.matcher ? ( + typeof this.matcher === 'function' ? serializeFn(this.matcher) : buildMetadataForMatcher(this.matcher, hmac) + ) : null; + const handlersMetadata = (typeof this.handlers === 'function') + ? serializeFn(this.handlers) + : Object.fromEntries( + Object.entries(this.handlers).map(([eventType, reducer]) => [eventType, serializeFn(reducer)]) + ); + return { + kind: 'projection', + name: this.name, + initialState: this.initialState, + matcher: matcherMetadata, + handlersKind: typeof this.handlers === 'function' ? 'function' : 'map', + handlers: handlersMetadata + }; + } + + static restore(name, options = {}) { + assert(typeof name === 'string' && name !== '', 'Projection.restore requires a projection name.'); + const fileName = options.fileName || `${name}.projection`; + return Projection.restoreFromFile(fileName, options); + } + + static restoreFromFile(fileName, options = {}) { + assert(fs.existsSync(fileName), `Projection file does not exist: ${fileName}`); + const metadata = JSON.parse(fs.readFileSync(fileName, 'utf8')); + return Projection.fromMetadata(metadata, { ...options, fileName }); + } + + static fromMetadata(metadata, options = {}) { + assert(metadata && typeof metadata === 'object', 'Invalid projection metadata.'); + if (metadata.kind === 'composite-projection') { + return CompositeProjection.fromMetadata(metadata, options); + } + assert(metadata.kind === 'projection', 'Invalid projection metadata kind.'); + const hmac = options.hmac; + const deserialize = (matcherMetadata) => { + if (!matcherMetadata) { + return undefined; + } + if (typeof matcherMetadata.matcher === 'string') { + assert(typeof hmac === 'function', 'Must provide options.hmac to restore function projections.'); + } + return buildMatcherFromMetadata(matcherMetadata, hmac); + }; + const handlers = metadata.handlersKind === 'function' + ? deserialize(metadata.handlers) + : Object.fromEntries( + Object.entries(metadata.handlers || {}).map(([eventType, reducerMetadata]) => [eventType, deserialize(reducerMetadata)]) + ); + const projection = new Projection(metadata.name, { + initialState: metadata.initialState, + matcher: deserialize(metadata.matcher), + handlers + }, { + ...options, + fileName: options.fileName || null + }); + projection.reset(); + return projection; + } + + static compose(name, projections, options = {}) { + return new CompositeProjection(name, projections, options); + } +} + +class CompositeProjection extends Projection { + + constructor(name, projections, options = {}) { + assert(projections && typeof projections === 'object' && !Array.isArray(projections), 'CompositeProjection requires an object map of projections.'); + const normalized = {}; + for (const [projectionName, projection] of Object.entries(projections)) { + normalized[projectionName] = projection instanceof Projection + ? projection + : new Projection(projectionName, projection, options); + } + super(name, { + initialState: Object.fromEntries( + Object.entries(normalized).map(([projectionName, projection]) => [projectionName, projection.initialState]) + ), + handlers: (state) => state, + matcher: options.matcher + }, options); + this.projections = normalized; + this.reset(); + } + + get types() { + const types = new Set(); + for (const projection of Object.values(this.projections)) { + for (const type of projection.types) { + types.add(type); + } + } + return [...types]; + } + + apply(state, event) { + if (!this.matches(event)) { + this.state = state; + return state; + } + const currentState = state || this.initialState; + const nextState = {}; + for (const [name, projection] of Object.entries(this.projections)) { + nextState[name] = projection.apply(currentState[name], event); + } + this.state = nextState; + return nextState; + } + + reset() { + for (const projection of Object.values(this.projections)) { + projection.reset(); + } + this.state = Object.fromEntries( + Object.entries(this.projections).map(([projectionName, projection]) => [projectionName, projection.state]) + ); + return this.state; + } + + toMetadata(hmac = this.hmac) { + return { + kind: 'composite-projection', + name: this.name, + matcher: this.matcher ? buildMetadataForMatcher(this.matcher, hmac) : null, + projections: Object.fromEntries( + Object.entries(this.projections).map(([name, projection]) => [name, projection.toMetadata(hmac)]) + ) + }; + } + + static fromMetadata(metadata, options = {}) { + const hmac = options.hmac; + const deserializeMatcher = (matcherMetadata) => { + if (!matcherMetadata) { + return undefined; + } + if (typeof matcherMetadata.matcher === 'string') { + assert(typeof hmac === 'function', 'Must provide options.hmac to restore function projections.'); + } + return buildMatcherFromMetadata(matcherMetadata, hmac); + }; + const projections = Object.fromEntries( + Object.entries(metadata.projections || {}).map(([name, projectionMetadata]) => [ + name, + Projection.fromMetadata(projectionMetadata, options) + ]) + ); + return new CompositeProjection(metadata.name, projections, { + ...options, + matcher: deserializeMatcher(metadata.matcher) + }); + } +} + +export default Projection; +export { CompositeProjection }; diff --git a/test/Consumer.spec.js b/test/Consumer.spec.js index db669117..47cf53b0 100644 --- a/test/Consumer.spec.js +++ b/test/Consumer.spec.js @@ -3,6 +3,7 @@ import fs from 'fs-extra'; import fsNative from 'fs'; import Storage from '../src/Storage.js'; import Consumer from '../src/Consumer.js'; +import Projection, { CompositeProjection } from '../src/Projection.js'; import { createHmac } from '../src/utils/metadataUtil.js'; import { fileURLToPath } from 'url'; @@ -562,6 +563,48 @@ describe('Consumer', function() { expect(() => new Consumer(storage, 'foobar', 'consumer-projection-hmac', {}, 0, { hmac: createHmac('wrong-secret') })).to.throwError(/Invalid HMAC/); }); + it('can attach a projection instance and restore it on reopen', function(done) { + consumer = new Consumer(storage, 'foobar', 'consumer-projection-instance', { count: 0 }); + const projection = new Projection('consumer-projection-instance', { + initialState: { count: 0 }, + handlers: { + Foobar: (state, event) => ({ ...state, count: state.count + event.id }) + } + }, { + hmac: createHmac('test-secret') + }); + consumer.project(projection); + consumer.on('caught-up', () => { + expect(consumer.state.count).to.be(6); + consumer.stop(); + consumer = new Consumer(storage, 'foobar', 'consumer-projection-instance', {}, 0, { hmac: createHmac('test-secret') }); + consumer.on('progress', () => { + if (consumer.state.count === 10) { + done(); + } + }); + storage.write({ type: 'Foobar', id: 4 }); + }); + storage.write({ type: 'Foobar', id: 1 }); + storage.write({ type: 'Foobar', id: 2 }); + storage.write({ type: 'Foobar', id: 3 }); + }); + + it('supports composite projections', function() { + const projection = new CompositeProjection('overview', { + count: { + initialState: 0, + handlers: { Foobar: (state) => state + 1 } + }, + last: { + initialState: null, + handlers: { Foobar: (state, event) => event.id || state } + } + }); + projection.handle([{ type: 'Foobar', id: 1 }, { type: 'Foobar', id: 2 }]); + expect(projection.state).to.eql({ count: 2, last: 2 }); + }); + it('can build consistency guards (aggregates)', function(done) { const guard = new Consumer(storage, 'foobar', 'unique-bar-guard'); guard.apply = function(event) { diff --git a/test/EventStore.spec.js b/test/EventStore.spec.js index 684aad50..0da72d57 100644 --- a/test/EventStore.spec.js +++ b/test/EventStore.spec.js @@ -4,6 +4,8 @@ import fsSync from 'fs'; import path from 'path'; import EventStore, { ExpectedVersion, OptimisticConcurrencyError, CommitCondition, LOCK_RECLAIM } from '../src/EventStore.js'; import Consumer from '../src/Consumer.js'; +import Projection from '../src/Projection.js'; +import { createHmac } from '../src/utils/metadataUtil.js'; import { fileURLToPath } from 'url'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -1402,6 +1404,75 @@ describe('EventStore', function() { eventstore.commit('foo', { foo: 'bar', id: 1 }); eventstore.commit('bar', { foo: 'baz', id: 2 }); }); + + it('restores a projected consumer with eventStore typeAccessor defaults', function(done) { + eventstore = new EventStore({ + storageDirectory, + typeAccessor: 'type', + storageConfig: { + hmac: createHmac('test-secret') + } + }); + eventstore.createEventStream('user-stream', (event) => event.stream === 'user-stream'); + + const consumer = eventstore.getConsumer('user-stream', 'user-counter', { count: 0 }); + consumer.project(new Projection('user-counter', { + initialState: { count: 0 }, + handlers: { + UserCreated: (state) => ({ ...state, count: state.count + 1 }) + } + })); + eventstore.commit('user-stream', [{ type: 'UserCreated', id: 1 }]); + eventstore.commit('user-stream', [{ type: 'UserCreated', id: 2 }]); + + consumer.on('progress', () => { + if (consumer.state.count !== 2) { + return; + } + eventstore.close(); + eventstore = new EventStore({ + storageDirectory, + typeAccessor: 'type', + storageConfig: { + hmac: createHmac('test-secret') + } + }); + const reopened = eventstore.getConsumer('user-stream', 'user-counter', { count: 0 }); + reopened.on('progress', () => { + if (reopened.state.count === 3) { + done(); + } + }); + eventstore.commit('user-stream', [{ type: 'UserCreated', id: 3 }]); + }); + }); + }); + + describe('getProjection', function() { + + it('creates and restores persisted projections with EventStore defaults', function() { + eventstore = new EventStore({ + storageDirectory, + typeAccessor: 'type', + storageConfig: { + hmac: createHmac('test-secret') + } + }); + const projection = eventstore.getProjection('user-count', { + initialState: 0, + handlers: { + UserCreated: (state) => state + 1 + } + }); + projection.persist(); + + const restored = eventstore.getProjection('user-count'); + const state = restored.handle([ + { payload: { type: 'UserCreated' } }, + { payload: { type: 'UserCreated' } } + ]); + expect(state).to.be(2); + }); }); describe('scanConsumers', function() { From 8a7e2d5560897ebe00586b2cf7b55fe4df5567ba Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 31 May 2026 11:06:30 +0000 Subject: [PATCH 35/40] Address projection API review feedback --- docs/api.md | 37 +++++---------------- docs/consumers.md | 17 ++-------- src/Consumer.js | 74 +++++------------------------------------ src/EventStore.js | 25 ++++++++------ src/Projection.js | 12 +++++-- test/Consumer.spec.js | 44 ++++++++++++++++++------ test/EventStore.spec.js | 6 ++-- 7 files changed, 80 insertions(+), 135 deletions(-) diff --git a/docs/api.md b/docs/api.md index d64c59b9..77fec781 100644 --- a/docs/api.md +++ b/docs/api.md @@ -280,31 +280,10 @@ Asynchronously scan all consumer state files and return their identifiers. --- -#### `consumer.createProjection(projectionFn, [options])` +#### `eventstore.getProjection(name, [definition])` ```javascript -consumer.createProjection(projectionFn [, options]) -``` - -Register a reducer-style projection as the consumer's `'data'` handler and persist it so it is auto-restored when reopening the same consumer. - -`projectionFn` can be either: - -- a reducer function: `(state, event) => state` -- an object map: `{ [eventType]: (state, event) => state }` - -Options: - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `options.hmac` | `function(string): string` | storage HMAC | HMAC function used to sign/verify serialized projections. Required when no storage-level HMAC is configured. | - ---- - -#### `eventstore.getProjection(name, [definition], [options])` - -```javascript -eventstore.getProjection(name [, definition [, options]]) → Projection +eventstore.getProjection(name [, definition]) → Projection ``` Create a `Projection` with EventStore defaults (`typeAccessor`, storage HMAC), or restore a previously persisted one when `definition` is omitted. @@ -321,23 +300,23 @@ Create a `Projection` with EventStore defaults (`typeAccessor`, storage HMAC), o --- -#### `consumer.project(projection, [options])` +#### `consumer.project(projection)` ```javascript -consumer.project(projection [, options]) +consumer.project(projection) ``` -Attach a `Projection` instance to a durable consumer and persist its definition for automatic restore when reopening the same consumer. +Attach a projection-like object (`apply(state, event)`) as the consumer `'data'` handler. --- -#### `projection.subscribe(consumer, [options])` +#### `projection.subscribe(consumer)` ```javascript -projection.subscribe(consumer [, options]) +projection.subscribe(consumer) ``` -Alias for `consumer.project(projection, options)`. +Attach this projection to the consumer and persist its definition next to the consumer state file so `eventstore.getConsumer(...)` can restore and reconnect it automatically. --- diff --git a/docs/consumers.md b/docs/consumers.md index 51b27835..adaf8849 100644 --- a/docs/consumers.md +++ b/docs/consumers.md @@ -74,28 +74,15 @@ consumer.setState((state) => ({ ...state, count: state.count + 1 })); Use a `Projection` to define *how* events are projected into state, then connect it to a `Consumer` for durable continuous updates. ```javascript -import crypto from 'crypto'; - -const hmac = (code) => crypto.createHmac('sha256', 'your-private-secret').update(code).digest('hex'); - const projection = eventstore.getProjection('orders-total', { initialState: { total: 0 }, handlers: { OrderCreated: (state, event) => ({ ...state, total: state.total + (event.payload.amount || 0) }) } -}, { hmac }); +}); const consumer = eventstore.getConsumer('orders', 'orders-projection', projection.initialState); -consumer.project(projection); -``` - -You can still use `consumer.createProjection(...)` for the shorter in-place API: - -```javascript -consumer.createProjection({ - OrderCreated: (state, event) => ({ ...state, orders: [...state.orders, event] }), - OrderCancelled: (state, event) => ({ ...state, cancelled: state.cancelled + 1 }) -}, { hmac }); +projection.subscribe(consumer); ``` Projections are composable via `CompositeProjection`: diff --git a/src/Consumer.js b/src/Consumer.js index 0ebcd6c9..abd6999d 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -4,7 +4,6 @@ import path from 'path'; import { assert } from './utils/util.js'; import { ensureDirectory } from './utils/fsUtil.js'; import { normalizeConsumerStateArgs } from './utils/apiHelpers.js'; -import Projection from './Projection.js'; import Storage from './Storage/ReadableStorage.js'; const MAX_CATCHUP_BATCH = 10; @@ -35,7 +34,7 @@ class Consumer extends stream.Readable { * @param {object} [initialState={}] The initial state of the consumer. * @param {number} [startFrom=0] The revision to start from within the index to consume. */ - constructor(storage, indexName, identifier, initialState = {}, startFrom = 0, options = {}) { + constructor(storage, indexName, identifier, initialState = {}, startFrom = 0) { super({ objectMode: true }); assert(storage instanceof Storage, 'Must provide a storage for the consumer.'); @@ -44,7 +43,6 @@ class Consumer extends stream.Readable { this.initializeStorage(storage, indexName, identifier); this.restoreState(initialState, startFrom); - this.restoreProjection(options); this.handler = this.handleNewDocument.bind(this); this.on('error', () => (this.handleDocument = false)); } @@ -62,7 +60,6 @@ class Consumer extends stream.Readable { this.identifier = identifier; const consumerDirectory = path.join(this.storage.indexDirectory, 'consumers'); this.fileName = path.join(consumerDirectory, this.storage.storageFile + '.' + indexName + '.' + identifier); - this.projectionFileName = this.fileName + '.projection'; if (ensureDirectory(consumerDirectory)) { this.cleanUpFailedWrites(); } @@ -74,16 +71,12 @@ class Consumer extends stream.Readable { */ cleanUpFailedWrites() { const consumerBaseName = path.basename(this.fileName); - const projectionBaseName = consumerBaseName + '.projection'; const escapedConsumerBaseName = consumerBaseName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); const failedStateFilePattern = new RegExp(`^${escapedConsumerBaseName}\\.\\d+$`); const consumerDirectory = path.dirname(this.fileName); const files = fs.readdirSync(consumerDirectory); for (let file of files) { - if (file === projectionBaseName) { - continue; - } - if (file === projectionBaseName + '.tmp' || failedStateFilePattern.test(file)) { + if (failedStateFilePattern.test(file)) { safeUnlink(path.join(consumerDirectory, file)); } } @@ -115,35 +108,12 @@ class Consumer extends stream.Readable { } /** - * Restore a persisted projection and register it as data handler. - * @private - * @param {object} [options={}] - * @param {function(string): string} [options.hmac] - */ - restoreProjection(options = {}) { - this.projection = null; - this.projectionHandler = null; - if (!this.projectionFileName || !fs.existsSync(this.projectionFileName)) { - return; - } - const projection = Projection.restoreFromFile(this.projectionFileName, { - hmac: options.hmac || this.storage.hmac, - typeAccessor: options.typeAccessor - }); - this.project(projection, { persist: false }); - } - - /** - * Register a projection function as `data` event handler. - * @private - * @param {Projection} projection - * @param {object} [options] - * @param {boolean} [options.persist=true] - * @param {function(string): string} [options.hmac] + * Register a projection as `data` event handler. + * @api + * @param {{ apply: function(object, object): object }} projection */ - project(projection, options = {}) { - assert(projection instanceof Projection, 'Projection must be an instance of Projection.'); - const { persist = true, hmac } = options; + project(projection) { + assert(projection && typeof projection.apply === 'function', 'Projection must implement apply(state, event).'); if (this.projectionHandler) { this.removeListener('data', this.projectionHandler); } @@ -152,35 +122,7 @@ class Consumer extends stream.Readable { this.setState(projection.apply(this.state, event)); }; this.on('data', this.projectionHandler); - if (persist) { - projection.persist({ - fileName: this.projectionFileName, - hmac: hmac || projection.hmac || this.storage.hmac - }); - } - return projection; - } - - /** - * Create and persist a projection reducer and register it as data handler. - * - * @api - * @param {function(object, object): object|object} projectionFn - * @param {object} [options] - * @param {function(string): string} [options.hmac] Required for function projections. - */ - createProjection(projectionFn, options = {}) { - const projection = projectionFn instanceof Projection - ? projectionFn - : new Projection(options.name || this.identifier, { - initialState: this.state, - matcher: options.matcher, - handlers: projectionFn - }, { - hmac: options.hmac || this.storage.hmac, - typeAccessor: options.typeAccessor - }); - return this.project(projection, options); + return this; } /** diff --git a/src/EventStore.js b/src/EventStore.js index f91b7655..f1cf25b3 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -122,6 +122,9 @@ class EventStore extends events.EventEmitter { } this.initialize(storeName, storageConfig); + this.projectionHmac = typeof storageConfig.hmac === 'function' + ? storageConfig.hmac + : this.storage.hmac; } /** @@ -819,10 +822,14 @@ class EventStore extends events.EventEmitter { const projectionTypeAccessor = this.typeAccessor ? (event) => this.typeAccessor(event?.payload || event) : undefined; - const consumer = new Consumer(this.storage, streamName === '_all' ? '_all' : 'stream-' + streamName, identifier, initialState, since, { - hmac: this.storage.hmac, - typeAccessor: projectionTypeAccessor - }); + const consumer = new Consumer(this.storage, streamName === '_all' ? '_all' : 'stream-' + streamName, identifier, initialState, since); + const consumerProjectionFileName = `${consumer.fileName}.projection`; + if (fs.existsSync(consumerProjectionFileName)) { + Projection.restoreFromFile(consumerProjectionFileName, { + hmac: this.projectionHmac, + typeAccessor: projectionTypeAccessor + }).subscribe(consumer); + } consumer.streamName = streamName; this.consumers.set(identifier, consumer); return consumer; @@ -833,19 +840,17 @@ class EventStore extends events.EventEmitter { * * @param {string} name Projection name. * @param {object} [definition] Projection definition. - * @param {object} [options] Projection options. * @returns {Projection} */ - getProjection(name, definition, options = {}) { + getProjection(name, definition) { assert(typeof name === 'string' && name !== '', 'Must provide a projection name.'); const projectionTypeAccessor = this.typeAccessor ? (event) => this.typeAccessor(event?.payload || event) - : options.typeAccessor; + : undefined; const projectionFileName = path.join(this.storage.indexDirectory, 'projections', this.storage.storageFile + '.' + name + '.projection'); const projectionOptions = { - ...options, - fileName: options.fileName || projectionFileName, - hmac: options.hmac || this.storage.hmac, + fileName: projectionFileName, + hmac: this.projectionHmac, typeAccessor: projectionTypeAccessor }; if (definition) { diff --git a/src/Projection.js b/src/Projection.js index 7b580b65..09c7e5b2 100644 --- a/src/Projection.js +++ b/src/Projection.js @@ -85,9 +85,17 @@ class Projection { return matches(event, this.matcher); } - subscribe(consumer, options = {}) { + subscribe(consumer) { assert(consumer && typeof consumer.project === 'function', 'Projection.subscribe expects a Consumer instance.'); - consumer.project(this, options); + const projectionFileName = consumer.fileName ? `${consumer.fileName}.projection` : null; + const isAlreadySubscribed = consumer.projection === this; + const isAlreadyPersisted = projectionFileName && this.fileName === projectionFileName && fs.existsSync(projectionFileName); + consumer.project(this); + if (!isAlreadySubscribed && !isAlreadyPersisted) { + this.persist({ + fileName: projectionFileName || this.fileName + }); + } return this; } diff --git a/test/Consumer.spec.js b/test/Consumer.spec.js index 47cf53b0..760315f3 100644 --- a/test/Consumer.spec.js +++ b/test/Consumer.spec.js @@ -521,9 +521,14 @@ describe('Consumer', function() { }); }); - it('can create projections from a reducer function', function(done) { + it('can attach projections from a reducer function', function(done) { consumer = new Consumer(storage, 'foobar', 'consumer-projection', { count: 0 }); - consumer.createProjection((state, event) => ({ ...state, count: state.count + event.id }), { hmac: createHmac('test-secret') }); + new Projection('consumer-projection', { + initialState: { count: 0 }, + handlers: (state, event) => ({ ...state, count: state.count + event.id }) + }, { + hmac: createHmac('test-secret') + }).subscribe(consumer); consumer.on('caught-up', () => { expect(consumer.state.count).to.be(6); done(); @@ -534,16 +539,23 @@ describe('Consumer', function() { storage.write({ type: 'Foobar', id: 3 }); }); - it('can create projections from event-type reducer maps and restore them on reopen', function(done) { + it('can attach and restore projections from event-type reducer maps', function(done) { consumer = new Consumer(storage, 'foobar', 'consumer-projection-map', { count: 0 }); - consumer.createProjection({ - Foobar: (state, event) => ({ ...state, count: state.count + event.id }), - Bazinga: (state) => state + const projection = new Projection('consumer-projection-map', { + initialState: { count: 0 }, + handlers: { + Foobar: (state, event) => ({ ...state, count: state.count + event.id }), + Bazinga: (state) => state + } }, { hmac: createHmac('test-secret') }); + projection.subscribe(consumer); consumer.on('caught-up', () => { consumer.stop(); - consumer = new Consumer(storage, 'foobar', 'consumer-projection-map', {}, 0, { hmac: createHmac('test-secret') }); + consumer = new Consumer(storage, 'foobar', 'consumer-projection-map', {}); + Projection.restoreFromFile(`${consumer.fileName}.projection`, { + hmac: createHmac('test-secret') + }).subscribe(consumer); consumer.on('progress', () => { if (consumer.state.count === 10) { done(); @@ -559,8 +571,15 @@ describe('Consumer', function() { it('throws if function projection is restored without trusted hmac', function() { consumer = new Consumer(storage, 'foobar', 'consumer-projection-hmac'); - consumer.createProjection((state, event) => ({ ...state, lastId: event.id }), { hmac: createHmac('test-secret') }); - expect(() => new Consumer(storage, 'foobar', 'consumer-projection-hmac', {}, 0, { hmac: createHmac('wrong-secret') })).to.throwError(/Invalid HMAC/); + new Projection('consumer-projection-hmac', { + initialState: {}, + handlers: (state, event) => ({ ...state, lastId: event.id }) + }, { + hmac: createHmac('test-secret') + }).subscribe(consumer); + expect(() => Projection.restoreFromFile(`${consumer.fileName}.projection`, { + hmac: createHmac('wrong-secret') + })).to.throwError(/Invalid HMAC/); }); it('can attach a projection instance and restore it on reopen', function(done) { @@ -573,11 +592,14 @@ describe('Consumer', function() { }, { hmac: createHmac('test-secret') }); - consumer.project(projection); + projection.subscribe(consumer); consumer.on('caught-up', () => { expect(consumer.state.count).to.be(6); consumer.stop(); - consumer = new Consumer(storage, 'foobar', 'consumer-projection-instance', {}, 0, { hmac: createHmac('test-secret') }); + consumer = new Consumer(storage, 'foobar', 'consumer-projection-instance', {}); + Projection.restoreFromFile(`${consumer.fileName}.projection`, { + hmac: createHmac('test-secret') + }).subscribe(consumer); consumer.on('progress', () => { if (consumer.state.count === 10) { done(); diff --git a/test/EventStore.spec.js b/test/EventStore.spec.js index 0da72d57..82c8a29f 100644 --- a/test/EventStore.spec.js +++ b/test/EventStore.spec.js @@ -1416,12 +1416,14 @@ describe('EventStore', function() { eventstore.createEventStream('user-stream', (event) => event.stream === 'user-stream'); const consumer = eventstore.getConsumer('user-stream', 'user-counter', { count: 0 }); - consumer.project(new Projection('user-counter', { + new Projection('user-counter', { initialState: { count: 0 }, handlers: { UserCreated: (state) => ({ ...state, count: state.count + 1 }) } - })); + }, { + hmac: createHmac('test-secret') + }).subscribe(consumer); eventstore.commit('user-stream', [{ type: 'UserCreated', id: 1 }]); eventstore.commit('user-stream', [{ type: 'UserCreated', id: 2 }]); From 240fbd29f24afae8af04079b90eff1ccca88b8d2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Jun 2026 17:06:07 +0000 Subject: [PATCH 36/40] Address projection API and utility review feedback --- docs/api.md | 19 +++++++------------ docs/consumers.md | 16 +++++++++------- src/Consumer.js | 20 ++------------------ src/EventStore.js | 31 +++++++++++++++---------------- src/Projection.js | 22 +++++----------------- src/utils/fsUtil.js | 39 +++++++++++++++++++++++++++++++++++++++ test/EventStore.spec.js | 15 ++++++--------- 7 files changed, 83 insertions(+), 79 deletions(-) diff --git a/docs/api.md b/docs/api.md index 77fec781..b2e7e379 100644 --- a/docs/api.md +++ b/docs/api.md @@ -280,23 +280,18 @@ Asynchronously scan all consumer state files and return their identifiers. --- -#### `eventstore.getProjection(name, [definition])` +#### `eventstore.getProjection(name, [handlers], [initialState], [matcher])` ```javascript -eventstore.getProjection(name [, definition]) → Projection +eventstore.getProjection(name [, handlers] [, initialState] [, matcher]) → Projection ``` -Create a `Projection` with EventStore defaults (`typeAccessor`, storage HMAC), or restore a previously persisted one when `definition` is omitted. +Create a `Projection` with EventStore defaults (`typeAccessor`, storage HMAC), or restore a previously persisted one when `handlers` is omitted. -`definition` shape: +- `handlers`: reducer function `(state, event) => state` or map `{ [eventType]: reducer }` +- `initialState`: initial projection state (default `{}`) +- `matcher`: optional object/function matcher (same shape as stream/query matchers) -```javascript -{ - initialState: any, - handlers: (state, event) => state | { [eventType]: (state, event) => state }, - matcher: object|function // optional -} -``` --- @@ -316,7 +311,7 @@ Attach a projection-like object (`apply(state, event)`) as the consumer `'data'` projection.subscribe(consumer) ``` -Attach this projection to the consumer and persist its definition next to the consumer state file so `eventstore.getConsumer(...)` can restore and reconnect it automatically. +Attach this projection to the consumer (same wiring behavior as `consumer.project(projection)`) and persist its definition next to the consumer state file so `eventstore.getConsumer(...)` can restore and reconnect it automatically. --- diff --git a/docs/consumers.md b/docs/consumers.md index adaf8849..9b50e866 100644 --- a/docs/consumers.md +++ b/docs/consumers.md @@ -75,11 +75,8 @@ Use a `Projection` to define *how* events are projected into state, then connect ```javascript const projection = eventstore.getProjection('orders-total', { - initialState: { total: 0 }, - handlers: { - OrderCreated: (state, event) => ({ ...state, total: state.total + (event.payload.amount || 0) }) - } -}); + OrderCreated: (state, event) => ({ ...state, total: state.total + (event.payload.amount || 0) }) +}, { total: 0 }); const consumer = eventstore.getConsumer('orders', 'orders-projection', projection.initialState); projection.subscribe(consumer); @@ -88,10 +85,15 @@ projection.subscribe(consumer); Projections are composable via `CompositeProjection`: ```javascript -import { CompositeProjection } from 'event-storage'; +import { CompositeProjection, Projection } from 'event-storage'; + +const count = new Projection('count', { + initialState: 0, + handlers: { OrderCreated: (state) => state + 1 } +}); const overview = new CompositeProjection('overview', { - count: { initialState: 0, handlers: { OrderCreated: (state) => state + 1 } }, + count, last: { initialState: null, handlers: { OrderCreated: (state, event) => event.payload } } }); ``` diff --git a/src/Consumer.js b/src/Consumer.js index abd6999d..8e850344 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -2,26 +2,11 @@ import stream from 'stream'; import fs from 'fs'; import path from 'path'; import { assert } from './utils/util.js'; -import { ensureDirectory } from './utils/fsUtil.js'; +import { ensureDirectory, safeUnlink, writeFileAtomic } from './utils/fsUtil.js'; import { normalizeConsumerStateArgs } from './utils/apiHelpers.js'; import Storage from './Storage/ReadableStorage.js'; const MAX_CATCHUP_BATCH = 10; -/** - * Safely unlink a file and ignore if it doesn't exist. - * @param {string} filename - */ -const safeUnlink = (filename) => { - /* c8 ignore next */ - try { - fs.unlinkSync(filename); - } catch (e) { - if (e.code !== "ENOENT") { - throw e; - } - } -}; - /** * Implements an event-driven durable Consumer that provides at-least-once delivery semantics or exactly-once processing semantics if only using setState(). */ @@ -194,9 +179,8 @@ class Consumer extends stream.Readable { throw new Error(`Trying to update consumer ${this.name} concurrently. Keep each single consumer within a single process.`); } try { - fs.writeFileSync(tmpFile, consumerData); // If the write fails (half-way), the consumer state file will not be corrupted - fs.renameSync(tmpFile, this.fileName); + writeFileAtomic(this.fileName, consumerData, { tmpFileName: tmpFile }); this.emit('persisted', consumerState); } catch (e) { /* c8 ignore next */ diff --git a/src/EventStore.js b/src/EventStore.js index f1cf25b3..e9251f8a 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -121,10 +121,10 @@ class EventStore extends events.EventEmitter { } } + this.projectionTypeAccessor = this.typeAccessor + ? (event) => this.typeAccessor(event?.payload || event) + : undefined; this.initialize(storeName, storageConfig); - this.projectionHmac = typeof storageConfig.hmac === 'function' - ? storageConfig.hmac - : this.storage.hmac; } /** @@ -819,15 +819,12 @@ class EventStore extends events.EventEmitter { // identifier already exists for another stream. existingConsumer.stop(); } - const projectionTypeAccessor = this.typeAccessor - ? (event) => this.typeAccessor(event?.payload || event) - : undefined; const consumer = new Consumer(this.storage, streamName === '_all' ? '_all' : 'stream-' + streamName, identifier, initialState, since); const consumerProjectionFileName = `${consumer.fileName}.projection`; if (fs.existsSync(consumerProjectionFileName)) { Projection.restoreFromFile(consumerProjectionFileName, { - hmac: this.projectionHmac, - typeAccessor: projectionTypeAccessor + hmac: this.storage.hmac, + typeAccessor: this.projectionTypeAccessor }).subscribe(consumer); } consumer.streamName = streamName; @@ -839,21 +836,23 @@ class EventStore extends events.EventEmitter { * Get or create a projection with EventStore defaults. * * @param {string} name Projection name. - * @param {object} [definition] Projection definition. + * @param {function(object, object): object|object} [handlers] Projection handlers (reducer fn or reducer map). + * @param {object} [initialState={}] Projection initial state. + * @param {object|function(object): boolean} [matcher] Optional projection matcher. * @returns {Projection} */ - getProjection(name, definition) { + getProjection(name, handlers, initialState = {}, matcher) { assert(typeof name === 'string' && name !== '', 'Must provide a projection name.'); - const projectionTypeAccessor = this.typeAccessor - ? (event) => this.typeAccessor(event?.payload || event) - : undefined; const projectionFileName = path.join(this.storage.indexDirectory, 'projections', this.storage.storageFile + '.' + name + '.projection'); const projectionOptions = { fileName: projectionFileName, - hmac: this.projectionHmac, - typeAccessor: projectionTypeAccessor + hmac: this.storage.hmac, + typeAccessor: this.projectionTypeAccessor }; - if (definition) { + if (handlers !== undefined) { + const definition = (handlers && typeof handlers === 'object' && !Array.isArray(handlers) && Object.prototype.hasOwnProperty.call(handlers, 'handlers')) + ? handlers + : { handlers, initialState, matcher }; return new Projection(name, definition, projectionOptions); } return Projection.restore(name, projectionOptions); diff --git a/src/Projection.js b/src/Projection.js index 09c7e5b2..742c2efc 100644 --- a/src/Projection.js +++ b/src/Projection.js @@ -1,20 +1,11 @@ import fs from 'fs'; import path from 'path'; import { assert } from './utils/util.js'; -import { ensureDirectory } from './utils/fsUtil.js'; +import { ensureDirectory, writeFileAtomic } from './utils/fsUtil.js'; import { buildMatcherFromMetadata, buildMetadataForMatcher, matches } from './utils/metadataUtil.js'; const DEFAULT_TYPE_ACCESSOR = (event) => event?.type || event?.payload?.type; -const safeUnlink = (filename) => { - try { - fs.unlinkSync(filename); - } catch (e) { - if (e.code !== "ENOENT") { - throw e; - } - } -}; class Projection { @@ -105,13 +96,10 @@ class Projection { const metadata = this.toMetadata(hmac); const tmpFile = fileName + '.tmp'; ensureDirectory(path.dirname(fileName)); - try { - fs.writeFileSync(tmpFile, JSON.stringify(metadata), 'utf8'); - fs.renameSync(tmpFile, fileName); - } catch (e) { - safeUnlink(tmpFile); - throw e; - } + writeFileAtomic(fileName, JSON.stringify(metadata), { + tmpFileName: tmpFile, + encoding: 'utf8' + }); this.fileName = fileName; this.hmac = hmac; return fileName; diff --git a/src/utils/fsUtil.js b/src/utils/fsUtil.js index 7c0df566..05856cf0 100644 --- a/src/utils/fsUtil.js +++ b/src/utils/fsUtil.js @@ -2,6 +2,43 @@ import fs from 'fs'; import path from 'path'; import { mkdirpSync } from 'mkdirp'; +/** + * Safely unlink a file and ignore ENOENT. + * @param {string} fileName + */ +function safeUnlink(fileName) { + try { + fs.unlinkSync(fileName); + } catch (e) { + if (e.code !== 'ENOENT') { + throw e; + } + } +} + +/** + * Atomically write a file by writing to a temporary file first and then renaming it. + * @param {string} fileName + * @param {string|Buffer} data + * @param {{ tmpFileName?: string, encoding?: BufferEncoding }} [options] + * @returns {string} + */ +function writeFileAtomic(fileName, data, options = {}) { + const tmpFileName = options.tmpFileName || `${fileName}.tmp`; + try { + if (options.encoding) { + fs.writeFileSync(tmpFileName, data, options.encoding); + } else { + fs.writeFileSync(tmpFileName, data); + } + fs.renameSync(tmpFileName, fileName); + } catch (e) { + safeUnlink(tmpFileName); + throw e; + } + return fileName; +} + /** * Ensure that the given directory exists. * @param {string} dirName Target directory. @@ -123,5 +160,7 @@ function scanForFiles(directory, regexPattern, onEach, onDone) { export { ensureDirectory, + safeUnlink, + writeFileAtomic, scanForFiles, }; diff --git a/test/EventStore.spec.js b/test/EventStore.spec.js index 82c8a29f..f4ac5f0b 100644 --- a/test/EventStore.spec.js +++ b/test/EventStore.spec.js @@ -1410,7 +1410,7 @@ describe('EventStore', function() { storageDirectory, typeAccessor: 'type', storageConfig: { - hmac: createHmac('test-secret') + hmacSecret: 'test-secret' } }); eventstore.createEventStream('user-stream', (event) => event.stream === 'user-stream'); @@ -1422,7 +1422,7 @@ describe('EventStore', function() { UserCreated: (state) => ({ ...state, count: state.count + 1 }) } }, { - hmac: createHmac('test-secret') + hmac: eventstore.storage.hmac }).subscribe(consumer); eventstore.commit('user-stream', [{ type: 'UserCreated', id: 1 }]); eventstore.commit('user-stream', [{ type: 'UserCreated', id: 2 }]); @@ -1436,7 +1436,7 @@ describe('EventStore', function() { storageDirectory, typeAccessor: 'type', storageConfig: { - hmac: createHmac('test-secret') + hmacSecret: 'test-secret' } }); const reopened = eventstore.getConsumer('user-stream', 'user-counter', { count: 0 }); @@ -1457,15 +1457,12 @@ describe('EventStore', function() { storageDirectory, typeAccessor: 'type', storageConfig: { - hmac: createHmac('test-secret') + hmacSecret: 'test-secret' } }); const projection = eventstore.getProjection('user-count', { - initialState: 0, - handlers: { - UserCreated: (state) => state + 1 - } - }); + UserCreated: (state) => state + 1 + }, 0); projection.persist(); const restored = eventstore.getProjection('user-count'); From 5029f61b1ec351375b0bb88e2f32b1f22bb74cc9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Jun 2026 17:09:08 +0000 Subject: [PATCH 37/40] Refine projection API docs and atomic fs helpers --- src/EventStore.js | 9 ++++++++- src/utils/fsUtil.js | 20 ++++---------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/src/EventStore.js b/src/EventStore.js index e9251f8a..18ca989e 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -850,7 +850,7 @@ class EventStore extends events.EventEmitter { typeAccessor: this.projectionTypeAccessor }; if (handlers !== undefined) { - const definition = (handlers && typeof handlers === 'object' && !Array.isArray(handlers) && Object.prototype.hasOwnProperty.call(handlers, 'handlers')) + const definition = isProjectionDefinitionObject(handlers) ? handlers : { handlers, initialState, matcher }; return new Projection(name, definition, projectionOptions); @@ -898,6 +898,13 @@ class EventStore extends events.EventEmitter { } +function isProjectionDefinitionObject(value) { + return value + && typeof value === 'object' + && !Array.isArray(value) + && Object.hasOwn(value, 'handlers'); +} + EventStore.Storage = Storage; EventStore.Index = Index; diff --git a/src/utils/fsUtil.js b/src/utils/fsUtil.js index 05856cf0..0c35f229 100644 --- a/src/utils/fsUtil.js +++ b/src/utils/fsUtil.js @@ -2,10 +2,7 @@ import fs from 'fs'; import path from 'path'; import { mkdirpSync } from 'mkdirp'; -/** - * Safely unlink a file and ignore ENOENT. - * @param {string} fileName - */ +// Best-effort cleanup for temporary files after interrupted/failed writes. function safeUnlink(fileName) { try { fs.unlinkSync(fileName); @@ -16,21 +13,12 @@ function safeUnlink(fileName) { } } -/** - * Atomically write a file by writing to a temporary file first and then renaming it. - * @param {string} fileName - * @param {string|Buffer} data - * @param {{ tmpFileName?: string, encoding?: BufferEncoding }} [options] - * @returns {string} - */ +// Prevent partially written persistence files from replacing the last valid state. function writeFileAtomic(fileName, data, options = {}) { const tmpFileName = options.tmpFileName || `${fileName}.tmp`; + const writeOptions = options.encoding ? { encoding: options.encoding } : undefined; try { - if (options.encoding) { - fs.writeFileSync(tmpFileName, data, options.encoding); - } else { - fs.writeFileSync(tmpFileName, data); - } + fs.writeFileSync(tmpFileName, data, writeOptions); fs.renameSync(tmpFileName, fileName); } catch (e) { safeUnlink(tmpFileName); From b1eba00b4639f77f6ab4f6bf763197fae585826a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Jun 2026 20:16:01 +0000 Subject: [PATCH 38/40] Address remaining projection and consumer review feedback --- docs/consumers.md | 1 + src/Consumer.js | 30 ++++++++---- src/EventStore.js | 12 ++--- src/Projection.js | 99 +++++++++++++++++++++++++++++++++++---- src/utils/fsUtil.js | 25 +++++++++- src/utils/metadataUtil.js | 1 + test/Consumer.spec.js | 28 +++++++++++ test/util.spec.js | 22 ++++++++- 8 files changed, 192 insertions(+), 26 deletions(-) diff --git a/docs/consumers.md b/docs/consumers.md index 9b50e866..d2e7c214 100644 --- a/docs/consumers.md +++ b/docs/consumers.md @@ -96,6 +96,7 @@ const overview = new CompositeProjection('overview', { count, last: { initialState: null, handlers: { OrderCreated: (state, event) => event.payload } } }); +// overview.state -> { count: number, last: object|null } ``` ## Resetting a Consumer diff --git a/src/Consumer.js b/src/Consumer.js index 8e850344..7d379942 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -2,7 +2,7 @@ import stream from 'stream'; import fs from 'fs'; import path from 'path'; import { assert } from './utils/util.js'; -import { ensureDirectory, safeUnlink, writeFileAtomic } from './utils/fsUtil.js'; +import { ensureDirectory, isSafeRelativeName, resolvePathWithinRoot, safeUnlink, writeFileAtomic } from './utils/fsUtil.js'; import { normalizeConsumerStateArgs } from './utils/apiHelpers.js'; import Storage from './Storage/ReadableStorage.js'; const MAX_CATCHUP_BATCH = 10; @@ -39,12 +39,15 @@ class Consumer extends stream.Readable { * @param {string} identifier The unique name to identify this consumer. */ initializeStorage(storage, indexName, identifier) { + assert(indexName === '_all' || isSafeRelativeName(indexName), `Invalid index name "${indexName}" for consumer.`); + assert(isSafeRelativeName(identifier), `Invalid identifier "${identifier}" for consumer.`); this.storage = storage; this.index = this.storage.openIndex(indexName); this.indexName = indexName; this.identifier = identifier; const consumerDirectory = path.join(this.storage.indexDirectory, 'consumers'); - this.fileName = path.join(consumerDirectory, this.storage.storageFile + '.' + indexName + '.' + identifier); + this.fileName = resolvePathWithinRoot(consumerDirectory, `${this.storage.storageFile}.${indexName}.${identifier}`); + ensureDirectory(path.dirname(this.fileName)); if (ensureDirectory(consumerDirectory)) { this.cleanUpFailedWrites(); } @@ -56,12 +59,14 @@ class Consumer extends stream.Readable { */ cleanUpFailedWrites() { const consumerBaseName = path.basename(this.fileName); - const escapedConsumerBaseName = consumerBaseName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - const failedStateFilePattern = new RegExp(`^${escapedConsumerBaseName}\\.\\d+$`); const consumerDirectory = path.dirname(this.fileName); const files = fs.readdirSync(consumerDirectory); for (let file of files) { - if (failedStateFilePattern.test(file)) { + if (!file.startsWith(consumerBaseName + '.')) { + continue; + } + const suffix = file.slice(consumerBaseName.length + 1); + if (/^\d+$/.test(suffix)) { safeUnlink(path.join(consumerDirectory, file)); } } @@ -99,6 +104,11 @@ class Consumer extends stream.Readable { */ project(projection) { assert(projection && typeof projection.apply === 'function', 'Projection must implement apply(state, event).'); + const projectionFileName = this.fileName ? `${this.fileName}.projection` : null; + const isAlreadySubscribed = this.projection === projection; + const isAlreadyPersisted = projectionFileName + && projection.fileName === projectionFileName + && fs.existsSync(projectionFileName); if (this.projectionHandler) { this.removeListener('data', this.projectionHandler); } @@ -107,6 +117,9 @@ class Consumer extends stream.Readable { this.setState(projection.apply(this.state, event)); }; this.on('data', this.projectionHandler); + if (!isAlreadySubscribed && !isAlreadyPersisted && typeof projection.persist === 'function') { + projection.persist({ fileName: projectionFileName || projection.fileName }); + } return this; } @@ -124,6 +137,9 @@ class Consumer extends stream.Readable { if (typeof newState === 'function') { newState = newState(this.state); } + if (this.state === newState) { + return; + } this.state = Object.freeze(newState); this.doPersist = persist; } @@ -179,9 +195,7 @@ class Consumer extends stream.Readable { throw new Error(`Trying to update consumer ${this.name} concurrently. Keep each single consumer within a single process.`); } try { - // If the write fails (half-way), the consumer state file will not be corrupted - writeFileAtomic(this.fileName, consumerData, { tmpFileName: tmpFile }); - this.emit('persisted', consumerState); + writeFileAtomic(this.fileName, consumerData, { tmpFileName: tmpFile }, () => this.emit('persisted', consumerState)); } catch (e) { /* c8 ignore next */ safeUnlink(tmpFile); diff --git a/src/EventStore.js b/src/EventStore.js index 18ca989e..d7a3443c 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -8,8 +8,8 @@ import Index from './Index.js'; import Consumer from './Consumer.js'; import Projection from './Projection.js'; import { assert, getPropertyAtPath } from './utils/util.js'; -import { ensureDirectory, scanForFiles } from './utils/fsUtil.js'; -import { buildTypeMatcherFn } from './utils/metadataUtil.js'; +import { ensureDirectory, isSafeRelativeName, scanForFiles } from './utils/fsUtil.js'; +import { buildTypeMatcherFn, isPlainObject } from './utils/metadataUtil.js'; import { fixCommitArgumentTypes, parseStreamFromIndexName, normalizePredicateRaw } from './utils/apiHelpers.js'; const ExpectedVersion = { @@ -21,7 +21,6 @@ const ExpectedVersion = { * Default matcher property paths mirroring the Storage default, used for index optimization. */ const DEFAULT_MATCHER_PROPERTIES = ['stream', 'payload.type']; -const STREAM_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_]*(?:[\/:@~+=\-#.][A-Za-z0-9_]+)*$/; const STORAGE_HOOK_EVENTS = new Set(['preCommit', 'preRead']); class OptimisticConcurrencyError extends Error {} @@ -448,7 +447,7 @@ class EventStore extends events.EventEmitter { return null; } assert(typeof type === 'string', 'typeAccessor must return a string.'); - assert(STREAM_NAME_PATTERN.test(type), `typeAccessor must return a valid stream name. Got: "${type}"`); + assert(isSafeRelativeName(type), `typeAccessor must return a valid stream name. Got: "${type}"`); return type; } @@ -899,10 +898,7 @@ class EventStore extends events.EventEmitter { function isProjectionDefinitionObject(value) { - return value - && typeof value === 'object' - && !Array.isArray(value) - && Object.hasOwn(value, 'handlers'); + return isPlainObject(value) && Object.hasOwn(value, 'handlers'); } EventStore.Storage = Storage; diff --git a/src/Projection.js b/src/Projection.js index 742c2efc..5425efe7 100644 --- a/src/Projection.js +++ b/src/Projection.js @@ -9,6 +9,11 @@ const DEFAULT_TYPE_ACCESSOR = (event) => event?.type || event?.payload?.type; class Projection { + /** + * @param {string} name Projection name. + * @param {{ initialState?: object, handlers: function(object, object): object|object, matcher?: object|function(object): boolean }} [definition] + * @param {{ hmac?: function(string): string, typeAccessor?: function(object): string, fileName?: string }} [options] + */ constructor(name, definition = {}, options = {}) { assert(typeof name === 'string' && name !== '', 'Projection must have a name.'); const { initialState = {}, handlers, matcher } = definition; @@ -35,6 +40,12 @@ class Projection { return Object.keys(this.handlers); } + /** + * Apply one event to the provided state and return the next state. + * @param {*} state + * @param {object} event + * @returns {*} + */ apply(state, event) { if (!this.matches(event)) { this.state = state; @@ -53,6 +64,11 @@ class Projection { return nextState; } + /** + * Reset to initialState and project all events from the given iterable stream. + * @param {Iterable} stream + * @returns {*} + */ handle(stream) { this.reset(); for (const event of stream) { @@ -61,11 +77,20 @@ class Projection { return this.state; } + /** + * Reset current projection state to its initial state. + * @returns {*} + */ reset() { this.state = this.initialState; return this.state; } + /** + * Check whether an event matches this projection's matcher definition. + * @param {object} event + * @returns {boolean} + */ matches(event) { if (!this.matcher) { return true; @@ -76,20 +101,22 @@ class Projection { return matches(event, this.matcher); } + /** + * Subscribe this projection to a consumer and persist when needed. + * @param {{ project: function(Projection): object }} consumer + * @returns {Projection} + */ subscribe(consumer) { assert(consumer && typeof consumer.project === 'function', 'Projection.subscribe expects a Consumer instance.'); - const projectionFileName = consumer.fileName ? `${consumer.fileName}.projection` : null; - const isAlreadySubscribed = consumer.projection === this; - const isAlreadyPersisted = projectionFileName && this.fileName === projectionFileName && fs.existsSync(projectionFileName); consumer.project(this); - if (!isAlreadySubscribed && !isAlreadyPersisted) { - this.persist({ - fileName: projectionFileName || this.fileName - }); - } return this; } + /** + * Persist projection definition metadata to disk. + * @param {{ hmac?: function(string): string, fileName?: string }} [options] + * @returns {string} Persisted file name. + */ persist(options = {}) { const hmac = options.hmac || this.hmac; const fileName = options.fileName || this.fileName || `${this.name}.projection`; @@ -105,6 +132,11 @@ class Projection { return fileName; } + /** + * Serialize this projection definition into trusted metadata. + * @param {function(string): string} [hmac] + * @returns {object} + */ toMetadata(hmac = this.hmac) { const serializeFn = (fn) => { assert(typeof hmac === 'function', 'Must provide options.hmac for function projections.'); @@ -128,18 +160,36 @@ class Projection { }; } + /** + * Restore a projection by name from default or configured file location. + * @param {string} name + * @param {{ fileName?: string, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {Projection} + */ static restore(name, options = {}) { assert(typeof name === 'string' && name !== '', 'Projection.restore requires a projection name.'); const fileName = options.fileName || `${name}.projection`; return Projection.restoreFromFile(fileName, options); } + /** + * Restore a projection from an explicit metadata file path. + * @param {string} fileName + * @param {{ hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {Projection} + */ static restoreFromFile(fileName, options = {}) { assert(fs.existsSync(fileName), `Projection file does not exist: ${fileName}`); const metadata = JSON.parse(fs.readFileSync(fileName, 'utf8')); return Projection.fromMetadata(metadata, { ...options, fileName }); } + /** + * Recreate a projection instance from serialized metadata. + * @param {object} metadata + * @param {{ fileName?: string, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {Projection} + */ static fromMetadata(metadata, options = {}) { assert(metadata && typeof metadata === 'object', 'Invalid projection metadata.'); if (metadata.kind === 'composite-projection') { @@ -173,6 +223,13 @@ class Projection { return projection; } + /** + * Compose multiple projections into one composite projection. + * @param {string} name + * @param {object} projections + * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {CompositeProjection} + */ static compose(name, projections, options = {}) { return new CompositeProjection(name, projections, options); } @@ -180,6 +237,11 @@ class Projection { class CompositeProjection extends Projection { + /** + * @param {string} name + * @param {object} projections + * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + */ constructor(name, projections, options = {}) { assert(projections && typeof projections === 'object' && !Array.isArray(projections), 'CompositeProjection requires an object map of projections.'); const normalized = {}; @@ -209,6 +271,12 @@ class CompositeProjection extends Projection { return [...types]; } + /** + * Apply one event across all child projections and return composed state. + * @param {object} state + * @param {object} event + * @returns {object} + */ apply(state, event) { if (!this.matches(event)) { this.state = state; @@ -223,6 +291,10 @@ class CompositeProjection extends Projection { return nextState; } + /** + * Reset all child projections and rebuild composed state. + * @returns {object} + */ reset() { for (const projection of Object.values(this.projections)) { projection.reset(); @@ -233,6 +305,11 @@ class CompositeProjection extends Projection { return this.state; } + /** + * Serialize composed projection metadata recursively. + * @param {function(string): string} [hmac] + * @returns {object} + */ toMetadata(hmac = this.hmac) { return { kind: 'composite-projection', @@ -244,6 +321,12 @@ class CompositeProjection extends Projection { }; } + /** + * Restore a composed projection from serialized metadata. + * @param {object} metadata + * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {CompositeProjection} + */ static fromMetadata(metadata, options = {}) { const hmac = options.hmac; const deserializeMatcher = (matcherMetadata) => { diff --git a/src/utils/fsUtil.js b/src/utils/fsUtil.js index 0c35f229..77d77c5d 100644 --- a/src/utils/fsUtil.js +++ b/src/utils/fsUtil.js @@ -2,6 +2,8 @@ import fs from 'fs'; import path from 'path'; import { mkdirpSync } from 'mkdirp'; +const SAFE_RELATIVE_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_]*(?:[\/:@~+=\-#.][A-Za-z0-9_]+)*$/; + // Best-effort cleanup for temporary files after interrupted/failed writes. function safeUnlink(fileName) { try { @@ -14,12 +16,15 @@ function safeUnlink(fileName) { } // Prevent partially written persistence files from replacing the last valid state. -function writeFileAtomic(fileName, data, options = {}) { +function writeFileAtomic(fileName, data, options = {}, onSuccess = null) { const tmpFileName = options.tmpFileName || `${fileName}.tmp`; const writeOptions = options.encoding ? { encoding: options.encoding } : undefined; try { fs.writeFileSync(tmpFileName, data, writeOptions); fs.renameSync(tmpFileName, fileName); + if (typeof onSuccess === 'function') { + onSuccess(); + } } catch (e) { safeUnlink(tmpFileName); throw e; @@ -27,6 +32,22 @@ function writeFileAtomic(fileName, data, options = {}) { return fileName; } +function isSafeRelativeName(name) { + return typeof name === 'string' + && name !== '' + && SAFE_RELATIVE_NAME_PATTERN.test(name); +} + +function resolvePathWithinRoot(rootDirectory, relativePath) { + const root = path.resolve(rootDirectory); + const resolvedPath = path.resolve(root, relativePath); + const rootRelativePath = path.relative(root, resolvedPath); + if (rootRelativePath.startsWith('..') || path.isAbsolute(rootRelativePath)) { + throw new Error(`Invalid relative path "${relativePath}".`); + } + return resolvedPath; +} + /** * Ensure that the given directory exists. * @param {string} dirName Target directory. @@ -151,4 +172,6 @@ export { safeUnlink, writeFileAtomic, scanForFiles, + isSafeRelativeName, + resolvePathWithinRoot, }; diff --git a/src/utils/metadataUtil.js b/src/utils/metadataUtil.js index 2c89b5fa..e12cf21d 100644 --- a/src/utils/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -508,6 +508,7 @@ function buildOperatorBufferMatcher(operatorObj) { export { createHmac, + isPlainObject, matches, buildMetadataHeader, buildMetadataForMatcher, diff --git a/test/Consumer.spec.js b/test/Consumer.spec.js index 760315f3..e53d8ddd 100644 --- a/test/Consumer.spec.js +++ b/test/Consumer.spec.js @@ -41,6 +41,14 @@ describe('Consumer', function() { expect(() => new Consumer(storage, 'foobar')).to.throwError(/identifier/); }); + it('throws for invalid consumer index names', function() { + expect(() => new Consumer(storage, '../foobar', 'consumer1')).to.throwError(/Invalid index name/); + }); + + it('throws for invalid consumer identifiers', function() { + expect(() => new Consumer(storage, 'foobar', '../consumer1')).to.throwError(/Invalid identifier/); + }); + it('creates consumer directory if not existing', function() { consumer = new Consumer(storage, 'foobar', 'consumer1'); expect(fs.existsSync(dataDirectory + '/consumers')).to.be(true); @@ -539,6 +547,26 @@ describe('Consumer', function() { storage.write({ type: 'Foobar', id: 3 }); }); + it('consumer.project persists a projection when attaching', function(done) { + consumer = new Consumer(storage, 'foobar', 'consumer-project-method', { count: 0 }); + const projection = new Projection('consumer-project-method', { + initialState: { count: 0 }, + handlers: { + Foobar: (state, event) => ({ ...state, count: state.count + event.id }) + } + }, { hmac: createHmac('test-secret') }); + + consumer.project(projection); + expect(fs.existsSync(`${consumer.fileName}.projection`)).to.be(true); + consumer.on('caught-up', () => { + expect(consumer.state.count).to.be(6); + done(); + }); + storage.write({ type: 'Foobar', id: 1 }); + storage.write({ type: 'Foobar', id: 2 }); + storage.write({ type: 'Foobar', id: 3 }); + }); + it('can attach and restore projections from event-type reducer maps', function(done) { consumer = new Consumer(storage, 'foobar', 'consumer-projection-map', { count: 0 }); const projection = new Projection('consumer-projection-map', { diff --git a/test/util.spec.js b/test/util.spec.js index b9adc01e..6124af08 100644 --- a/test/util.spec.js +++ b/test/util.spec.js @@ -2,7 +2,7 @@ import expect from 'expect.js'; import fs from 'fs-extra'; import path from 'path'; import { iterate } from '../src/utils/util.js'; -import { scanForFiles } from '../src/utils/fsUtil.js'; +import { isSafeRelativeName, resolvePathWithinRoot, scanForFiles } from '../src/utils/fsUtil.js'; import { fileURLToPath } from 'url'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -90,6 +90,26 @@ describe('util', function() { }); }); + describe('file path helpers', function() { + + it('detects safe relative names', function() { + expect(isSafeRelativeName('stream-orders')).to.be(true); + expect(isSafeRelativeName('../orders')).to.be(false); + }); + + it('resolves paths within a root directory', function() { + const root = path.join(testDir, 'root'); + const resolved = resolvePathWithinRoot(root, 'a/b.file'); + expect(resolved.startsWith(path.resolve(root))).to.be(true); + }); + + it('rejects traversal paths outside of root directory', function() { + const root = path.join(testDir, 'root'); + expect(() => resolvePathWithinRoot(root, '../outside.file')).to.throwError(/Invalid relative path/); + }); + + }); + }); }); From 90c6e5f68594be1a216b4d880f8874cb55ab0608 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 5 Jun 2026 19:23:07 +0000 Subject: [PATCH 39/40] refactor projection composition and simplify consumer persist --- src/CompositeProjection.js | 129 +++++++++++++++++++++++++++++++++++++ src/Consumer.js | 7 +- src/Projection.js | 117 +-------------------------------- test/Consumer.spec.js | 18 +++--- 4 files changed, 142 insertions(+), 129 deletions(-) create mode 100644 src/CompositeProjection.js diff --git a/src/CompositeProjection.js b/src/CompositeProjection.js new file mode 100644 index 00000000..91ff9b9f --- /dev/null +++ b/src/CompositeProjection.js @@ -0,0 +1,129 @@ +import { assert } from './utils/util.js'; +import { buildMatcherFromMetadata, buildMetadataForMatcher } from './utils/metadataUtil.js'; + +/** + * Build the CompositeProjection class on top of the provided Projection base class. + * This avoids cyclic imports between Projection and CompositeProjection modules. + * + * @param {typeof import('./Projection.js').default} Projection + * @returns {typeof import('./Projection.js').CompositeProjection} + */ +function createCompositeProjectionClass(Projection) { + return class CompositeProjection extends Projection { + + /** + * @param {string} name + * @param {object} projections + * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + */ + constructor(name, projections, options = {}) { + assert(projections && typeof projections === 'object' && !Array.isArray(projections), 'CompositeProjection requires an object map of projections.'); + const normalized = {}; + for (const [projectionName, projection] of Object.entries(projections)) { + normalized[projectionName] = projection instanceof Projection + ? projection + : new Projection(projectionName, projection, options); + } + super(name, { + initialState: Object.fromEntries( + Object.entries(normalized).map(([projectionName, projection]) => [projectionName, projection.initialState]) + ), + handlers: (state) => state, + matcher: options.matcher + }, options); + this.projections = normalized; + this.reset(); + } + + get types() { + const types = new Set(); + for (const projection of Object.values(this.projections)) { + for (const type of projection.types) { + types.add(type); + } + } + return [...types]; + } + + /** + * Apply one event across all child projections and return composed state. + * @param {object} state + * @param {object} event + * @returns {object} + */ + apply(state, event) { + if (!this.matches(event)) { + this.state = state; + return state; + } + const currentState = state || this.initialState; + const nextState = {}; + for (const [name, projection] of Object.entries(this.projections)) { + nextState[name] = projection.apply(currentState[name], event); + } + this.state = nextState; + return nextState; + } + + /** + * Reset all child projections and rebuild composed state. + * @returns {object} + */ + reset() { + for (const projection of Object.values(this.projections)) { + projection.reset(); + } + this.state = Object.fromEntries( + Object.entries(this.projections).map(([projectionName, projection]) => [projectionName, projection.state]) + ); + return this.state; + } + + /** + * Serialize composed projection metadata recursively. + * @param {function(string): string} [hmac] + * @returns {object} + */ + toMetadata(hmac = this.hmac) { + return { + kind: 'composite-projection', + name: this.name, + matcher: this.matcher ? buildMetadataForMatcher(this.matcher, hmac) : null, + projections: Object.fromEntries( + Object.entries(this.projections).map(([name, projection]) => [name, projection.toMetadata(hmac)]) + ) + }; + } + + /** + * Restore a composed projection from serialized metadata. + * @param {object} metadata + * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {CompositeProjection} + */ + static fromMetadata(metadata, options = {}) { + const hmac = options.hmac; + const deserializeMatcher = (matcherMetadata) => { + if (!matcherMetadata) { + return undefined; + } + if (typeof matcherMetadata.matcher === 'string') { + assert(typeof hmac === 'function', 'Must provide options.hmac to restore function projections.'); + } + return buildMatcherFromMetadata(matcherMetadata, hmac); + }; + const projections = Object.fromEntries( + Object.entries(metadata.projections || {}).map(([name, projectionMetadata]) => [ + name, + Projection.fromMetadata(projectionMetadata, options) + ]) + ); + return new this(metadata.name, projections, { + ...options, + matcher: deserializeMatcher(metadata.matcher) + }); + } + }; +} + +export { createCompositeProjectionClass }; diff --git a/src/Consumer.js b/src/Consumer.js index 7d379942..18c15550 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -194,12 +194,7 @@ class Consumer extends stream.Readable { if (fs.existsSync(tmpFile)) { throw new Error(`Trying to update consumer ${this.name} concurrently. Keep each single consumer within a single process.`); } - try { - writeFileAtomic(this.fileName, consumerData, { tmpFileName: tmpFile }, () => this.emit('persisted', consumerState)); - } catch (e) { - /* c8 ignore next */ - safeUnlink(tmpFile); - } + writeFileAtomic(this.fileName, consumerData, { tmpFileName: tmpFile }, () => this.emit('persisted', consumerState)); }); } diff --git a/src/Projection.js b/src/Projection.js index 5425efe7..a61b5569 100644 --- a/src/Projection.js +++ b/src/Projection.js @@ -3,6 +3,7 @@ import path from 'path'; import { assert } from './utils/util.js'; import { ensureDirectory, writeFileAtomic } from './utils/fsUtil.js'; import { buildMatcherFromMetadata, buildMetadataForMatcher, matches } from './utils/metadataUtil.js'; +import { createCompositeProjectionClass } from './CompositeProjection.js'; const DEFAULT_TYPE_ACCESSOR = (event) => event?.type || event?.payload?.type; @@ -235,121 +236,7 @@ class Projection { } } -class CompositeProjection extends Projection { - - /** - * @param {string} name - * @param {object} projections - * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] - */ - constructor(name, projections, options = {}) { - assert(projections && typeof projections === 'object' && !Array.isArray(projections), 'CompositeProjection requires an object map of projections.'); - const normalized = {}; - for (const [projectionName, projection] of Object.entries(projections)) { - normalized[projectionName] = projection instanceof Projection - ? projection - : new Projection(projectionName, projection, options); - } - super(name, { - initialState: Object.fromEntries( - Object.entries(normalized).map(([projectionName, projection]) => [projectionName, projection.initialState]) - ), - handlers: (state) => state, - matcher: options.matcher - }, options); - this.projections = normalized; - this.reset(); - } - - get types() { - const types = new Set(); - for (const projection of Object.values(this.projections)) { - for (const type of projection.types) { - types.add(type); - } - } - return [...types]; - } - - /** - * Apply one event across all child projections and return composed state. - * @param {object} state - * @param {object} event - * @returns {object} - */ - apply(state, event) { - if (!this.matches(event)) { - this.state = state; - return state; - } - const currentState = state || this.initialState; - const nextState = {}; - for (const [name, projection] of Object.entries(this.projections)) { - nextState[name] = projection.apply(currentState[name], event); - } - this.state = nextState; - return nextState; - } - - /** - * Reset all child projections and rebuild composed state. - * @returns {object} - */ - reset() { - for (const projection of Object.values(this.projections)) { - projection.reset(); - } - this.state = Object.fromEntries( - Object.entries(this.projections).map(([projectionName, projection]) => [projectionName, projection.state]) - ); - return this.state; - } - - /** - * Serialize composed projection metadata recursively. - * @param {function(string): string} [hmac] - * @returns {object} - */ - toMetadata(hmac = this.hmac) { - return { - kind: 'composite-projection', - name: this.name, - matcher: this.matcher ? buildMetadataForMatcher(this.matcher, hmac) : null, - projections: Object.fromEntries( - Object.entries(this.projections).map(([name, projection]) => [name, projection.toMetadata(hmac)]) - ) - }; - } - - /** - * Restore a composed projection from serialized metadata. - * @param {object} metadata - * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] - * @returns {CompositeProjection} - */ - static fromMetadata(metadata, options = {}) { - const hmac = options.hmac; - const deserializeMatcher = (matcherMetadata) => { - if (!matcherMetadata) { - return undefined; - } - if (typeof matcherMetadata.matcher === 'string') { - assert(typeof hmac === 'function', 'Must provide options.hmac to restore function projections.'); - } - return buildMatcherFromMetadata(matcherMetadata, hmac); - }; - const projections = Object.fromEntries( - Object.entries(metadata.projections || {}).map(([name, projectionMetadata]) => [ - name, - Projection.fromMetadata(projectionMetadata, options) - ]) - ); - return new CompositeProjection(metadata.name, projections, { - ...options, - matcher: deserializeMatcher(metadata.matcher) - }); - } -} +const CompositeProjection = createCompositeProjectionClass(Projection); export default Projection; export { CompositeProjection }; diff --git a/test/Consumer.spec.js b/test/Consumer.spec.js index e53d8ddd..f8d36528 100644 --- a/test/Consumer.spec.js +++ b/test/Consumer.spec.js @@ -355,26 +355,28 @@ describe('Consumer', function() { }, 15); }); - it('swallows persistence write errors and removes temp files', function(done) { + it('rethrows persistence write errors and removes temp files', function() { consumer = new Consumer(storage, 'foobar', 'consumer-1'); consumer.position = 1; consumer.state = Object.freeze({ foo: 1 }); const tmpFile = consumer.fileName + '.1'; const originalWriteFileSync = fsNative.writeFileSync; + const originalSetImmediate = global.setImmediate; + global.setImmediate = (fn) => { + fn(); + return null; + }; fsNative.writeFileSync = () => { const error = new Error('disk full'); error.code = 'ENOSPC'; throw error; }; - consumer.persist(); - - setTimeout(() => { - fsNative.writeFileSync = originalWriteFileSync; - expect(fs.existsSync(tmpFile)).to.be(false); - done(); - }, 15); + expect(() => consumer.persist()).to.throwError(/disk full/); + fsNative.writeFileSync = originalWriteFileSync; + global.setImmediate = originalSetImmediate; + expect(fs.existsSync(tmpFile)).to.be(false); }); it('restores state after reopening', function(done) { From 89f689991918a7a9c1a25538e73be7cb77430656 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Jun 2026 13:24:21 +0000 Subject: [PATCH 40/40] Rebase branch onto latest main and resolve conflicts --- src/EventStore.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/EventStore.js b/src/EventStore.js index d7a3443c..b7a306fd 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -257,7 +257,7 @@ class EventStore extends events.EventEmitter { makeReadOnly(callback) { if (this.storage instanceof ReadOnlyStorage) { callback?.(); - return + return; } for (const consumer of this.consumers.values()) { consumer.stop();