diff --git a/.gitignore b/.gitignore index e877ba4..913f6d6 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,6 @@ node_modules /data *.tgz /event-storage-ui -/event-storage-http \ No newline at end of file +/event-storage-http +/Platform.md +/*/node_modules diff --git a/AGENTS.md b/AGENTS.md index 26f3969..ed5b095 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,12 +3,15 @@ ## General guidelines for interaction Do not repeat yourself. Be concise and precise in your answers. No paraphrasing of previous information unless specifically asked for. +If you run into a decision point, maybe because the instructions given were contradictory or incomplete, try to identify the underlying principle or intent and unless that is absolutely clear, ask for clarification and/or a decision. Explain the issue you stumbled upon and the options you see, and ask for guidance on how to proceed. +The code and documentation language is English, even if the user is communicating in another language. +Read the Platform.md file if it exists to know which platform you are running on and what capabilities you have, instead of trying to run linux tools on a Windows machine. **Keep this file up-to-date**: after any code review that surfaces new architectural insights or a new principle, compact this file and integrate what was learned. ## Principles (in priority order) 1. **Clean API surface** — usage for common cases should be straightforward and well-documented. Avoid breaking changes unless they significantly improve usability. -2. **Understandable code** — cyclomatic complexity per method should stay around or below current levels. Higher-level methods should delegate to clearly-named helpers so the logic reads like pseudo-code. Use utility functions (e.g. `kWayMerge`, binary search) for generic algorithms. +2. **Understandable code** — cyclomatic complexity per method should stay around or below current levels (~15). Higher-level methods should delegate to clearly-named helpers so the logic reads like pseudo-code. Use utility functions (e.g. `kWayMerge`, binary search) for generic algorithms. 3. **Performance** — maintain good performance across all code paths. The Index and Partition layers are most performance-sensitive and may prefer performance over readability at the margin. Elsewhere, prefer simplicity; simpler code is often faster. ### Performance trade-off lessons @@ -18,8 +21,17 @@ Do not repeat yourself. Be concise and precise in your answers. No paraphrasing - **Pass cheap prefilter results as hints to expensive scans**: when a cheap pass (e.g. `Buffer.indexOf`) already locates a candidate position, carry that result forward as a hint to the costlier depth- or context-aware pass rather than letting it re-scan from the start. Eliminates a full O(n) scan on every call. - **Custom implementations only when benchmarks justify the complexity**: a hand-rolled solution may edge out a standard library call by 10–15 %, but the maintenance cost is rarely worth it unless the gain is substantial and measurable at realistic data sizes (e.g. a custom numeric parser vs. `JSON.parse` with try/catch). - **Prefer a flag on a shared helper over near-duplicate functions**: when two functions differ only in a single behavioral detail, add a boolean parameter to the shared function rather than maintaining two near-identical copies. Keep the flag's semantics explicit and limited to one axis of variation. +- **Inline single-use hot-path helpers once the surrounding flow becomes simpler**: if a helper is only called from one place and its logic can be embedded while keeping the caller below the local complexity/return-count budget, prefer the inline version. Removes call indirection and often makes the hot path easier to read as straight-line pseudo-code. +### Code Quality standards + +- Keep cyclomatic complexity per method around 15 or below. If a method exceeds that, look for ways to break it down into smaller, clearly-named helper methods. Exceptions can be made for very performance-sensitive code paths, but prefer readability in general. +- Keep number of return statements per method below 6. Multiple return points can be acceptable if they significantly improve readability, but watch for methods that become hard to follow due to too many exit points. +- Avoid deep nesting of conditionals. If you find yourself nesting more than 3 levels deep, consider refactoring to flatten the structure, such as by using guard clauses or extracting nested logic into separate methods. +- Use descriptive method and variable names to make the code self-documenting. Higher level methods should read like pseudo-code, delegating to well-named helpers that encapsulate specific logic or steps in the process. +- In general, optimize the code for readability first, and only introduce complexity when there is a clear performance benefit that has been measured and justified. + ## Architecture ``` diff --git a/bench/bench-matcher.js b/bench/bench-matcher.js index 7b91256..a146603 100644 --- a/bench/bench-matcher.js +++ b/bench/bench-matcher.js @@ -9,17 +9,24 @@ const WARMUP_RUNS = 1; const MEASURED_RUNS = 5; const matcherObjects = [ - { name: 'flat', matcher: { type: 'Foo' } }, - { name: 'scoped', matcher: { payload: { type: 'Foo' } } }, - { name: 'multi', matcher: { payload: { type: ['Foo', 'Baz'] } } }, - { name: '$gt', matcher: { payload: { value: { $gt: 100 } } } }, - { name: '$eq', matcher: { payload: { type: { $eq: 'Foo' } } } } + { name: 'flat', matcher: { type: 'Foo' } }, + { name: 'scoped', matcher: { payload: { type: 'Foo' } } }, + { name: 'anyOf2', matcher: { payload: { type: ['BazingaHappened', 'Foo'] } } }, + { name: 'anyOf4', matcher: { payload: { type: ['BazingaHappened', 'BarxingaHappened', 'QuuxingaHappened', 'Foo'] } } }, + { name: '$gt', matcher: { payload: { value: { $gt: 100 } } } }, + { name: '$eq', matcher: { payload: { type: { $eq: 'Foo' } } } }, + // ── multi-operator numeric: two numeric ops → fast path (no JSON.parse) ── + { name: '$gte+$lt', matcher: { payload: { value: { $gte: 100, $lt: 2000 } } } }, + // ── two numeric + string: numeric ops on value field + string op on type field ── + { name: '$gte+$lt+str', matcher: { payload: { value: { $gte: 100, $lt: 2000 }, type: 'Foo' } } } ]; const implementations = item => [ { name: `${item.name}-obj`, impl: 'obj', fn: (doc) => matches(doc, item.matcher) }, { name: `${item.name}-raw`, impl: 'raw', fn: buildRawBufferMatcher(item.matcher) }, - { name: `${item.name}-+obj`, impl: '+obj', fn: (buffer) => matches(JSON.parse(buffer.toString('utf8')), item.matcher) } + // The following is a reference benchmark against realistic usage, where an obj matcher requires a deserialization, which is costly. + // Do NOT remove the commented implementation case. + //{ name: `${item.name}-+obj`, impl: '+obj', fn: (buffer) => matches(JSON.parse(buffer.toString('utf8')), item.matcher) } ]; const implementationsCount = implementations(matcherObjects[0]).length; // assuming all have same count diff --git a/docs/api.md b/docs/api.md index 33a7041..b2e7e37 100644 --- a/docs/api.md +++ b/docs/api.md @@ -47,6 +47,19 @@ Close the event store and free all resources. --- +#### `makeReadOnly([callback])` ✅ Stable + +```javascript +eventstore.makeReadOnly([callback]) +``` + +Flush pending writes, close the writable storage, and reopen the store in read-only mode. +The optional `callback` is called after the new read-only storage has emitted `'opened'`. + +This is mainly useful for deployment handover workflows where one process stops writing and the same process switches into read-only mode; for most applications, `eventstore.close(); new EventStore(..., { readOnly: true })` is the simpler option. + +--- + #### `commit(streamName, events, [expectedVersion], [metadata], [callback])` ✅ Stable ```javascript @@ -267,6 +280,41 @@ Asynchronously scan all consumer state files and return their identifiers. --- +#### `eventstore.getProjection(name, [handlers], [initialState], [matcher])` + +```javascript +eventstore.getProjection(name [, handlers] [, initialState] [, matcher]) → Projection +``` + +Create a `Projection` with EventStore defaults (`typeAccessor`, storage HMAC), or restore a previously persisted one when `handlers` is omitted. + +- `handlers`: reducer function `(state, event) => state` or map `{ [eventType]: reducer }` +- `initialState`: initial projection state (default `{}`) +- `matcher`: optional object/function matcher (same shape as stream/query matchers) + + +--- + +#### `consumer.project(projection)` + +```javascript +consumer.project(projection) +``` + +Attach a projection-like object (`apply(state, event)`) as the consumer `'data'` handler. + +--- + +#### `projection.subscribe(consumer)` + +```javascript +projection.subscribe(consumer) +``` + +Attach this projection to the consumer (same wiring behavior as `consumer.project(projection)`) and persist its definition next to the consumer state file so `eventstore.getConsumer(...)` can restore and reconnect it automatically. + +--- + ### Events emitted | Event | Payload | Description | diff --git a/docs/changelog.md b/docs/changelog.md index f5df7a7..01ea047 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,18 @@ # Changelog +## 1.3.2 + +- Added `EventStore.makeReadOnly([callback])` as a niche handover API for switching a running writer process into read-only mode without restarting it. For the common case, creating a fresh read-only `EventStore` instance remains the recommended approach. +- Added public `VERSION` export at the package root and mirrored it on `EventStore.VERSION` so dependent layers can read the `event-storage` version directly without resolving/parsing `package.json` at runtime. +- Fixed `ReadOnlyStorage.open()` to keep the same opened-event semantics as the readable and writable storage variants, so read-only reopen paths and the `makeReadOnly()` callback now work consistently. + +## 1.3.1 + +- Added published TypeScript declarations (`index.d.ts`) and package-level `types` metadata so consumers can reference `EventStore` and related public APIs from `event-storage`. +- `EventStream` now exposes `where()` for matcher-based filtering. +- `EventStream.filter()` now supports `Readable.filter(callback, options)` delegation when options are provided. +- Matcher-style `EventStream.filter(matcher)` remains supported for backward compatibility but now emits a deprecation warning and should be migrated to `where()`. + ## 1.3.0 New in 1.3: diff --git a/docs/consumers.md b/docs/consumers.md index 9b24b4b..d2e7c21 100644 --- a/docs/consumers.md +++ b/docs/consumers.md @@ -69,6 +69,36 @@ consumer.setState({ count: 0 }); consumer.setState((state) => ({ ...state, count: state.count + 1 })); ``` +## Projections + +Use a `Projection` to define *how* events are projected into state, then connect it to a `Consumer` for durable continuous updates. + +```javascript +const projection = eventstore.getProjection('orders-total', { + OrderCreated: (state, event) => ({ ...state, total: state.total + (event.payload.amount || 0) }) +}, { total: 0 }); + +const consumer = eventstore.getConsumer('orders', 'orders-projection', projection.initialState); +projection.subscribe(consumer); +``` + +Projections are composable via `CompositeProjection`: + +```javascript +import { CompositeProjection, Projection } from 'event-storage'; + +const count = new Projection('count', { + initialState: 0, + handlers: { OrderCreated: (state) => state + 1 } +}); + +const overview = new CompositeProjection('overview', { + count, + last: { initialState: null, handlers: { OrderCreated: (state, event) => event.payload } } +}); +// overview.state -> { count: number, last: object|null } +``` + ## Resetting a Consumer Force the consumer to reprocess events from a given position: @@ -144,4 +174,6 @@ eventstore.on('ready', () => { - Multiple read-only instances can run simultaneously. - This enables a multi-process projection pattern: one writer process + N reader processes building different projections. +If you need to hand the *current* process over from writing to reading during a deployment, use `EventStore.makeReadOnly([callback])` instead of starting a new read-only instance. + > In principle this also works across machines sharing a common filesystem (e.g. NFS), as long as the Node.js file watcher functions correctly on that filesystem. diff --git a/docs/performance.md b/docs/performance.md index 462d707..d544fc4 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -128,3 +128,34 @@ These benchmarks measure pure **read operations** (no writes). The range scan re | range scan | stable | 453,288 | ±0.28% | 86 | | range scan | latest | 423,291 | ±0.45% | 81 | | range scan | latest(raw) | 886,578 | ±0.59% | 90 | + +### Raw Buffer Operator Matcher Benchmark + +This benchmark compares the same matcher definitions in two execution modes: + +- **`obj` (baseline):** run `matches(document, matcher)` on already deserialized events +- **`raw`:** run `buildRawBufferMatcher(matcher)` directly on NDJSON buffers + +Matcher objects used in `bench/bench-matcher.js`: + +```js +{ type: 'Foo' } +{ payload: { type: 'Foo' } } +{ payload: { type: ['BazingaHappened', 'Foo'] } } +{ payload: { type: ['BazingaHappened', 'BarxingaHappened', 'QuuxingaHappened', 'Foo'] } } +{ payload: { value: { $gt: 100 } } } +{ payload: { type: { $eq: 'Foo' } } } +{ payload: { value: { $gte: 100, $lt: 2000 } } } +{ payload: { value: { $gte: 100, $lt: 2000 }, type: 'Foo' } } +``` + +Observed median delta (`raw` vs `obj`) from the benchmark run: + +At **ANY match ratio**, raw is roughly **40–55% slower** than object mode. +At **ANY match ratio**, raw is roughly **50–60% faster** than object mode when deserializing the event first. + +Interpretation: + +- When events are **already deserialized**, object matchers are the correct baseline and typically faster. +- In workloads where data is still in **raw buffer form** (streaming), raw matchers avoid `JSON.parse` and can outperform a deserialize+object-match pipeline by roughly **~50%** in practice. + diff --git a/index.d.ts b/index.d.ts new file mode 100644 index 0000000..6ef5403 --- /dev/null +++ b/index.d.ts @@ -0,0 +1,145 @@ +import { EventEmitter } from 'node:events'; +import { Readable } from 'node:stream'; + +export type EventMatcher = Record | ((payload: any, metadata?: any) => boolean); +export type EventPredicate = EventMatcher | ((buffer: Buffer) => boolean) | null; + +export interface StreamIndexInfo { + length: number; + metadata?: Record | null; +} + +export interface StreamInfo { + index: StreamIndexInfo; + matcher?: EventMatcher; + closed?: boolean; +} + +export class CommitCondition { + constructor(types: string[], matcher: EventMatcher | null | undefined, noneMatchAfter: number, raw?: boolean); + types: string[]; + matcher: EventMatcher | null; + raw: boolean; + noneMatchAfter: number; +} + +export class OptimisticConcurrencyError extends Error {} + +export const ExpectedVersion: { + Any: -1; + EmptyStream: 0; +}; + +export const VERSION: string; + +export class EventStream extends Readable { + constructor(name: string, eventStore: EventStore, minRevision?: number, maxRevision?: number, predicate?: EventPredicate, raw?: boolean); + name: string; + raw: boolean; + version: number; + minRevision: number; + maxRevision: number; + streamIndex: StreamIndexInfo; + predicate: EventPredicate; + + from(revision: number): this; + until(revision: number): this; + fromStart(): this; + fromEnd(): this; + previous(amount: number): this; + following(amount: number): this; + backwards(): this; + forwards(): this; + where(predicate?: EventPredicate): this; + filter(predicate?: EventPredicate): this; + filter( + predicate: (chunk: any, options?: { signal?: AbortSignal }) => boolean | Promise, + options: { concurrency?: number; signal?: AbortSignal } + ): Readable; + next(): any | false; +} + +export class Consumer extends Readable { + constructor(storage: Storage, indexName: string, identifier: string, initialState?: Record, startFrom?: number); + streamName: string; + position: number; + state: Record; + start(): this; + stop(): void; + reset(initialState?: Record, startFrom?: number): this; + setState(state: Record): this; +} + +export class Storage extends EventEmitter { + static ReadOnly: typeof Storage; + static StorageLockedError: typeof StorageLockedError; + static LOCK_THROW: number; + static LOCK_RECLAIM: number; + + initialized: boolean | null; + length: number; + indexDirectory: string; + storageFile: string; +} + +export class StorageLockedError extends Error {} + +export class Index { + static ReadOnly: typeof Index; + static Entry: unknown; + + length: number; + metadata?: Record | null; +} + +export class EventStore extends EventEmitter { + static Storage: typeof Storage; + static Index: typeof Index; + static VERSION: string; + + storage: Storage; + streams: Record; + consumers: Map; + length: number; + + constructor(storeName?: string, config?: Record); + + open(callback?: (err?: Error | null) => void): void; + close(callback?: (err?: Error | null) => void): void; + + getStreamVersion(streamName: string): number; + getEventStream(streamName: string, minRevision?: number, maxRevision?: number, predicate?: EventPredicate, raw?: boolean): EventStream | false; + getAllEvents(minRevision?: number, maxRevision?: number, predicate?: EventPredicate, raw?: boolean): EventStream; + fromStreams(streamName: string, streamNames: string[], minRevision?: number, maxRevision?: number, predicate?: EventPredicate, raw?: boolean): EventStream; + getEventStreamForCategory(categoryName: string, minRevision?: number, maxRevision?: number, predicate?: EventPredicate, raw?: boolean): EventStream; + + query(types: string[], matcher?: EventMatcher | null, minRevision?: number, raw?: boolean): { stream: EventStream; condition: CommitCondition }; + createEventStream(streamName: string, matcher: EventMatcher, reindex?: boolean): EventStream; + + commit( + streamName: string, + events: any[], + expectedVersion: number | CommitCondition, + metadata: Record, + callback: (error: Error | null, commit?: Record) => void + ): void; + + getConsumer(streamName: string, identifier: string, initialState?: Record, since?: number): Consumer; + getConsumer(identifier: string): Consumer | null; + scanConsumers( + callback: (error: Error | null, consumers: Array<{ name: string; stream: string; identifier: string }>) => void, + autoStart?: boolean + ): void; +} + +export const LOCK_THROW: number; +export const LOCK_RECLAIM: number; + +export function matches(document: Record, matcher: Record): boolean; +export function buildRawBufferMatcher(matcher: Record): (buffer: Buffer, startOffset?: number) => boolean; + +export default EventStore; + + + + diff --git a/index.js b/index.js index b6d72de..2f8e0df 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,31 @@ -export { default as EventStore, default, ExpectedVersion, OptimisticConcurrencyError, CommitCondition, LOCK_THROW, LOCK_RECLAIM } from './src/EventStore.js'; -export { default as EventStream } from './src/EventStream.js'; -export { default as Storage, StorageLockedError } from './src/Storage.js'; -export { default as Index } from './src/Index.js'; -export { default as Consumer } from './src/Consumer.js'; -export { matches, buildRawBufferMatcher } from './src/utils/metadataUtil.js'; +import packageJson from './package.json' with { type: 'json' }; +import EventStore, { ExpectedVersion, OptimisticConcurrencyError, CommitCondition, LOCK_THROW, LOCK_RECLAIM } from './src/EventStore.js'; +import EventStream from './src/EventStream.js'; +import Storage, { StorageLockedError } from './src/Storage.js'; +import Index from './src/Index.js'; +import Consumer from './src/Consumer.js'; +import Projection, { CompositeProjection } from './src/Projection.js'; +import { matches, buildRawBufferMatcher } from './src/utils/metadataUtil.js'; + +const VERSION = packageJson.version; +EventStore.VERSION = VERSION; + +export { + EventStore, + EventStore as default, + ExpectedVersion, + OptimisticConcurrencyError, + CommitCondition, + LOCK_THROW, + LOCK_RECLAIM, + VERSION, + EventStream, + Storage, + StorageLockedError, + Index, + Consumer, + Projection, + CompositeProjection, + matches, + buildRawBufferMatcher +}; diff --git a/package-lock.json b/package-lock.json index 181a112..390a8a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "event-storage", - "version": "1.2.0", + "version": "1.3.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "event-storage", - "version": "1.2.0", + "version": "1.3.2", "license": "MIT", "dependencies": { "mkdirp": "^3.0.1" diff --git a/package.json b/package.json index 1db79c7..3af33b3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "event-storage", - "version": "1.3.0", + "version": "1.3.2", "type": "module", "description": "An optimized embedded event store for node.js", "keywords": [ @@ -24,6 +24,7 @@ "exports": { ".": "./index.js" }, + "types": "./index.d.ts", "scripts": { "test": "c8 --reporter=lcov --reporter=text mocha test/*.spec.js", "test:grep": "c8 --reporter=lcov --reporter=text mocha test/*.spec.js --grep", @@ -45,6 +46,7 @@ "src/Storage/*.js", "src/WatchesFile.js", "index.js", + "index.d.ts", "src/utils/*.js" ], "license": "MIT", diff --git a/src/CompositeProjection.js b/src/CompositeProjection.js new file mode 100644 index 0000000..91ff9b9 --- /dev/null +++ b/src/CompositeProjection.js @@ -0,0 +1,129 @@ +import { assert } from './utils/util.js'; +import { buildMatcherFromMetadata, buildMetadataForMatcher } from './utils/metadataUtil.js'; + +/** + * Build the CompositeProjection class on top of the provided Projection base class. + * This avoids cyclic imports between Projection and CompositeProjection modules. + * + * @param {typeof import('./Projection.js').default} Projection + * @returns {typeof import('./Projection.js').CompositeProjection} + */ +function createCompositeProjectionClass(Projection) { + return class CompositeProjection extends Projection { + + /** + * @param {string} name + * @param {object} projections + * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + */ + constructor(name, projections, options = {}) { + assert(projections && typeof projections === 'object' && !Array.isArray(projections), 'CompositeProjection requires an object map of projections.'); + const normalized = {}; + for (const [projectionName, projection] of Object.entries(projections)) { + normalized[projectionName] = projection instanceof Projection + ? projection + : new Projection(projectionName, projection, options); + } + super(name, { + initialState: Object.fromEntries( + Object.entries(normalized).map(([projectionName, projection]) => [projectionName, projection.initialState]) + ), + handlers: (state) => state, + matcher: options.matcher + }, options); + this.projections = normalized; + this.reset(); + } + + get types() { + const types = new Set(); + for (const projection of Object.values(this.projections)) { + for (const type of projection.types) { + types.add(type); + } + } + return [...types]; + } + + /** + * Apply one event across all child projections and return composed state. + * @param {object} state + * @param {object} event + * @returns {object} + */ + apply(state, event) { + if (!this.matches(event)) { + this.state = state; + return state; + } + const currentState = state || this.initialState; + const nextState = {}; + for (const [name, projection] of Object.entries(this.projections)) { + nextState[name] = projection.apply(currentState[name], event); + } + this.state = nextState; + return nextState; + } + + /** + * Reset all child projections and rebuild composed state. + * @returns {object} + */ + reset() { + for (const projection of Object.values(this.projections)) { + projection.reset(); + } + this.state = Object.fromEntries( + Object.entries(this.projections).map(([projectionName, projection]) => [projectionName, projection.state]) + ); + return this.state; + } + + /** + * Serialize composed projection metadata recursively. + * @param {function(string): string} [hmac] + * @returns {object} + */ + toMetadata(hmac = this.hmac) { + return { + kind: 'composite-projection', + name: this.name, + matcher: this.matcher ? buildMetadataForMatcher(this.matcher, hmac) : null, + projections: Object.fromEntries( + Object.entries(this.projections).map(([name, projection]) => [name, projection.toMetadata(hmac)]) + ) + }; + } + + /** + * Restore a composed projection from serialized metadata. + * @param {object} metadata + * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {CompositeProjection} + */ + static fromMetadata(metadata, options = {}) { + const hmac = options.hmac; + const deserializeMatcher = (matcherMetadata) => { + if (!matcherMetadata) { + return undefined; + } + if (typeof matcherMetadata.matcher === 'string') { + assert(typeof hmac === 'function', 'Must provide options.hmac to restore function projections.'); + } + return buildMatcherFromMetadata(matcherMetadata, hmac); + }; + const projections = Object.fromEntries( + Object.entries(metadata.projections || {}).map(([name, projectionMetadata]) => [ + name, + Projection.fromMetadata(projectionMetadata, options) + ]) + ); + return new this(metadata.name, projections, { + ...options, + matcher: deserializeMatcher(metadata.matcher) + }); + } + }; +} + +export { createCompositeProjectionClass }; diff --git a/src/Consumer.js b/src/Consumer.js index 58411b5..18c1555 100644 --- a/src/Consumer.js +++ b/src/Consumer.js @@ -2,26 +2,11 @@ import stream from 'stream'; import fs from 'fs'; import path from 'path'; import { assert } from './utils/util.js'; -import { ensureDirectory } from './utils/fsUtil.js'; +import { ensureDirectory, isSafeRelativeName, resolvePathWithinRoot, safeUnlink, writeFileAtomic } from './utils/fsUtil.js'; import { normalizeConsumerStateArgs } from './utils/apiHelpers.js'; import Storage from './Storage/ReadableStorage.js'; const MAX_CATCHUP_BATCH = 10; -/** - * Safely unlink a file and ignore if it doesn't exist. - * @param {string} filename - */ -const safeUnlink = (filename) => { - /* c8 ignore next */ - try { - fs.unlinkSync(filename); - } catch (e) { - if (e.code !== "ENOENT") { - throw e; - } - } -}; - /** * Implements an event-driven durable Consumer that provides at-least-once delivery semantics or exactly-once processing semantics if only using setState(). */ @@ -54,11 +39,15 @@ class Consumer extends stream.Readable { * @param {string} identifier The unique name to identify this consumer. */ initializeStorage(storage, indexName, identifier) { + assert(indexName === '_all' || isSafeRelativeName(indexName), `Invalid index name "${indexName}" for consumer.`); + assert(isSafeRelativeName(identifier), `Invalid identifier "${identifier}" for consumer.`); this.storage = storage; this.index = this.storage.openIndex(indexName); this.indexName = indexName; + this.identifier = identifier; const consumerDirectory = path.join(this.storage.indexDirectory, 'consumers'); - this.fileName = path.join(consumerDirectory, this.storage.storageFile + '.' + indexName + '.' + identifier); + this.fileName = resolvePathWithinRoot(consumerDirectory, `${this.storage.storageFile}.${indexName}.${identifier}`); + ensureDirectory(path.dirname(this.fileName)); if (ensureDirectory(consumerDirectory)) { this.cleanUpFailedWrites(); } @@ -69,11 +58,15 @@ class Consumer extends stream.Readable { * @private */ cleanUpFailedWrites() { - const consumerNamePrefix = path.basename(this.fileName) + '.'; + const consumerBaseName = path.basename(this.fileName); const consumerDirectory = path.dirname(this.fileName); const files = fs.readdirSync(consumerDirectory); for (let file of files) { - if (file.startsWith(consumerNamePrefix)) { + if (!file.startsWith(consumerBaseName + '.')) { + continue; + } + const suffix = file.slice(consumerBaseName.length + 1); + if (/^\d+$/.test(suffix)) { safeUnlink(path.join(consumerDirectory, file)); } } @@ -104,6 +97,32 @@ class Consumer extends stream.Readable { this.consuming = false; } + /** + * Register a projection as `data` event handler. + * @api + * @param {{ apply: function(object, object): object }} projection + */ + project(projection) { + assert(projection && typeof projection.apply === 'function', 'Projection must implement apply(state, event).'); + const projectionFileName = this.fileName ? `${this.fileName}.projection` : null; + const isAlreadySubscribed = this.projection === projection; + const isAlreadyPersisted = projectionFileName + && projection.fileName === projectionFileName + && fs.existsSync(projectionFileName); + if (this.projectionHandler) { + this.removeListener('data', this.projectionHandler); + } + this.projection = projection; + this.projectionHandler = (event) => { + this.setState(projection.apply(this.state, event)); + }; + this.on('data', this.projectionHandler); + if (!isAlreadySubscribed && !isAlreadyPersisted && typeof projection.persist === 'function') { + projection.persist({ fileName: projectionFileName || projection.fileName }); + } + return this; + } + /** * Update the state of this consumer transactionally with the position. * May only be called from within the document handling callback. @@ -118,6 +137,9 @@ class Consumer extends stream.Readable { if (typeof newState === 'function') { newState = newState(this.state); } + if (this.state === newState) { + return; + } this.state = Object.freeze(newState); this.doPersist = persist; } @@ -172,15 +194,7 @@ class Consumer extends stream.Readable { if (fs.existsSync(tmpFile)) { throw new Error(`Trying to update consumer ${this.name} concurrently. Keep each single consumer within a single process.`); } - try { - fs.writeFileSync(tmpFile, consumerData); - // If the write fails (half-way), the consumer state file will not be corrupted - fs.renameSync(tmpFile, this.fileName); - this.emit('persisted', consumerState); - } catch (e) { - /* c8 ignore next */ - safeUnlink(tmpFile); - } + writeFileAtomic(this.fileName, consumerData, { tmpFileName: tmpFile }, () => this.emit('persisted', consumerState)); }); } diff --git a/src/EventStore.js b/src/EventStore.js index 848fc9e..b7a306f 100644 --- a/src/EventStore.js +++ b/src/EventStore.js @@ -6,9 +6,10 @@ import events from 'events'; import Storage, { ReadOnly as ReadOnlyStorage, LOCK_THROW, LOCK_RECLAIM } from './Storage.js'; import Index from './Index.js'; import Consumer from './Consumer.js'; +import Projection from './Projection.js'; import { assert, getPropertyAtPath } from './utils/util.js'; -import { ensureDirectory, scanForFiles } from './utils/fsUtil.js'; -import { buildTypeMatcherFn } from './utils/metadataUtil.js'; +import { ensureDirectory, isSafeRelativeName, scanForFiles } from './utils/fsUtil.js'; +import { buildTypeMatcherFn, isPlainObject } from './utils/metadataUtil.js'; import { fixCommitArgumentTypes, parseStreamFromIndexName, normalizePredicateRaw } from './utils/apiHelpers.js'; const ExpectedVersion = { @@ -20,7 +21,6 @@ const ExpectedVersion = { * Default matcher property paths mirroring the Storage default, used for index optimization. */ const DEFAULT_MATCHER_PROPERTIES = ['stream', 'payload.type']; -const STREAM_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_]*(?:[\/:@~+=\-#.][A-Za-z0-9_]+)*$/; const STORAGE_HOOK_EVENTS = new Set(['preCommit', 'preRead']); class OptimisticConcurrencyError extends Error {} @@ -40,7 +40,7 @@ class OptimisticConcurrencyError extends Error {} class CommitCondition { /** * @param {string[]} types - * @param {function(object, object): boolean|object|null} [matcher] + * @param {(function(object, object): boolean)|object|null} [matcher] * @param {number} noneMatchAfter * @param {boolean} [raw=false] */ @@ -120,6 +120,9 @@ class EventStore extends events.EventEmitter { } } + this.projectionTypeAccessor = this.typeAccessor + ? (event) => this.typeAccessor(event?.payload || event) + : undefined; this.initialize(storeName, storageConfig); } @@ -129,24 +132,34 @@ class EventStore extends events.EventEmitter { * @param {object} storageConfig */ initialize(storeName, storageConfig) { + this.storageConfig = storageConfig; this.streamsDirectory = path.resolve(storageConfig.indexDirectory); - this.storeName = storeName; - this.storage = (storageConfig.readOnly === true) ? - new ReadOnlyStorage(storeName, storageConfig) - : new Storage(storeName, storageConfig); - this.streams = Object.create(null); - this.streams._all = { index: this.storage.index }; this.consumers = new Map(); - this.storage.on('index-created', this.registerStream.bind(this)); + const storage = storageConfig.readOnly === true + ? new ReadOnlyStorage(storeName, storageConfig) + : new Storage(storeName, storageConfig); - this.storage.on('opened', () => { + this.mountStorage(storage, () => { this.checkUnfinishedCommits(); this.emit('ready'); }); + } - this.storage.open(); + /** + * Wire up a storage instance: reset the streams map, attach the index-created listener, + * register a one-shot opened handler, then open the storage. + * @private + * @param {ReadableStorage} storage + * @param {function} [onOpened] Called once when the storage is opened. + */ + mountStorage(storage, onOpened) { + this.storage = storage; + this.streams = Object.create(null); + this.streams._all = { index: this.storage.index }; + this.storage.on('index-created', this.registerStream.bind(this)); + this.storage.open(onOpened); } /** @@ -231,6 +244,35 @@ class EventStore extends events.EventEmitter { this.storage.close(); } + /** + * Flush all pending writes, then re-open the store in read-only mode. + * Any registered consumers are stopped. The `callback` is invoked once the + * new read-only storage has finished opening. + * + * Does not re-emit `'ready'` — use the callback to react to the transition. + * + * @api + * @param {function} [callback] Called when the store is ready in read-only mode. + */ + makeReadOnly(callback) { + if (this.storage instanceof ReadOnlyStorage) { + callback?.(); + return; + } + for (const consumer of this.consumers.values()) { + consumer.stop(); + } + this.consumers.clear(); + + this.storage.flush(); + this.storage.close(); + + const readOnlyConfig = Object.assign({}, this.storageConfig, { readOnly: true }); + this.storageConfig = readOnlyConfig; + + this.mountStorage(new ReadOnlyStorage(this.storeName, readOnlyConfig), callback); + } + /** * Override EventEmitter.on() to delegate 'preCommit' and 'preRead' event registrations * to the underlying storage, so that `eventstore.on('preCommit', handler)` works naturally. @@ -405,7 +447,7 @@ class EventStore extends events.EventEmitter { return null; } assert(typeof type === 'string', 'typeAccessor must return a string.'); - assert(STREAM_NAME_PATTERN.test(type), `typeAccessor must return a valid stream name. Got: "${type}"`); + assert(isSafeRelativeName(type), `typeAccessor must return a valid stream name. Got: "${type}"`); return type; } @@ -777,11 +819,44 @@ class EventStore extends events.EventEmitter { existingConsumer.stop(); } const consumer = new Consumer(this.storage, streamName === '_all' ? '_all' : 'stream-' + streamName, identifier, initialState, since); + const consumerProjectionFileName = `${consumer.fileName}.projection`; + if (fs.existsSync(consumerProjectionFileName)) { + Projection.restoreFromFile(consumerProjectionFileName, { + hmac: this.storage.hmac, + typeAccessor: this.projectionTypeAccessor + }).subscribe(consumer); + } consumer.streamName = streamName; this.consumers.set(identifier, consumer); return consumer; } + /** + * Get or create a projection with EventStore defaults. + * + * @param {string} name Projection name. + * @param {function(object, object): object|object} [handlers] Projection handlers (reducer fn or reducer map). + * @param {object} [initialState={}] Projection initial state. + * @param {object|function(object): boolean} [matcher] Optional projection matcher. + * @returns {Projection} + */ + getProjection(name, handlers, initialState = {}, matcher) { + assert(typeof name === 'string' && name !== '', 'Must provide a projection name.'); + const projectionFileName = path.join(this.storage.indexDirectory, 'projections', this.storage.storageFile + '.' + name + '.projection'); + const projectionOptions = { + fileName: projectionFileName, + hmac: this.storage.hmac, + typeAccessor: this.projectionTypeAccessor + }; + if (handlers !== undefined) { + const definition = isProjectionDefinitionObject(handlers) + ? handlers + : { handlers, initialState, matcher }; + return new Projection(name, definition, projectionOptions); + } + return Projection.restore(name, projectionOptions); + } + /** * Scan the existing consumers on this EventStore and asynchronously invoke a callback with the parsed list. * @@ -822,6 +897,10 @@ class EventStore extends events.EventEmitter { } +function isProjectionDefinitionObject(value) { + return isPlainObject(value) && Object.hasOwn(value, 'handlers'); +} + EventStore.Storage = Storage; EventStore.Index = Index; diff --git a/src/EventStream.js b/src/EventStream.js index 6865eb3..97a5f2b 100644 --- a/src/EventStream.js +++ b/src/EventStream.js @@ -2,8 +2,10 @@ import stream from 'stream'; import { assert } from './utils/util.js'; import { buildRawBufferMatcher, matches } from './utils/metadataUtil.js'; import { normalizeRevision, normalizeMaxRevision } from './utils/apiHelpers.js'; +import { emitDeprecationWarningOnce } from './utils/deprecations.js'; const NDJSON_NEWLINE = Buffer.from('\n'); +const FILTER_MATCHER_DEPRECATION_CODE = 'EVENTSTREAM_FILTER_MATCHER_DEPRECATED'; /** * An event stream is a simple wrapper around an iterator over storage documents. @@ -245,16 +247,15 @@ class EventStream extends stream.Readable { } /** - * Apply a filter predicate to this stream. Only events for which `predicate(payload, metadata)` - * returns a truthy value will be yielded. The predicate is stored as a first-class property - * of the stream and applied in {@link EventStream#next}. + * Apply matcher semantics to this EventStream instance (legacy `filter()` behavior). * * @api - * @param {function(object, object): boolean} predicate A function receiving `(payload, metadata)`. - * Events for which the predicate returns falsy are skipped. + * @param {(function(object, object): boolean)|(function(Buffer): boolean)|object|null} [predicate] + * Matcher function/object for object mode, or buffer matcher function/object for raw mode. + * Omit to clear the current predicate. * @returns {EventStream} `this` */ - filter(predicate) { + where(predicate) { this.predicate = predicate || null; this.rawMatcher = null; this._iterator = null; @@ -262,6 +263,28 @@ class EventStream extends stream.Readable { return this; } + /** + * Readable-compatible filter entry point. + * + * - `filter(callback, options)` delegates to Node's `Readable.filter(...)`. + * - `filter(matcher)` keeps legacy EventStream matcher behavior and is deprecated. + * + * @api + * @param {function|object|null} [predicate] + * @param {{concurrency?: number, signal?: AbortSignal}|undefined} [options] + * @returns {stream.Readable|EventStream} + */ + filter(predicate, options) { + if (options !== undefined) { + return super.filter(predicate, options); + } + emitDeprecationWarningOnce( + FILTER_MATCHER_DEPRECATION_CODE, + 'EventStream.filter() with matcher semantics is deprecated. Use EventStream.where() for matcher/object predicates or pass options to filter() to use Readable.filter().' + ); + return this.where(predicate); + } + matchesPredicate(entry) { if (!this.predicate) { return true; diff --git a/src/Projection.js b/src/Projection.js new file mode 100644 index 0000000..a61b556 --- /dev/null +++ b/src/Projection.js @@ -0,0 +1,242 @@ +import fs from 'fs'; +import path from 'path'; +import { assert } from './utils/util.js'; +import { ensureDirectory, writeFileAtomic } from './utils/fsUtil.js'; +import { buildMatcherFromMetadata, buildMetadataForMatcher, matches } from './utils/metadataUtil.js'; +import { createCompositeProjectionClass } from './CompositeProjection.js'; + +const DEFAULT_TYPE_ACCESSOR = (event) => event?.type || event?.payload?.type; + + +class Projection { + + /** + * @param {string} name Projection name. + * @param {{ initialState?: object, handlers: function(object, object): object|object, matcher?: object|function(object): boolean }} [definition] + * @param {{ hmac?: function(string): string, typeAccessor?: function(object): string, fileName?: string }} [options] + */ + constructor(name, definition = {}, options = {}) { + assert(typeof name === 'string' && name !== '', 'Projection must have a name.'); + const { initialState = {}, handlers, matcher } = definition; + assert((typeof handlers === 'function') || (handlers && typeof handlers === 'object' && !Array.isArray(handlers)), 'Projection handlers must be a function or an object map of functions.'); + if (typeof handlers === 'object') { + for (const reducer of Object.values(handlers)) { + assert(typeof reducer === 'function', 'Projection handler maps must contain reducer functions.'); + } + } + this.name = name; + this.initialState = Object.freeze(initialState); + this.handlers = handlers; + this.matcher = matcher; + this.hmac = options.hmac || null; + this.typeAccessor = options.typeAccessor || DEFAULT_TYPE_ACCESSOR; + this.fileName = options.fileName || null; + this.state = this.initialState; + } + + get types() { + if (typeof this.handlers === 'function') { + return []; + } + return Object.keys(this.handlers); + } + + /** + * Apply one event to the provided state and return the next state. + * @param {*} state + * @param {object} event + * @returns {*} + */ + apply(state, event) { + if (!this.matches(event)) { + this.state = state; + return state; + } + let reducer = this.handlers; + if (typeof this.handlers === 'object') { + reducer = this.handlers[this.typeAccessor(event)]; + if (typeof reducer !== 'function') { + this.state = state; + return state; + } + } + const nextState = reducer(state, event); + this.state = nextState; + return nextState; + } + + /** + * Reset to initialState and project all events from the given iterable stream. + * @param {Iterable} stream + * @returns {*} + */ + handle(stream) { + this.reset(); + for (const event of stream) { + this.state = this.apply(this.state, event); + } + return this.state; + } + + /** + * Reset current projection state to its initial state. + * @returns {*} + */ + reset() { + this.state = this.initialState; + return this.state; + } + + /** + * Check whether an event matches this projection's matcher definition. + * @param {object} event + * @returns {boolean} + */ + matches(event) { + if (!this.matcher) { + return true; + } + if (typeof this.matcher === 'function') { + return this.matcher(event); + } + return matches(event, this.matcher); + } + + /** + * Subscribe this projection to a consumer and persist when needed. + * @param {{ project: function(Projection): object }} consumer + * @returns {Projection} + */ + subscribe(consumer) { + assert(consumer && typeof consumer.project === 'function', 'Projection.subscribe expects a Consumer instance.'); + consumer.project(this); + return this; + } + + /** + * Persist projection definition metadata to disk. + * @param {{ hmac?: function(string): string, fileName?: string }} [options] + * @returns {string} Persisted file name. + */ + persist(options = {}) { + const hmac = options.hmac || this.hmac; + const fileName = options.fileName || this.fileName || `${this.name}.projection`; + const metadata = this.toMetadata(hmac); + const tmpFile = fileName + '.tmp'; + ensureDirectory(path.dirname(fileName)); + writeFileAtomic(fileName, JSON.stringify(metadata), { + tmpFileName: tmpFile, + encoding: 'utf8' + }); + this.fileName = fileName; + this.hmac = hmac; + return fileName; + } + + /** + * Serialize this projection definition into trusted metadata. + * @param {function(string): string} [hmac] + * @returns {object} + */ + toMetadata(hmac = this.hmac) { + const serializeFn = (fn) => { + assert(typeof hmac === 'function', 'Must provide options.hmac for function projections.'); + return buildMetadataForMatcher(fn, hmac); + }; + const matcherMetadata = this.matcher ? ( + typeof this.matcher === 'function' ? serializeFn(this.matcher) : buildMetadataForMatcher(this.matcher, hmac) + ) : null; + const handlersMetadata = (typeof this.handlers === 'function') + ? serializeFn(this.handlers) + : Object.fromEntries( + Object.entries(this.handlers).map(([eventType, reducer]) => [eventType, serializeFn(reducer)]) + ); + return { + kind: 'projection', + name: this.name, + initialState: this.initialState, + matcher: matcherMetadata, + handlersKind: typeof this.handlers === 'function' ? 'function' : 'map', + handlers: handlersMetadata + }; + } + + /** + * Restore a projection by name from default or configured file location. + * @param {string} name + * @param {{ fileName?: string, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {Projection} + */ + static restore(name, options = {}) { + assert(typeof name === 'string' && name !== '', 'Projection.restore requires a projection name.'); + const fileName = options.fileName || `${name}.projection`; + return Projection.restoreFromFile(fileName, options); + } + + /** + * Restore a projection from an explicit metadata file path. + * @param {string} fileName + * @param {{ hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {Projection} + */ + static restoreFromFile(fileName, options = {}) { + assert(fs.existsSync(fileName), `Projection file does not exist: ${fileName}`); + const metadata = JSON.parse(fs.readFileSync(fileName, 'utf8')); + return Projection.fromMetadata(metadata, { ...options, fileName }); + } + + /** + * Recreate a projection instance from serialized metadata. + * @param {object} metadata + * @param {{ fileName?: string, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {Projection} + */ + static fromMetadata(metadata, options = {}) { + assert(metadata && typeof metadata === 'object', 'Invalid projection metadata.'); + if (metadata.kind === 'composite-projection') { + return CompositeProjection.fromMetadata(metadata, options); + } + assert(metadata.kind === 'projection', 'Invalid projection metadata kind.'); + const hmac = options.hmac; + const deserialize = (matcherMetadata) => { + if (!matcherMetadata) { + return undefined; + } + if (typeof matcherMetadata.matcher === 'string') { + assert(typeof hmac === 'function', 'Must provide options.hmac to restore function projections.'); + } + return buildMatcherFromMetadata(matcherMetadata, hmac); + }; + const handlers = metadata.handlersKind === 'function' + ? deserialize(metadata.handlers) + : Object.fromEntries( + Object.entries(metadata.handlers || {}).map(([eventType, reducerMetadata]) => [eventType, deserialize(reducerMetadata)]) + ); + const projection = new Projection(metadata.name, { + initialState: metadata.initialState, + matcher: deserialize(metadata.matcher), + handlers + }, { + ...options, + fileName: options.fileName || null + }); + projection.reset(); + return projection; + } + + /** + * Compose multiple projections into one composite projection. + * @param {string} name + * @param {object} projections + * @param {{ matcher?: object|function(object): boolean, hmac?: function(string): string, typeAccessor?: function(object): string }} [options] + * @returns {CompositeProjection} + */ + static compose(name, projections, options = {}) { + return new CompositeProjection(name, projections, options); + } +} + +const CompositeProjection = createCompositeProjectionClass(Projection); + +export default Projection; +export { CompositeProjection }; diff --git a/src/Storage/ReadOnlyStorage.js b/src/Storage/ReadOnlyStorage.js index 87d7cf4..560abf6 100644 --- a/src/Storage/ReadOnlyStorage.js +++ b/src/Storage/ReadOnlyStorage.js @@ -31,14 +31,16 @@ class ReadOnlyStorage extends ReadableStorage { * Will emit an 'opened' event if finished. * * @api + * @param {function(): void} [callback] Called after indexes open, before `'opened'` is emitted. + * Can be used as a synchronous alternative to listening to the `'opened'` event. * @returns {boolean} */ - open() { + open(callback) { if (!this.watcher) { this.watcher = new Watcher([this.dataDirectory, this.indexDirectory], this.storageFilesFilter); this.watcher.on('rename', this.onStorageFileChanged); } - return super.open(); + return super.open(callback); } /** diff --git a/src/utils/apiHelpers.js b/src/utils/apiHelpers.js index 93a4098..d6f18aa 100644 --- a/src/utils/apiHelpers.js +++ b/src/utils/apiHelpers.js @@ -1,3 +1,14 @@ +/** + * Normalize commit overloads into a single argument object. + * + * @param {object|object[]} events Event or event list. + * @param {number|object|function} expectedVersion Expected version, CommitCondition, or already metadata/callback. + * @param {object|function} metadata Commit metadata or callback. + * @param {function} callback Completion callback. + * @param {number} ExpectedVersionAny Fallback value for "any" expectedVersion. + * @param {Function} CommitConditionClass Class used for CommitCondition checks. + * @returns {{events: object[], expectedVersion: number|object, metadata: object, callback: function}} Normalized commit arguments. + */ function fixCommitArgumentTypes(events, expectedVersion, metadata, callback, ExpectedVersionAny, CommitConditionClass) { if (!(events instanceof Array)) { events = [events]; @@ -17,6 +28,12 @@ function fixCommitArgumentTypes(events, expectedVersion, metadata, callback, Exp return { events, expectedVersion, metadata, callback }; } +/** + * Derive the stream name from an index name. + * + * @param {string} indexName Index file/index name. + * @returns {string} Corresponding stream name. + */ function parseStreamFromIndexName(indexName) { if (indexName === '_all') { return '_all'; @@ -27,6 +44,13 @@ function parseStreamFromIndexName(indexName) { return indexName; } +/** + * Support predicate/raw shorthand (`predicate=true`). + * + * @param {object|function|boolean|null} predicate Filter predicate or raw shorthand. + * @param {boolean} raw Raw flag from the call signature. + * @returns {{predicate: object|function|null, raw: boolean}} Normalized predicate/raw pair. + */ function normalizePredicateRaw(predicate, raw) { if (typeof predicate === 'boolean' && raw === false) { return { predicate: null, raw: predicate }; @@ -34,6 +58,14 @@ function normalizePredicateRaw(predicate, raw) { return { predicate, raw }; } +/** + * Support constructor overloads with optional `name`. + * + * @param {string|object} name Name or already the options object. + * @param {object} options Options object when `name` is provided. + * @param {string|undefined} [fallbackName=undefined] Fallback name when no explicit `name` is passed. + * @returns {{name: string|undefined, options: object}} Normalized name/options pair. + */ function normalizeNamedCtorArgs(name, options, fallbackName = undefined) { if (typeof name !== 'string') { return { name: fallbackName, options: name }; @@ -41,14 +73,35 @@ function normalizeNamedCtorArgs(name, options, fallbackName = undefined) { return { name, options }; } +/** + * Normalize negative revisions relative to stream length. + * + * @param {number} version Requested revision. + * @param {number} length Current stream length. + * @returns {number} Resolved revision. + */ function normalizeRevision(version, length) { return version < 0 ? version + length + 1 : version; } +/** + * Clamp and normalize maxRevision, including negative values. + * + * @param {number} length Current stream length. + * @param {number} maxRevision Requested max revision. + * @returns {number} Effective max revision in valid range. + */ function normalizeMaxRevision(length, maxRevision) { return Math.min(length, maxRevision < 0 ? length + maxRevision + 1 : maxRevision); } +/** + * Support the consumer overload where the first argument is a numeric start offset. + * + * @param {object|number} initialState Initial state or numeric start offset. + * @param {number} startFrom Start offset from the call signature. + * @returns {{initialState: object, startFrom: number}} Normalized consumer initialization values. + */ function normalizeConsumerStateArgs(initialState, startFrom) { if (typeof initialState === 'number') { return { initialState: {}, startFrom: initialState }; diff --git a/src/utils/deprecations.js b/src/utils/deprecations.js new file mode 100644 index 0000000..faed970 --- /dev/null +++ b/src/utils/deprecations.js @@ -0,0 +1,19 @@ +const emittedDeprecationWarnings = new Set(); + +/** + * Emit a deprecation warning only once per warning code. + * + * @param {string} code Unique warning code. + * @param {string} message Warning message. + * @returns {void} + */ +function emitDeprecationWarningOnce(code, message) { + if (emittedDeprecationWarnings.has(code)) { + return; + } + emittedDeprecationWarnings.add(code); + process.emitWarning(message, 'DeprecationWarning', code); +} + +export { emitDeprecationWarningOnce }; + diff --git a/src/utils/fsUtil.js b/src/utils/fsUtil.js index f84bde5..77d77c5 100644 --- a/src/utils/fsUtil.js +++ b/src/utils/fsUtil.js @@ -2,10 +2,56 @@ import fs from 'fs'; import path from 'path'; import { mkdirpSync } from 'mkdirp'; +const SAFE_RELATIVE_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_]*(?:[\/:@~+=\-#.][A-Za-z0-9_]+)*$/; + +// Best-effort cleanup for temporary files after interrupted/failed writes. +function safeUnlink(fileName) { + try { + fs.unlinkSync(fileName); + } catch (e) { + if (e.code !== 'ENOENT') { + throw e; + } + } +} + +// Prevent partially written persistence files from replacing the last valid state. +function writeFileAtomic(fileName, data, options = {}, onSuccess = null) { + const tmpFileName = options.tmpFileName || `${fileName}.tmp`; + const writeOptions = options.encoding ? { encoding: options.encoding } : undefined; + try { + fs.writeFileSync(tmpFileName, data, writeOptions); + fs.renameSync(tmpFileName, fileName); + if (typeof onSuccess === 'function') { + onSuccess(); + } + } catch (e) { + safeUnlink(tmpFileName); + throw e; + } + return fileName; +} + +function isSafeRelativeName(name) { + return typeof name === 'string' + && name !== '' + && SAFE_RELATIVE_NAME_PATTERN.test(name); +} + +function resolvePathWithinRoot(rootDirectory, relativePath) { + const root = path.resolve(rootDirectory); + const resolvedPath = path.resolve(root, relativePath); + const rootRelativePath = path.relative(root, resolvedPath); + if (rootRelativePath.startsWith('..') || path.isAbsolute(rootRelativePath)) { + throw new Error(`Invalid relative path "${relativePath}".`); + } + return resolvedPath; +} + /** * Ensure that the given directory exists. - * @param {string} dirName - * @return {boolean} true if the directory existed already + * @param {string} dirName Target directory. + * @returns {boolean} True when the directory already existed. */ function ensureDirectory(dirName) { if (!fs.existsSync(dirName)) { @@ -21,9 +67,10 @@ function ensureDirectory(dirName) { /** * Invoke `onEach` if `relativePath` matches `regexPattern`, passing the first capture group or the full match. * - * @param {string} relativePath - * @param {RegExp} regexPattern - * @param {function(string)} onEach + * @param {string} relativePath Relative file path. + * @param {RegExp} regexPattern Regex used for path matching. + * @param {function(string): void} onEach Callback invoked per match. + * @returns {void} */ function visitMatchingPath(relativePath, regexPattern, onEach) { const match = relativePath.match(regexPattern); @@ -35,11 +82,11 @@ function visitMatchingPath(relativePath, regexPattern, onEach) { /** * Classify `entries` into matching files (visited via `onEach`) and subdirectory names (returned). * - * @param {fs.Dirent[]} entries - * @param {string} relativePrefix - * @param {RegExp} regexPattern - * @param {function(string)} onEach - * @returns {string[]} names of subdirectory entries + * @param {fs.Dirent[]} entries Directory entries from one level. + * @param {string} relativePrefix Relative prefix for child entries. + * @param {RegExp} regexPattern Regex for file paths. + * @param {function(string): void} onEach Callback for matching files. + * @returns {string[]} Names of subdirectory entries. */ function classifyEntries(entries, relativePrefix, regexPattern, onEach) { const subdirs = []; @@ -56,12 +103,13 @@ function classifyEntries(entries, relativePrefix, regexPattern, onEach) { /** * Sequentially scan each name in `subdirs`, calling `done` when all are complete or on first error. * - * @param {string[]} subdirs - * @param {string} dir - * @param {string} relativePrefix - * @param {RegExp} regexPattern - * @param {function(string)} onEach - * @param {function(Error?)} done + * @param {string[]} subdirs Subdirectory names. + * @param {string} dir Absolute parent path. + * @param {string} relativePrefix Relative prefix used during recursion. + * @param {RegExp} regexPattern Regex for file paths. + * @param {function(string): void} onEach Callback for matching files. + * @param {function(Error=): void} done Completion callback. + * @returns {void} */ function scanSubdirs(subdirs, dir, relativePrefix, regexPattern, onEach, done) { let i = 0; @@ -79,12 +127,13 @@ function scanSubdirs(subdirs, dir, relativePrefix, regexPattern, onEach, done) { /** * Asynchronously scan one directory level, then recurse into subdirectories sequentially. * - * @param {string} dir - * @param {string} relativePrefix - * @param {boolean} isRoot - * @param {RegExp} regexPattern - * @param {function(string)} onEach - * @param {function(Error?)} done + * @param {string} dir Absolute directory path. + * @param {string} relativePrefix Relative prefix for match paths. + * @param {boolean} isRoot True for the initial call. + * @param {RegExp} regexPattern Regex for file paths. + * @param {function(string): void} onEach Callback for matching files. + * @param {function(Error=): void} done Completion callback. + * @returns {void} */ function scanDir(dir, relativePrefix, isRoot, regexPattern, onEach, done) { fs.readdir(dir, { withFileTypes: true }, (err, entries) => { @@ -112,6 +161,7 @@ function scanDir(dir, relativePrefix, isRoot, regexPattern, onEach, done) { * @param {RegExp} regexPattern The pattern to match relative file paths against. * @param {function(string)} onEach Called with the first capturing group (or full match) for each matching path. * @param {function(Error?)} onDone Called when the scan is complete, or with an error if one occurred. + * @returns {void} */ function scanForFiles(directory, regexPattern, onEach, onDone) { scanDir(directory, '', true, regexPattern, onEach, onDone); @@ -119,5 +169,9 @@ function scanForFiles(directory, regexPattern, onEach, onDone) { export { ensureDirectory, + safeUnlink, + writeFileAtomic, scanForFiles, + isSafeRelativeName, + resolvePathWithinRoot, }; diff --git a/src/utils/jsonUtil.js b/src/utils/jsonUtil.js index 2937297..77688cd 100644 --- a/src/utils/jsonUtil.js +++ b/src/utils/jsonUtil.js @@ -5,10 +5,16 @@ const BYTE_CLOSE_OBJECT = 0x7d; const BYTE_OPEN_ARRAY = 0x5b; const BYTE_CLOSE_ARRAY = 0x5d; const BYTE_COMMA = 0x2c; +const BYTE_SIGN_MINUS = 0x2d; +const BYTE_DECIMAL_SEP = 0x2e; /** * Advance past a JSON string whose opening `"` is at `i`. * Returns the position after the closing `"`, or -1 if the string is unterminated. + * + * @param {Buffer} buffer Source JSON buffer. + * @param {number} i Offset of the opening quote. + * @returns {number} Offset after the closing quote, or -1 if unterminated. */ function skipString(buffer, i) { let j = i + 1; @@ -28,33 +34,44 @@ function skipString(buffer, i) { /** * Check if a character byte is a valid JSON value delimiter (comma, closing brace, or closing bracket). - * @param {number} char - * @returns {boolean} + * @param {number} char Byte value to test. + * @returns {boolean} True when `char` is `,`, `}` or `]`. */ function isDelimiter(char) { return (char === BYTE_COMMA || char === BYTE_CLOSE_OBJECT || char === BYTE_CLOSE_ARRAY); } /** - * @param {number} char - * @returns {boolean} + * @param {number} char Byte value to test. + * @returns {boolean} True when `char` opens an object or array. */ function isOpeningBracket(char) { return char === BYTE_OPEN_OBJECT || char === BYTE_OPEN_ARRAY; } /** - * @param {number} char - * @returns {boolean} + * @param {number} char Byte value to test. + * @returns {boolean} True when `char` closes an object or array. */ function isClosingBracket(char) { return char === BYTE_CLOSE_OBJECT || char === BYTE_CLOSE_ARRAY; } +/** + * @param {number} char Byte value to test. + * @returns {boolean} True when `char` is `{`. + */ function isOpeningObject(char) { return char === BYTE_OPEN_OBJECT; } +/** + * @param {Buffer} buffer Source JSON buffer. + * @param {Buffer} pattern Pattern to find. + * @param {number} startOffset Search start offset. + * @param {number|undefined} lastMatchPosition Optional cached candidate position. + * @returns {number} Match position or -1. + */ function nextIndexOf(buffer, pattern, startOffset, lastMatchPosition) { if (lastMatchPosition === undefined || lastMatchPosition < startOffset) { return buffer.indexOf(pattern, startOffset); @@ -72,6 +89,13 @@ function nextIndexOf(buffer, pattern, startOffset, lastMatchPosition) { * For key patterns (`"key":`) pass `isKeyPattern=true` to skip that trailing delimiter check. * Returns -1 when no such position exists before the end of the buffer or when a closing brace * reduces depth below zero (the top-level object has ended). + * + * @param {Buffer} buffer Source JSON buffer. + * @param {Buffer} pattern Serialized key/value pattern. + * @param {number} [startOffset=0] Offset where scanning begins. + * @param {number|undefined} [matchPosition=undefined] Optional cached candidate position. + * @param {boolean} [isKeyPattern=false] Skip value-delimiter validation for key-only patterns. + * @returns {number} Match position at the same JSON depth, or -1. */ function indexOfSameLevel(buffer, pattern, startOffset = 0, matchPosition = undefined, isKeyPattern = false) { let depth = 0; @@ -123,10 +147,11 @@ function indexOfSameLevel(buffer, pattern, startOffset = 0, matchPosition = unde } /** - * Extract the end position (exclusive) of a JSON scalar value starting at `offset`. - * `offset` should point to the first character of the value ('"' for strings, digit/-/true/false/null for others). - * Returns the position after the value ends (past the closing quote for strings, past the last digit/char for others). - * Returns -1 if the buffer is malformed. + * Find the end of a scalar JSON value so operator matching can parse only the relevant slice. + * + * @param {Buffer} buffer Source JSON buffer. + * @param {number} offset Offset where the scalar starts. + * @returns {number} End offset (exclusive), or -1 for invalid start. */ function findJsonValueEnd(buffer, offset) { /* c8 ignore next 3 */ @@ -145,19 +170,133 @@ function findJsonValueEnd(buffer, offset) { } /** - * Convert a JSON scalar value buffer to a comparable JavaScript value for range operators. - * Supports strings, numbers, booleans, and null. + * Parse one scalar JSON slice only after a byte-level match has already narrowed the candidate. + * + * @param {Buffer} buffer Source JSON buffer. + * @param {number} startOffset Start offset (inclusive). + * @param {number} endOffset End offset (exclusive). + * @returns {string|number|boolean|null|undefined} Parsed scalar, or `undefined` on parse failure. */ function parseJsonValue(buffer, startOffset, endOffset) { - const valueStr = buffer.toString('utf8', startOffset, endOffset).trim(); - if (!valueStr) { - return undefined; - } try { + const valueStr = buffer.toString('utf8', startOffset, endOffset); return JSON.parse(valueStr); } catch { return undefined; } } -export { BYTE_OPEN_OBJECT, BYTE_CLOSE_OBJECT, isOpeningObject, indexOfSameLevel, findJsonValueEnd, parseJsonValue }; +/** + * @param {number} byte + * @returns {boolean} True when `byte` is an ASCII digit. + */ +function isAsciiDigit(byte) { + return byte >= 0x30 && byte <= 0x39; +} + +/** + * Compare a contiguous ASCII digit sequence in `buffer` against expected digits. + * For JSON.stringify()-normalized numbers this is enough for both integer and fraction parts. + * Returns only the ordering; when ordering === 0, callers can compute the consumed length as startOffset + expectedDigits.length. + * + * @param {Buffer} buffer + * @param {number} startOffset + * @param {string} expectedDigits + * @returns {-1|0|1} + */ +function compareDigits(buffer, startOffset, expectedDigits) { + let index = startOffset; + let position = 0; + let ordering = 0; + const expectedLength = expectedDigits.length; + + while (index < buffer.length && isAsciiDigit(buffer[index])) { + if (ordering === 0 && position < expectedLength) { + const expectedByte = expectedDigits.charCodeAt(position); + if (buffer[index] !== expectedByte) { + ordering = buffer[index] < expectedByte ? -1 : 1; + } + } + position++; + index++; + } + + if (position !== expectedLength) { + ordering = position > expectedLength ? 1 : -1; + } + + return ordering; +} + +/** + * Compare a compact JSON numeric token in `buffer` against a precompiled expected number, + * using one linear pass over the buffer slice and no `parseJsonValue` call. + * + * @param {Buffer} buffer + * @param {number} startOffset + * @param {{isNegative: boolean, integerPart: string, fractionPart: string}} expected + * @returns {-1|0|1|null} Ordering (`actual` vs `expected`) or null for invalid/non-numeric token. + */ +function compareNumeric(buffer, startOffset, expected) { + let index = startOffset; + const firstByte = buffer[index]; + if (firstByte !== BYTE_SIGN_MINUS && !isAsciiDigit(firstByte)) { + return null; + } + + const isNegative = firstByte === BYTE_SIGN_MINUS; + if (isNegative !== expected.isNegative) { + return isNegative ? -1 : 1; + } + + if (isNegative) { + index++; + } + + let result = compareDigits(buffer, index, expected.integerPart); + if (result === 0) { + index += expected.integerPart.length; + + const hasFraction = index < buffer.length && buffer[index] === BYTE_DECIMAL_SEP; + const expectedHasFraction = expected.fractionPart.length > 0; + if (hasFraction !== expectedHasFraction) { + result = hasFraction ? 1 : -1; + } else if (hasFraction) { + result = compareDigits(buffer, index + 1, expected.fractionPart); + } + } + return isNegative ? -result : result; +} + +/** + * Compare a matched key's scalar value against pre-serialized candidates without reparsing JSON. + * + * @param {Buffer} buffer Source JSON buffer. + * @param {number} valueStart Offset where the candidate scalar begins. + * @param {Buffer[]} patterns Pre-serialized scalar candidates. + * @returns {boolean} True when any candidate matches exactly and is delimiter-terminated. + */ +function matchesAnyValuePattern(buffer, valueStart, patterns) { + for (const pattern of patterns) { + const valueEnd = valueStart + pattern.length; + if (valueEnd > buffer.length) { + continue; + } + let matches = true; + for (let i = 0; i < pattern.length; i++) { + if (buffer[valueStart + i] !== pattern[i]) { + matches = false; + break; + } + } + if (!matches) { + continue; + } + if (isDelimiter(buffer[valueEnd])) { + return true; + } + } + return false; +} + +export { isOpeningObject, indexOfSameLevel, findJsonValueEnd, parseJsonValue, compareNumeric, matchesAnyValuePattern }; diff --git a/src/utils/metadataUtil.js b/src/utils/metadataUtil.js index 3316bf3..e12cf21 100644 --- a/src/utils/metadataUtil.js +++ b/src/utils/metadataUtil.js @@ -1,17 +1,37 @@ import crypto from 'crypto'; -import { assert, assertEqual } from './util.js'; -import { BYTE_OPEN_OBJECT, indexOfSameLevel, findJsonValueEnd, parseJsonValue } from './jsonUtil.js'; +import {assert, assertEqual} from './util.js'; +import { + indexOfSameLevel, + findJsonValueEnd, + parseJsonValue, + matchesAnyValuePattern, + isOpeningObject, + compareNumeric +} from './jsonUtil.js'; const compiledOperatorMatcherCache = new WeakMap(); +/** + * @param {any} value Value to classify. + * @returns {boolean} True when `value` is a non-null object. + */ +function isObject(value) { + return value !== null && typeof value === 'object'; +} + +/** + * @param {any} value Value to classify. + * @returns {boolean} True when `value` is a non-array object. + */ function isPlainObject(value) { return value !== null && typeof value === 'object' && !Array.isArray(value); } +/** + * @param {object} obj Candidate matcher object. + * @returns {boolean} True when all keys are operator keys (`$...`). + */ function isOperatorObject(obj) { - if (!isPlainObject(obj)) { - return false; - } const keys = Object.keys(obj); return keys.length > 0 && keys.every(key => key.startsWith('$')); } @@ -19,13 +39,17 @@ function isOperatorObject(obj) { /** * Dispatch between array (OR), operator object, nested object, and scalar equality matching * so callers don't need to know the shape of `matcherValue`. + * + * @param {any} documentValue Value from the document. + * @param {any} matcherValue Value from the matcher definition. + * @returns {boolean} True when both values match under matcher semantics. */ function propertyMatchesValue(documentValue, matcherValue) { - if (Array.isArray(matcherValue)) { - return matcherValue.includes(documentValue); - } else if (matcherValue && typeof matcherValue === 'object') { - const operatorChecks = getCompiledOperatorChecks(matcherValue); - if (operatorChecks) { + if (isObject(matcherValue)) { + if (Array.isArray(matcherValue)) { + return matcherValue.includes(documentValue); + } else if (isOperatorObject(matcherValue)) { + const operatorChecks = getCompiledOperatorChecks(matcherValue); return matchesCompiledOperators(documentValue, operatorChecks); } return matches(documentValue, matcherValue); @@ -36,6 +60,9 @@ function propertyMatchesValue(documentValue, matcherValue) { /** * Pre-compile an operator object into an array of comparison closures so the hot path avoids * repeated `Object.entries` + switch dispatch per matched document. + * + * @param {object} operatorObj Object containing operator/value pairs. + * @returns {Array} Compiled predicate checks in evaluation order. */ function buildOperatorChecks(operatorObj) { const checks = []; @@ -69,21 +96,25 @@ function buildOperatorChecks(operatorObj) { /** * Return cached compiled checks for `operatorObj`, compiling and caching on first access. * Using WeakMap keeps operator objects GC-eligible and avoids mutating user-supplied objects. + * + * @param {object} operatorObj Object containing operator/value pairs. + * @returns {Array} Cached or newly compiled operator checks. */ function getCompiledOperatorChecks(operatorObj) { const cachedChecks = compiledOperatorMatcherCache.get(operatorObj); if (cachedChecks) { return cachedChecks; } - /* c8 ignore next */ - if (!isOperatorObject(operatorObj)) { - return null; - } const checks = buildOperatorChecks(operatorObj); compiledOperatorMatcherCache.set(operatorObj, checks); return checks; } +/** + * @param {any} documentValue Parsed scalar value from the document. + * @param {Array} checks Compiled operator checks. + * @returns {boolean} True when all checks pass. + */ function matchesCompiledOperators(documentValue, checks) { if (documentValue === undefined) { return false; @@ -123,10 +154,10 @@ function buildMetadataHeader(magic, metadata) { * @returns {function(string)} A function that calculates the HMAC for a given string */ const createHmac = secret => string => { - const hmac = crypto.createHmac('sha256', secret); - hmac.update(string); - return hmac.digest('hex'); - }; + const hmac = crypto.createHmac('sha256', secret); + hmac.update(string); + return hmac.digest('hex'); +}; /** * @typedef {object|function(object):boolean} Matcher @@ -157,15 +188,15 @@ function matches(document, matcher) { * @returns {{matcher: string|object, hmac?: string}} */ function buildMetadataForMatcher(matcher, hmac) { - /* c8 ignore next */ - if (!matcher) { - return undefined; - } + /* c8 ignore next 2 */ + if (!matcher) { + return undefined; + } if (typeof matcher === 'object') { - return { matcher }; + return {matcher}; } const matcherString = matcher.toString(); - return { matcher: matcherString, hmac: hmac(matcherString) }; + return {matcher: matcherString, hmac: hmac(matcherString)}; } /** @@ -174,12 +205,12 @@ function buildMetadataForMatcher(matcher, hmac) { * @returns {Matcher} The matcher object or function. */ function buildMatcherFromMetadata(matcherMetadata, hmac) { - let matcher; - if (typeof matcherMetadata.matcher === 'object') { - matcher = matcherMetadata.matcher; - } else { - /* c8 ignore next */ - assert(matcherMetadata.hmac === hmac(matcherMetadata.matcher), 'Invalid HMAC for matcher.'); + let matcher; + if (typeof matcherMetadata.matcher === 'object') { + matcher = matcherMetadata.matcher; + } else { + /* c8 ignore next 1 */ + assert(matcherMetadata.hmac === hmac(matcherMetadata.matcher), 'Invalid HMAC for matcher.'); matcher = eval('(' + matcherMetadata.matcher + ')').bind({}); // jshint ignore:line } @@ -195,34 +226,35 @@ function buildMatcherFromMetadata(matcherMetadata, hmac) { */ function buildTypeMatcherFn(payloadPath) { const parts = payloadPath.split('.'); - return function(typeValue) { + return function (typeValue) { let obj = typeValue; for (let i = parts.length - 1; i >= 0; i--) { - obj = { [parts[i]]: obj }; + obj = {[parts[i]]: obj}; } - return { payload: obj }; + return {payload: obj}; }; } /** - * Builds a raw-buffer matcher. - * It expects the Buffer to contain compact stringified JSON - * and supports matcher objects with sub properties and multi-value matches (OR/any of). + * Compile an object matcher into a raw-buffer predicate so raw-mode reads can filter compact + * JSON without parsing every document first. * - * @param {object} matcher Object matcher. - * @returns {function(Buffer): boolean} + * @param {object} matcher Object matcher to compile. + * @param {{enableOperatorBufferMatcher?: boolean}} [options] Raw matcher build options. + * @returns {function(Buffer): boolean} Predicate over compact JSON buffers. */ -function buildRawBufferMatcher(matcher = {}) { +function buildRawBufferMatcher(matcher = {}, options = {}) { assert(isPlainObject(matcher), 'Matcher must be an object.', TypeError); + const enableOperatorBufferMatcher = options.enableOperatorBufferMatcher !== false; - const root = buildMatcherTree(matcher); + const root = buildMatcherTree(matcher, enableOperatorBufferMatcher); /* c8 ignore next 3 */ if (root.children.length === 0) { return () => true; } return function matchesRawBuffer(buffer) { - if (buffer[0] !== BYTE_OPEN_OBJECT) { + if (!isOpeningObject(buffer[0])) { return false; } if (!preCheck(buffer, 1, root)) { @@ -233,127 +265,131 @@ function buildRawBufferMatcher(matcher = {}) { } /** - * Optimization pass: check that every required byte pattern is present anywhere in the buffer - * before spending the more expensive per-depth scan in `matchesNode`. - */ -function preCheck(buffer, startOffset, node) { - for (const child of node.children) { - if (child.valuePatterns && !child.valuePatterns.some((pattern, i) => { - child.valueMatches[i] = buffer.indexOf(pattern, startOffset); - return child.valueMatches[i] !== -1; - })) { - return false; - } - if (child.operators) { - const keyPos = buffer.indexOf(child.operatorKeyPrefix, startOffset); - if (keyPos === -1) { - return false; - } - child.operatorMatch = keyPos; - } - if (child.objectPattern) { - const objectMatch = buffer.indexOf(child.objectPattern, startOffset); - if (objectMatch === -1) { - return false; - } - child.objMatch = objectMatch; - if (!preCheck(buffer, objectMatch, child.node)) { - return false; - } - } - } - return true; -} - -/** - * Pre-compile a plain object matcher into a tree of byte patterns so `matchesNode` can scan - * raw JSON buffers without deserializing them. + * Compile a matcher object into a tree whose children each carry one primary byte pattern plus + * optional follow-up checks for nested objects, operators, or multi-value scalars. + * Fast scalar-equality children are placed first so preCheck and matchesNode short-circuit early. + * + * @param {object} matcher Matcher object for this tree level. + * @returns {{children: Array}} Compiled child descriptors for this level. */ function buildMatcherTree(matcher) { - const node = { children: [] }; + const fast = []; + const slow = []; for (const [key, value] of Object.entries(matcher)) { - node.children.push(buildMatcherTreeChild(key, value)); + const child = buildMatcherTreeChild(key, value); + // A child with only one byte pattern and no follow-up matcher cannot be outperformed by any extra matcher logic. + (!child.matches ? fast : slow).push(child); } - return node; + return {children: [...fast, ...slow]}; } /** - * Decide which matching strategy to use for a key/value pair and build the corresponding - * child node with pre-computed byte patterns or compiled operator checks. + * Normalize one matcher property into the cheapest raw-buffer strategy for that value shape. + * Children that compile to a plain byte-equality pattern leave `matches` as null. + * + * @param {string} key Property name at this matcher level. + * @param {any} value Matcher value for `key`. + * @returns {{pattern: Buffer, isKeyPattern: boolean, matches: ((function(Buffer, number): boolean)|null), node: ({children: Array}|null), lastMatch: number}} Compiled descriptor consumed by preCheck/matchesNode. */ function buildMatcherTreeChild(key, value) { const keyPrefix = Buffer.from(`${JSON.stringify(key)}:`, 'utf8'); - const operatorChecks = isPlainObject(value) - ? getCompiledOperatorChecks(value) - : null; const child = { - objectPattern: null, - valuePatterns: null, - operators: null, - operatorChecks: null, - operatorKeyPrefix: null, - operatorMatch: null, + pattern: null, + isKeyPattern: false, + matches: null, node: null, - objMatch: null, - valueMatches: [] + lastMatch: -1 }; - if (Array.isArray(value)) { - if (value.some(item => item && typeof item === 'object')) { - throw new TypeError('Array matcher values must be scalars.'); - } - child.valuePatterns = value.map(item => buildValuePattern(keyPrefix, item)); - return child; - } else if (operatorChecks) { - // A lone $eq is semantically identical to a scalar equality check — fold it into a - // value pattern at compile time so the buffer scan takes the same fast path as { key: value }. - if ('$eq' in value && Object.keys(value).length === 1) { - child.valuePatterns = [buildValuePattern(keyPrefix, value['$eq'])]; - return child; + if (isObject(value)) { + if (Array.isArray(value)) { + assert(!value.some(isObject), 'Array matcher values must be scalars.', TypeError); + if (value.length === 1) { + child.pattern = buildKeyValuePattern(keyPrefix, value[0]); + } else { + child.isKeyPattern = true; + child.pattern = keyPrefix; + const valuePatterns = value.map(item => Buffer.from(JSON.stringify(item), 'utf8')); + child.matches = (buffer, startOffset) => matchesAnyValuePattern(buffer, startOffset, valuePatterns); + } + } else if ('$eq' in value && Object.keys(value).length === 1) { + // A lone $eq is semantically identical to a scalar equality check — fold it into a + // value pattern at compile time so the buffer scan takes the same fast path as { key: value }. + child.pattern = buildKeyValuePattern(keyPrefix, value['$eq']); + } else if ('$ne' in value && Object.keys(value).length === 1) { + // A lone $ne is the logical negation of $eq: confirm the key exists at this level, + // then reject only when the value byte-matches the excluded pattern. + child.isKeyPattern = true; + child.pattern = keyPrefix; + const nePattern = [Buffer.from(JSON.stringify(value['$ne']), 'utf8')]; + child.matches = (buffer, valueStart) => !matchesAnyValuePattern(buffer, valueStart, nePattern); + } else if (isOperatorObject(value)) { + child.isKeyPattern = true; + child.pattern = keyPrefix; + child.matches = buildOperatorBufferMatcher(value); + } else { + child.pattern = Buffer.concat([keyPrefix, Buffer.from('{', 'utf8')]); + child.node = buildMatcherTree(value); + child.matches = (buffer, startOffset) => matchesNode(buffer, startOffset, child.node); } - child.operators = value; - child.operatorChecks = operatorChecks; - child.operatorKeyPrefix = keyPrefix; - return child; - } else if (value && typeof value === 'object') { - child.objectPattern = Buffer.concat([keyPrefix, Buffer.from('{', 'utf8')]); - child.node = buildMatcherTree(value); - return child; + } else { + child.pattern = buildKeyValuePattern(keyPrefix, value); } - child.valuePatterns = [buildValuePattern(keyPrefix, value)]; return child; } -function buildValuePattern(keyPrefix, value) { +/** + * @param {Buffer} keyPrefix Serialized key prefix (`"key":`). + * @param {any} value Scalar value to append. + * @returns {Buffer} Full serialized `"key":value` pattern. + */ +function buildKeyValuePattern(keyPrefix, value) { return Buffer.concat([keyPrefix, Buffer.from(JSON.stringify(value), 'utf8')]); } /** - * Verify that each required byte pattern in the tree is present at the correct JSON nesting - * depth so values inside nested objects don't satisfy a top-level match requirement. + * Cheap pass: confirm each child's primary pattern exists somewhere and cache that position as a + * hint for the depth-aware pass that follows. + * + * @param {Buffer} buffer Compact JSON document buffer. + * @param {number} startOffset Start offset within `buffer`. + * @param {{children: Array}} node Compiled matcher node for this level. + * @returns {boolean} True when every child pattern exists somewhere from `startOffset`. */ -function matchesNode(buffer, startOffset, node) { +function preCheck(buffer, startOffset, node) { for (const child of node.children) { - if (child.valuePatterns && !child.valuePatterns.some((pattern, i) => { - return indexOfSameLevel(buffer, pattern, startOffset, child.valueMatches[i]) !== -1; - })) { + child.lastMatch = buffer.indexOf(child.pattern, startOffset); + if (child.lastMatch === -1) { return false; } + if (child.node && !preCheck(buffer, child.lastMatch, child.node)) { + return false; + } + } + return true; +} - if (child.operators && !matchesOperatorInBuffer(buffer, startOffset, child)) { +/** + * Confirm that each prechecked child really matches at the requested JSON level and then run the + * optional value-specific follow-up checks from the compiled tree. + * + * @param {Buffer} buffer Compact JSON document buffer. + * @param {number} startOffset Start offset within `buffer`. + * @param {{children: Array}} node Compiled matcher node for this level. + * @returns {boolean} True when all children match at the requested JSON level. + */ +function matchesNode(buffer, startOffset, node) { + for (const child of node.children) { + const matchPosition = indexOfSameLevel(buffer, child.pattern, startOffset, child.lastMatch, child.isKeyPattern); + if (matchPosition === -1) { return false; } - if (child.node) { - const objectIndex = indexOfSameLevel(buffer, child.objectPattern, startOffset, child.objMatch); - if (objectIndex === -1) { - return false; - } - if (!matchesNode(buffer, objectIndex + child.objectPattern.length, child.node)) { - return false; - } + const valueStart = matchPosition + child.pattern.length; + if (child.matches && !child.matches(buffer, valueStart)) { + return false; } } @@ -361,32 +397,118 @@ function matchesNode(buffer, startOffset, node) { } /** - * Check if a value at the given key-offset matches the specified operators. - * Finds the key pattern, extracts the value, parses it, and applies operators. + * Parse the scalar at a matched key position only when operator objects require a real JavaScript + * comparison instead of a pure byte-pattern check. + * + * @param {Buffer} buffer Compact JSON document buffer. + * @param {number} startOffset Offset of the scalar value to parse. + * @param {Array} operatorChecks Compiled operator checks. + * @returns {boolean} True when the parsed scalar satisfies all operators. */ -function matchesOperatorInBuffer(buffer, startOffset, child) { - const keyPos = indexOfSameLevel(buffer, child.operatorKeyPrefix, startOffset, child.operatorMatch, true); - if (keyPos === -1) { - return false; - } +function matchesOperatorInBuffer(buffer, startOffset, operatorChecks) { + const valueStart = startOffset; + const valueEnd = findJsonValueEnd(buffer, valueStart); + /* c8 ignore next 2 */ + if (valueEnd === -1 || valueEnd <= valueStart) { + return false; + } - const valueStart = keyPos + child.operatorKeyPrefix.length; - if (valueStart >= buffer.length) { - return false; - } + const parsedValue = parseJsonValue(buffer, valueStart, valueEnd); - const valueEnd = findJsonValueEnd(buffer, valueStart); - if (valueEnd === -1 || valueEnd <= valueStart) { - return false; + return matchesCompiledOperators(parsedValue, operatorChecks); +} + +/** + * Map pre-computed ordering information to operator semantics. + * ordering: -1 (actual < expected), 0 (equal), 1 (actual > expected) + * + * @param {string} operator + * @param {-1|0|1} ordering + * @returns {boolean} + */ +function matchesOrdering(operator, ordering) { + switch (operator) { + case '$gt': + return ordering === 1; + case '$gte': + return ordering >= 0; + case '$lt': + return ordering === -1; + case '$lte': + return ordering <= 0; + case '$eq': + return ordering === 0; + case '$ne': + return ordering !== 0; + /* c8 ignore next 1 */ + default: + return false; + } + } + +/** + * @param {Array<[string, any]>} entries + * @returns {Array<{operator: string, expectedNumeric: {isNegative: boolean, integerPart: string, fractionPart: string}}>|null} + */ +function buildNumericOperatorComparisons(entries) { + const comparisons = []; + for (const [operator, expectedValue] of entries) { + if (typeof expectedValue !== 'number') { + return null; + } + const expectedStr = JSON.stringify(expectedValue); + const expectedIsNegative = expectedStr[0] === '-'; + const intStart = expectedIsNegative ? 1 : 0; + const [expectedIntegerPart, expectedFractionPart = ''] = expectedStr.substring(intStart).split('.'); + comparisons.push({ + operator, + expectedNumeric: { + isNegative: expectedIsNegative, + integerPart: expectedIntegerPart, + fractionPart: expectedFractionPart + } + }); } + return comparisons; +} - const parsedValue = parseJsonValue(buffer, valueStart, valueEnd); +/** + * Build a specialized buffer-based operator comparator by pre-compiling operator-specific + * byte shortcuts at matcher build time. This avoids runtime dispatch and enables aggressive + * short-circuit evaluation for sign mismatches and digit-count differences. + * + * Assumes no scientific notation in the JSON buffer. + * + * @param {object} operatorObj Operator object (e.g., { $gt: 100 }). + * @returns {function(Buffer, number): boolean} Predicate that reads the buffer at given offset. + */ +function buildOperatorBufferMatcher(operatorObj) { + const entries = Object.entries(operatorObj); + const numericComparisons = buildNumericOperatorComparisons(entries); + if (numericComparisons) { + return (buffer, startOffset) => { + for (const comparison of numericComparisons) { + const ordering = compareNumeric(buffer, startOffset, comparison.expectedNumeric); + /* c8 ignore next 2 */ + if (ordering === null) { + return false; + } + if (!matchesOrdering(comparison.operator, ordering)) { + return false; + } + } + return true; + }; + } - return matchesCompiledOperators(parsedValue, child.operatorChecks); + // Non-numeric expected value: use generic operator checks. + const operatorChecks = getCompiledOperatorChecks(operatorObj); + return (buffer, startOffset) => matchesOperatorInBuffer(buffer, startOffset, operatorChecks); } export { createHmac, + isPlainObject, matches, buildMetadataHeader, buildMetadataForMatcher, diff --git a/src/utils/util.js b/src/utils/util.js index c39a79a..1b78737 100644 --- a/src/utils/util.js +++ b/src/utils/util.js @@ -1,9 +1,10 @@ /** * Assert that actual and expected match or throw an Error with the given message appended by information about expected and actual value. * - * @param {*} actual - * @param {*} expected - * @param {string} message + * @param {*} actual Actual value. + * @param {*} expected Expected value. + * @param {string} message Error message prefix. + * @returns {void} */ function assertEqual(actual, expected, message) { if (actual !== expected) { @@ -14,9 +15,10 @@ function assertEqual(actual, expected, message) { /** * Assert that the condition holds and if not, throw an error with the given message. * - * @param {boolean} condition - * @param {string} message - * @param {typeof Error} ErrorType + * @param {boolean} condition Condition to verify. + * @param {string} message Error message when the condition fails. + * @param {typeof Error} ErrorType Error class to throw. + * @returns {void} */ function assert(condition, message, ErrorType = Error) { if (!condition) { @@ -27,9 +29,9 @@ function assert(condition, message, ErrorType = Error) { /** * Return the amount required to align value to the given alignment. * It calculates the difference of the alignment and the modulo of value by alignment. - * @param {number} value - * @param {number} alignment - * @returns {number} + * @param {number} value Source value. + * @param {number} alignment Target alignment. + * @returns {number} Additional offset needed to reach the next aligned value. */ function alignTo(value, alignment) { return (alignment - (value % alignment)) % alignment; @@ -38,8 +40,8 @@ function alignTo(value, alignment) { /** * Method for hashing a string (e.g. a partition name) to a 32-bit unsigned integer. * - * @param {string} str - * @returns {number} + * @param {string} str Input string. + * @returns {number} 32-bit unsigned hash value. */ function hash(str) { /* c8 ignore next 3 */ @@ -115,8 +117,9 @@ function wrapAndCheck(index, length) { /** * Iterate an array-like list in forward or reverse order. * - * @param {Iterable} entries - * @param {boolean} forwards + * @param {Iterable|ArrayLike} entries Entries to iterate. + * @param {boolean} forwards Iteration direction. + * @returns {Generator} */ function* iterate(entries, forwards) { if (forwards) { @@ -141,7 +144,7 @@ function* iterate(entries, forwards) { * @param {boolean} [ascending=true] When true, yields items in ascending key order (min-merge). * When false, yields in descending key order (max-merge). * @param {function(*): *} [visit] Optional extractor for the yielded value. Defaults to identity. - * @returns {Generator<*>} + * @returns {Generator<*>} Merged sequence in key order. */ function *kWayMerge(iterables, getSortKey, ascending = true, visit = v => v) { const states = []; @@ -175,9 +178,9 @@ function *kWayMerge(iterables, getSortKey, ascending = true, visit = v => v) { * Read a scalar value at a dot-notation path from an object. * Returns `undefined` if any path segment is absent or an intermediate value is not an object. * - * @param {object} obj + * @param {object} obj Source object. * @param {string} dotPath Dot-separated property path, e.g. `'payload.type'`. - * @returns {*} + * @returns {*} Value at the path or `undefined`. */ function getPropertyAtPath(obj, dotPath) { let current = obj; diff --git a/test/Consumer.spec.js b/test/Consumer.spec.js index a5de02b..f8d3652 100644 --- a/test/Consumer.spec.js +++ b/test/Consumer.spec.js @@ -3,6 +3,8 @@ import fs from 'fs-extra'; import fsNative from 'fs'; import Storage from '../src/Storage.js'; import Consumer from '../src/Consumer.js'; +import Projection, { CompositeProjection } from '../src/Projection.js'; +import { createHmac } from '../src/utils/metadataUtil.js'; import { fileURLToPath } from 'url'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -39,6 +41,14 @@ describe('Consumer', function() { expect(() => new Consumer(storage, 'foobar')).to.throwError(/identifier/); }); + it('throws for invalid consumer index names', function() { + expect(() => new Consumer(storage, '../foobar', 'consumer1')).to.throwError(/Invalid index name/); + }); + + it('throws for invalid consumer identifiers', function() { + expect(() => new Consumer(storage, 'foobar', '../consumer1')).to.throwError(/Invalid identifier/); + }); + it('creates consumer directory if not existing', function() { consumer = new Consumer(storage, 'foobar', 'consumer1'); expect(fs.existsSync(dataDirectory + '/consumers')).to.be(true); @@ -345,26 +355,28 @@ describe('Consumer', function() { }, 15); }); - it('swallows persistence write errors and removes temp files', function(done) { + it('rethrows persistence write errors and removes temp files', function() { consumer = new Consumer(storage, 'foobar', 'consumer-1'); consumer.position = 1; consumer.state = Object.freeze({ foo: 1 }); const tmpFile = consumer.fileName + '.1'; const originalWriteFileSync = fsNative.writeFileSync; + const originalSetImmediate = global.setImmediate; + global.setImmediate = (fn) => { + fn(); + return null; + }; fsNative.writeFileSync = () => { const error = new Error('disk full'); error.code = 'ENOSPC'; throw error; }; - consumer.persist(); - - setTimeout(() => { - fsNative.writeFileSync = originalWriteFileSync; - expect(fs.existsSync(tmpFile)).to.be(false); - done(); - }, 15); + expect(() => consumer.persist()).to.throwError(/disk full/); + fsNative.writeFileSync = originalWriteFileSync; + global.setImmediate = originalSetImmediate; + expect(fs.existsSync(tmpFile)).to.be(false); }); it('restores state after reopening', function(done) { @@ -519,6 +531,132 @@ describe('Consumer', function() { }); }); + it('can attach projections from a reducer function', function(done) { + consumer = new Consumer(storage, 'foobar', 'consumer-projection', { count: 0 }); + new Projection('consumer-projection', { + initialState: { count: 0 }, + handlers: (state, event) => ({ ...state, count: state.count + event.id }) + }, { + hmac: createHmac('test-secret') + }).subscribe(consumer); + consumer.on('caught-up', () => { + expect(consumer.state.count).to.be(6); + done(); + }); + + storage.write({ type: 'Foobar', id: 1 }); + storage.write({ type: 'Foobar', id: 2 }); + storage.write({ type: 'Foobar', id: 3 }); + }); + + it('consumer.project persists a projection when attaching', function(done) { + consumer = new Consumer(storage, 'foobar', 'consumer-project-method', { count: 0 }); + const projection = new Projection('consumer-project-method', { + initialState: { count: 0 }, + handlers: { + Foobar: (state, event) => ({ ...state, count: state.count + event.id }) + } + }, { hmac: createHmac('test-secret') }); + + consumer.project(projection); + expect(fs.existsSync(`${consumer.fileName}.projection`)).to.be(true); + consumer.on('caught-up', () => { + expect(consumer.state.count).to.be(6); + done(); + }); + storage.write({ type: 'Foobar', id: 1 }); + storage.write({ type: 'Foobar', id: 2 }); + storage.write({ type: 'Foobar', id: 3 }); + }); + + it('can attach and restore projections from event-type reducer maps', function(done) { + consumer = new Consumer(storage, 'foobar', 'consumer-projection-map', { count: 0 }); + const projection = new Projection('consumer-projection-map', { + initialState: { count: 0 }, + handlers: { + Foobar: (state, event) => ({ ...state, count: state.count + event.id }), + Bazinga: (state) => state + } + }, { hmac: createHmac('test-secret') }); + projection.subscribe(consumer); + + consumer.on('caught-up', () => { + consumer.stop(); + consumer = new Consumer(storage, 'foobar', 'consumer-projection-map', {}); + Projection.restoreFromFile(`${consumer.fileName}.projection`, { + hmac: createHmac('test-secret') + }).subscribe(consumer); + consumer.on('progress', () => { + if (consumer.state.count === 10) { + done(); + } + }); + storage.write({ type: 'Foobar', id: 4 }); + }); + + storage.write({ type: 'Foobar', id: 1 }); + storage.write({ type: 'Foobar', id: 2 }); + storage.write({ type: 'Foobar', id: 3 }); + }); + + it('throws if function projection is restored without trusted hmac', function() { + consumer = new Consumer(storage, 'foobar', 'consumer-projection-hmac'); + new Projection('consumer-projection-hmac', { + initialState: {}, + handlers: (state, event) => ({ ...state, lastId: event.id }) + }, { + hmac: createHmac('test-secret') + }).subscribe(consumer); + expect(() => Projection.restoreFromFile(`${consumer.fileName}.projection`, { + hmac: createHmac('wrong-secret') + })).to.throwError(/Invalid HMAC/); + }); + + it('can attach a projection instance and restore it on reopen', function(done) { + consumer = new Consumer(storage, 'foobar', 'consumer-projection-instance', { count: 0 }); + const projection = new Projection('consumer-projection-instance', { + initialState: { count: 0 }, + handlers: { + Foobar: (state, event) => ({ ...state, count: state.count + event.id }) + } + }, { + hmac: createHmac('test-secret') + }); + projection.subscribe(consumer); + consumer.on('caught-up', () => { + expect(consumer.state.count).to.be(6); + consumer.stop(); + consumer = new Consumer(storage, 'foobar', 'consumer-projection-instance', {}); + Projection.restoreFromFile(`${consumer.fileName}.projection`, { + hmac: createHmac('test-secret') + }).subscribe(consumer); + consumer.on('progress', () => { + if (consumer.state.count === 10) { + done(); + } + }); + storage.write({ type: 'Foobar', id: 4 }); + }); + storage.write({ type: 'Foobar', id: 1 }); + storage.write({ type: 'Foobar', id: 2 }); + storage.write({ type: 'Foobar', id: 3 }); + }); + + it('supports composite projections', function() { + const projection = new CompositeProjection('overview', { + count: { + initialState: 0, + handlers: { Foobar: (state) => state + 1 } + }, + last: { + initialState: null, + handlers: { Foobar: (state, event) => event.id || state } + } + }); + projection.handle([{ type: 'Foobar', id: 1 }, { type: 'Foobar', id: 2 }]); + expect(projection.state).to.eql({ count: 2, last: 2 }); + }); + it('can build consistency guards (aggregates)', function(done) { const guard = new Consumer(storage, 'foobar', 'unique-bar-guard'); guard.apply = function(event) { diff --git a/test/EventStore.spec.js b/test/EventStore.spec.js index 970f47e..7c15d4b 100644 --- a/test/EventStore.spec.js +++ b/test/EventStore.spec.js @@ -4,6 +4,8 @@ import fsSync from 'fs'; import path from 'path'; import EventStore, { ExpectedVersion, OptimisticConcurrencyError, CommitCondition, LOCK_RECLAIM } from '../src/EventStore.js'; import Consumer from '../src/Consumer.js'; +import Projection from '../src/Projection.js'; +import { createHmac } from '../src/utils/metadataUtil.js'; import { fileURLToPath } from 'url'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -1378,6 +1380,74 @@ describe('EventStore', function() { eventstore.commit('bar', { foo: 'baz', id: 2 }); }); + it('restores a projected consumer with eventStore typeAccessor defaults', function(done) { + eventstore = new EventStore({ + storageDirectory, + typeAccessor: 'type', + storageConfig: { + hmacSecret: 'test-secret' + } + }); + eventstore.createEventStream('user-stream', (event) => event.stream === 'user-stream'); + + const consumer = eventstore.getConsumer('user-stream', 'user-counter', { count: 0 }); + new Projection('user-counter', { + initialState: { count: 0 }, + handlers: { + UserCreated: (state) => ({ ...state, count: state.count + 1 }) + } + }, { + hmac: eventstore.storage.hmac + }).subscribe(consumer); + eventstore.commit('user-stream', [{ type: 'UserCreated', id: 1 }]); + eventstore.commit('user-stream', [{ type: 'UserCreated', id: 2 }]); + + consumer.on('progress', () => { + if (consumer.state.count !== 2) { + return; + } + eventstore.close(); + eventstore = new EventStore({ + storageDirectory, + typeAccessor: 'type', + storageConfig: { + hmacSecret: 'test-secret' + } + }); + const reopened = eventstore.getConsumer('user-stream', 'user-counter', { count: 0 }); + reopened.on('progress', () => { + if (reopened.state.count === 3) { + done(); + } + }); + eventstore.commit('user-stream', [{ type: 'UserCreated', id: 3 }]); + }); + }); + }); + + describe('getProjection', function() { + + it('creates and restores persisted projections with EventStore defaults', function() { + eventstore = new EventStore({ + storageDirectory, + typeAccessor: 'type', + storageConfig: { + hmacSecret: 'test-secret' + } + }); + const projection = eventstore.getProjection('user-count', { + UserCreated: (state) => state + 1 + }, 0); + projection.persist(); + + const restored = eventstore.getProjection('user-count'); + const state = restored.handle([ + { payload: { type: 'UserCreated' } }, + { payload: { type: 'UserCreated' } } + ]); + expect(state).to.be(2); + }); + it('rebinds an existing identifier to a different stream when requested', function(done) { eventstore = new EventStore({ storageDirectory @@ -1402,6 +1472,74 @@ describe('EventStore', function() { eventstore.commit('foo', { foo: 'bar', id: 1 }); eventstore.commit('bar', { foo: 'baz', id: 2 }); }); + + it('restores a projected consumer with eventStore typeAccessor defaults', function(done) { + eventstore = new EventStore({ + storageDirectory, + typeAccessor: 'type', + storageConfig: { + hmacSecret: 'test-secret' + } + }); + eventstore.createEventStream('user-stream', (event) => event.stream === 'user-stream'); + + const consumer = eventstore.getConsumer('user-stream', 'user-counter', { count: 0 }); + new Projection('user-counter', { + initialState: { count: 0 }, + handlers: { + UserCreated: (state) => ({ ...state, count: state.count + 1 }) + } + }, { + hmac: eventstore.storage.hmac + }).subscribe(consumer); + eventstore.commit('user-stream', [{ type: 'UserCreated', id: 1 }]); + eventstore.commit('user-stream', [{ type: 'UserCreated', id: 2 }]); + + consumer.on('progress', () => { + if (consumer.state.count !== 2) { + return; + } + eventstore.close(); + eventstore = new EventStore({ + storageDirectory, + typeAccessor: 'type', + storageConfig: { + hmacSecret: 'test-secret' + } + }); + const reopened = eventstore.getConsumer('user-stream', 'user-counter', { count: 0 }); + reopened.on('progress', () => { + if (reopened.state.count === 3) { + done(); + } + }); + eventstore.commit('user-stream', [{ type: 'UserCreated', id: 3 }]); + }); + }); + }); + + describe('getProjection', function() { + + it('creates and restores persisted projections with EventStore defaults', function() { + eventstore = new EventStore({ + storageDirectory, + typeAccessor: 'type', + storageConfig: { + hmacSecret: 'test-secret' + } + }); + const projection = eventstore.getProjection('user-count', { + UserCreated: (state) => state + 1 + }, 0); + projection.persist(); + + const restored = eventstore.getProjection('user-count'); + const state = restored.handle([ + { payload: { type: 'UserCreated' } }, + { payload: { type: 'UserCreated' } } + ]); + expect(state).to.be(2); + }); }); describe('scanConsumers', function() { @@ -2210,4 +2348,75 @@ describe('EventStore', function() { }); + describe('makeReadOnly()', function() { + + it('switches to read-only mode and calls the callback', function(done) { + eventstore = new EventStore({ storageDirectory }); + eventstore.on('ready', () => { + eventstore.commit('foo', [{ x: 1 }], () => { + eventstore.makeReadOnly(() => { + expect(eventstore.storage.constructor.name).to.be('ReadOnlyStorage'); + expect(() => eventstore.commit('foo', [{ x: 2 }])).to.throwError(/read-only/); + done(); + }); + }); + }); + }); + + it('can switch to read-only outside a commit callback and keeps already written events', function(done) { + eventstore = new EventStore({ storageDirectory }); + eventstore.on('ready', () => { + eventstore.commit('deployment-stream', [{ step: 'before-switch' }]); + eventstore.makeReadOnly(() => { + expect(eventstore.storage.constructor.name).to.be('ReadOnlyStorage'); + + const stream = eventstore.getEventStream('deployment-stream'); + expect(stream).to.not.be(false); + expect(stream.events.length).to.be(1); + expect(stream.events[0].step).to.be('before-switch'); + + expect(() => eventstore.commit('deployment-stream', [{ step: 'after-switch' }])).to.throwError(/read-only/); + done(); + }); + }); + }); + + it('preserves existing events after switching to read-only', function(done) { + eventstore = new EventStore({ storageDirectory }); + eventstore.on('ready', () => { + eventstore.commit('bar', [{ y: 1 }, { y: 2 }], () => { + eventstore.makeReadOnly(() => { + const stream = eventstore.getEventStream('bar'); + const events = stream.events; + expect(events.length).to.be(2); + expect(events[0].y).to.be(1); + expect(events[1].y).to.be(2); + done(); + }); + }); + }); + }); + + it('prevents new commits after switching to read-only', function(done) { + eventstore = new EventStore({ storageDirectory }); + eventstore.on('ready', () => { + eventstore.makeReadOnly(() => { + expect(() => eventstore.commit('baz', [{ z: 1 }])).to.throwError(/read-only/); + done(); + }); + }); + }); + + it('prevents createEventStream after switching to read-only', function(done) { + eventstore = new EventStore({ storageDirectory }); + eventstore.on('ready', () => { + eventstore.makeReadOnly(() => { + expect(() => eventstore.createEventStream('new-stream', { stream: 'new-stream' })).to.throwError(/read-only/); + done(); + }); + }); + }); + + }); + }); diff --git a/test/EventStream.spec.js b/test/EventStream.spec.js index cb263d0..e58effb 100644 --- a/test/EventStream.spec.js +++ b/test/EventStream.spec.js @@ -248,6 +248,27 @@ describe('EventStream', function() { expect(stream.events).to.eql(events); }); + it('delegates to Readable.filter() when called with options', async function() { + const filteredReadable = stream.filter((payload) => payload === 'foo', { concurrency: 1 }); + expect(filteredReadable).not.to.be(stream); + + const values = []; + for await (const value of filteredReadable) { + values.push(value); + } + expect(values).to.eql(['foo']); + }); + + }); + + describe('where()', function() { + + it('applies matcher semantics and resets the iterator', function() { + const filtered = stream.where({ payload: 'foo' }); + expect(filtered).to.be(stream); + expect(stream.events).to.eql(['foo']); + }); + }); describe('object matcher (non-raw mode)', function() { diff --git a/test/Exports.spec.js b/test/Exports.spec.js new file mode 100644 index 0000000..890e127 --- /dev/null +++ b/test/Exports.spec.js @@ -0,0 +1,11 @@ +import expect from 'expect.js'; +import EventStore, { VERSION } from '../index.js'; + +describe('Public exports', function() { + it('exports VERSION and mirrors it on EventStore.VERSION', function() { + expect(VERSION).to.be.a('string'); + expect(VERSION.length > 0).to.be(true); + expect(EventStore.VERSION).to.be(VERSION); + }); +}); + diff --git a/test/jsonUtil.spec.js b/test/jsonUtil.spec.js index bcff6b2..4fcd8c9 100644 --- a/test/jsonUtil.spec.js +++ b/test/jsonUtil.spec.js @@ -1,5 +1,5 @@ import expect from 'expect.js'; -import { indexOfSameLevel, findJsonValueEnd, parseJsonValue } from '../src/utils/jsonUtil.js'; +import { indexOfSameLevel, findJsonValueEnd, parseJsonValue, matchesAnyValuePattern } from '../src/utils/jsonUtil.js'; describe('jsonUtil', function() { @@ -84,6 +84,34 @@ describe('jsonUtil', function() { }); + describe('matchesAnyValuePattern', function() { + + it('matches any exact scalar candidate at the given offset', function() { + const buffer = Buffer.from('{"type":"Foo","id":1}', 'utf8'); + const valueStart = buffer.indexOf(Buffer.from('"Foo"', 'utf8')); + const patterns = [Buffer.from('"Bar"', 'utf8'), Buffer.from('"Foo"', 'utf8')]; + + expect(matchesAnyValuePattern(buffer, valueStart, patterns)).to.be(true); + }); + + it('rejects prefix matches like 5 for an actual value 50', function() { + const buffer = Buffer.from('{"amount":50}', 'utf8'); + const valueStart = buffer.indexOf(Buffer.from('50', 'utf8')); + const patterns = [Buffer.from('5', 'utf8')]; + + expect(matchesAnyValuePattern(buffer, valueStart, patterns)).to.be(false); + }); + + it('returns false when none of the candidates match at the exact offset', function() { + const buffer = Buffer.from('{"type":"Foo"}', 'utf8'); + const valueStart = buffer.indexOf(Buffer.from('"Foo"', 'utf8')); + const patterns = [Buffer.from('"Bar"', 'utf8'), Buffer.from('"Baz"', 'utf8')]; + + expect(matchesAnyValuePattern(buffer, valueStart, patterns)).to.be(false); + }); + + }); + }); diff --git a/test/metadataUtil.spec.js b/test/metadataUtil.spec.js index 99fe91f..0db4a63 100644 --- a/test/metadataUtil.spec.js +++ b/test/metadataUtil.spec.js @@ -9,58 +9,58 @@ import { createHmac } from '../src/utils/metadataUtil.js'; -describe('metadataUtil', function() { +describe('metadataUtil', function () { - describe('buildRawBufferMatcher', function() { + describe('buildRawBufferMatcher', function () { - it('matches top-level scalar properties in raw JSON buffers', function() { - const matchesTypeFoo = buildRawBufferMatcher({ type: 'Foo' }); + it('matches top-level scalar properties in raw JSON buffers', function () { + const matchesTypeFoo = buildRawBufferMatcher({type: 'Foo'}); const buffer = Buffer.from('{"type":"Foo","id":1}', 'utf8'); expect(matchesTypeFoo(buffer)).to.be(true); }); - it('does not match nested values with the same key/value', function() { - const matchesTypeFoo = buildRawBufferMatcher({ type: 'Foo' }); + it('does not match nested values with the same key/value', function () { + const matchesTypeFoo = buildRawBufferMatcher({type: 'Foo'}); const buffer = Buffer.from('{"payload":{"type":"Foo"},"type":"Bar"}', 'utf8'); expect(matchesTypeFoo(buffer)).to.be(false); }); - it('supports matchers with multiple top-level properties', function() { - const matcher = buildRawBufferMatcher({ type: 'Foo', version: 3 }); + it('supports matchers with multiple top-level properties', function () { + const matcher = buildRawBufferMatcher({type: 'Foo', version: 3}); const buffer = Buffer.from('{"type":"Foo","version":3,"payload":"ok"}', 'utf8'); expect(matcher(buffer)).to.be(true); }); - it('requires compact key-value separators without whitespace', function() { - const matcher = buildRawBufferMatcher({ type: 'Foo', enabled: true }); + it('requires compact key-value separators without whitespace', function () { + const matcher = buildRawBufferMatcher({type: 'Foo', enabled: true}); const buffer = Buffer.from('{ "type" : "Foo", "enabled" : true }', 'utf8'); expect(matcher(buffer)).to.be(false); }); - it('supports nested matcher objects via the scoped fallback', function() { - const matcher = buildRawBufferMatcher({ payload: { type: 'Foo' } }); + it('supports nested matcher objects via the scoped fallback', function () { + const matcher = buildRawBufferMatcher({payload: {type: 'Foo'}}); const buffer = Buffer.from('{"payload":{"type":"Foo"},"id":1}', 'utf8'); expect(matcher(buffer)).to.be(true); }); }); - describe('buildRawBufferMatcher (nested/object-array mode)', function() { + describe('buildRawBufferMatcher (nested/object-array mode)', function () { - it('matches one-level nested object properties', function() { - const matcher = buildRawBufferMatcher({ payload: { type: 'Foo' } }); + it('matches one-level nested object properties', function () { + const matcher = buildRawBufferMatcher({payload: {type: 'Foo'}}); const buffer = Buffer.from('{"payload":{"type":"Foo"},"id":1}', 'utf8'); expect(matcher(buffer)).to.be(true); }); - it('does not match when nested value differs', function() { - const matcher = buildRawBufferMatcher({ payload: { type: 'Foo' } }); + it('does not match when nested value differs', function () { + const matcher = buildRawBufferMatcher({payload: {type: 'Foo'}}); const buffer = Buffer.from('{"payload":{"type":"Bar"},"id":1}', 'utf8'); expect(matcher(buffer)).to.be(false); }); - it('matches one of two allowed nested scalar values', function() { - const matcher = buildRawBufferMatcher({ payload: { type: ['Foo', 'Bar'] } }); + it('matches one of two allowed nested scalar values', function () { + const matcher = buildRawBufferMatcher({payload: {type: ['Foo', 'Bar']}}); const fooBuffer = Buffer.from('{"payload":{"type":"Foo"},"id":1}', 'utf8'); const barBuffer = Buffer.from('{"payload":{"type":"Bar"},"id":2}', 'utf8'); const bazBuffer = Buffer.from('{"payload":{"type":"Baz"},"id":3}', 'utf8'); @@ -70,35 +70,47 @@ describe('metadataUtil', function() { expect(matcher(bazBuffer)).to.be(false); }); - it('throws when matcher is not an object', function() { + it('matches with array containing single scalar value', function () { + const matcher = buildRawBufferMatcher({status: ['active']}); + const buffer = Buffer.from('{"status":"active"}', 'utf8'); + expect(matcher(buffer)).to.be(true); + }); + + it('matches nested array with single value', function () { + const matcher = buildRawBufferMatcher({payload: {type: ['Foo']}}); + const buffer = Buffer.from('{"payload":{"type":"Foo"}}', 'utf8'); + expect(matcher(buffer)).to.be(true); + }); + + it('throws when matcher is not an object', function () { expect(() => buildRawBufferMatcher(null)).to.throwError(TypeError); expect(() => buildRawBufferMatcher(['Foo'])).to.throwError(TypeError); expect(() => buildRawBufferMatcher('Foo')).to.throwError(TypeError); }); - it('throws when an array matcher contains object values', function() { - expect(() => buildRawBufferMatcher({ type: [{ value: 'Foo' }] })).to.throwError(TypeError); + it('throws when an array matcher contains object values', function () { + expect(() => buildRawBufferMatcher({type: [{value: 'Foo'}]})).to.throwError(TypeError); }); - it('matches every object when matcher is empty', function() { + it('matches every object when matcher is empty', function () { const matcher = buildRawBufferMatcher({}); expect(matcher(Buffer.from('{"type":"Foo"}', 'utf8'))).to.be(true); expect(matcher(Buffer.from('{"nested":{"value":1}}', 'utf8'))).to.be(true); }); - it('does not match non-object JSON buffers', function() { - const matcher = buildRawBufferMatcher({ type: 'Foo' }); + it('does not match non-object JSON buffers', function () { + const matcher = buildRawBufferMatcher({type: 'Foo'}); expect(matcher(Buffer.from('[{"type":"Foo"}]', 'utf8'))).to.be(false); }); - it('does not match when the required nested object key is missing', function() { - const matcher = buildRawBufferMatcher({ payload: { type: 'Foo' } }); + it('does not match when the required nested object key is missing', function () { + const matcher = buildRawBufferMatcher({payload: {type: 'Foo'}}); const buffer = Buffer.from('{"meta":{"type":"Foo"},"id":1}', 'utf8'); expect(matcher(buffer)).to.be(false); }); - it('matches string values containing escaped quotes', function() { - const matcher = buildRawBufferMatcher({ payload: { text: 'a"b' } }); + it('matches string values containing escaped quotes', function () { + const matcher = buildRawBufferMatcher({payload: {text: 'a"b'}}); const matchingBuffer = Buffer.from('{"payload":{"text":"a\\\"b"}}', 'utf8'); const nonMatchingBuffer = Buffer.from('{"payload":{"text":"ab"}}', 'utf8'); @@ -106,217 +118,408 @@ describe('metadataUtil', function() { expect(matcher(nonMatchingBuffer)).to.be(false); }); - it('matches when escaped characters appear before the matched key on the same level', function() { - const matcher = buildRawBufferMatcher({ type: 'Foo' }); + it('matches when escaped characters appear before the matched key on the same level', function () { + const matcher = buildRawBufferMatcher({type: 'Foo'}); const buffer = Buffer.from('{"note":"a\\\\b and \\\"quoted\\\"","type":"Foo"}', 'utf8'); expect(matcher(buffer)).to.be(true); }); - it('does not match malformed objects with a premature closing brace before the key', function() { - const matcher = buildRawBufferMatcher({ type: 'Foo' }); + it('does not match malformed objects with a premature closing brace before the key', function () { + const matcher = buildRawBufferMatcher({type: 'Foo'}); const buffer = Buffer.from('{"payload":{"id":1}},"type":"Foo"}', 'utf8'); expect(matcher(buffer)).to.be(false); }); - it('does not match nested object keys that are only present deeper than the requested level', function() { - const matcher = buildRawBufferMatcher({ payload: { source: { kind: 'A' } } }); + it('does not match nested object keys that are only present deeper than the requested level', function () { + const matcher = buildRawBufferMatcher({payload: {source: {kind: 'A'}}}); const buffer = Buffer.from('{"payload":{"wrapper":{"source":{"kind":"A"}}}}', 'utf8'); expect(matcher(buffer)).to.be(false); }); - it('does not match when a nested object exists but its nested value differs', function() { - const matcher = buildRawBufferMatcher({ payload: { source: { kind: 'A' } } }); + it('does not match when a nested object exists but its nested value differs', function () { + const matcher = buildRawBufferMatcher({payload: {source: {kind: 'A'}}}); const buffer = Buffer.from('{"payload":{"source":{"kind":"B"}}}', 'utf8'); expect(matcher(buffer)).to.be(false); }); - it('does not match scalar prefixes like 30 for an expected value of 3', function() { - const matcher = buildRawBufferMatcher({ version: 3 }); + it('does not match scalar prefixes like 30 for an expected value of 3', function () { + const matcher = buildRawBufferMatcher({version: 3}); const buffer = Buffer.from('{"version":30}', 'utf8'); expect(matcher(buffer)).to.be(false); }); - }); - - describe('operators: $gt, $gte, $lt, $lte', function() { - - it('matches values greater than threshold with $gt', function() { - const matcher = buildRawBufferMatcher({ amount: { $gt: 100 } }); - const buffer = Buffer.from('{"amount":150}', 'utf8'); - expect(matcher(buffer)).to.be(true); - }); - - it('does not match values equal or less than threshold with $gt', function() { - const matcher = buildRawBufferMatcher({ amount: { $gt: 100 } }); - expect(matcher(Buffer.from('{"amount":100}', 'utf8'))).to.be(false); - expect(matcher(Buffer.from('{"amount":50}', 'utf8'))).to.be(false); - }); - - it('matches values greater or equal to threshold with $gte', function() { - const matcher = buildRawBufferMatcher({ amount: { $gte: 100 } }); - expect(matcher(Buffer.from('{"amount":150}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"amount":100}', 'utf8'))).to.be(true); - }); - - it('does not match values less than threshold with $gte', function() { - const matcher = buildRawBufferMatcher({ amount: { $gte: 100 } }); - expect(matcher(Buffer.from('{"amount":99}', 'utf8'))).to.be(false); - }); - - it('matches string values with $gte', function() { - const matcher = buildRawBufferMatcher({ status: { $gte: 'pending' } }); - expect(matcher(Buffer.from('{"status":"pending"}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"status":"submitted"}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"status":"draft"}', 'utf8'))).to.be(false); - }); - - it('combines multiple operators in one matcher', function() { - const matcher = buildRawBufferMatcher({ version: { $gte: 2, $lt: 5 } }); - expect(matcher(Buffer.from('{"version":2}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"version":3}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"version":4}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"version":5}', 'utf8'))).to.be(false); - expect(matcher(Buffer.from('{"version":1}', 'utf8'))).to.be(false); - }); - - it('works with nested objects', function() { - const matcher = buildRawBufferMatcher({ payload: { amount: { $gte: 50 } } }); - expect(matcher(Buffer.from('{"payload":{"amount":50}}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"payload":{"amount":100}}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"payload":{"amount":25}}', 'utf8'))).to.be(false); - }); - - it('works with floating point numbers', function() { - const matcher = buildRawBufferMatcher({ price: { $gt: 19.99 } }); - expect(matcher(Buffer.from('{"price":20.0}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"price":19.99}', 'utf8'))).to.be(false); - expect(matcher(Buffer.from('{"price":19.98}', 'utf8'))).to.be(false); - }); - - it('works with negative numbers', function() { - const matcher = buildRawBufferMatcher({ temperature: { $gte: -10 } }); - expect(matcher(Buffer.from('{"temperature":-5}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"temperature":-10}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"temperature":-15}', 'utf8'))).to.be(false); - }); - - it('works with scientific notation numbers', function() { - const matcher = buildRawBufferMatcher({ amount: { $gt: 1000 } }); - expect(matcher(Buffer.from('{"amount":1.1e3}', 'utf8'))).to.be(true); - expect(matcher(Buffer.from('{"amount":1e3}', 'utf8'))).to.be(false); - }); - - it('does not match malformed numeric values for numeric operators', function() { - const matcher = buildRawBufferMatcher({ amount: { $gte: 10 } }); - expect(matcher(Buffer.from('{"amount":10abc}', 'utf8'))).to.be(false); - }); - - }); - - describe('matches with operators ($gt, $gte, $lt, $lte, $eq, $ne)', function() { - - it('matches with $gt', function() { - expect(matches({ amount: 150 }, { amount: { $gt: 100 } })).to.be(true); - expect(matches({ amount: 100 }, { amount: { $gt: 100 } })).to.be(false); - expect(matches({ amount: 50 }, { amount: { $gt: 100 } })).to.be(false); - }); - - it('matches with $gte', function() { - expect(matches({ amount: 150 }, { amount: { $gte: 100 } })).to.be(true); - expect(matches({ amount: 100 }, { amount: { $gte: 100 } })).to.be(true); - expect(matches({ amount: 99 }, { amount: { $gte: 100 } })).to.be(false); - }); - - it('matches with $lt', function() { - expect(matches({ amount: 50 }, { amount: { $lt: 100 } })).to.be(true); - expect(matches({ amount: 100 }, { amount: { $lt: 100 } })).to.be(false); - expect(matches({ amount: 150 }, { amount: { $lt: 100 } })).to.be(false); - }); - - it('matches with $lte', function() { - expect(matches({ amount: 50 }, { amount: { $lte: 100 } })).to.be(true); - expect(matches({ amount: 100 }, { amount: { $lte: 100 } })).to.be(true); - expect(matches({ amount: 101 }, { amount: { $lte: 100 } })).to.be(false); - }); - - it('matches with $eq', function() { - expect(matches({ status: 'active' }, { status: { $eq: 'active' } })).to.be(true); - expect(matches({ status: 'inactive' }, { status: { $eq: 'active' } })).to.be(false); - }); - - it('matches with $ne', function() { - expect(matches({ status: 'inactive' }, { status: { $ne: 'active' } })).to.be(true); - expect(matches({ status: 'active' }, { status: { $ne: 'active' } })).to.be(false); - }); - - it('combines multiple operators', function() { - expect(matches({ version: 3 }, { version: { $gte: 2, $lt: 5 } })).to.be(true); - expect(matches({ version: 1 }, { version: { $gte: 2, $lt: 5 } })).to.be(false); - expect(matches({ version: 5 }, { version: { $gte: 2, $lt: 5 } })).to.be(false); - }); - - it('throws on unknown operator', function() { - expect(() => matches({ x: 1 }, { x: { $unknown: 1 } })).to.throwError(TypeError); - }); - - it('does not confuse operator objects with plain nested matchers', function() { - // Plain nested object (no $ keys) → deep match - expect(matches({ meta: { kind: 'A' } }, { meta: { kind: 'A' } })).to.be(true); - // Operator object → range comparison - expect(matches({ amount: 50 }, { amount: { $gt: 10 } })).to.be(true); - }); - - it('reuses compiled operator checks on repeated matcher objects', function() { - const matcher = { amount: { $gte: 10 } }; - expect(matches({ amount: 12 }, matcher)).to.be(true); - expect(matches({ amount: 9 }, matcher)).to.be(false); - expect(matches({ amount: 11 }, matcher)).to.be(true); - }); - - }); - - describe('matcher metadata helpers', function() { - - it('returns undefined when no matcher is provided', function() { - expect(buildMetadataForMatcher(undefined, createHmac('secret'))).to.be(undefined); - }); - - it('serializes and restores function matchers with hmac', function() { - const hmac = createHmac('secret'); - const matcher = event => event.type === 'Foo'; - const metadata = buildMetadataForMatcher(matcher, hmac); - - expect(metadata.matcher).to.be.a('string'); - expect(metadata.hmac).to.be(hmac(metadata.matcher)); - - const restored = buildMatcherFromMetadata(metadata, hmac); - expect(restored({ type: 'Foo' })).to.be(true); - expect(restored({ type: 'Bar' })).to.be(false); - }); - - it('passes through object matchers unchanged in metadata', function() { - const matcher = { payload: { type: 'Foo' } }; - const metadata = buildMetadataForMatcher(matcher, createHmac('secret')); - const restored = buildMatcherFromMetadata(metadata, createHmac('secret')); - - expect(metadata).to.eql({ matcher }); - expect(restored).to.eql(matcher); - }); - - it('builds type matcher functions for nested paths', function() { - const typeMatcher = buildTypeMatcherFn('meta.kind'); - expect(typeMatcher('OrderPlaced')).to.eql({ payload: { meta: { kind: 'OrderPlaced' } } }); - }); - - it('builds a padded metadata header', function() { - const header = buildMetadataHeader('MAGICHDR', { version: 1 }); - - expect(header.slice(0, 8).toString('utf8')).to.be('MAGICHDR'); - expect(header.length % 16).to.be(0); - expect(header.readUInt32BE(8)).to.be.greaterThan(0); - }); - - }); + }); + + describe('operators: $gt, $gte, $lt, $lte', function () { + + it('matches values greater than threshold with $gt', function () { + const matcher = buildRawBufferMatcher({amount: {$gt: 100}}); + const buffer = Buffer.from('{"amount":150}', 'utf8'); + expect(matcher(buffer)).to.be(true); + }); + + it('does not match values equal or less than threshold with $gt', function () { + const matcher = buildRawBufferMatcher({amount: {$gt: 100}}); + expect(matcher(Buffer.from('{"amount":100}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"amount":50}', 'utf8'))).to.be(false); + }); + + it('matches values greater or equal to threshold with $gte', function () { + const matcher = buildRawBufferMatcher({amount: {$gte: 100}}); + expect(matcher(Buffer.from('{"amount":150}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"amount":100}', 'utf8'))).to.be(true); + }); + + it('does not match values less than threshold with $gte', function () { + const matcher = buildRawBufferMatcher({amount: {$gte: 100}}); + expect(matcher(Buffer.from('{"amount":99}', 'utf8'))).to.be(false); + }); + + it('matches string values with $gte', function () { + const matcher = buildRawBufferMatcher({status: {$gte: 'pending'}}); + expect(matcher(Buffer.from('{"status":"pending"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"submitted"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"draft"}', 'utf8'))).to.be(false); + }); + + it('combines multiple operators in one matcher', function () { + const matcher = buildRawBufferMatcher({version: {$gte: 2, $lt: 5}}); + expect(matcher(Buffer.from('{"version":2}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"version":3}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"version":4}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"version":5}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"version":1}', 'utf8'))).to.be(false); + }); + + it('combines multiple numeric operators including $ne', function () { + const matcher = buildRawBufferMatcher({version: {$gt: 1, $lt: 5, $ne: 3}}); + expect(matcher(Buffer.from('{"version":2}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"version":3}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"version":4}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"version":5}', 'utf8'))).to.be(false); + }); + + it('supports multiple string operators via generic fallback', function () { + const matcher = buildRawBufferMatcher({status: {$gte: 'b', $lt: 'd'}}); + expect(matcher(Buffer.from('{"status":"b"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"c"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"d"}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"status":"a"}', 'utf8'))).to.be(false); + }); + + it('matches ISO datetime strings in a nested range', function () { + const matcher = buildRawBufferMatcher({ + payload: { + at: { + $gte: '2021-06-09T21:03:28.297Z', + $lt: '2021-06-09T21:41:53.775Z' + } + } + }); + + const events = [ + {payload: {at: '2021-06-09T21:03:28.297Z'}}, + {payload: {at: '2021-06-09T21:20:00.000Z'}}, + {payload: {at: '2021-06-09T21:41:53.774Z'}}, + {payload: {at: '2021-06-09T21:03:28.296Z'}}, + {payload: {at: '2021-06-09T21:41:53.775Z'}} + ]; + + const matching = events.filter(event => matcher(Buffer.from(JSON.stringify(event), 'utf8'))); + expect(matching).to.eql([ + {payload: {at: '2021-06-09T21:03:28.297Z'}}, + {payload: {at: '2021-06-09T21:20:00.000Z'}}, + {payload: {at: '2021-06-09T21:41:53.774Z'}} + ]); + }); + + it('works with nested objects', function () { + const matcher = buildRawBufferMatcher({payload: {amount: {$gte: 50}}}); + expect(matcher(Buffer.from('{"payload":{"amount":50}}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"payload":{"amount":100}}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"payload":{"amount":25}}', 'utf8'))).to.be(false); + }); + + it('works with floating point numbers', function () { + const matcher = buildRawBufferMatcher({price: {$gt: 19.99}}); + expect(matcher(Buffer.from('{"price":20.0}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"price":19.99}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"price":19.98}', 'utf8'))).to.be(false); + }); + + it('works with negative numbers', function () { + const matcher = buildRawBufferMatcher({temperature: {$gte: -10}}); + expect(matcher(Buffer.from('{"temperature":-5}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"temperature":-10}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"temperature":-15}', 'utf8'))).to.be(false); + }); + + it('matches values less than threshold with $lt', function () { + const matcher = buildRawBufferMatcher({amount: {$lt: 100}}); + expect(matcher(Buffer.from('{"amount":50}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"amount":100}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"amount":150}', 'utf8'))).to.be(false); + }); + + it('matches values less or equal to threshold with $lte', function () { + const matcher = buildRawBufferMatcher({amount: {$lte: 100}}); + expect(matcher(Buffer.from('{"amount":50}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"amount":100}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"amount":101}', 'utf8'))).to.be(false); + }); + + it('matches with standalone $eq operator', function () { + const matcher = buildRawBufferMatcher({status: {$eq: 'active'}}); + expect(matcher(Buffer.from('{"status":"active"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"inactive"}', 'utf8'))).to.be(false); + }); + + it('matches with $ne operator', function () { + const matcher = buildRawBufferMatcher({status: {$ne: 'active'}}); + expect(matcher(Buffer.from('{"status":"inactive"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"active"}', 'utf8'))).to.be(false); + }); + + it('lone $ne does not match when the key is absent', function () { + const matcher = buildRawBufferMatcher({status: {$ne: 'active'}}); + expect(matcher(Buffer.from('{"other":"active"}', 'utf8'))).to.be(false); + }); + + it('lone $ne ignores matching value at the wrong nesting level', function () { + // "status":"active" appears only inside payload — must not match the top-level $ne check + const matcher = buildRawBufferMatcher({status: {$ne: 'active'}}); + expect(matcher(Buffer.from('{"payload":{"status":"active"},"status":"inactive"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"payload":{"status":"inactive"},"status":"active"}', 'utf8'))).to.be(false); + }); + + it('lone $ne matches a numeric value that differs', function () { + const matcher = buildRawBufferMatcher({version: {$ne: 3}}); + expect(matcher(Buffer.from('{"version":4}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"version":3}', 'utf8'))).to.be(false); + }); + + it('handles fractional numbers with operators', function () { + const matcher = buildRawBufferMatcher({price: {$lte: 19.99}}); + expect(matcher(Buffer.from('{"price":19.99}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"price":19.98}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"price":20.0}', 'utf8'))).to.be(false); + }); + + it('matches with $eq and $ne on string values', function () { + const eqMatcher = buildRawBufferMatcher({type: {$eq: 'EventType'}}); + expect(eqMatcher(Buffer.from('{"type":"EventType"}', 'utf8'))).to.be(true); + expect(eqMatcher(Buffer.from('{"type":"OtherType"}', 'utf8'))).to.be(false); + }); + + it('matches when multiple operators and properties are combined', function () { + const matcher = buildRawBufferMatcher({ + status: {$ne: 'draft'}, + priority: {$gte: 5} + }); + expect(matcher(Buffer.from('{"status":"published","priority":7}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"draft","priority":7}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"status":"published","priority":3}', 'utf8'))).to.be(false); + }); + + it('uses generic operator checks when operator buffer matcher is disabled', function () { + const matcher = buildRawBufferMatcher( + {amount: {$gt: 100}}, + {enableOperatorBufferMatcher: false} + ); + expect(matcher(Buffer.from('{"amount":150}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"amount":100}', 'utf8'))).to.be(false); + }); + + it('rejects early via fast scalar pattern when string equals precedes operator', function () { + // status:'active' is a fast scalar pattern; amount:{$gte:50} is slow. + // With reordering, the status check runs first in both preCheck and matchesNode, + // so documents with wrong status never reach the numeric operator. + const matcher = buildRawBufferMatcher({status: 'active', amount: {$gte: 50}}); + expect(matcher(Buffer.from('{"status":"active","amount":100}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"status":"inactive","amount":100}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"status":"active","amount":10}', 'utf8'))).to.be(false); + }); + + it('rejects early when type equality is listed after a multi-operator in source order', function () { + // Matcher keys: amount first (slow), type second (fast). + // After reordering, type:"Foo" moves to front and acts as a prefilter. + const matcher = buildRawBufferMatcher({amount: {$gt: 10, $lt: 100}, type: 'Foo'}); + expect(matcher(Buffer.from('{"amount":50,"type":"Foo"}', 'utf8'))).to.be(true); + expect(matcher(Buffer.from('{"amount":50,"type":"Bar"}', 'utf8'))).to.be(false); + expect(matcher(Buffer.from('{"amount":150,"type":"Foo"}', 'utf8'))).to.be(false); + }); + + }); + + describe('matches with operators ($gt, $gte, $lt, $lte, $eq, $ne)', function () { + + it('matches with $gt', function () { + expect(matches({amount: 150}, {amount: {$gt: 100}})).to.be(true); + expect(matches({amount: 100}, {amount: {$gt: 100}})).to.be(false); + expect(matches({amount: 50}, {amount: {$gt: 100}})).to.be(false); + }); + + it('matches with $gte', function () { + expect(matches({amount: 150}, {amount: {$gte: 100}})).to.be(true); + expect(matches({amount: 100}, {amount: {$gte: 100}})).to.be(true); + expect(matches({amount: 99}, {amount: {$gte: 100}})).to.be(false); + }); + + it('matches with $lt', function () { + expect(matches({amount: 50}, {amount: {$lt: 100}})).to.be(true); + expect(matches({amount: 100}, {amount: {$lt: 100}})).to.be(false); + expect(matches({amount: 150}, {amount: {$lt: 100}})).to.be(false); + }); + + it('matches with $lte', function () { + expect(matches({amount: 50}, {amount: {$lte: 100}})).to.be(true); + expect(matches({amount: 100}, {amount: {$lte: 100}})).to.be(true); + expect(matches({amount: 101}, {amount: {$lte: 100}})).to.be(false); + }); + + it('matches with $eq', function () { + expect(matches({status: 'active'}, {status: {$eq: 'active'}})).to.be(true); + expect(matches({status: 'inactive'}, {status: {$eq: 'active'}})).to.be(false); + }); + + it('matches with $ne', function () { + expect(matches({status: 'inactive'}, {status: {$ne: 'active'}})).to.be(true); + expect(matches({status: 'active'}, {status: {$ne: 'active'}})).to.be(false); + }); + + it('combines multiple operators', function () { + expect(matches({version: 3}, {version: {$gte: 2, $lt: 5}})).to.be(true); + expect(matches({version: 1}, {version: {$gte: 2, $lt: 5}})).to.be(false); + expect(matches({version: 5}, {version: {$gte: 2, $lt: 5}})).to.be(false); + }); + + it('throws on unknown operator', function () { + expect(() => matches({x: 1}, {x: {$unknown: 1}})).to.throwError(TypeError); + }); + + it('does not confuse operator objects with plain nested matchers', function () { + // Plain nested object (no $ keys) → deep match + expect(matches({meta: {kind: 'A'}}, {meta: {kind: 'A'}})).to.be(true); + // Operator object → range comparison + expect(matches({amount: 50}, {amount: {$gt: 10}})).to.be(true); + }); + + it('reuses compiled operator checks on repeated matcher objects', function () { + const matcher = {amount: {$gte: 10}}; + expect(matches({amount: 12}, matcher)).to.be(true); + expect(matches({amount: 9}, matcher)).to.be(false); + expect(matches({amount: 11}, matcher)).to.be(true); + }); + + it('matches with $lt', function () { + expect(matches({amount: 50}, {amount: {$lt: 100}})).to.be(true); + expect(matches({amount: 100}, {amount: {$lt: 100}})).to.be(false); + expect(matches({amount: 150}, {amount: {$lt: 100}})).to.be(false); + }); + + it('matches with $lte', function () { + expect(matches({amount: 50}, {amount: {$lte: 100}})).to.be(true); + expect(matches({amount: 100}, {amount: {$lte: 100}})).to.be(true); + expect(matches({amount: 101}, {amount: {$lte: 100}})).to.be(false); + }); + + it('matches with $ne', function () { + expect(matches({status: 'inactive'}, {status: {$ne: 'active'}})).to.be(true); + expect(matches({status: 'active'}, {status: {$ne: 'active'}})).to.be(false); + }); + + it('matches when operator value is undefined (matches all non-undefined values)', function () { + expect(matches({amount: 50}, {amount: undefined})).to.be(true); + expect(matches({amount: undefined}, {amount: undefined})).to.be(true); + }); + + it('matches when document property is undefined', function () { + expect(matches({}, {missing: undefined})).to.be(true); + expect(matches({other: 1}, {missing: {$gt: 10}})).to.be(false); + }); + + it('handles array equality matching', function () { + expect(matches({status: 'active'}, {status: ['active', 'pending']})).to.be(true); + expect(matches({status: 'completed'}, {status: ['active', 'pending']})).to.be(false); + }); + + it('handles nested object matching without operators', function () { + expect(matches({meta: {kind: 'A', version: 1}}, {meta: {kind: 'A'}})).to.be(true); + expect(matches({meta: {kind: 'B'}}, {meta: {kind: 'A'}})).to.be(false); + }); + + }); + + describe('matcher metadata helpers', function () { + + it('returns undefined when no matcher is provided', function () { + expect(buildMetadataForMatcher(undefined, createHmac('secret'))).to.be(undefined); + }); + + it('serializes and restores function matchers with hmac', function () { + const hmac = createHmac('secret'); + const matcher = event => event.type === 'Foo'; + const metadata = buildMetadataForMatcher(matcher, hmac); + + expect(metadata.matcher).to.be.a('string'); + expect(metadata.hmac).to.be(hmac(metadata.matcher)); + + const restored = buildMatcherFromMetadata(metadata, hmac); + expect(restored({type: 'Foo'})).to.be(true); + expect(restored({type: 'Bar'})).to.be(false); + }); + + it('passes through object matchers unchanged in metadata', function () { + const matcher = {payload: {type: 'Foo'}}; + const metadata = buildMetadataForMatcher(matcher, createHmac('secret')); + const restored = buildMatcherFromMetadata(metadata, createHmac('secret')); + + expect(metadata).to.eql({matcher}); + expect(restored).to.eql(matcher); + }); + + it('builds type matcher functions for single-level paths', function () { + const typeMatcher = buildTypeMatcherFn('type'); + expect(typeMatcher('OrderPlaced')).to.eql({payload: {type: 'OrderPlaced'}}); + }); + + it('builds type matcher functions for nested paths', function () { + const typeMatcher = buildTypeMatcherFn('meta.kind'); + expect(typeMatcher('OrderPlaced')).to.eql({payload: {meta: {kind: 'OrderPlaced'}}}); + }); + + it('builds type matcher functions for deeply nested paths', function () { + const typeMatcher = buildTypeMatcherFn('deeply.nested.type'); + expect(typeMatcher('MyEvent')).to.eql({ + payload: {deeply: {nested: {type: 'MyEvent'}}} + }); + }); + + it('builds a padded metadata header', function () { + const header = buildMetadataHeader('MAGICHDR', {version: 1}); + + expect(header.slice(0, 8).toString('utf8')).to.be('MAGICHDR'); + expect(header.length % 16).to.be(0); + expect(header.readUInt32BE(8)).to.be.greaterThan(0); + }); + + it('builds metadata header with large metadata object', function () { + const largeMetadata = {key: 'value'.repeat(50)}; + const header = buildMetadataHeader('MAGICHDR', largeMetadata); + + expect(header.slice(0, 8).toString('utf8')).to.be('MAGICHDR'); + expect(header.length % 16).to.be(0); + }); + + it('creates different HMACs for different secrets', function () { + const hmac1 = createHmac('secret1'); + const hmac2 = createHmac('secret2'); + const testString = 'test'; + + expect(hmac1(testString)).not.to.be(hmac2(testString)); + }); + + }); }); diff --git a/test/util.spec.js b/test/util.spec.js index b9adc01..6124af0 100644 --- a/test/util.spec.js +++ b/test/util.spec.js @@ -2,7 +2,7 @@ import expect from 'expect.js'; import fs from 'fs-extra'; import path from 'path'; import { iterate } from '../src/utils/util.js'; -import { scanForFiles } from '../src/utils/fsUtil.js'; +import { isSafeRelativeName, resolvePathWithinRoot, scanForFiles } from '../src/utils/fsUtil.js'; import { fileURLToPath } from 'url'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -90,6 +90,26 @@ describe('util', function() { }); }); + describe('file path helpers', function() { + + it('detects safe relative names', function() { + expect(isSafeRelativeName('stream-orders')).to.be(true); + expect(isSafeRelativeName('../orders')).to.be(false); + }); + + it('resolves paths within a root directory', function() { + const root = path.join(testDir, 'root'); + const resolved = resolvePathWithinRoot(root, 'a/b.file'); + expect(resolved.startsWith(path.resolve(root))).to.be(true); + }); + + it('rejects traversal paths outside of root directory', function() { + const root = path.join(testDir, 'root'); + expect(() => resolvePathWithinRoot(root, '../outside.file')).to.throwError(/Invalid relative path/); + }); + + }); + }); });