From 04e4e2945f1faad70239e2dd16037eea1411f381 Mon Sep 17 00:00:00 2001 From: ABHAY PANDEY Date: Sat, 5 Sep 2026 10:27:38 +0530 Subject: [PATCH 1/2] feat(settings): store user overrides in PostgreSQL Add settings_overrides table and SETTINGS_BACKEND=db opt-in so operator overrides live in the database instead of editable YAML files. Existing file overrides are imported on first bootstrap when the table is empty. Admin PATCH and restore save through a single async restore path. --- .changeset/settings-db-overrides.md | 5 + CONFIGURATION.md | 1 + ..._120000_create_settings_overrides_table.js | 11 +++ .../admin/patch-settings-controller.ts | 4 +- .../admin/post-settings-restore-controller.ts | 2 +- src/index.ts | 13 ++- .../settings-override-repository.ts | 44 +++++++++ src/utils/settings-config.ts | 59 +++++++++++- src/utils/settings-store.ts | 62 ++++++++++++ src/utils/settings.ts | 4 + .../settings-overrides-migration.spec.ts | 13 +++ .../settings-override-repository.spec.ts | 96 +++++++++++++++++++ test/unit/utils/settings-flatten.spec.ts | 28 ++++++ test/unit/utils/settings-store.spec.ts | 27 ++++++ 14 files changed, 362 insertions(+), 7 deletions(-) create mode 100644 .changeset/settings-db-overrides.md create mode 100644 migrations/20260905_120000_create_settings_overrides_table.js create mode 100644 src/repositories/settings-override-repository.ts create mode 100644 src/utils/settings-store.ts create mode 100644 test/unit/migrations/settings-overrides-migration.spec.ts create mode 100644 test/unit/repositories/settings-override-repository.spec.ts create mode 100644 test/unit/utils/settings-flatten.spec.ts create mode 100644 test/unit/utils/settings-store.spec.ts diff --git a/.changeset/settings-db-overrides.md b/.changeset/settings-db-overrides.md new file mode 100644 index 000000000..466c634e3 --- /dev/null +++ b/.changeset/settings-db-overrides.md @@ -0,0 +1,5 @@ +--- +"nostream": minor +--- + +feat: store relay settings overrides in PostgreSQL when SETTINGS_BACKEND=db diff --git a/CONFIGURATION.md b/CONFIGURATION.md index f47212af4..9732650be 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -61,6 +61,7 @@ The following environment variables can be set: | ADMIN_DEPENDENCY_PING_TIMEOUT_MS | Timeout for admin DB/Redis dependency pings (ms) | 3000 | | GRAFANA_URL | Grafana base URL for admin dashboard embeds | http://127.0.0.1:7777 | | NOSTR_CONFIG_DIR | Configuration directory | /.nostr/ | +| SETTINGS_BACKEND | Where operator overrides are stored: `file` (default) or `db` | file | | MONITOR_PRIVATE_KEY | Hex-encoded private key for the NIP-66 monitor identity that will sign kind 30166/10166 events. Configure via environment variable, not settings.yaml. | | | DEBUG | Debugging filter | | | ZEBEDEE_API_KEY | Zebedee Project API Key | | diff --git a/migrations/20260905_120000_create_settings_overrides_table.js b/migrations/20260905_120000_create_settings_overrides_table.js new file mode 100644 index 000000000..e91fe7711 --- /dev/null +++ b/migrations/20260905_120000_create_settings_overrides_table.js @@ -0,0 +1,11 @@ +exports.up = async function (knex) { + await knex.schema.createTable('settings_overrides', (table) => { + table.text('path').primary() + table.jsonb('value').notNullable() + table.timestamp('updated_at', { useTz: true }).notNullable().defaultTo(knex.fn.now()) + }) +} + +exports.down = async function (knex) { + await knex.schema.dropTableIfExists('settings_overrides') +} diff --git a/src/controllers/admin/patch-settings-controller.ts b/src/controllers/admin/patch-settings-controller.ts index 7914d3b8e..8d82c8757 100644 --- a/src/controllers/admin/patch-settings-controller.ts +++ b/src/controllers/admin/patch-settings-controller.ts @@ -8,11 +8,11 @@ import { getByPath, loadMergedSettings, loadUserSettings, - saveSettings, setByPath, validatePathAgainstDefaults, validateSettings, } from '../../utils/settings-config' +import { saveUserSettingsOverrides } from '../../utils/settings-store' import { isWriteProtectedSettingsPath, redactSettingsValue, @@ -61,7 +61,7 @@ export class PatchAdminSettingsController implements IController { return } - saveSettings(nextUserSettings as unknown as Settings) + await saveUserSettingsOverrides(nextUserSettings as unknown as Settings) const updatedChanges = changes.map(({ path }) => ({ path, value: redactSettingsValue(path, getByPath(nextUserSettings, path)), diff --git a/src/controllers/admin/post-settings-restore-controller.ts b/src/controllers/admin/post-settings-restore-controller.ts index aadcca090..45ae23a79 100644 --- a/src/controllers/admin/post-settings-restore-controller.ts +++ b/src/controllers/admin/post-settings-restore-controller.ts @@ -14,7 +14,7 @@ export class PostAdminSettingsRestoreController implements IController { } try { - restoreSettingsBackup(validation.value.filename) + await restoreSettingsBackup(validation.value.filename) appendSettingsAuditLog({ action: 'settings.restored', filename: validation.value.filename, diff --git a/src/index.ts b/src/index.ts index a1f9d1a99..43a3bb681 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,12 +1,16 @@ import cluster from 'cluster' import { appFactory } from './factories/app-factory' +import { createLogger } from './factories/logger-factory' import { dvmOrchestratorWorkerFactory } from './factories/dvm-orchestrator-worker-factory' import { maintenanceWorkerFactory } from './factories/maintenance-worker-factory' import { relayMonitorWorkerFactory } from './factories/relay-monitor-worker-factory' import { staticMirroringWorkerFactory } from './factories/static-mirroring.worker-factory' import { workerFactory } from './factories/worker-factory' import { initializeMetricsTelemetry } from './telemetry/metrics' +import { bootstrapSettingsStore } from './utils/settings-store' + +const logger = createLogger('index') export const getRunner = () => { if (cluster.isPrimary) { @@ -32,5 +36,12 @@ export const getRunner = () => { if (require.main === module) { initializeMetricsTelemetry() - getRunner().run() + void bootstrapSettingsStore() + .then(() => { + getRunner().run() + }) + .catch((error) => { + logger.error('failed to bootstrap settings store:', error) + process.exit(1) + }) } diff --git a/src/repositories/settings-override-repository.ts b/src/repositories/settings-override-repository.ts new file mode 100644 index 000000000..0f4422aed --- /dev/null +++ b/src/repositories/settings-override-repository.ts @@ -0,0 +1,44 @@ +import { DatabaseClient } from '../@types/base' +import { Settings } from '../@types/settings' +import { createLogger } from '../factories/logger-factory' +import { flattenSettingsToPaths, setByPath } from '../utils/settings-config' + +const logger = createLogger('settings-override-repository') + +export class SettingsOverrideRepository { + public constructor(private readonly dbClient: DatabaseClient) {} + + public async loadOverrides(): Promise { + const rows = await this.dbClient('settings_overrides').select('path', 'value') + + let settings: Record = {} + for (const row of rows) { + settings = setByPath(settings, row.path, row.value) + } + + return settings as unknown as Settings + } + + public async replaceOverrides(overrides: Settings): Promise { + const entries = flattenSettingsToPaths(overrides) + + await this.dbClient.transaction(async (transaction) => { + await transaction('settings_overrides').delete() + + if (entries.length === 0) { + return + } + + const now = new Date() + await transaction('settings_overrides').insert( + entries.map((entry) => ({ + path: entry.path, + value: entry.value, + updated_at: now, + })), + ) + }) + + logger('replaced %d settings override paths', entries.length) + } +} diff --git a/src/utils/settings-config.ts b/src/utils/settings-config.ts index d6b83e725..d97584009 100644 --- a/src/utils/settings-config.ts +++ b/src/utils/settings-config.ts @@ -303,7 +303,7 @@ export const loadDefaults = (): Settings => { return yaml.load(defaultsRaw) as Settings } -export const loadUserSettings = (): Settings => { +export const loadUserSettingsFromFiles = (): Settings => { ensureSettingsExists() const settingsPath = getSettingsFilePath() @@ -323,6 +323,17 @@ export const loadUserSettings = (): Settings => { return {} as Settings } +export const loadUserSettings = (): Settings => { + if (process.env.SETTINGS_BACKEND === 'db') { + // Lazy import avoids a circular dependency with settings-store bootstrap. + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { getDbOverridesCache } = require('./settings-store') as typeof import('./settings-store') + return getDbOverridesCache() + } + + return loadUserSettingsFromFiles() +} + export const loadMergedSettings = (): Settings => { return mergeDeepRight(loadDefaults(), loadUserSettings()) as Settings } @@ -366,6 +377,40 @@ export const filterSettingsAgainstDefaults = (settings: unknown, defaults: unkno return settings } +export const flattenSettingsToPaths = ( + value: unknown, + prefix = '', +): Array<{ path: string; value: unknown }> => { + if (Array.isArray(value)) { + return value.flatMap((entry, index) => { + const nextPrefix = prefix ? `${prefix}[${index}]` : `[${index}]` + return flattenSettingsToPaths(entry, nextPrefix) + }) + } + + if (isPlainObject(value)) { + const entries = Object.entries(value) + if (entries.length === 0 && prefix) { + return [{ path: prefix, value: {} }] + } + + return entries.flatMap(([key, entry]) => { + const nextPrefix = prefix ? `${prefix}.${key}` : key + if (isPlainObject(entry) || Array.isArray(entry)) { + return flattenSettingsToPaths(entry, nextPrefix) + } + + return [{ path: nextPrefix, value: entry }] + }) + } + + if (prefix) { + return [{ path: prefix, value }] + } + + return [] +} + export const saveSettings = (settings: Settings): void => { ensureSettingsExists() const serialized = yaml.dump(toSerializable(settings), { lineWidth: 120 }) @@ -428,7 +473,7 @@ export const listSettingsBackups = (): SettingsBackupInfo[] => { .sort((left, right) => right.createdAt.localeCompare(left.createdAt)) } -export const restoreSettingsBackup = (filename: string): Settings => { +const loadValidatedSettingsBackup = (filename: string): Settings => { if (!isSafeBackupFilename(filename)) { throw new Error('Invalid backup filename') } @@ -444,7 +489,15 @@ export const restoreSettingsBackup = (filename: string): Settings => { throw new Error(`Backup failed validation: ${issues.map((issue) => issue.path).join(', ')}`) } - saveSettings(restored) + return restored +} + +export const restoreSettingsBackup = async (filename: string): Promise => { + const restored = loadValidatedSettingsBackup(filename) + + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { saveUserSettingsOverrides } = require('./settings-store') as typeof import('./settings-store') + await saveUserSettingsOverrides(restored) return restored } diff --git a/src/utils/settings-store.ts b/src/utils/settings-store.ts new file mode 100644 index 000000000..5767a7350 --- /dev/null +++ b/src/utils/settings-store.ts @@ -0,0 +1,62 @@ +import { Settings } from '../@types/settings' +import { getMasterDbClient } from '../database/client' +import { createLogger } from '../factories/logger-factory' +import { SettingsOverrideRepository } from '../repositories/settings-override-repository' +import { SettingsStatic } from './settings' +import { loadUserSettingsFromFiles, saveSettings } from './settings-config' + +const logger = createLogger('settings-store') + +let dbOverridesCache: Settings = {} as Settings +let bootstrapComplete = false + +export type SettingsBackend = 'file' | 'db' + +export const getSettingsBackend = (): SettingsBackend => { + return process.env.SETTINGS_BACKEND === 'db' ? 'db' : 'file' +} + +export const getDbOverridesCache = (): Settings => structuredClone(dbOverridesCache) + +const isEmptyOverrides = (settings: Settings): boolean => { + return Object.keys(settings as object).length === 0 +} + +export const bootstrapSettingsStore = async (): Promise => { + if (getSettingsBackend() !== 'db') { + bootstrapComplete = true + return + } + + const repository = new SettingsOverrideRepository(getMasterDbClient()) + let overrides = await repository.loadOverrides() + + if (isEmptyOverrides(overrides)) { + const fileOverrides = loadUserSettingsFromFiles() + if (!isEmptyOverrides(fileOverrides)) { + logger.info('importing settings overrides from file into database') + await repository.replaceOverrides(fileOverrides) + overrides = fileOverrides + } + } + + dbOverridesCache = overrides + bootstrapComplete = true + SettingsStatic.invalidateCache() +} + +export const saveUserSettingsOverrides = async (settings: Settings): Promise => { + if (getSettingsBackend() === 'db') { + if (!bootstrapComplete) { + throw new Error('settings store has not been bootstrapped') + } + + const repository = new SettingsOverrideRepository(getMasterDbClient()) + await repository.replaceOverrides(settings) + dbOverridesCache = structuredClone(settings) + SettingsStatic.invalidateCache() + return + } + + saveSettings(settings) +} diff --git a/src/utils/settings.ts b/src/utils/settings.ts index c47aa321a..665a6ff4a 100644 --- a/src/utils/settings.ts +++ b/src/utils/settings.ts @@ -95,6 +95,10 @@ export class SettingsStatic { } } + public static invalidateCache(): void { + SettingsStatic._settings = undefined + } + public static saveSettings(path: string, settings: Settings) { logger('saving settings to %s: %o', path, settings) return fs.writeFileSync(join(path, 'settings.yaml'), yaml.dump(settings), { encoding: 'utf-8' }) diff --git a/test/unit/migrations/settings-overrides-migration.spec.ts b/test/unit/migrations/settings-overrides-migration.spec.ts new file mode 100644 index 000000000..bd197646f --- /dev/null +++ b/test/unit/migrations/settings-overrides-migration.spec.ts @@ -0,0 +1,13 @@ +import { createRequire } from 'node:module' + +import { expect } from 'chai' + +const requireFromHere = createRequire(__filename) +const migration = requireFromHere('../../../migrations/20260905_120000_create_settings_overrides_table.js') + +describe('migrations/20260905_120000_create_settings_overrides_table', () => { + it('exports up and down', () => { + expect(migration.up).to.be.a('function') + expect(migration.down).to.be.a('function') + }) +}) diff --git a/test/unit/repositories/settings-override-repository.spec.ts b/test/unit/repositories/settings-override-repository.spec.ts new file mode 100644 index 000000000..2d5cd1dfd --- /dev/null +++ b/test/unit/repositories/settings-override-repository.spec.ts @@ -0,0 +1,96 @@ +import * as chai from 'chai' +import * as sinon from 'sinon' +import sinonChai from 'sinon-chai' + +import { SettingsOverrideRepository } from '../../../src/repositories/settings-override-repository' + +chai.use(sinonChai) + +const { expect } = chai + +describe('SettingsOverrideRepository', () => { + let queryBuilder: { + select: sinon.SinonStub + delete: sinon.SinonStub + insert: sinon.SinonStub + } + let transactionBuilder: { + delete: sinon.SinonStub + insert: sinon.SinonStub + } + let dbClient: any + let repository: SettingsOverrideRepository + + beforeEach(() => { + queryBuilder = { + select: sinon.stub().resolves([ + { path: 'info.name', value: 'Test Relay' }, + { path: 'nip66.enabled', value: true }, + ]), + delete: sinon.stub(), + insert: sinon.stub(), + } + + transactionBuilder = { + delete: sinon.stub().resolves(), + insert: sinon.stub().resolves(), + } + + const transactionFn = sinon.stub() + transactionFn.withArgs('settings_overrides').returns(transactionBuilder) + + dbClient = sinon.stub() + dbClient.callsFake((table: string) => { + if (table === 'settings_overrides') { + return queryBuilder + } + + throw new Error(`unexpected table ${table}`) + }) + dbClient.transaction = sinon.stub().callsFake(async (callback: (trx: typeof transactionFn) => Promise) => { + await callback(transactionFn) + }) + + repository = new SettingsOverrideRepository(dbClient) + }) + + it('loads overrides into a nested settings object', async () => { + const overrides = await repository.loadOverrides() + + expect(overrides).to.deep.equal({ + info: { name: 'Test Relay' }, + nip66: { enabled: true }, + }) + }) + + it('replaces all override rows from a nested settings object', async () => { + await repository.replaceOverrides({ + info: { name: 'Updated Relay' }, + admin: { enabled: true }, + } as any) + + expect(dbClient.transaction).to.have.been.calledOnce + expect(transactionBuilder.delete).to.have.been.calledOnce + expect(transactionBuilder.insert).to.have.been.calledOnce + + const insertedRows = transactionBuilder.insert.firstCall.args[0] + expect(insertedRows).to.have.length(2) + expect(insertedRows[0]).to.deep.include({ + path: 'info.name', + value: 'Updated Relay', + }) + expect(insertedRows[0].updated_at).to.be.instanceOf(Date) + expect(insertedRows[1]).to.deep.include({ + path: 'admin.enabled', + value: true, + }) + expect(insertedRows[1].updated_at).to.be.instanceOf(Date) + }) + + it('clears all rows when overrides are empty', async () => { + await repository.replaceOverrides({} as any) + + expect(transactionBuilder.delete).to.have.been.calledOnce + expect(transactionBuilder.insert).not.to.have.been.called + }) +}) diff --git a/test/unit/utils/settings-flatten.spec.ts b/test/unit/utils/settings-flatten.spec.ts new file mode 100644 index 000000000..3994a4584 --- /dev/null +++ b/test/unit/utils/settings-flatten.spec.ts @@ -0,0 +1,28 @@ +import { expect } from 'chai' + +import { flattenSettingsToPaths } from '../../../src/utils/settings-config' + +describe('flattenSettingsToPaths', () => { + it('flattens nested objects into dot paths', () => { + expect( + flattenSettingsToPaths({ + info: { name: 'Relay', relay_url: 'wss://relay.example.com' }, + nip66: { enabled: true }, + }), + ).to.deep.equal([ + { path: 'info.name', value: 'Relay' }, + { path: 'info.relay_url', value: 'wss://relay.example.com' }, + { path: 'nip66.enabled', value: true }, + ]) + }) + + it('flattens indexed array paths', () => { + expect( + flattenSettingsToPaths({ + nip66: { + targets: ['wss://relay.example.com'], + }, + }), + ).to.deep.equal([{ path: 'nip66.targets[0]', value: 'wss://relay.example.com' }]) + }) +}) diff --git a/test/unit/utils/settings-store.spec.ts b/test/unit/utils/settings-store.spec.ts new file mode 100644 index 000000000..aedd91f5b --- /dev/null +++ b/test/unit/utils/settings-store.spec.ts @@ -0,0 +1,27 @@ +import { expect } from 'chai' + +import { getSettingsBackend } from '../../../src/utils/settings-store' + +describe('settings-store backend selection', () => { + const originalBackend = process.env.SETTINGS_BACKEND + + afterEach(() => { + if (originalBackend === undefined) { + delete process.env.SETTINGS_BACKEND + } else { + process.env.SETTINGS_BACKEND = originalBackend + } + }) + + it('defaults to file backend', () => { + delete process.env.SETTINGS_BACKEND + + expect(getSettingsBackend()).to.equal('file') + }) + + it('selects db backend when SETTINGS_BACKEND=db', () => { + process.env.SETTINGS_BACKEND = 'db' + + expect(getSettingsBackend()).to.equal('db') + }) +}) From a351a517f5887cb5f021d3cf066296e7bb4abf7e Mon Sep 17 00:00:00 2001 From: ABHAY PANDEY Date: Sat, 24 Oct 2026 09:33:38 +0530 Subject: [PATCH 2/2] fix(settings): address db override persistence and reload gaps JSON-encode jsonb values, preserve dotted map keys and empty arrays, reload overrides across workers, route CLI and notifications through the db store, and backup before database replaces. --- src/cli/commands/config.ts | 9 +- .../admin/patch-notifications-controller.ts | 4 +- .../settings-override-repository.ts | 2 +- src/utils/settings-config.ts | 86 +++++++++++++++++-- src/utils/settings-paths.ts | 2 + src/utils/settings-store.ts | 70 ++++++++++++++- .../patch-notifications-controller.spec.ts | 11 +-- .../settings-override-repository.spec.ts | 4 +- test/unit/utils/settings-flatten.spec.ts | 27 ++++++ 9 files changed, 194 insertions(+), 21 deletions(-) diff --git a/src/cli/commands/config.ts b/src/cli/commands/config.ts index c851ff7ce..5d61520a2 100644 --- a/src/cli/commands/config.ts +++ b/src/cli/commands/config.ts @@ -6,7 +6,6 @@ import { loadMergedSettings, loadUserSettings, parseTypedValue, - saveSettings, setByPath, toCategoryLabel, validatePathAgainstDefaults, @@ -21,6 +20,7 @@ import { validateEnvPair, validateEnvValues, } from '../utils/env-config' +import { bootstrapSettingsStore, saveUserSettingsOverrides } from '../../utils/settings-store' import { logError, logInfo } from '../utils/output' import { runStart } from './start' import { runStop } from './stop' @@ -77,6 +77,7 @@ const restartRelay = async (): Promise => { } export const runConfigList = async (options: { json?: boolean } = {}): Promise => { + await bootstrapSettingsStore() const settings = loadMergedSettings() if (options.json) { @@ -89,6 +90,7 @@ export const runConfigList = async (options: { json?: boolean } = {}): Promise => { + await bootstrapSettingsStore() const settings = loadMergedSettings() as unknown as Record const value = getByPath(settings, path) @@ -122,6 +124,8 @@ export const runConfigSet = async ( ): Promise => { const valueType = options.valueType ?? 'inferred' + await bootstrapSettingsStore() + const pathIssues = validatePathAgainstDefaults(path) if (pathIssues.length > 0) { logError(pathIssues[0].message) @@ -146,7 +150,7 @@ export const runConfigSet = async ( } } - saveSettings(next as any) + await saveUserSettingsOverrides(next as any) logInfo(`Updated ${path}`) @@ -158,6 +162,7 @@ export const runConfigSet = async ( } export const runConfigValidate = async (): Promise => { + await bootstrapSettingsStore() const settings = loadMergedSettings() const issues = validateSettings(settings) diff --git a/src/controllers/admin/patch-notifications-controller.ts b/src/controllers/admin/patch-notifications-controller.ts index 2e3545d49..fdfbd83a1 100644 --- a/src/controllers/admin/patch-notifications-controller.ts +++ b/src/controllers/admin/patch-notifications-controller.ts @@ -17,9 +17,9 @@ import { loadDefaults, loadMergedSettings, loadUserSettings, - saveSettings, validateSettings, } from '../../utils/settings-config' +import { saveUserSettingsOverrides } from '../../utils/settings-store' import { validateSchema } from '../../utils/validation' import { recordAdminAuditFromRequest } from '../../utils/record-admin-audit' @@ -55,7 +55,7 @@ export class PatchAdminNotificationsController implements IController { return } - saveSettings(nextUserSettings) + await saveUserSettingsOverrides(nextUserSettings) appendSettingsAuditLog({ action: 'settings.updated', changes: [{ path: 'admin.notifications', reload: 'hot-reload' }], diff --git a/src/repositories/settings-override-repository.ts b/src/repositories/settings-override-repository.ts index 0f4422aed..a52016e20 100644 --- a/src/repositories/settings-override-repository.ts +++ b/src/repositories/settings-override-repository.ts @@ -33,7 +33,7 @@ export class SettingsOverrideRepository { await transaction('settings_overrides').insert( entries.map((entry) => ({ path: entry.path, - value: entry.value, + value: JSON.stringify(entry.value), updated_at: now, })), ) diff --git a/src/utils/settings-config.ts b/src/utils/settings-config.ts index ee8a5fa6f..882b5dd13 100644 --- a/src/utils/settings-config.ts +++ b/src/utils/settings-config.ts @@ -72,6 +72,49 @@ const hasOwn = (object: Record, key: string): boolean => { return Object.prototype.hasOwnProperty.call(object, key) } +const QUOTED_PATH_KEY = /^\["((?:\\.|[^"\\])*)"\]$/ + +export const formatSettingsPathKey = (key: string): string => { + if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) { + return key + } + + return `["${key.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"]` +} + +const tokenizePathSegments = (input: string): string[] => { + const segments: string[] = [] + let index = 0 + + while (index < input.length) { + if (input[index] === '.') { + index += 1 + continue + } + + if (input[index] === '[') { + const closing = input.indexOf('"]', index + 1) + if (closing < 0) { + throw new Error(`Invalid path segment in: ${input}`) + } + + segments.push(input.slice(index, closing + 2)) + index = closing + 2 + continue + } + + let next = index + while (next < input.length && input[next] !== '.') { + next += 1 + } + + segments.push(input.slice(index, next)) + index = next + } + + return segments.filter(Boolean) +} + const parsePath = (path: string): PathToken[] => { const input = path.trim() @@ -80,11 +123,14 @@ const parsePath = (path: string): PathToken[] => { } const tokens: PathToken[] = [] - const segments = input.split('.').map((part) => part.trim()) - for (const segment of segments) { - if (!segment) { - throw new Error(`Invalid path segment in: ${path}`) + for (const segment of tokenizePathSegments(input)) { + const quoted = segment.match(QUOTED_PATH_KEY) + if (quoted) { + const key = quoted[1].replace(/\\"/g, '"').replace(/\\\\/g, '\\') + assertSafePathKey(key, segment) + tokens.push({ type: 'key', key }) + continue } const match = segment.match(/^([A-Za-z_][A-Za-z0-9_]*)(\[(\d+)\])*$/) @@ -114,7 +160,8 @@ const formatPathTokens = (tokens: PathToken[]): string => { for (const token of tokens) { if (token.type === 'key') { - out = out ? `${out}.${token.key}` : token.key + const formatted = formatSettingsPathKey(token.key) + out = out ? `${out}.${formatted}` : formatted continue } @@ -395,6 +442,10 @@ export const flattenSettingsToPaths = ( prefix = '', ): Array<{ path: string; value: unknown }> => { if (Array.isArray(value)) { + if (value.length === 0 && prefix) { + return [{ path: prefix, value: [] }] + } + return value.flatMap((entry, index) => { const nextPrefix = prefix ? `${prefix}[${index}]` : `[${index}]` return flattenSettingsToPaths(entry, nextPrefix) @@ -408,7 +459,12 @@ export const flattenSettingsToPaths = ( } return entries.flatMap(([key, entry]) => { - const nextPrefix = prefix ? `${prefix}.${key}` : key + const keySegment = formatSettingsPathKey(key) + const nextPrefix = prefix + ? keySegment.startsWith('[') + ? `${prefix}${keySegment}` + : `${prefix}.${keySegment}` + : keySegment if (isPlainObject(entry) || Array.isArray(entry)) { return flattenSettingsToPaths(entry, nextPrefix) } @@ -463,7 +519,23 @@ export type SettingsBackupInfo = { } const isSafeBackupFilename = (filename: string): boolean => { - return /^settings\.\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z\.yaml$/.test(filename) + return /^settings(?:-db)?\.\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z\.yaml$/.test(filename) +} + +export const backupUserSettingsOverrides = (overrides: Settings): string => { + ensureSettingsExists() + const backupDirectory = getSettingsBackupDir() + const timestamp = new Date().toISOString().replace(/[:.]/g, '-') + const filename = `settings-db.${timestamp}.yaml` + + fs.mkdirSync(backupDirectory, { recursive: true }) + fs.writeFileSync( + join(backupDirectory, filename), + yaml.dump(toSerializable(overrides), { lineWidth: 120 }), + { encoding: 'utf-8', mode: 0o600 }, + ) + + return filename } export const listSettingsBackups = (): SettingsBackupInfo[] => { diff --git a/src/utils/settings-paths.ts b/src/utils/settings-paths.ts index d55743195..6bc154f75 100644 --- a/src/utils/settings-paths.ts +++ b/src/utils/settings-paths.ts @@ -12,3 +12,5 @@ export const getDefaultSettingsFilePath = (): string => join(process.cwd(), 'res export const getSettingsBackupDir = (): string => join(getConfigBaseDir(), 'backups') export const getSettingsAuditLogPath = (): string => join(getConfigBaseDir(), 'settings-audit.jsonl') + +export const getSettingsDbImportMarkerPath = (): string => join(getConfigBaseDir(), '.settings-db-import-complete') diff --git a/src/utils/settings-store.ts b/src/utils/settings-store.ts index 5767a7350..efb6ab846 100644 --- a/src/utils/settings-store.ts +++ b/src/utils/settings-store.ts @@ -1,14 +1,26 @@ +import fs from 'fs' + +import cluster from 'cluster' + import { Settings } from '../@types/settings' import { getMasterDbClient } from '../database/client' import { createLogger } from '../factories/logger-factory' import { SettingsOverrideRepository } from '../repositories/settings-override-repository' import { SettingsStatic } from './settings' -import { loadUserSettingsFromFiles, saveSettings } from './settings-config' +import { + backupUserSettingsOverrides, + loadUserSettingsFromFiles, + saveSettings, +} from './settings-config' +import { getSettingsDbImportMarkerPath } from './settings-paths' const logger = createLogger('settings-store') +const SETTINGS_OVERRIDES_RELOAD_EVENT = 'settings-overrides-reload' + let dbOverridesCache: Settings = {} as Settings let bootstrapComplete = false +let reloadListenerRegistered = false export type SettingsBackend = 'file' | 'db' @@ -22,7 +34,54 @@ const isEmptyOverrides = (settings: Settings): boolean => { return Object.keys(settings as object).length === 0 } +const markSettingsFileImportComplete = (): void => { + fs.writeFileSync(getSettingsDbImportMarkerPath(), `${new Date().toISOString()}\n`, { encoding: 'utf-8', mode: 0o600 }) +} + +const hasCompletedSettingsFileImport = (): boolean => { + return fs.existsSync(getSettingsDbImportMarkerPath()) +} + +export const reloadDbOverridesFromDatabase = async (): Promise => { + if (getSettingsBackend() !== 'db') { + return + } + + const repository = new SettingsOverrideRepository(getMasterDbClient()) + dbOverridesCache = await repository.loadOverrides() + SettingsStatic.invalidateCache() +} + +const registerSettingsOverridesReloadListener = (): void => { + if (reloadListenerRegistered || getSettingsBackend() !== 'db') { + return + } + + reloadListenerRegistered = true + process.on('message', (message: { eventName?: string }) => { + if (message?.eventName !== SETTINGS_OVERRIDES_RELOAD_EVENT) { + return + } + + void reloadDbOverridesFromDatabase().catch((error) => { + logger.error('failed to reload database settings overrides: %o', error) + }) + }) +} + +const broadcastSettingsOverridesReload = (): void => { + if (getSettingsBackend() !== 'db' || !cluster.isPrimary || !cluster.workers) { + return + } + + for (const worker of Object.values(cluster.workers)) { + worker?.send({ eventName: SETTINGS_OVERRIDES_RELOAD_EVENT }) + } +} + export const bootstrapSettingsStore = async (): Promise => { + registerSettingsOverridesReloadListener() + if (getSettingsBackend() !== 'db') { bootstrapComplete = true return @@ -31,13 +90,14 @@ export const bootstrapSettingsStore = async (): Promise => { const repository = new SettingsOverrideRepository(getMasterDbClient()) let overrides = await repository.loadOverrides() - if (isEmptyOverrides(overrides)) { + if (!hasCompletedSettingsFileImport() && isEmptyOverrides(overrides)) { const fileOverrides = loadUserSettingsFromFiles() if (!isEmptyOverrides(fileOverrides)) { logger.info('importing settings overrides from file into database') await repository.replaceOverrides(fileOverrides) overrides = fileOverrides } + markSettingsFileImportComplete() } dbOverridesCache = overrides @@ -52,9 +112,15 @@ export const saveUserSettingsOverrides = async (settings: Settings): Promise { sandbox.stub(settingsConfig, 'loadMergedSettings').returns(baseMergedSettings as any) sandbox.stub(settingsConfig, 'loadUserSettings').returns({ admin: { notifications: baseNotifications } } as any) - sandbox.stub(settingsConfig, 'saveSettings') + sandbox.stub(settingsStore, 'saveUserSettingsOverrides').resolves() sandbox.stub(settingsConfig, 'appendSettingsAuditLog') sandbox.stub(adminNotificationsSettings, 'getRedactedAdminNotifications').returns({ ...baseNotifications, @@ -93,7 +94,7 @@ describe('PatchAdminNotificationsController', () => { expect(response.status.calledOnceWithExactly(400)).to.equal(true) expect(response.send.firstCall.args[0].error).to.equal('Validation failed') expect(response.send.firstCall.args[0].issues).to.be.an('array').that.is.not.empty - expect(settingsConfig.saveSettings).to.not.have.been.called + expect(settingsStore.saveUserSettingsOverrides).to.not.have.been.called expect(outboxRepository.enqueue).to.not.have.been.called }) @@ -118,8 +119,8 @@ describe('PatchAdminNotificationsController', () => { ) expect(response.status.calledOnceWithExactly(200)).to.equal(true) - expect(settingsConfig.saveSettings).to.have.been.calledOnce - const saved = (settingsConfig.saveSettings as Sinon.SinonStub).firstCall.args[0] + expect(settingsStore.saveUserSettingsOverrides).to.have.been.calledOnce + const saved = (settingsStore.saveUserSettingsOverrides as Sinon.SinonStub).firstCall.args[0] expect(saved.admin.notifications.enabled).to.equal(true) expect(saved.admin.notifications.targets[0].enabled).to.equal(false) expect(saved.admin.notifications.targets[0].url).to.equal('https://discord.com/api/webhooks/secret') @@ -137,6 +138,6 @@ describe('PatchAdminNotificationsController', () => { await controller.handleRequest({ body: { enabled: true }, ip: '127.0.0.1' } as any, response as any) expect(response.status.calledOnceWithExactly(200)).to.equal(true) - expect(settingsConfig.saveSettings).to.have.been.calledOnce + expect(settingsStore.saveUserSettingsOverrides).to.have.been.calledOnce }) }) diff --git a/test/unit/repositories/settings-override-repository.spec.ts b/test/unit/repositories/settings-override-repository.spec.ts index 2d5cd1dfd..dddad30e7 100644 --- a/test/unit/repositories/settings-override-repository.spec.ts +++ b/test/unit/repositories/settings-override-repository.spec.ts @@ -77,12 +77,12 @@ describe('SettingsOverrideRepository', () => { expect(insertedRows).to.have.length(2) expect(insertedRows[0]).to.deep.include({ path: 'info.name', - value: 'Updated Relay', + value: JSON.stringify('Updated Relay'), }) expect(insertedRows[0].updated_at).to.be.instanceOf(Date) expect(insertedRows[1]).to.deep.include({ path: 'admin.enabled', - value: true, + value: JSON.stringify(true), }) expect(insertedRows[1].updated_at).to.be.instanceOf(Date) }) diff --git a/test/unit/utils/settings-flatten.spec.ts b/test/unit/utils/settings-flatten.spec.ts index 3994a4584..88435deab 100644 --- a/test/unit/utils/settings-flatten.spec.ts +++ b/test/unit/utils/settings-flatten.spec.ts @@ -16,6 +16,33 @@ describe('flattenSettingsToPaths', () => { ]) }) + it('preserves notification event keys that contain dots', () => { + expect( + flattenSettingsToPaths({ + admin: { + notifications: { + events: { 'settings.changed': true, 'mirror.disconnected': false }, + }, + }, + }), + ).to.deep.equal([ + { path: 'admin.notifications.events["settings.changed"]', value: true }, + { path: 'admin.notifications.events["mirror.disconnected"]', value: false }, + ]) + }) + + it('stores empty arrays as explicit override rows', () => { + expect( + flattenSettingsToPaths({ + limits: { + message: { + ipWhitelist: [], + }, + }, + }), + ).to.deep.equal([{ path: 'limits.message.ipWhitelist', value: [] }]) + }) + it('flattens indexed array paths', () => { expect( flattenSettingsToPaths({