From ba68343d894ff1994e50716372f093b8cebe0248 Mon Sep 17 00:00:00 2001 From: mameikagou Date: Wed, 9 Sep 2026 13:44:17 +0800 Subject: [PATCH 1/3] fix: expose legacy migration with configured embeddings --- docker-compose.yml | 6 ++++ src/cli/cli_app.ts | 4 ++- src/cli/commands/migrate.ts | 43 +++++++++++++++++++++++++++++ src/core/embeddings/environment.ts | 8 ++++-- src/core/migration/legacy_mapper.ts | 7 +++-- 5 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 src/cli/commands/migrate.ts diff --git a/docker-compose.yml b/docker-compose.yml index 43398f59..f250c086 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,13 @@ services: LONGMEMORY_RATE_LIMIT_MAX_REQUESTS: ${LONGMEMORY_RATE_LIMIT_MAX_REQUESTS:-100} LONGMEMORY_EMBEDDING_PROVIDER: ${LONGMEMORY_EMBEDDING_PROVIDER:-} LONGMEMORY_EMBEDDING_TIER: ${LONGMEMORY_EMBEDDING_TIER:-deep} + LONGMEMORY_EMBEDDING_FALLBACK: ${LONGMEMORY_EMBEDDING_FALLBACK:-synthetic} LONGMEMORY_EMBEDDING_DIMENSION: ${LONGMEMORY_EMBEDDING_DIMENSION:-1536} + LONGMEMORY_EMBEDDING_TIMEOUT_MS: ${LONGMEMORY_EMBEDDING_TIMEOUT_MS:-30000} + LONGMEMORY_EMBEDDING_MAX_RETRIES: ${LONGMEMORY_EMBEDDING_MAX_RETRIES:-2} + LONGMEMORY_EMBEDDING_RETRY_BASE_MS: ${LONGMEMORY_EMBEDDING_RETRY_BASE_MS:-250} + LONGMEMORY_OPENAI_BASE_URL: ${LONGMEMORY_OPENAI_BASE_URL:-https://api.openai.com/v1} + LONGMEMORY_OPENAI_EMBEDDING_MODEL: ${LONGMEMORY_OPENAI_EMBEDDING_MODEL:-text-embedding-3-small} LONGMEMORY_OLLAMA_URL: ${LONGMEMORY_OLLAMA_URL:-http://host.docker.internal:11434} LONGMEMORY_OLLAMA_EMBEDDING_MODEL: ${LONGMEMORY_OLLAMA_EMBEDDING_MODEL:-nomic-embed-text} OPENAI_API_KEY: ${OPENAI_API_KEY:-} diff --git a/src/cli/cli_app.ts b/src/cli/cli_app.ts index f0850bba..d5f3129b 100644 --- a/src/cli/cli_app.ts +++ b/src/cli/cli_app.ts @@ -28,6 +28,7 @@ const commands = new Map([ ['doctor', async () => (await import('./commands/doctor.js')).doctor_command], ['serve', async () => (await import('./commands/serve.js')).serve_command], ['mcp', async () => (await import('./commands/mcp.js')).mcp_command], + ['migrate', async () => (await import('./commands/migrate.js')).migrate_command], ['ingest', async () => (await import('./commands/ingest.js')).ingest_command], ['recall', async () => (await import('./commands/recall.js')).recall_command], ['explain', async () => (await import('./commands/explain.js')).explain_command], @@ -82,6 +83,7 @@ const help = { global_flags: ['--db ', '--project ', '--user ', '--json', '--jsonl', '--pretty', '--compact', '--no-color', '--silent', '--interactive', '--dry-run', '--token-budget ', '--cwd '], commands: [ 'status', 'init', 'doctor', 'serve [--host ] [--port ] [--mcp-http]', 'mcp [--read-only]', + 'migrate --from --to [--report ]', 'ingest "memory" [--stdin] [--type ] [--source ]', 'recall "query" [--mode ]', 'explain ', 'timeline ', 'memory list [--limit ] [--status ]', 'maintenance decay [--limit ] [--all]', 'maintenance reinforce ', @@ -135,4 +137,4 @@ export async function run_cli_app(argv = process.argv.slice(2), env: NodeJS.Proc } export const registered_commands = () => [...commands.keys()]; -export const register_cli_command = (name: string, command: cli_command) => commands.set(name, async () => command); \ No newline at end of file +export const register_cli_command = (name: string, command: cli_command) => commands.set(name, async () => command); diff --git a/src/cli/commands/migrate.ts b/src/cli/commands/migrate.ts new file mode 100644 index 00000000..40b4929c --- /dev/null +++ b/src/cli/commands/migrate.ts @@ -0,0 +1,43 @@ +/* +* __ __ ___ +* / / ____ ____ ____ _/ |/ /__ ____ ___ ____ _______ __ +* / / / __ \/ __ \/ __ `/ /|_/ / _ \/ __ `__ \/ __ \/ ___/ / / / +* / /___/ /_/ / / / / /_/ / / / / __/ / / / / / /_/ / / / /_/ / +* /_____/\____/_/ /_/\__, /_/ /_/\___/_/ /_/ /_/\____/_/ \__, / + /____/ /____/ + * + * cavira oss (c) 2026 - nullure (c) 2026 + * ---------------------------------------------------------- + * file : src/cli/commands/migrate.ts + * usage : implements the LongMemory legacy migration command + */ + + +import type { memory_config as longmemory_config } from '../../core/create_memory.js'; +import { migrate_legacy } from '../../core/migration/legacy_mapper.js'; +import { write_migration_report } from '../../core/migration/migration_report.js'; +import type { cli_command } from '../context/cli_context.js'; +import { command_flags, flag, memory_config, require_value } from '../context/cli_context.js'; +import { emit } from '../output/pretty.js'; +import { panel } from '../output/panel.js'; + +export const migrate_command: cli_command = async (context) => { + command_flags(context, ['from', 'to', 'report']); + const from = require_value(flag(context, 'from'), '--from'); + const to = require_value(flag(context, 'to'), '--to'); + const configured = memory_config(context); + const migration_config: Omit = {}; + if (configured.embedding_provider) migration_config.embedding_provider = configured.embedding_provider; + if (configured.embedding_dimension !== undefined) migration_config.embedding_dimension = configured.embedding_dimension; + if (configured.multilingual_embedding_provider) migration_config.multilingual_embedding_provider = configured.multilingual_embedding_provider; + const report = await migrate_legacy({ from, to, memory_config: migration_config }); + const report_path = write_migration_report(report, flag(context, 'report') ?? `${to}.migration-report.json`); + const result = { ok: report.benchmark_result.passed && report.errors.length === 0, report_path, ...report }; + emit(context, result, () => panel('', context.colors, { + title: 'Legacy migration', kind: result.ok ? 'success' : 'warning', width: context.terminal_width, rows: [ + ['Source', report.source_path], ['Destination', report.destination_path], ['Imported nodes', report.imported_nodes], + ['Imported edges', report.imported_edges], ['Skipped records', report.skipped_records.length], + ['Errors', report.errors.length], ['Integrity benchmark', report.benchmark_result.passed], ['Report', report_path], + ], + })); +}; diff --git a/src/core/embeddings/environment.ts b/src/core/embeddings/environment.ts index b8349134..c4dc3773 100644 --- a/src/core/embeddings/environment.ts +++ b/src/core/embeddings/environment.ts @@ -19,6 +19,10 @@ import { create_embedding_stack } from './stack.js'; const providers = new Set(['openai', 'gemini', 'aws', 'ollama', 'local', 'siray', 'synthetic']); const tiers = new Set(['fast', 'smart', 'deep', 'hybrid']); const value = (env: NodeJS.ProcessEnv, ...keys: string[]) => keys.map((key) => env[key]?.trim()).find(Boolean); +const explicit_value = (env: NodeJS.ProcessEnv, ...keys: string[]) => { + const key = keys.find((candidate) => Object.prototype.hasOwnProperty.call(env, candidate)); + return key === undefined ? undefined : env[key]?.trim(); +}; const number_value = (env: NodeJS.ProcessEnv, keys: string[], fallback: number, min = 0) => { const raw = value(env, ...keys); if (!raw) return fallback; @@ -42,7 +46,7 @@ export function load_embedding_environment(env: NodeJS.ProcessEnv = process.env) const default_dimension = tier === 'smart' ? 384 : tier === 'deep' ? 1536 : 256; return { provider: provider(selected, 'synthetic'), - fallback: (value(env, 'LONGMEMORY_EMBEDDING_FALLBACK', 'OM_EMBEDDING_FALLBACK') ?? 'synthetic').split(',').map((name) => provider(name.trim(), 'synthetic')), + fallback: (explicit_value(env, 'LONGMEMORY_EMBEDDING_FALLBACK', 'OM_EMBEDDING_FALLBACK') ?? 'synthetic').split(',').map((name) => name.trim()).filter(Boolean).map((name) => provider(name, 'synthetic')), tier, dimension: number_value(env, ['LONGMEMORY_EMBEDDING_DIMENSION', 'OM_VEC_DIM', 'OM_MAX_VECTOR_DIM'], default_dimension, 1), timeout_ms: number_value(env, ['LONGMEMORY_EMBEDDING_TIMEOUT_MS', 'OM_EMBED_TIMEOUT_MS'], 30_000, 1), @@ -75,4 +79,4 @@ export function create_embedding_environment(env: NodeJS.ProcessEnv = process.en embed: (text: string, language?: string) => embedding_provider.embed(text, { language, purpose: 'document' }), }; return { config, embedding_provider, multilingual_embedding_provider, embedding_dimension: config.dimension }; -} \ No newline at end of file +} diff --git a/src/core/migration/legacy_mapper.ts b/src/core/migration/legacy_mapper.ts index 3bef9392..54f2e05f 100644 --- a/src/core/migration/legacy_mapper.ts +++ b/src/core/migration/legacy_mapper.ts @@ -16,7 +16,7 @@ import { existsSync, mkdirSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import Database from 'better-sqlite3'; -import { create_memory } from '../create_memory.js'; +import { create_memory, type memory_config } from '../create_memory.js'; import { create_hydro_edge } from '../memory/durable_graph.js'; import { manual_provenance } from '../types/provenance.js'; import { SqliteStore } from '../../stores/sqlite/sqlite_store.js'; @@ -28,6 +28,7 @@ export type legacy_migration_options = { from: string; to: string; overwrite?: boolean; + memory_config?: Omit; }; const supported_relations = new Set(['contains', 'refers_to', 'same_as', 'supports', 'contradicts', 'supersedes', 'derived_from', 'grounds', 'semantic_shift']); @@ -120,7 +121,7 @@ export async function migrate_legacy(options: legacy_migration_options): Promise const skipped = [...clean.skipped]; const errors = [...clean.errors]; let contradictions_found = 0; - const memory = create_memory({ store: 'sqlite', db_path: to, enable_consolidation: true }); + const memory = create_memory({ ...options.memory_config, store: 'sqlite', db_path: to, enable_consolidation: true }); try { for (const item of clean.records) { try { @@ -211,4 +212,4 @@ export async function migrate_legacy(options: legacy_migration_options): Promise benchmark_result: await benchmark_migration(to, [...imported_node_ids]), }; return report; -} \ No newline at end of file +} From f152f22f5d6ec506d615f3d38735fcbd23dd864a Mon Sep 17 00:00:00 2001 From: mameikagou Date: Wed, 9 Sep 2026 13:54:23 +0800 Subject: [PATCH 2/3] fix: keep migration verification read-only --- src/core/migration/legacy_mapper.ts | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/core/migration/legacy_mapper.ts b/src/core/migration/legacy_mapper.ts index 54f2e05f..37b79e67 100644 --- a/src/core/migration/legacy_mapper.ts +++ b/src/core/migration/legacy_mapper.ts @@ -49,11 +49,15 @@ const relation_type = (value: string) => { return supported_relations.has(mapped) ? mapped : 'refers_to'; }; -async function benchmark_migration(path: string, imported_node_ids: string[]): Promise { +async function benchmark_migration( + path: string, + imported_node_ids: string[], + memory_config?: Omit, +): Promise { const store = new SqliteStore(path, { startup_integrity_check: true }); const integrity = store.check_integrity(); store.close(); - const memory = create_memory({ store: 'sqlite', db_path: path }); + const memory = create_memory({ ...memory_config, store: 'sqlite', db_path: path, readonly: true }); try { const stats = await memory.getStats(); const hydration = imported_node_ids.length === 0 || (await memory.explain(imported_node_ids[0])).node !== null; @@ -68,10 +72,15 @@ async function benchmark_migration(path: string, imported_node_ids: string[]): P } } -async function copy_hydrograph(from: string, to: string, started_at: number): Promise { +async function copy_hydrograph( + from: string, + to: string, + started_at: number, + memory_config?: Omit, +): Promise { const source = new Database(from, { readonly: true, fileMustExist: true }); try { await source.backup(to); } finally { source.close(); } - const memory = create_memory({ store: 'sqlite', db_path: to }); + const memory = create_memory({ ...memory_config, store: 'sqlite', db_path: to, readonly: true }); const stats = await memory.getStats(); await memory.close(); const store = new SqliteStore(to); @@ -98,7 +107,7 @@ async function copy_hydrograph(from: string, to: string, started_at: number): Pr contradictions_found: 0, skipped_records: [], errors: [], - benchmark_result: await benchmark_migration(to, node_ids), + benchmark_result: await benchmark_migration(to, node_ids, memory_config), }; } @@ -111,7 +120,7 @@ export async function migrate_legacy(options: legacy_migration_options): Promise if (existsSync(to) && !options.overwrite) throw new Error(`migration destination already exists: ${to}`); mkdirSync(dirname(to), { recursive: true }); const read = read_legacy_source(from); - if (read.format === 'hydrograph') return copy_hydrograph(from, to, started_at); + if (read.format === 'hydrograph') return copy_hydrograph(from, to, started_at, options.memory_config); const clean = clean_legacy_data(read); const imported_node_ids = new Set(); const imported_edge_ids = new Set(); @@ -209,7 +218,7 @@ export async function migrate_legacy(options: legacy_migration_options): Promise contradictions_found, skipped_records: skipped, errors, - benchmark_result: await benchmark_migration(to, [...imported_node_ids]), + benchmark_result: await benchmark_migration(to, [...imported_node_ids], options.memory_config), }; return report; } From d70fcfb5df6806a7f16b806ff2d88138966329af Mon Sep 17 00:00:00 2001 From: mameikagou Date: Wed, 9 Sep 2026 14:13:26 +0800 Subject: [PATCH 3/3] fix: migrate legacy worlds into project hierarchy --- src/core/migration/legacy_mapper.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/core/migration/legacy_mapper.ts b/src/core/migration/legacy_mapper.ts index 37b79e67..0811d6da 100644 --- a/src/core/migration/legacy_mapper.ts +++ b/src/core/migration/legacy_mapper.ts @@ -18,6 +18,7 @@ import { dirname, resolve } from 'node:path'; import Database from 'better-sqlite3'; import { create_memory, type memory_config } from '../create_memory.js'; import { create_hydro_edge } from '../memory/durable_graph.js'; +import { project_memory } from '../project/project_memory.js'; import { manual_provenance } from '../types/provenance.js'; import { SqliteStore } from '../../stores/sqlite/sqlite_store.js'; import { clean_legacy_data } from './legacy_cleaner.js'; @@ -132,6 +133,12 @@ export async function migrate_legacy(options: legacy_migration_options): Promise let contradictions_found = 0; const memory = create_memory({ ...options.memory_config, store: 'sqlite', db_path: to, enable_consolidation: true }); try { + const projects = new project_memory({ memory, tenant_id: 'default', project_id: 'legacy-migration', name: 'Legacy migration' }); + const document_worlds = new Map(); + for (const project_id of new Set(clean.records.map((item) => item.world))) { + const project = await projects.createProject({ tenant_id: 'default', project_id, name: project_id }); + document_worlds.set(project_id, project.world_ids.documents); + } for (const item of clean.records) { try { const result = await memory.ingest({ @@ -142,13 +149,13 @@ export async function migrate_legacy(options: legacy_migration_options): Promise observed_at: item.observed_at, valid_from: item.valid_from, valid_to: item.valid_to, - world: item.world, + world_id: document_worlds.get(item.world), tags: item.tags, facet_hint: item.facet, external: item.source !== null, source: item.source ?? undefined, contract: item.source ? undefined : { requires_grounding: false, source_required: false }, - metadata: item.metadata, + metadata: { ...item.metadata, project_id: item.world, canonical_project: item.world }, }); node_by_source.set(item.source_id, result.node.id); result.diff.created_node_ids.forEach((id) => imported_node_ids.add(id));