diff --git a/app/scripts/nmr-cli/Dockerfile b/app/scripts/nmr-cli/Dockerfile index f7877d5..45b9abf 100644 --- a/app/scripts/nmr-cli/Dockerfile +++ b/app/scripts/nmr-cli/Dockerfile @@ -32,6 +32,9 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +# Downgrade to Node 22 for nmr-processing compatibility +RUN npm install -g n && n 22 && hash -r + COPY package.json ./ RUN npm install @@ -40,9 +43,11 @@ COPY . ./ RUN npm run build +# Install nmr-cli as a global package +RUN npm install -g . -# Install the nmr-cli as a global package +# Usage examples (run after `docker run -it nmr-cli bash`): +# nmr-cli parse-spectra -u https://s3.uni-jena.de/nmrxiv/production/archive/1368b8fd-3af7-48f7-8b6d-13de9c1ecf8b/492-62-6.zip -s # For example, nmr-cli parse-spectra -u https://s3.uni-jena.de/nmrxiv/production/archive/d5ec45db-d984-46e9-bb72-c43fd4a463fa/Sali_Eth.zip -s # For example, nmr-cli parse-spectra -u https://cheminfo.github.io/bruker-data-test/data/zipped/aspirin-1h.zip # nmr-cli predict -n "1H" --id 1 --type "nmr;1H;1d" --shifts "1" --solvent "Dimethylsulphoxide-D6 (DMSO-D6, C2D6SO)" -m $"\n Ketcher 6122516162D 1 1.00000 0.00000 0\n\n 16 17 0 0 0 0 0 0 0 0999 V2000\n 1.1954 -4.6484 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 2.9258 -4.6479 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 2.0622 -4.1483 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 2.9258 -5.6488 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 1.1954 -5.6533 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 2.0644 -6.1483 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 3.7902 -4.1495 0.0000 N 0 0 0 0 0 0 0 0 0 0 0 0\n 4.6574 -4.6498 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 3.7964 -6.1512 0.0000 N 0 0 0 0 0 0 0 0 0 0 0 0\n 4.6596 -5.6458 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 5.5228 -4.1488 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 5.5277 -6.1421 0.0000 O 0 0 0 0 0 0 0 0 0 0 0 0\n 6.3895 -4.6477 0.0000 O 0 0 0 0 0 0 0 0 0 0 0 0\n 5.5216 -3.1488 0.0000 O 0 0 0 0 0 0 0 0 0 0 0 0\n 7.2548 -4.1466 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 8.1215 -4.6455 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 3 1 2 0 0 0 0\n 1 5 1 0 0 0 0\n 5 6 2 0 0 0 0\n 6 4 1 0 0 0 0\n 4 2 1 0 0 0 0\n 2 3 1 0 0 0 0\n 4 9 1 0 0 0 0\n 9 10 2 0 0 0 0\n 10 8 1 0 0 0 0\n 8 7 2 0 0 0 0\n 7 2 1 0 0 0 0\n 8 11 1 0 0 0 0\n 10 12 1 0 0 0 0\n 11 13 1 0 0 0 0\n 11 14 2 0 0 0 0\n 13 15 1 0 0 0 0\n 15 16 1 0 0 0 0\nM END" -RUN npm install . -g diff --git a/app/scripts/nmr-cli/package-lock.json b/app/scripts/nmr-cli/package-lock.json index 6f47fd6..3ae8706 100644 --- a/app/scripts/nmr-cli/package-lock.json +++ b/app/scripts/nmr-cli/package-lock.json @@ -29,7 +29,7 @@ }, "devDependencies": { "@types/lodash.merge": "^4.6.9", - "@types/node": "^26.2.0", + "@types/node": "^26.4.0", "@types/yargs": "^17.0.35", "ts-node": "^10.9.2", "typescript": "^5.9.3" @@ -128,9 +128,9 @@ } }, "node_modules/@types/node": { - "version": "26.2.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.2.0.tgz", - "integrity": "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==", + "version": "26.4.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.4.0.tgz", + "integrity": "sha512-faiGnoIrLH/V8cibOMEAZ8pMw6oXqSukl29ra4mN8GdaB2ZewzeaLj+INpV5N+Z1eKWzY+IzaIZH2EIR6YZRNQ==", "dev": true, "license": "MIT", "dependencies": { diff --git a/app/scripts/nmr-cli/package.json b/app/scripts/nmr-cli/package.json index c795b3a..09dafd2 100644 --- a/app/scripts/nmr-cli/package.json +++ b/app/scripts/nmr-cli/package.json @@ -32,7 +32,7 @@ }, "devDependencies": { "@types/lodash.merge": "^4.6.9", - "@types/node": "^26.2.0", + "@types/node": "^26.4.0", "@types/yargs": "^17.0.35", "ts-node": "^10.9.2", "typescript": "^5.9.3" diff --git a/app/scripts/nmr-cli/src/parse/browser-manager.ts b/app/scripts/nmr-cli/src/parse/browser-manager.ts new file mode 100644 index 0000000..4a070c4 --- /dev/null +++ b/app/scripts/nmr-cli/src/parse/browser-manager.ts @@ -0,0 +1,163 @@ +import playwright from 'playwright'; +import type { FifoLogger } from 'fifo-logger'; +import type { SpectrumWorker } from './spectrum-worker'; +import { Spectrum } from '@zakodium/nmr-types'; +import { Snapshot } from './spectrum-snapshot'; +import { toMessage } from './utility/toMessage'; + +// How long to wait for the NMRium "Loading" indicator to appear/disappear +// before giving up on a single spectrum's snapshot. Prevents one stuck page +// from hanging the entire pipeline. +export const SNAPSHOT_LOADING_TIMEOUT_MS = 30_000; + +// A single flaky failure (a slow spectrum tipping over the loading timeout, +// a one-off page hiccup) shouldn't permanently cost a spectrum its snapshot, +// so each capture gets one retry on a fresh page before giving up. +export const SNAPSHOT_MAX_ATTEMPTS = 2; + +// Owns the single shared Firefox process. Multiple lanes each get their own +// context/page from it. `reset` is guarded so that if several lanes hit a +// dead browser at once, only the first one actually relaunches — the rest +// just see `current` has already changed and pick up the fresh instance. +export class BrowserManager { + private current: Promise | null = null; + + async get(): Promise { + if (!this.current) { + this.current = playwright.firefox.launch(); + } + return this.current; + } + + async reset(stale: playwright.Browser): Promise { + if (this.current && (await this.current) === stale) { + const toClose = this.current; + this.current = null; + await (await toClose).close().catch(() => { }); + } + } + + async closeAll(): Promise { + if (this.current) { + const toClose = this.current; + this.current = null; + await (await toClose).close().catch(() => { }); + } + } +} + +// One snapshot lane = one browser tab, reused across every spectrum it's +// assigned. Launching the browser/context happens once per lane; every +// spectrum after the first gets a `page.reload()` (not a new context or +// browser) before its `nmr-wrapper:load` message, so each snapshot starts +// from a genuinely empty NMRium instance instead of relying on `load` +// merging vs. replacing the previous spectrum's state. A reload of an +// already-booted SPA is far cheaper than relaunching the browser/context, +// so this keeps the speed win while removing the state-leak risk. +export class SnapshotLane { + private context: playwright.BrowserContext | null = null; + private page: playwright.Page | null = null; + private hasLoadedSpectrum = false; + + constructor(private manager: BrowserManager, private url: string) { } + + private async ensurePage(): Promise { + if (this.page && !this.page.isClosed()) { + if (this.hasLoadedSpectrum) { + await this.page.reload(); + await this.page.locator('text=Loading').waitFor({ state: 'hidden', timeout: SNAPSHOT_LOADING_TIMEOUT_MS }); + } + return this.page; + } + const browser = await this.manager.get(); + this.context = await browser.newContext(playwright.devices['Desktop Chrome HiDPI']); + this.page = await this.context.newPage(); + await this.page.goto(this.url); + await this.page.locator('text=Loading').waitFor({ state: 'hidden', timeout: SNAPSHOT_LOADING_TIMEOUT_MS }); + this.hasLoadedSpectrum = false; + return this.page; + } + + // Discards this lane's page/context so the next attempt (or the next + // spectrum, if we're giving up) starts from a clean page instead of + // whatever broken state caused the failure. Only escalates to a full + // browser relaunch if the browser process itself is gone. + private async recover(): Promise { + await this.context?.close().catch(() => { }); + this.context = null; + this.page = null; + this.hasLoadedSpectrum = false; + + const browser = await this.manager.get(); + if (!browser.isConnected()) { + await this.manager.reset(browser); + } + } + + private async attemptCapture( + spectrum: Spectrum, + version: unknown, + spectrumWorker: SpectrumWorker + ): Promise { + const page = await this.ensurePage(); + + const stringObject = await spectrumWorker.run('serialize', spectrum, version); + + // Passed as a Playwright function argument rather than spliced into an + // evaluated script string, so a backtick or `${...}` sequence anywhere + // in the spectrum data can't break (or hijack) the script. + await page.evaluate( + ({ data }) => { + window.postMessage({ type: 'nmr-wrapper:load', data: { data, type: 'nmrium' } }, '*'); + }, + { data: JSON.parse(stringObject) } + ); + + await page.locator('text=Loading').waitFor({ state: 'hidden', timeout: SNAPSHOT_LOADING_TIMEOUT_MS }); + + const snapshot = await page.locator('#nmrSVG .container').screenshot(); + this.hasLoadedSpectrum = true; + return snapshot.toString('base64'); + } + + async capture( + spectrum: Spectrum, + id: string, + version: unknown, + spectrumWorker: SpectrumWorker, + logger: FifoLogger + ): Promise { + let lastError: unknown; + + for (let attempt = 1; attempt <= SNAPSHOT_MAX_ATTEMPTS; attempt++) { + const start = Date.now(); + try { + const image = await this.attemptCapture(spectrum, version, spectrumWorker); + logger.info( + { id, stage: 'snapshot', attempt, durationMs: Date.now() - start }, + `Captured snapshot for spectrum: ${id}` + ); + return { id, image }; + } catch (e) { + lastError = e; + logger.error( + { id, stage: 'snapshot', attempt, durationMs: Date.now() - start, details: toMessage(e) }, + `Snapshot attempt ${attempt}/${SNAPSHOT_MAX_ATTEMPTS} failed for spectrum: ${id}` + ); + await this.recover(); + } + } + + logger.error( + { id, stage: 'snapshot', details: toMessage(lastError) }, + `Giving up on snapshot for spectrum: ${id} after ${SNAPSHOT_MAX_ATTEMPTS} attempts` + ); + return { id, image: null }; + } + + async dispose(): Promise { + await this.context?.close().catch(() => { }); + this.context = null; + this.page = null; + } +} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/prase-spectra.ts b/app/scripts/nmr-cli/src/parse/prase-spectra.ts index d4d24c4..011b2bd 100644 --- a/app/scripts/nmr-cli/src/parse/prase-spectra.ts +++ b/app/scripts/nmr-cli/src/parse/prase-spectra.ts @@ -1,166 +1,26 @@ -import { join, isAbsolute } from 'path' -import { NmriumData, ParsingOptions, type NmriumState } from '@zakodium/nmrium-core' -import init from '@zakodium/nmrium-core-plugins' -import playwright from 'playwright' -import { FileCollection } from 'file-collection' -import { FileOptionsArgs } from '..' -import { isSpectrum2D } from './data/data2d/isSpectrum2D' -import { initiateDatum2D } from './data/data2d/initiateDatum2D' -import { initiateDatum1D } from './data/data1D/initiateDatum1D' -import { detectZones } from './data/data2d/detectZones' -import { detectRanges } from './data/data1D/detectRanges' -import { Filters1DManager, Filters2DManager } from 'nmr-processing' -import yargs from 'yargs' -import { createWriteStream } from 'fs' -import { JsonStreamStringify } from 'json-stream-stringify'; -import { FifoLogger } from 'fifo-logger' +import { join, isAbsolute } from 'node:path'; +import type { ParsingOptions, NmriumState } from '@zakodium/nmrium-core'; +import init from '@zakodium/nmrium-core-plugins'; +import { FileCollection } from 'file-collection'; +import yargs from 'yargs'; +import { FifoLogger } from 'fifo-logger'; +import { FileOptionsArgs } from '..'; +import { runSpectraPipeline } from './run-pipeline'; +import { outputResult } from './utility/outputResult'; +import { toMessage } from './utility/toMessage'; -type RequiredKey = Omit & Required>; - -function toMessage(e: unknown): string { - return e instanceof Error ? e.message : String(e) -} - - -const parsingOptions: ParsingOptions = { - onLoadProcessing: { autoProcessing: true }, - selector: { general: { dataSelection: 'preferFT' } }, - experimentalFeatures: true, -}; - - -interface Snapshot { - id: string; - image: string | null; -} - -const core = init() - -function generateNMRiumURL() { - const baseURL = process.env['BASE_NMRIUM_URL'] || '' - const url = new URL(baseURL) - url.searchParams.append('workspace', 'embedded') - return url.toString() -} - -async function launchBrowser() { - return playwright.firefox.launch(); -} - -async function captureSpectraViewAsBase64(nmriumState: Partial, logger: FifoLogger): Promise { - const { data: { spectra } = { spectra: [] }, version } = nmriumState; - - if (!spectra?.length) return []; - - const url = generateNMRiumURL(); - const snapshots: Snapshot[] = []; - let browser = await launchBrowser(); - - for (const spectrum of spectra) { - let context = null; - - try { - // recreate browser if it has crashed - if (!browser.isConnected()) { - browser = await launchBrowser(); - } - - context = await browser.newContext(playwright.devices['Desktop Chrome HiDPI']); - const page = await context.newPage(); - - await page.goto(url); - await page.locator('text=Loading').waitFor({ state: 'hidden' }); - - const stringObject = JSON.stringify( - { version, data: { spectra: [{ ...spectrum }] } }, - (key, value: unknown) => ArrayBuffer.isView(value) ? Array.from(value as unknown as Iterable) : value - ); - - await page.evaluate(` - window.postMessage({ type: "nmr-wrapper:load", data: { data: ${stringObject}, type: "nmrium" } }, '*'); - `); - - await page.locator('text=Loading').waitFor({ state: 'hidden' }); - - const snapshot = await page.locator('#nmrSVG .container').screenshot(); - snapshots.push({ id: spectrum.id, image: snapshot.toString('base64') }); - - } catch (e) { - logger.error({ id: spectrum.id, stage: 'snapshot', details: toMessage(e) }, `Failed to capture snapshot for spectrum: ${spectrum.id}`); - // browser crashed — close and recreate for next spectrum - await browser.close().catch(() => { }); - browser = await launchBrowser(); - - } finally { - await context?.close().catch(() => { }); - } - } - - await browser.close().catch(() => { }); - return snapshots; -} - - -interface ProcessSpectraOptions { - autoDetection: boolean; autoProcessing: boolean; -} - -function processSpectra(data: NmriumData, options: ProcessSpectraOptions, logger: FifoLogger) { - - const { autoDetection = false, autoProcessing = false } = options - for (let index = 0; index < data.spectra.length; index++) { - const inputSpectrum = data.spectra[index] - const is2D = isSpectrum2D(inputSpectrum); - let spectrum = null; +const core = init(); - try { - spectrum = is2D ? initiateDatum2D(inputSpectrum) : initiateDatum1D(inputSpectrum); - } catch (e) { - logger.error({ id: inputSpectrum.id, stage: 'parsing', details: toMessage(e) }, `Failed to parse spectrum: ${inputSpectrum.id}`); - continue; - } - - if (autoProcessing) { - try { - - isSpectrum2D(spectrum) ? Filters2DManager.reapplyFilters(spectrum) : Filters1DManager.reapplyFilters(spectrum); - logger.info({ id: inputSpectrum.id, stage: 'processing' }, `Processed spectrum: ${inputSpectrum.id}`); - - } catch (e) { - logger.error({ id: inputSpectrum.id, stage: 'processing', details: toMessage(e) }, `Failed to process spectrum: ${inputSpectrum.id}`); - } - } - - if (autoDetection && spectrum.info.isFt) { - try { - isSpectrum2D(spectrum) ? detectZones(spectrum) : detectRanges(spectrum); - logger.info({ id: inputSpectrum.id, stage: 'detection' }, `Detected peaks for spectrum: ${inputSpectrum.id}`); - } catch (e) { - logger.error({ id: inputSpectrum.id, stage: 'detection', details: toMessage(e) }, `Failed to detect peaks for spectrum: ${inputSpectrum.id}`); - } - } - - if (!spectrum) continue; - - data.spectra[index] = spectrum; - } - - -} +type RequiredKey = Omit & Required>; -function outputResult(result: any, outputPath?: string) { - const stream = new JsonStreamStringify(result); - if (outputPath) { - const writeStream = createWriteStream(outputPath); - stream.pipe(writeStream); - writeStream.on('finish', () => { - process.stderr.write(`Output written to: ${outputPath}\n`); - }); - } else { - stream.pipe(process.stdout); - } +function getParsingOptions(autoProcessing: boolean): ParsingOptions { + return { + onLoadProcessing: { autoProcessing }, + selector: { general: { dataSelection: 'preferFT' } }, + experimentalFeatures: true, + }; } async function processAndSerialize( @@ -170,18 +30,11 @@ async function processAndSerialize( ) { const { s: enableSnapshot = false, p: autoProcessing = false, d: autoDetection = false, o, r } = options; - if (nmriumState.data) { - processSpectra(nmriumState.data, { autoDetection, autoProcessing }, logger); - } - - const images: Snapshot[] = enableSnapshot - ? await captureSpectraViewAsBase64(nmriumState, logger) - : []; + const images = await runSpectraPipeline(nmriumState, { autoProcessing, autoDetection, enableSnapshot }, logger); const { data, version } = core.serializeNmriumState( nmriumState as NmriumState, - { includeData: r ? 'rawData' : 'dataSource', }, - + { includeData: r ? 'rawData' : 'dataSource' }, ); // include the meta and info object in case of serialize as dataSource @@ -189,72 +42,67 @@ async function processAndSerialize( if (!r) { for (let i = 0; i < spectra.length; i++) { const { info = {}, meta = {} } = nmriumState.data?.spectra[i] || {}; - spectra[i] = { ...spectra[i], info, meta } + spectra[i] = { ...spectra[i], info, meta }; } } + // Drop the raw processed spectra (typed arrays, filter history) now that + // everything needed from them has been copied into `spectra` above — + // otherwise they stay resident in memory alongside the serialized copy + // for the rest of the (potentially large, streamed) output write. + if (nmriumState.data) nmriumState.data.spectra = []; const logs = logger.getLogs(); - outputResult({ nmriumState: { data, version }, images, logs }, o); + await outputResult({ nmriumState: { data, version }, images, logs }, o); } async function loadSpectrumFromURL(options: RequiredKey, logger: FifoLogger) { const { u: url, include, exclude } = options; - const { pathname: relativePath, origin: baseURL } = new URL(url) + const { pathname: relativePath, origin: baseURL } = new URL(url); const source = { - entries: [ - { - relativePath, - }, - ], + entries: [{ relativePath }], baseURL, - } - + }; - const { state } = await core.readFromWebSource(source, { ...parsingOptions, fileFilter: { include, exclude }, logger }); - - processAndSerialize(state, options, logger) + const { state } = await core.readFromWebSource(source, { ...getParsingOptions(true), fileFilter: { include, exclude }, logger }); + await processAndSerialize(state, options, logger); } async function loadSpectrumFromFilePath(options: RequiredKey, logger: FifoLogger) { const { dir: path, include, exclude } = options; - const dirPath = isAbsolute(path) ? path : join(process.cwd(), path) + const dirPath = isAbsolute(path) ? path : join(process.cwd(), path); const fileCollection = await FileCollection.fromPath(dirPath, { unzip: { zipExtensions: ['zip', 'nmredata'] }, filter: { include, exclude }, - }) - - const { - state - } = await core.read(fileCollection, { ...parsingOptions, logger }) + }); - processAndSerialize(state, options, logger) + const { state } = await core.read(fileCollection, { ...getParsingOptions(true), logger }); + await processAndSerialize(state, options, logger); } - -function parseSpectra(argv: yargs.ArgumentsCamelCase -) { +async function parseSpectra(argv: yargs.ArgumentsCamelCase) { const logger = new FifoLogger(); - const { u, dir } = argv; - // Handle parsing the spectra file logic based on argv options - if (u) { - loadSpectrumFromURL({ u, ...argv }, logger); - } - - if (dir) { - loadSpectrumFromFilePath({ dir, ...argv }, logger); + try { + // Branches are mutually exclusive and awaited so a rejection is caught + // here instead of becoming an unhandled promise rejection, and so -u + // and --dir can't race to write the same output. + if (u) { + await loadSpectrumFromURL({ u, ...argv }, logger); + } else if (dir) { + await loadSpectrumFromFilePath({ dir, ...argv }, logger); + } else { + throw new Error('Either --u (URL) or --dir (directory) must be provided.'); + } + } catch (e) { + logger.error({ stage: 'fatal', details: toMessage(e) }, `Pipeline failed: ${toMessage(e)}`); + process.stderr.write(`${toMessage(e)}\n`); + process.exitCode = 1; } - - - } - - - -export { loadSpectrumFromFilePath, loadSpectrumFromURL, parseSpectra } +export { loadSpectrumFromFilePath, loadSpectrumFromURL, parseSpectra }; \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/run-concurrency.ts b/app/scripts/nmr-cli/src/parse/run-concurrency.ts new file mode 100644 index 0000000..9d1f615 --- /dev/null +++ b/app/scripts/nmr-cli/src/parse/run-concurrency.ts @@ -0,0 +1,39 @@ +import os from 'node:os'; + +// --- Concurrency tuning ----------------------------------------------- +// CPU-bound work (auto-processing / peak detection) is parallelized across +// worker threads: one worker per lane, reused for every spectrum that lane +// handles (spawning a worker reloads the whole nmr-processing module graph, +// so lanes are long-lived, not spawned per spectrum). +// +// Snapshot capture is parallelized across browser tabs, but each tab is far +// more memory-hungry than a worker thread, so it gets its own (lower) cap +// regardless of CPU count. +// +// Both are overridable via env vars for tuning on a given machine. +export const CPU_CONCURRENCY = Number(process.env['NMR_CLI_CPU_CONCURRENCY']) || Math.max(1, os.cpus().length - 1); +export const SNAPSHOT_CONCURRENCY = Number(process.env['NMR_CLI_SNAPSHOT_CONCURRENCY']) || 3; + +// Runs `handler` over `items` using a fixed number of long-lived "lanes" +// rather than firing off one promise per item. `laneIndex` is stable for the +// lifetime of a lane, so handlers can lazily attach an expensive, reusable +// resource (a worker thread, a browser page) to a given lane instead of +// creating one per item. +export async function runWithConcurrency( + items: T[], + concurrency: number, + handler: (item: T, laneIndex: number) => Promise +): Promise { + if (items.length === 0) return; + let cursor = 0; + const laneCount = Math.max(1, Math.min(concurrency, items.length)); + const lanes = Array.from({ length: laneCount }, (_, laneIndex) => + (async () => { + while (cursor < items.length) { + const item = items[cursor++]; + await handler(item, laneIndex); + } + })() + ); + await Promise.all(lanes); +} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/run-pipeline.ts b/app/scripts/nmr-cli/src/parse/run-pipeline.ts new file mode 100644 index 0000000..688a2af --- /dev/null +++ b/app/scripts/nmr-cli/src/parse/run-pipeline.ts @@ -0,0 +1,143 @@ +import { isSpectrum2D, type NmriumState } from '@zakodium/nmrium-core'; +import type { FifoLogger } from 'fifo-logger'; +import { runWithConcurrency, CPU_CONCURRENCY, SNAPSHOT_CONCURRENCY } from './run-concurrency'; +import { SpectrumWorker } from './spectrum-worker'; +import { BrowserManager, SnapshotLane } from './browser-manager'; +import { Snapshot } from './spectrum-snapshot'; +import { toMessage } from './utility/toMessage'; +import { SpectrumPipelineOptions } from './type/SpectrumPipelineOptions'; +import { ParsedSpectrum } from './type/ParsedSpectrum'; +import { initiateDatum2D } from './data/data2d/initiateDatum2D'; +import { initiateDatum1D } from './data/data1D/initiateDatum1D'; + +function getDurationMs(start: number): number { + return Date.now() - start; +} + +function generateNMRiumURL(): string { + const baseURL = process.env['BASE_NMRIUM_URL'] || ''; + const url = new URL(baseURL); + url.searchParams.append('workspace', 'embedded'); + return url.toString(); +} + +// Each spectrum flows through the same stages — process -> detect -> snapshot +// — but stages now run across a fixed number of concurrent "lanes" instead +// of one spectrum at a time. Every lane owns its own worker thread (CPU +// stages) and, if snapshots are enabled, its own persistent browser page. +// +// Every stage logs `durationMs` alongside its existing pass/fail log, so a +// run's logs can be aggregated afterwards to see where time actually goes +// (parse vs. process vs. detect vs. snapshot) instead of guessing. +export async function runSpectraPipeline( + nmriumState: Partial, + options: SpectrumPipelineOptions, + logger: FifoLogger +): Promise { + const data = nmriumState.data; + const { version } = nmriumState; + if (!data) return []; + + const parsed = parseSpectraList(data.spectra, logger); + + // Write the parsed set back so serialization later only sees spectra that + // actually parsed successfully. + data.spectra = parsed.map((d) => d.spectrum); + const indexById = new Map(parsed.map((d, i) => [d.id, i])); + + const { autoProcessing, autoDetection, enableSnapshot } = options; + const snapshots: Snapshot[] = []; + if (parsed.length === 0) return snapshots; + + const concurrency = enableSnapshot ? SNAPSHOT_CONCURRENCY : CPU_CONCURRENCY; + const browserManager = enableSnapshot ? new BrowserManager() : null; + const url = enableSnapshot ? generateNMRiumURL() : ''; + + const spectrumWorkers: SpectrumWorker[] = []; + const snapshotLanes: SnapshotLane[] = []; + + const getWorker = (laneIndex: number): SpectrumWorker => { + if (!spectrumWorkers[laneIndex]) spectrumWorkers[laneIndex] = new SpectrumWorker(); + return spectrumWorkers[laneIndex]; + }; + const getSnapshotLane = (laneIndex: number): SnapshotLane => { + if (!snapshotLanes[laneIndex]) snapshotLanes[laneIndex] = new SnapshotLane(browserManager!, url); + return snapshotLanes[laneIndex]; + }; + + try { + await runWithConcurrency(parsed, concurrency, async ({ id, spectrum: initial }, laneIndex) => { + const spectrumWorker = getWorker(laneIndex); + let spectrum = initial; + + if (autoProcessing) { + const start = Date.now(); + try { + spectrum = await spectrumWorker.run('process', spectrum); + logger.info({ id, stage: 'processing', durationMs: getDurationMs(start) }, `Processed spectrum: ${id}`); + } catch (e) { + logger.error( + { id, stage: 'processing', durationMs: getDurationMs(start), details: toMessage(e) }, + `Failed to process spectrum: ${id}` + ); + } + } + + if (autoDetection && spectrum.info.isFt) { + const start = Date.now(); + try { + spectrum = await spectrumWorker.run('detect', spectrum); + logger.info({ id, stage: 'detection', durationMs: getDurationMs(start) }, `Detected peaks for spectrum: ${id}`); + } catch (e) { + logger.error( + { id, stage: 'detection', durationMs: getDurationMs(start), details: toMessage(e) }, + `Failed to detect peaks for spectrum: ${id}` + ); + } + } + + data.spectra[indexById.get(id)!] = spectrum; + + if (enableSnapshot) { + const lane = getSnapshotLane(laneIndex); + // Timing + retry for this stage live inside SnapshotLane.capture + // itself, since a retry needs its own per-attempt timing. + const snapshot = await lane.capture(spectrum, id, version, spectrumWorker, logger); + snapshots.push(snapshot); + } + }); + } finally { + await Promise.all(spectrumWorkers.filter(Boolean).map((w) => w.terminate())); + await Promise.all(snapshotLanes.filter(Boolean).map((l) => l.dispose())); + await browserManager?.closeAll(); + } + + return snapshots; +} + + + + + + + +function parseSpectraList(rawSpectra: any[], logger: FifoLogger): ParsedSpectrum[] { + const parsed: ParsedSpectrum[] = []; + for (const inputSpectrum of rawSpectra) { + const id = inputSpectrum.id; + const start = Date.now(); + try { + const spectrum = isSpectrum2D(inputSpectrum) + ? initiateDatum2D(inputSpectrum) + : initiateDatum1D(inputSpectrum); + logger.info({ id, stage: 'parsing', durationMs: Date.now() - start }, `Parsed spectrum: ${id}`); + parsed.push({ id, spectrum }); + } catch (e) { + logger.error( + { id, stage: 'parsing', durationMs: Date.now() - start, details: toMessage(e) }, + `Failed to parse spectrum: ${id}` + ); + } + } + return parsed; +} diff --git a/app/scripts/nmr-cli/src/parse/spectrum-snapshot.ts b/app/scripts/nmr-cli/src/parse/spectrum-snapshot.ts new file mode 100644 index 0000000..400f9da --- /dev/null +++ b/app/scripts/nmr-cli/src/parse/spectrum-snapshot.ts @@ -0,0 +1,117 @@ +import playwright, { + type Browser, + type BrowserContext, + type Page, +} from 'playwright' + +import { SpectrumWorker } from './spectrum-worker' + +export interface Snapshot { + id: string + image: string | null +} + +const SNAPSHOT_LOADING_TIMEOUT_MS = 30_000 + +function generateNMRiumURL(): string { + const baseURL = process.env.BASE_NMRIUM_URL + + if (!baseURL) { + throw new Error( + 'BASE_NMRIUM_URL environment variable is not defined', + ) + } + + const url = new URL(baseURL) + url.searchParams.set('workspace', 'embedded') + + return url.toString() +} + +async function waitForNMRium(page: Page): Promise { + await page.locator('text=Loading').waitFor({ + state: 'hidden', + timeout: SNAPSHOT_LOADING_TIMEOUT_MS, + }) +} + +export class SpectrumSnapshot { + private browser: Browser | null = null + private context: BrowserContext | null = null + private page: Page | null = null + + async start(): Promise { + if (this.page) return + + this.browser = await playwright.firefox.launch() + + this.context = await this.browser.newContext( + playwright.devices['Desktop Chrome HiDPI'], + ) + + this.page = await this.context.newPage() + + await this.page.goto(generateNMRiumURL()) + + await waitForNMRium(this.page) + } + + async capture( + id: string, + spectrum: any, + version: unknown, + worker: SpectrumWorker, + ): Promise { + if (!this.page) { + throw new Error( + 'SpectrumSnapshot has not been started', + ) + } + + const stringObject = await worker.run( + 'serialize', + spectrum, + version, + ) + + /* + * Parse the serialized data here rather than injecting + * the JSON string into JavaScript source code. + */ + const data = JSON.parse(stringObject) + + await this.page.evaluate((nmriumData) => { + window.postMessage( + { + type: 'nmr-wrapper:load', + data: { + data: nmriumData, + type: 'nmrium', + }, + }, + '*', + ) + }, data) + + await waitForNMRium(this.page) + + const image = await this.page + .locator('#nmrSVG .container') + .screenshot() + + return { + id, + image: image.toString('base64'), + } + } + + async close(): Promise { + await this.context?.close().catch(() => { }) + await this.browser?.close().catch(() => { }) + + this.page = null + this.context = null + this.browser = null + } +} + diff --git a/app/scripts/nmr-cli/src/parse/spectrum-worker.ts b/app/scripts/nmr-cli/src/parse/spectrum-worker.ts new file mode 100644 index 0000000..bb7f715 --- /dev/null +++ b/app/scripts/nmr-cli/src/parse/spectrum-worker.ts @@ -0,0 +1,71 @@ +import { Spectrum } from '@zakodium/nmr-types'; +import { join } from 'node:path'; +import { Worker } from 'node:worker_threads'; +import { WorkerResponse } from './type/WorkerResponse'; + +// Runs processing/detection/serialization in a separate worker thread (its +// own V8 heap) so a heap-OOM there only terminates the worker instead of the +// whole CLI process. One instance is created per concurrency lane (see +// run-pipeline.ts). + +export class SpectrumWorker { + private worker: Worker | null = null; + + private spawn(): Worker { + const worker = new Worker(join(__dirname, 'worker-entry.js'), { + resourceLimits: { maxOldGenerationSizeMb: 3072 }, + }); + worker.on('error', () => { this.worker = null; }); + worker.on('exit', () => { this.worker = null; }); + return worker; + } + + private get(): Worker { + if (!this.worker) this.worker = this.spawn(); + return this.worker; + } + + run(task: 'process' | 'detect', spectrum: Spectrum): Promise; + run(task: 'serialize', spectrum: Spectrum, version: unknown): Promise; + run(task: 'process' | 'detect' | 'serialize', spectrum: Spectrum, version?: unknown): Promise { + return new Promise((resolve, reject) => { + const worker = this.get(); + let settled = false; + + const onMessage = (msg: WorkerResponse) => { + settled = true; + cleanup(); + if (msg.error) reject(new Error(msg.error)); + else resolve(task === 'serialize' ? msg.stringObject : msg.spectrum); + }; + const onError = (err: Error) => { + settled = true; + this.worker = null; + cleanup(); + reject(err); + }; + const onExit = (code: number) => { + this.worker = null; + if (!settled && code !== 0) { + cleanup(); + reject(new Error(`Worker exited with code ${code} (likely out of memory)`)); + } + }; + const cleanup = () => { + worker.off('message', onMessage); + worker.off('error', onError); + worker.off('exit', onExit); + }; + + worker.on('message', onMessage); + worker.on('error', onError); + worker.on('exit', onExit); + worker.postMessage({ spectrum, task, version }); + }); + } + + async terminate() { + await this.worker?.terminate().catch(() => { }); + this.worker = null; + } +} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/type/MoleculeExtended.ts b/app/scripts/nmr-cli/src/parse/type/MoleculeExtended.ts index 7e71a00..601a66f 100644 --- a/app/scripts/nmr-cli/src/parse/type/MoleculeExtended.ts +++ b/app/scripts/nmr-cli/src/parse/type/MoleculeExtended.ts @@ -1,11 +1,12 @@ -import { StateMolecule } from "@zakodium/nmrium-core"; +import { StateMolecule } from '@zakodium/nmrium-core' export interface MoleculeExtended - extends Required>, + extends + Required>, Omit { - mf: string; - em: number; - mw: number; - svg: string; - atoms: Record; -} \ No newline at end of file + mf: string + em: number + mw: number + svg: string + atoms: Record +} diff --git a/app/scripts/nmr-cli/src/parse/type/ParsedSpectrum.ts b/app/scripts/nmr-cli/src/parse/type/ParsedSpectrum.ts new file mode 100644 index 0000000..911550c --- /dev/null +++ b/app/scripts/nmr-cli/src/parse/type/ParsedSpectrum.ts @@ -0,0 +1,6 @@ +import { Spectrum } from "@zakodium/nmr-types"; + +export interface ParsedSpectrum { + id: string; + spectrum: Spectrum; +} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/type/Snapshot.ts b/app/scripts/nmr-cli/src/parse/type/Snapshot.ts new file mode 100644 index 0000000..0ed48cb --- /dev/null +++ b/app/scripts/nmr-cli/src/parse/type/Snapshot.ts @@ -0,0 +1,4 @@ +export interface Snapshot { + id: string; + image: string | null; +} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/type/SpectrumPipelineOptions.ts b/app/scripts/nmr-cli/src/parse/type/SpectrumPipelineOptions.ts new file mode 100644 index 0000000..001b4e7 --- /dev/null +++ b/app/scripts/nmr-cli/src/parse/type/SpectrumPipelineOptions.ts @@ -0,0 +1,5 @@ +export interface SpectrumPipelineOptions { + autoProcessing: boolean; + autoDetection: boolean; + enableSnapshot: boolean; +} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/type/WorkerRequest.ts b/app/scripts/nmr-cli/src/parse/type/WorkerRequest.ts new file mode 100644 index 0000000..fb39179 --- /dev/null +++ b/app/scripts/nmr-cli/src/parse/type/WorkerRequest.ts @@ -0,0 +1,8 @@ +import { Spectrum } from "@zakodium/nmr-types"; + +// Message contract shared with detectWorkerEntry.ts — keep both in sync. +export interface WorkerRequest { + spectrum: Spectrum; + task: 'process' | 'detect' | 'serialize'; + version?: unknown; +} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/type/WorkerResponse.ts b/app/scripts/nmr-cli/src/parse/type/WorkerResponse.ts new file mode 100644 index 0000000..7caaa46 --- /dev/null +++ b/app/scripts/nmr-cli/src/parse/type/WorkerResponse.ts @@ -0,0 +1,7 @@ +import { Spectrum } from "@zakodium/nmr-types"; + +export interface WorkerResponse { + spectrum?: Spectrum; + stringObject?: string; + error?: string; +} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/utility/getAtom.ts b/app/scripts/nmr-cli/src/parse/utility/getAtom.ts index 68e66ec..da09e78 100644 --- a/app/scripts/nmr-cli/src/parse/utility/getAtom.ts +++ b/app/scripts/nmr-cli/src/parse/utility/getAtom.ts @@ -1,3 +1,3 @@ export default function getAtom(nucleus: string): string { - return nucleus?.replaceAll(/\d/g, '') || ''; + return nucleus?.replaceAll(/\d/g, '') || '' } diff --git a/app/scripts/nmr-cli/src/parse/utility/isProton.ts b/app/scripts/nmr-cli/src/parse/utility/isProton.ts index 85f6e6e..00f3e6e 100644 --- a/app/scripts/nmr-cli/src/parse/utility/isProton.ts +++ b/app/scripts/nmr-cli/src/parse/utility/isProton.ts @@ -1,3 +1,3 @@ export function isProton(nucleus: string) { - return nucleus === '1H'; + return nucleus === '1H' } diff --git a/app/scripts/nmr-cli/src/parse/utility/outputResult.ts b/app/scripts/nmr-cli/src/parse/utility/outputResult.ts new file mode 100644 index 0000000..40de4b6 --- /dev/null +++ b/app/scripts/nmr-cli/src/parse/utility/outputResult.ts @@ -0,0 +1,24 @@ + +import { createWriteStream } from 'node:fs'; +import { JsonStreamStringify } from 'json-stream-stringify'; + +export function outputResult(result: any, outputPath?: string): Promise { + return new Promise((resolve, reject) => { + const stream = new JsonStreamStringify(result); + + if (outputPath) { + const writeStream = createWriteStream(outputPath); + stream.pipe(writeStream); + writeStream.on('finish', () => { + process.stderr.write(`Output written to: ${outputPath}\n`); + resolve(); + }); + writeStream.on('error', reject); + stream.on('error', reject); + } else { + stream.pipe(process.stdout); + stream.on('end', () => resolve()); + stream.on('error', reject); + } + }); +} diff --git a/app/scripts/nmr-cli/src/parse/utility/toMessage.ts b/app/scripts/nmr-cli/src/parse/utility/toMessage.ts new file mode 100644 index 0000000..6516c2c --- /dev/null +++ b/app/scripts/nmr-cli/src/parse/utility/toMessage.ts @@ -0,0 +1,3 @@ +export function toMessage(e: unknown): string { + return e instanceof Error ? e.message : String(e); +} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/worker-entry.ts b/app/scripts/nmr-cli/src/parse/worker-entry.ts new file mode 100644 index 0000000..b78df71 --- /dev/null +++ b/app/scripts/nmr-cli/src/parse/worker-entry.ts @@ -0,0 +1,70 @@ +import { parentPort } from 'node:worker_threads'; +import { Filters1DManager, Filters2DManager } from 'nmr-processing'; +import { detectZones } from './data/data2d/detectZones'; +import { detectRanges } from './data/data1D/detectRanges'; +import { Spectrum } from '@zakodium/nmr-types'; +import { isSpectrum2D } from '@zakodium/nmrium-core'; +import { WorkerRequest } from './type/WorkerRequest'; +import { WorkerResponse } from './type/WorkerResponse'; + +if (!parentPort) { + throw new Error('detectWorkerEntry must be run inside a worker_threads Worker'); +} + +// Both mutate the spectrum in place (same as the original single-threaded +// implementation) — the return is just for a uniform call signature. +async function runAutoProcessing(spectrum: Spectrum): Promise { + if (isSpectrum2D(spectrum)) { + Filters2DManager.reapplyFilters(spectrum); + } else { + Filters1DManager.reapplyFilters(spectrum); + } + return spectrum; +} + +async function runAutoDetection(spectrum: Spectrum): Promise { + if (isSpectrum2D(spectrum)) { + detectZones(spectrum); + } else { + detectRanges(spectrum); + } + return spectrum; +} + +// Spectra hold typed-array data (Float64Array etc.), which JSON.stringify +// otherwise mangles into `{0: ..., 1: ...}` objects instead of arrays — the +// replacer below converts any ArrayBuffer view to a plain array first. No +// nmrium-core-plugins init needed here: this is a plain JSON encoding of +// { version, data: { spectra: [spectrum] } }, matching exactly what the +// browser's `nmr-wrapper:load` message expects. +async function serializeSpectrum(spectrum: Spectrum, version: unknown): Promise { + return JSON.stringify( + { version, data: { spectra: [spectrum] } }, + (_key, value: unknown) => (ArrayBuffer.isView(value) ? Array.from(value as unknown as Iterable) : value) + ); +} + +parentPort.on('message', async (msg: WorkerRequest) => { + const { spectrum, task, version } = msg; + const reply = (response: WorkerResponse) => parentPort!.postMessage(response); + + try { + switch (task) { + case 'process': + reply({ spectrum: await runAutoProcessing(spectrum) }); + break; + case 'detect': + reply({ spectrum: await runAutoDetection(spectrum) }); + break; + case 'serialize': + reply({ stringObject: await serializeSpectrum(spectrum, version) }); + break; + default: { + const exhaustive: never = task; + throw new Error(`Unknown task: ${exhaustive}`); + } + } + } catch (e) { + reply({ error: e instanceof Error ? e.message : String(e) }); + } +}); \ No newline at end of file