diff --git a/app/scripts/nmr-cli/Dockerfile b/app/scripts/nmr-cli/Dockerfile index 45b9abf..f7877d5 100644 --- a/app/scripts/nmr-cli/Dockerfile +++ b/app/scripts/nmr-cli/Dockerfile @@ -32,9 +32,6 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -# Downgrade to Node 22 for nmr-processing compatibility -RUN npm install -g n && n 22 && hash -r - COPY package.json ./ RUN npm install @@ -43,11 +40,9 @@ COPY . ./ RUN npm run build -# Install nmr-cli as a global package -RUN npm install -g . -# Usage examples (run after `docker run -it nmr-cli bash`): -# nmr-cli parse-spectra -u https://s3.uni-jena.de/nmrxiv/production/archive/1368b8fd-3af7-48f7-8b6d-13de9c1ecf8b/492-62-6.zip -s +# Install the nmr-cli as a global package # For example, nmr-cli parse-spectra -u https://s3.uni-jena.de/nmrxiv/production/archive/d5ec45db-d984-46e9-bb72-c43fd4a463fa/Sali_Eth.zip -s # For example, nmr-cli parse-spectra -u https://cheminfo.github.io/bruker-data-test/data/zipped/aspirin-1h.zip # nmr-cli predict -n "1H" --id 1 --type "nmr;1H;1d" --shifts "1" --solvent "Dimethylsulphoxide-D6 (DMSO-D6, C2D6SO)" -m $"\n Ketcher 6122516162D 1 1.00000 0.00000 0\n\n 16 17 0 0 0 0 0 0 0 0999 V2000\n 1.1954 -4.6484 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 2.9258 -4.6479 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 2.0622 -4.1483 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 2.9258 -5.6488 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 1.1954 -5.6533 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 2.0644 -6.1483 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 3.7902 -4.1495 0.0000 N 0 0 0 0 0 0 0 0 0 0 0 0\n 4.6574 -4.6498 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 3.7964 -6.1512 0.0000 N 0 0 0 0 0 0 0 0 0 0 0 0\n 4.6596 -5.6458 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 5.5228 -4.1488 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 5.5277 -6.1421 0.0000 O 0 0 0 0 0 0 0 0 0 0 0 0\n 6.3895 -4.6477 0.0000 O 0 0 0 0 0 0 0 0 0 0 0 0\n 5.5216 -3.1488 0.0000 O 0 0 0 0 0 0 0 0 0 0 0 0\n 7.2548 -4.1466 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 8.1215 -4.6455 0.0000 C 0 0 0 0 0 0 0 0 0 0 0 0\n 3 1 2 0 0 0 0\n 1 5 1 0 0 0 0\n 5 6 2 0 0 0 0\n 6 4 1 0 0 0 0\n 4 2 1 0 0 0 0\n 2 3 1 0 0 0 0\n 4 9 1 0 0 0 0\n 9 10 2 0 0 0 0\n 10 8 1 0 0 0 0\n 8 7 2 0 0 0 0\n 7 2 1 0 0 0 0\n 8 11 1 0 0 0 0\n 10 12 1 0 0 0 0\n 11 13 1 0 0 0 0\n 11 14 2 0 0 0 0\n 13 15 1 0 0 0 0\n 15 16 1 0 0 0 0\nM END" +RUN npm install . -g diff --git a/app/scripts/nmr-cli/package-lock.json b/app/scripts/nmr-cli/package-lock.json index 3ae8706..6f47fd6 100644 --- a/app/scripts/nmr-cli/package-lock.json +++ b/app/scripts/nmr-cli/package-lock.json @@ -29,7 +29,7 @@ }, "devDependencies": { "@types/lodash.merge": "^4.6.9", - "@types/node": "^26.4.0", + "@types/node": "^26.2.0", "@types/yargs": "^17.0.35", "ts-node": "^10.9.2", "typescript": "^5.9.3" @@ -128,9 +128,9 @@ } }, "node_modules/@types/node": { - "version": "26.4.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.4.0.tgz", - "integrity": "sha512-faiGnoIrLH/V8cibOMEAZ8pMw6oXqSukl29ra4mN8GdaB2ZewzeaLj+INpV5N+Z1eKWzY+IzaIZH2EIR6YZRNQ==", + "version": "26.2.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.2.0.tgz", + "integrity": "sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==", "dev": true, "license": "MIT", "dependencies": { diff --git a/app/scripts/nmr-cli/package.json b/app/scripts/nmr-cli/package.json index 09dafd2..c795b3a 100644 --- a/app/scripts/nmr-cli/package.json +++ b/app/scripts/nmr-cli/package.json @@ -32,7 +32,7 @@ }, "devDependencies": { "@types/lodash.merge": "^4.6.9", - "@types/node": "^26.4.0", + "@types/node": "^26.2.0", "@types/yargs": "^17.0.35", "ts-node": "^10.9.2", "typescript": "^5.9.3" diff --git a/app/scripts/nmr-cli/src/parse/browser-manager.ts b/app/scripts/nmr-cli/src/parse/browser-manager.ts deleted file mode 100644 index 4a070c4..0000000 --- a/app/scripts/nmr-cli/src/parse/browser-manager.ts +++ /dev/null @@ -1,163 +0,0 @@ -import playwright from 'playwright'; -import type { FifoLogger } from 'fifo-logger'; -import type { SpectrumWorker } from './spectrum-worker'; -import { Spectrum } from '@zakodium/nmr-types'; -import { Snapshot } from './spectrum-snapshot'; -import { toMessage } from './utility/toMessage'; - -// How long to wait for the NMRium "Loading" indicator to appear/disappear -// before giving up on a single spectrum's snapshot. Prevents one stuck page -// from hanging the entire pipeline. -export const SNAPSHOT_LOADING_TIMEOUT_MS = 30_000; - -// A single flaky failure (a slow spectrum tipping over the loading timeout, -// a one-off page hiccup) shouldn't permanently cost a spectrum its snapshot, -// so each capture gets one retry on a fresh page before giving up. -export const SNAPSHOT_MAX_ATTEMPTS = 2; - -// Owns the single shared Firefox process. Multiple lanes each get their own -// context/page from it. `reset` is guarded so that if several lanes hit a -// dead browser at once, only the first one actually relaunches — the rest -// just see `current` has already changed and pick up the fresh instance. -export class BrowserManager { - private current: Promise | null = null; - - async get(): Promise { - if (!this.current) { - this.current = playwright.firefox.launch(); - } - return this.current; - } - - async reset(stale: playwright.Browser): Promise { - if (this.current && (await this.current) === stale) { - const toClose = this.current; - this.current = null; - await (await toClose).close().catch(() => { }); - } - } - - async closeAll(): Promise { - if (this.current) { - const toClose = this.current; - this.current = null; - await (await toClose).close().catch(() => { }); - } - } -} - -// One snapshot lane = one browser tab, reused across every spectrum it's -// assigned. Launching the browser/context happens once per lane; every -// spectrum after the first gets a `page.reload()` (not a new context or -// browser) before its `nmr-wrapper:load` message, so each snapshot starts -// from a genuinely empty NMRium instance instead of relying on `load` -// merging vs. replacing the previous spectrum's state. A reload of an -// already-booted SPA is far cheaper than relaunching the browser/context, -// so this keeps the speed win while removing the state-leak risk. -export class SnapshotLane { - private context: playwright.BrowserContext | null = null; - private page: playwright.Page | null = null; - private hasLoadedSpectrum = false; - - constructor(private manager: BrowserManager, private url: string) { } - - private async ensurePage(): Promise { - if (this.page && !this.page.isClosed()) { - if (this.hasLoadedSpectrum) { - await this.page.reload(); - await this.page.locator('text=Loading').waitFor({ state: 'hidden', timeout: SNAPSHOT_LOADING_TIMEOUT_MS }); - } - return this.page; - } - const browser = await this.manager.get(); - this.context = await browser.newContext(playwright.devices['Desktop Chrome HiDPI']); - this.page = await this.context.newPage(); - await this.page.goto(this.url); - await this.page.locator('text=Loading').waitFor({ state: 'hidden', timeout: SNAPSHOT_LOADING_TIMEOUT_MS }); - this.hasLoadedSpectrum = false; - return this.page; - } - - // Discards this lane's page/context so the next attempt (or the next - // spectrum, if we're giving up) starts from a clean page instead of - // whatever broken state caused the failure. Only escalates to a full - // browser relaunch if the browser process itself is gone. - private async recover(): Promise { - await this.context?.close().catch(() => { }); - this.context = null; - this.page = null; - this.hasLoadedSpectrum = false; - - const browser = await this.manager.get(); - if (!browser.isConnected()) { - await this.manager.reset(browser); - } - } - - private async attemptCapture( - spectrum: Spectrum, - version: unknown, - spectrumWorker: SpectrumWorker - ): Promise { - const page = await this.ensurePage(); - - const stringObject = await spectrumWorker.run('serialize', spectrum, version); - - // Passed as a Playwright function argument rather than spliced into an - // evaluated script string, so a backtick or `${...}` sequence anywhere - // in the spectrum data can't break (or hijack) the script. - await page.evaluate( - ({ data }) => { - window.postMessage({ type: 'nmr-wrapper:load', data: { data, type: 'nmrium' } }, '*'); - }, - { data: JSON.parse(stringObject) } - ); - - await page.locator('text=Loading').waitFor({ state: 'hidden', timeout: SNAPSHOT_LOADING_TIMEOUT_MS }); - - const snapshot = await page.locator('#nmrSVG .container').screenshot(); - this.hasLoadedSpectrum = true; - return snapshot.toString('base64'); - } - - async capture( - spectrum: Spectrum, - id: string, - version: unknown, - spectrumWorker: SpectrumWorker, - logger: FifoLogger - ): Promise { - let lastError: unknown; - - for (let attempt = 1; attempt <= SNAPSHOT_MAX_ATTEMPTS; attempt++) { - const start = Date.now(); - try { - const image = await this.attemptCapture(spectrum, version, spectrumWorker); - logger.info( - { id, stage: 'snapshot', attempt, durationMs: Date.now() - start }, - `Captured snapshot for spectrum: ${id}` - ); - return { id, image }; - } catch (e) { - lastError = e; - logger.error( - { id, stage: 'snapshot', attempt, durationMs: Date.now() - start, details: toMessage(e) }, - `Snapshot attempt ${attempt}/${SNAPSHOT_MAX_ATTEMPTS} failed for spectrum: ${id}` - ); - await this.recover(); - } - } - - logger.error( - { id, stage: 'snapshot', details: toMessage(lastError) }, - `Giving up on snapshot for spectrum: ${id} after ${SNAPSHOT_MAX_ATTEMPTS} attempts` - ); - return { id, image: null }; - } - - async dispose(): Promise { - await this.context?.close().catch(() => { }); - this.context = null; - this.page = null; - } -} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/prase-spectra.ts b/app/scripts/nmr-cli/src/parse/prase-spectra.ts index 011b2bd..d4d24c4 100644 --- a/app/scripts/nmr-cli/src/parse/prase-spectra.ts +++ b/app/scripts/nmr-cli/src/parse/prase-spectra.ts @@ -1,26 +1,166 @@ -import { join, isAbsolute } from 'node:path'; -import type { ParsingOptions, NmriumState } from '@zakodium/nmrium-core'; -import init from '@zakodium/nmrium-core-plugins'; -import { FileCollection } from 'file-collection'; -import yargs from 'yargs'; -import { FifoLogger } from 'fifo-logger'; -import { FileOptionsArgs } from '..'; -import { runSpectraPipeline } from './run-pipeline'; -import { outputResult } from './utility/outputResult'; -import { toMessage } from './utility/toMessage'; +import { join, isAbsolute } from 'path' +import { NmriumData, ParsingOptions, type NmriumState } from '@zakodium/nmrium-core' +import init from '@zakodium/nmrium-core-plugins' +import playwright from 'playwright' +import { FileCollection } from 'file-collection' +import { FileOptionsArgs } from '..' +import { isSpectrum2D } from './data/data2d/isSpectrum2D' +import { initiateDatum2D } from './data/data2d/initiateDatum2D' +import { initiateDatum1D } from './data/data1D/initiateDatum1D' +import { detectZones } from './data/data2d/detectZones' +import { detectRanges } from './data/data1D/detectRanges' +import { Filters1DManager, Filters2DManager } from 'nmr-processing' +import yargs from 'yargs' +import { createWriteStream } from 'fs' +import { JsonStreamStringify } from 'json-stream-stringify'; +import { FifoLogger } from 'fifo-logger' -const core = init(); +type RequiredKey = Omit & Required>; +function toMessage(e: unknown): string { + return e instanceof Error ? e.message : String(e) +} + + +const parsingOptions: ParsingOptions = { + onLoadProcessing: { autoProcessing: true }, + selector: { general: { dataSelection: 'preferFT' } }, + experimentalFeatures: true, +}; -type RequiredKey = Omit & Required>; +interface Snapshot { + id: string; + image: string | null; +} + +const core = init() + +function generateNMRiumURL() { + const baseURL = process.env['BASE_NMRIUM_URL'] || '' + const url = new URL(baseURL) + url.searchParams.append('workspace', 'embedded') + return url.toString() +} -function getParsingOptions(autoProcessing: boolean): ParsingOptions { - return { - onLoadProcessing: { autoProcessing }, - selector: { general: { dataSelection: 'preferFT' } }, - experimentalFeatures: true, - }; +async function launchBrowser() { + return playwright.firefox.launch(); +} + +async function captureSpectraViewAsBase64(nmriumState: Partial, logger: FifoLogger): Promise { + const { data: { spectra } = { spectra: [] }, version } = nmriumState; + + if (!spectra?.length) return []; + + const url = generateNMRiumURL(); + const snapshots: Snapshot[] = []; + let browser = await launchBrowser(); + + for (const spectrum of spectra) { + let context = null; + + try { + // recreate browser if it has crashed + if (!browser.isConnected()) { + browser = await launchBrowser(); + } + + context = await browser.newContext(playwright.devices['Desktop Chrome HiDPI']); + const page = await context.newPage(); + + await page.goto(url); + await page.locator('text=Loading').waitFor({ state: 'hidden' }); + + const stringObject = JSON.stringify( + { version, data: { spectra: [{ ...spectrum }] } }, + (key, value: unknown) => ArrayBuffer.isView(value) ? Array.from(value as unknown as Iterable) : value + ); + + await page.evaluate(` + window.postMessage({ type: "nmr-wrapper:load", data: { data: ${stringObject}, type: "nmrium" } }, '*'); + `); + + await page.locator('text=Loading').waitFor({ state: 'hidden' }); + + const snapshot = await page.locator('#nmrSVG .container').screenshot(); + snapshots.push({ id: spectrum.id, image: snapshot.toString('base64') }); + + } catch (e) { + logger.error({ id: spectrum.id, stage: 'snapshot', details: toMessage(e) }, `Failed to capture snapshot for spectrum: ${spectrum.id}`); + // browser crashed — close and recreate for next spectrum + await browser.close().catch(() => { }); + browser = await launchBrowser(); + + } finally { + await context?.close().catch(() => { }); + } + } + + await browser.close().catch(() => { }); + return snapshots; +} + + +interface ProcessSpectraOptions { + autoDetection: boolean; autoProcessing: boolean; +} + +function processSpectra(data: NmriumData, options: ProcessSpectraOptions, logger: FifoLogger) { + + const { autoDetection = false, autoProcessing = false } = options + for (let index = 0; index < data.spectra.length; index++) { + const inputSpectrum = data.spectra[index] + const is2D = isSpectrum2D(inputSpectrum); + let spectrum = null; + + try { + + spectrum = is2D ? initiateDatum2D(inputSpectrum) : initiateDatum1D(inputSpectrum); + } catch (e) { + logger.error({ id: inputSpectrum.id, stage: 'parsing', details: toMessage(e) }, `Failed to parse spectrum: ${inputSpectrum.id}`); + continue; + } + + if (autoProcessing) { + try { + + isSpectrum2D(spectrum) ? Filters2DManager.reapplyFilters(spectrum) : Filters1DManager.reapplyFilters(spectrum); + logger.info({ id: inputSpectrum.id, stage: 'processing' }, `Processed spectrum: ${inputSpectrum.id}`); + + } catch (e) { + logger.error({ id: inputSpectrum.id, stage: 'processing', details: toMessage(e) }, `Failed to process spectrum: ${inputSpectrum.id}`); + } + } + + if (autoDetection && spectrum.info.isFt) { + try { + isSpectrum2D(spectrum) ? detectZones(spectrum) : detectRanges(spectrum); + logger.info({ id: inputSpectrum.id, stage: 'detection' }, `Detected peaks for spectrum: ${inputSpectrum.id}`); + } catch (e) { + logger.error({ id: inputSpectrum.id, stage: 'detection', details: toMessage(e) }, `Failed to detect peaks for spectrum: ${inputSpectrum.id}`); + } + } + + if (!spectrum) continue; + + data.spectra[index] = spectrum; + } + + +} + +function outputResult(result: any, outputPath?: string) { + const stream = new JsonStreamStringify(result); + + if (outputPath) { + const writeStream = createWriteStream(outputPath); + stream.pipe(writeStream); + writeStream.on('finish', () => { + process.stderr.write(`Output written to: ${outputPath}\n`); + }); + } else { + stream.pipe(process.stdout); + } } async function processAndSerialize( @@ -30,11 +170,18 @@ async function processAndSerialize( ) { const { s: enableSnapshot = false, p: autoProcessing = false, d: autoDetection = false, o, r } = options; - const images = await runSpectraPipeline(nmriumState, { autoProcessing, autoDetection, enableSnapshot }, logger); + if (nmriumState.data) { + processSpectra(nmriumState.data, { autoDetection, autoProcessing }, logger); + } + + const images: Snapshot[] = enableSnapshot + ? await captureSpectraViewAsBase64(nmriumState, logger) + : []; const { data, version } = core.serializeNmriumState( nmriumState as NmriumState, - { includeData: r ? 'rawData' : 'dataSource' }, + { includeData: r ? 'rawData' : 'dataSource', }, + ); // include the meta and info object in case of serialize as dataSource @@ -42,67 +189,72 @@ async function processAndSerialize( if (!r) { for (let i = 0; i < spectra.length; i++) { const { info = {}, meta = {} } = nmriumState.data?.spectra[i] || {}; - spectra[i] = { ...spectra[i], info, meta }; + spectra[i] = { ...spectra[i], info, meta } } } - // Drop the raw processed spectra (typed arrays, filter history) now that - // everything needed from them has been copied into `spectra` above — - // otherwise they stay resident in memory alongside the serialized copy - // for the rest of the (potentially large, streamed) output write. - if (nmriumState.data) nmriumState.data.spectra = []; const logs = logger.getLogs(); - await outputResult({ nmriumState: { data, version }, images, logs }, o); + outputResult({ nmriumState: { data, version }, images, logs }, o); } async function loadSpectrumFromURL(options: RequiredKey, logger: FifoLogger) { const { u: url, include, exclude } = options; - const { pathname: relativePath, origin: baseURL } = new URL(url); + const { pathname: relativePath, origin: baseURL } = new URL(url) const source = { - entries: [{ relativePath }], + entries: [ + { + relativePath, + }, + ], baseURL, - }; + } + - const { state } = await core.readFromWebSource(source, { ...getParsingOptions(true), fileFilter: { include, exclude }, logger }); + const { state } = await core.readFromWebSource(source, { ...parsingOptions, fileFilter: { include, exclude }, logger }); + + processAndSerialize(state, options, logger) - await processAndSerialize(state, options, logger); } async function loadSpectrumFromFilePath(options: RequiredKey, logger: FifoLogger) { const { dir: path, include, exclude } = options; - const dirPath = isAbsolute(path) ? path : join(process.cwd(), path); + const dirPath = isAbsolute(path) ? path : join(process.cwd(), path) const fileCollection = await FileCollection.fromPath(dirPath, { unzip: { zipExtensions: ['zip', 'nmredata'] }, filter: { include, exclude }, - }); + }) + + const { + state + } = await core.read(fileCollection, { ...parsingOptions, logger }) - const { state } = await core.read(fileCollection, { ...getParsingOptions(true), logger }); + processAndSerialize(state, options, logger) - await processAndSerialize(state, options, logger); } -async function parseSpectra(argv: yargs.ArgumentsCamelCase) { + +function parseSpectra(argv: yargs.ArgumentsCamelCase +) { const logger = new FifoLogger(); + const { u, dir } = argv; + // Handle parsing the spectra file logic based on argv options + if (u) { + loadSpectrumFromURL({ u, ...argv }, logger); + } - try { - // Branches are mutually exclusive and awaited so a rejection is caught - // here instead of becoming an unhandled promise rejection, and so -u - // and --dir can't race to write the same output. - if (u) { - await loadSpectrumFromURL({ u, ...argv }, logger); - } else if (dir) { - await loadSpectrumFromFilePath({ dir, ...argv }, logger); - } else { - throw new Error('Either --u (URL) or --dir (directory) must be provided.'); - } - } catch (e) { - logger.error({ stage: 'fatal', details: toMessage(e) }, `Pipeline failed: ${toMessage(e)}`); - process.stderr.write(`${toMessage(e)}\n`); - process.exitCode = 1; + + if (dir) { + loadSpectrumFromFilePath({ dir, ...argv }, logger); } + + + } -export { loadSpectrumFromFilePath, loadSpectrumFromURL, parseSpectra }; \ No newline at end of file + + + +export { loadSpectrumFromFilePath, loadSpectrumFromURL, parseSpectra } diff --git a/app/scripts/nmr-cli/src/parse/run-concurrency.ts b/app/scripts/nmr-cli/src/parse/run-concurrency.ts deleted file mode 100644 index 9d1f615..0000000 --- a/app/scripts/nmr-cli/src/parse/run-concurrency.ts +++ /dev/null @@ -1,39 +0,0 @@ -import os from 'node:os'; - -// --- Concurrency tuning ----------------------------------------------- -// CPU-bound work (auto-processing / peak detection) is parallelized across -// worker threads: one worker per lane, reused for every spectrum that lane -// handles (spawning a worker reloads the whole nmr-processing module graph, -// so lanes are long-lived, not spawned per spectrum). -// -// Snapshot capture is parallelized across browser tabs, but each tab is far -// more memory-hungry than a worker thread, so it gets its own (lower) cap -// regardless of CPU count. -// -// Both are overridable via env vars for tuning on a given machine. -export const CPU_CONCURRENCY = Number(process.env['NMR_CLI_CPU_CONCURRENCY']) || Math.max(1, os.cpus().length - 1); -export const SNAPSHOT_CONCURRENCY = Number(process.env['NMR_CLI_SNAPSHOT_CONCURRENCY']) || 3; - -// Runs `handler` over `items` using a fixed number of long-lived "lanes" -// rather than firing off one promise per item. `laneIndex` is stable for the -// lifetime of a lane, so handlers can lazily attach an expensive, reusable -// resource (a worker thread, a browser page) to a given lane instead of -// creating one per item. -export async function runWithConcurrency( - items: T[], - concurrency: number, - handler: (item: T, laneIndex: number) => Promise -): Promise { - if (items.length === 0) return; - let cursor = 0; - const laneCount = Math.max(1, Math.min(concurrency, items.length)); - const lanes = Array.from({ length: laneCount }, (_, laneIndex) => - (async () => { - while (cursor < items.length) { - const item = items[cursor++]; - await handler(item, laneIndex); - } - })() - ); - await Promise.all(lanes); -} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/run-pipeline.ts b/app/scripts/nmr-cli/src/parse/run-pipeline.ts deleted file mode 100644 index 688a2af..0000000 --- a/app/scripts/nmr-cli/src/parse/run-pipeline.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { isSpectrum2D, type NmriumState } from '@zakodium/nmrium-core'; -import type { FifoLogger } from 'fifo-logger'; -import { runWithConcurrency, CPU_CONCURRENCY, SNAPSHOT_CONCURRENCY } from './run-concurrency'; -import { SpectrumWorker } from './spectrum-worker'; -import { BrowserManager, SnapshotLane } from './browser-manager'; -import { Snapshot } from './spectrum-snapshot'; -import { toMessage } from './utility/toMessage'; -import { SpectrumPipelineOptions } from './type/SpectrumPipelineOptions'; -import { ParsedSpectrum } from './type/ParsedSpectrum'; -import { initiateDatum2D } from './data/data2d/initiateDatum2D'; -import { initiateDatum1D } from './data/data1D/initiateDatum1D'; - -function getDurationMs(start: number): number { - return Date.now() - start; -} - -function generateNMRiumURL(): string { - const baseURL = process.env['BASE_NMRIUM_URL'] || ''; - const url = new URL(baseURL); - url.searchParams.append('workspace', 'embedded'); - return url.toString(); -} - -// Each spectrum flows through the same stages — process -> detect -> snapshot -// — but stages now run across a fixed number of concurrent "lanes" instead -// of one spectrum at a time. Every lane owns its own worker thread (CPU -// stages) and, if snapshots are enabled, its own persistent browser page. -// -// Every stage logs `durationMs` alongside its existing pass/fail log, so a -// run's logs can be aggregated afterwards to see where time actually goes -// (parse vs. process vs. detect vs. snapshot) instead of guessing. -export async function runSpectraPipeline( - nmriumState: Partial, - options: SpectrumPipelineOptions, - logger: FifoLogger -): Promise { - const data = nmriumState.data; - const { version } = nmriumState; - if (!data) return []; - - const parsed = parseSpectraList(data.spectra, logger); - - // Write the parsed set back so serialization later only sees spectra that - // actually parsed successfully. - data.spectra = parsed.map((d) => d.spectrum); - const indexById = new Map(parsed.map((d, i) => [d.id, i])); - - const { autoProcessing, autoDetection, enableSnapshot } = options; - const snapshots: Snapshot[] = []; - if (parsed.length === 0) return snapshots; - - const concurrency = enableSnapshot ? SNAPSHOT_CONCURRENCY : CPU_CONCURRENCY; - const browserManager = enableSnapshot ? new BrowserManager() : null; - const url = enableSnapshot ? generateNMRiumURL() : ''; - - const spectrumWorkers: SpectrumWorker[] = []; - const snapshotLanes: SnapshotLane[] = []; - - const getWorker = (laneIndex: number): SpectrumWorker => { - if (!spectrumWorkers[laneIndex]) spectrumWorkers[laneIndex] = new SpectrumWorker(); - return spectrumWorkers[laneIndex]; - }; - const getSnapshotLane = (laneIndex: number): SnapshotLane => { - if (!snapshotLanes[laneIndex]) snapshotLanes[laneIndex] = new SnapshotLane(browserManager!, url); - return snapshotLanes[laneIndex]; - }; - - try { - await runWithConcurrency(parsed, concurrency, async ({ id, spectrum: initial }, laneIndex) => { - const spectrumWorker = getWorker(laneIndex); - let spectrum = initial; - - if (autoProcessing) { - const start = Date.now(); - try { - spectrum = await spectrumWorker.run('process', spectrum); - logger.info({ id, stage: 'processing', durationMs: getDurationMs(start) }, `Processed spectrum: ${id}`); - } catch (e) { - logger.error( - { id, stage: 'processing', durationMs: getDurationMs(start), details: toMessage(e) }, - `Failed to process spectrum: ${id}` - ); - } - } - - if (autoDetection && spectrum.info.isFt) { - const start = Date.now(); - try { - spectrum = await spectrumWorker.run('detect', spectrum); - logger.info({ id, stage: 'detection', durationMs: getDurationMs(start) }, `Detected peaks for spectrum: ${id}`); - } catch (e) { - logger.error( - { id, stage: 'detection', durationMs: getDurationMs(start), details: toMessage(e) }, - `Failed to detect peaks for spectrum: ${id}` - ); - } - } - - data.spectra[indexById.get(id)!] = spectrum; - - if (enableSnapshot) { - const lane = getSnapshotLane(laneIndex); - // Timing + retry for this stage live inside SnapshotLane.capture - // itself, since a retry needs its own per-attempt timing. - const snapshot = await lane.capture(spectrum, id, version, spectrumWorker, logger); - snapshots.push(snapshot); - } - }); - } finally { - await Promise.all(spectrumWorkers.filter(Boolean).map((w) => w.terminate())); - await Promise.all(snapshotLanes.filter(Boolean).map((l) => l.dispose())); - await browserManager?.closeAll(); - } - - return snapshots; -} - - - - - - - -function parseSpectraList(rawSpectra: any[], logger: FifoLogger): ParsedSpectrum[] { - const parsed: ParsedSpectrum[] = []; - for (const inputSpectrum of rawSpectra) { - const id = inputSpectrum.id; - const start = Date.now(); - try { - const spectrum = isSpectrum2D(inputSpectrum) - ? initiateDatum2D(inputSpectrum) - : initiateDatum1D(inputSpectrum); - logger.info({ id, stage: 'parsing', durationMs: Date.now() - start }, `Parsed spectrum: ${id}`); - parsed.push({ id, spectrum }); - } catch (e) { - logger.error( - { id, stage: 'parsing', durationMs: Date.now() - start, details: toMessage(e) }, - `Failed to parse spectrum: ${id}` - ); - } - } - return parsed; -} diff --git a/app/scripts/nmr-cli/src/parse/spectrum-snapshot.ts b/app/scripts/nmr-cli/src/parse/spectrum-snapshot.ts deleted file mode 100644 index 400f9da..0000000 --- a/app/scripts/nmr-cli/src/parse/spectrum-snapshot.ts +++ /dev/null @@ -1,117 +0,0 @@ -import playwright, { - type Browser, - type BrowserContext, - type Page, -} from 'playwright' - -import { SpectrumWorker } from './spectrum-worker' - -export interface Snapshot { - id: string - image: string | null -} - -const SNAPSHOT_LOADING_TIMEOUT_MS = 30_000 - -function generateNMRiumURL(): string { - const baseURL = process.env.BASE_NMRIUM_URL - - if (!baseURL) { - throw new Error( - 'BASE_NMRIUM_URL environment variable is not defined', - ) - } - - const url = new URL(baseURL) - url.searchParams.set('workspace', 'embedded') - - return url.toString() -} - -async function waitForNMRium(page: Page): Promise { - await page.locator('text=Loading').waitFor({ - state: 'hidden', - timeout: SNAPSHOT_LOADING_TIMEOUT_MS, - }) -} - -export class SpectrumSnapshot { - private browser: Browser | null = null - private context: BrowserContext | null = null - private page: Page | null = null - - async start(): Promise { - if (this.page) return - - this.browser = await playwright.firefox.launch() - - this.context = await this.browser.newContext( - playwright.devices['Desktop Chrome HiDPI'], - ) - - this.page = await this.context.newPage() - - await this.page.goto(generateNMRiumURL()) - - await waitForNMRium(this.page) - } - - async capture( - id: string, - spectrum: any, - version: unknown, - worker: SpectrumWorker, - ): Promise { - if (!this.page) { - throw new Error( - 'SpectrumSnapshot has not been started', - ) - } - - const stringObject = await worker.run( - 'serialize', - spectrum, - version, - ) - - /* - * Parse the serialized data here rather than injecting - * the JSON string into JavaScript source code. - */ - const data = JSON.parse(stringObject) - - await this.page.evaluate((nmriumData) => { - window.postMessage( - { - type: 'nmr-wrapper:load', - data: { - data: nmriumData, - type: 'nmrium', - }, - }, - '*', - ) - }, data) - - await waitForNMRium(this.page) - - const image = await this.page - .locator('#nmrSVG .container') - .screenshot() - - return { - id, - image: image.toString('base64'), - } - } - - async close(): Promise { - await this.context?.close().catch(() => { }) - await this.browser?.close().catch(() => { }) - - this.page = null - this.context = null - this.browser = null - } -} - diff --git a/app/scripts/nmr-cli/src/parse/spectrum-worker.ts b/app/scripts/nmr-cli/src/parse/spectrum-worker.ts deleted file mode 100644 index bb7f715..0000000 --- a/app/scripts/nmr-cli/src/parse/spectrum-worker.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { Spectrum } from '@zakodium/nmr-types'; -import { join } from 'node:path'; -import { Worker } from 'node:worker_threads'; -import { WorkerResponse } from './type/WorkerResponse'; - -// Runs processing/detection/serialization in a separate worker thread (its -// own V8 heap) so a heap-OOM there only terminates the worker instead of the -// whole CLI process. One instance is created per concurrency lane (see -// run-pipeline.ts). - -export class SpectrumWorker { - private worker: Worker | null = null; - - private spawn(): Worker { - const worker = new Worker(join(__dirname, 'worker-entry.js'), { - resourceLimits: { maxOldGenerationSizeMb: 3072 }, - }); - worker.on('error', () => { this.worker = null; }); - worker.on('exit', () => { this.worker = null; }); - return worker; - } - - private get(): Worker { - if (!this.worker) this.worker = this.spawn(); - return this.worker; - } - - run(task: 'process' | 'detect', spectrum: Spectrum): Promise; - run(task: 'serialize', spectrum: Spectrum, version: unknown): Promise; - run(task: 'process' | 'detect' | 'serialize', spectrum: Spectrum, version?: unknown): Promise { - return new Promise((resolve, reject) => { - const worker = this.get(); - let settled = false; - - const onMessage = (msg: WorkerResponse) => { - settled = true; - cleanup(); - if (msg.error) reject(new Error(msg.error)); - else resolve(task === 'serialize' ? msg.stringObject : msg.spectrum); - }; - const onError = (err: Error) => { - settled = true; - this.worker = null; - cleanup(); - reject(err); - }; - const onExit = (code: number) => { - this.worker = null; - if (!settled && code !== 0) { - cleanup(); - reject(new Error(`Worker exited with code ${code} (likely out of memory)`)); - } - }; - const cleanup = () => { - worker.off('message', onMessage); - worker.off('error', onError); - worker.off('exit', onExit); - }; - - worker.on('message', onMessage); - worker.on('error', onError); - worker.on('exit', onExit); - worker.postMessage({ spectrum, task, version }); - }); - } - - async terminate() { - await this.worker?.terminate().catch(() => { }); - this.worker = null; - } -} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/type/MoleculeExtended.ts b/app/scripts/nmr-cli/src/parse/type/MoleculeExtended.ts index 601a66f..7e71a00 100644 --- a/app/scripts/nmr-cli/src/parse/type/MoleculeExtended.ts +++ b/app/scripts/nmr-cli/src/parse/type/MoleculeExtended.ts @@ -1,12 +1,11 @@ -import { StateMolecule } from '@zakodium/nmrium-core' +import { StateMolecule } from "@zakodium/nmrium-core"; export interface MoleculeExtended - extends - Required>, + extends Required>, Omit { - mf: string - em: number - mw: number - svg: string - atoms: Record -} + mf: string; + em: number; + mw: number; + svg: string; + atoms: Record; +} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/type/ParsedSpectrum.ts b/app/scripts/nmr-cli/src/parse/type/ParsedSpectrum.ts deleted file mode 100644 index 911550c..0000000 --- a/app/scripts/nmr-cli/src/parse/type/ParsedSpectrum.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { Spectrum } from "@zakodium/nmr-types"; - -export interface ParsedSpectrum { - id: string; - spectrum: Spectrum; -} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/type/Snapshot.ts b/app/scripts/nmr-cli/src/parse/type/Snapshot.ts deleted file mode 100644 index 0ed48cb..0000000 --- a/app/scripts/nmr-cli/src/parse/type/Snapshot.ts +++ /dev/null @@ -1,4 +0,0 @@ -export interface Snapshot { - id: string; - image: string | null; -} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/type/SpectrumPipelineOptions.ts b/app/scripts/nmr-cli/src/parse/type/SpectrumPipelineOptions.ts deleted file mode 100644 index 001b4e7..0000000 --- a/app/scripts/nmr-cli/src/parse/type/SpectrumPipelineOptions.ts +++ /dev/null @@ -1,5 +0,0 @@ -export interface SpectrumPipelineOptions { - autoProcessing: boolean; - autoDetection: boolean; - enableSnapshot: boolean; -} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/type/WorkerRequest.ts b/app/scripts/nmr-cli/src/parse/type/WorkerRequest.ts deleted file mode 100644 index fb39179..0000000 --- a/app/scripts/nmr-cli/src/parse/type/WorkerRequest.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { Spectrum } from "@zakodium/nmr-types"; - -// Message contract shared with detectWorkerEntry.ts — keep both in sync. -export interface WorkerRequest { - spectrum: Spectrum; - task: 'process' | 'detect' | 'serialize'; - version?: unknown; -} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/type/WorkerResponse.ts b/app/scripts/nmr-cli/src/parse/type/WorkerResponse.ts deleted file mode 100644 index 7caaa46..0000000 --- a/app/scripts/nmr-cli/src/parse/type/WorkerResponse.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Spectrum } from "@zakodium/nmr-types"; - -export interface WorkerResponse { - spectrum?: Spectrum; - stringObject?: string; - error?: string; -} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/utility/getAtom.ts b/app/scripts/nmr-cli/src/parse/utility/getAtom.ts index da09e78..68e66ec 100644 --- a/app/scripts/nmr-cli/src/parse/utility/getAtom.ts +++ b/app/scripts/nmr-cli/src/parse/utility/getAtom.ts @@ -1,3 +1,3 @@ export default function getAtom(nucleus: string): string { - return nucleus?.replaceAll(/\d/g, '') || '' + return nucleus?.replaceAll(/\d/g, '') || ''; } diff --git a/app/scripts/nmr-cli/src/parse/utility/isProton.ts b/app/scripts/nmr-cli/src/parse/utility/isProton.ts index 00f3e6e..85f6e6e 100644 --- a/app/scripts/nmr-cli/src/parse/utility/isProton.ts +++ b/app/scripts/nmr-cli/src/parse/utility/isProton.ts @@ -1,3 +1,3 @@ export function isProton(nucleus: string) { - return nucleus === '1H' + return nucleus === '1H'; } diff --git a/app/scripts/nmr-cli/src/parse/utility/outputResult.ts b/app/scripts/nmr-cli/src/parse/utility/outputResult.ts deleted file mode 100644 index 40de4b6..0000000 --- a/app/scripts/nmr-cli/src/parse/utility/outputResult.ts +++ /dev/null @@ -1,24 +0,0 @@ - -import { createWriteStream } from 'node:fs'; -import { JsonStreamStringify } from 'json-stream-stringify'; - -export function outputResult(result: any, outputPath?: string): Promise { - return new Promise((resolve, reject) => { - const stream = new JsonStreamStringify(result); - - if (outputPath) { - const writeStream = createWriteStream(outputPath); - stream.pipe(writeStream); - writeStream.on('finish', () => { - process.stderr.write(`Output written to: ${outputPath}\n`); - resolve(); - }); - writeStream.on('error', reject); - stream.on('error', reject); - } else { - stream.pipe(process.stdout); - stream.on('end', () => resolve()); - stream.on('error', reject); - } - }); -} diff --git a/app/scripts/nmr-cli/src/parse/utility/toMessage.ts b/app/scripts/nmr-cli/src/parse/utility/toMessage.ts deleted file mode 100644 index 6516c2c..0000000 --- a/app/scripts/nmr-cli/src/parse/utility/toMessage.ts +++ /dev/null @@ -1,3 +0,0 @@ -export function toMessage(e: unknown): string { - return e instanceof Error ? e.message : String(e); -} \ No newline at end of file diff --git a/app/scripts/nmr-cli/src/parse/worker-entry.ts b/app/scripts/nmr-cli/src/parse/worker-entry.ts deleted file mode 100644 index b78df71..0000000 --- a/app/scripts/nmr-cli/src/parse/worker-entry.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { parentPort } from 'node:worker_threads'; -import { Filters1DManager, Filters2DManager } from 'nmr-processing'; -import { detectZones } from './data/data2d/detectZones'; -import { detectRanges } from './data/data1D/detectRanges'; -import { Spectrum } from '@zakodium/nmr-types'; -import { isSpectrum2D } from '@zakodium/nmrium-core'; -import { WorkerRequest } from './type/WorkerRequest'; -import { WorkerResponse } from './type/WorkerResponse'; - -if (!parentPort) { - throw new Error('detectWorkerEntry must be run inside a worker_threads Worker'); -} - -// Both mutate the spectrum in place (same as the original single-threaded -// implementation) — the return is just for a uniform call signature. -async function runAutoProcessing(spectrum: Spectrum): Promise { - if (isSpectrum2D(spectrum)) { - Filters2DManager.reapplyFilters(spectrum); - } else { - Filters1DManager.reapplyFilters(spectrum); - } - return spectrum; -} - -async function runAutoDetection(spectrum: Spectrum): Promise { - if (isSpectrum2D(spectrum)) { - detectZones(spectrum); - } else { - detectRanges(spectrum); - } - return spectrum; -} - -// Spectra hold typed-array data (Float64Array etc.), which JSON.stringify -// otherwise mangles into `{0: ..., 1: ...}` objects instead of arrays — the -// replacer below converts any ArrayBuffer view to a plain array first. No -// nmrium-core-plugins init needed here: this is a plain JSON encoding of -// { version, data: { spectra: [spectrum] } }, matching exactly what the -// browser's `nmr-wrapper:load` message expects. -async function serializeSpectrum(spectrum: Spectrum, version: unknown): Promise { - return JSON.stringify( - { version, data: { spectra: [spectrum] } }, - (_key, value: unknown) => (ArrayBuffer.isView(value) ? Array.from(value as unknown as Iterable) : value) - ); -} - -parentPort.on('message', async (msg: WorkerRequest) => { - const { spectrum, task, version } = msg; - const reply = (response: WorkerResponse) => parentPort!.postMessage(response); - - try { - switch (task) { - case 'process': - reply({ spectrum: await runAutoProcessing(spectrum) }); - break; - case 'detect': - reply({ spectrum: await runAutoDetection(spectrum) }); - break; - case 'serialize': - reply({ stringObject: await serializeSpectrum(spectrum, version) }); - break; - default: { - const exhaustive: never = task; - throw new Error(`Unknown task: ${exhaustive}`); - } - } - } catch (e) { - reply({ error: e instanceof Error ? e.message : String(e) }); - } -}); \ No newline at end of file