diff --git a/CHANGELOG.md b/CHANGELOG.md index 58081e59..7a568bd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ All notable changes to this project will be documented in this file. +## [Unreleased] + +### Added +- `overviewMode: 'spectrogram'` (and `player.setOverviewMode()`): the overview + strip shows the whole recording as a spectrogram, max-pooled so short calls + stay visible; the waveform stands in while the spectrogram is computed. + The labeling app offers the choice under Display → Overview. +- Recording start time from AudioMoth, GUANO, BWF `bext` and WAV INFO metadata + or a date and time in the file name (adapted from the BirdNET website demo, + MIT): shown in the status bar, `player.recordingStart`, + `player.clockTimeAt(seconds)`, and Raven selection tables get + Begin Date / Begin Clock Time / End Clock Time. +- Microphone recording: `player.startRecording()` / `stopRecording()` show a + live spectrogram that flies in from the right while recording, then load + the recording (a WAV with its start time in the INFO chunk). Events + `recordingstart`, `recordingcomplete`, `recordingstop`. The labeling app + adds "Record from microphone" under Add recordings. + ## [1.1.3] - 2026-11-14 ### Added diff --git a/demo/labeling-app.html b/demo/labeling-app.html index 5d4c0371..cee74a68 100644 --- a/demo/labeling-app.html +++ b/demo/labeling-app.html @@ -378,6 +378,9 @@

Recording not found

+
@@ -820,8 +823,9 @@

Tags

} // ── Player ──────────────────────────────────────────────────────── - // Waveform above the spectrogram is optional (the overview below always shows one); applied on load + // Waveform above the spectrogram is optional; applied on load. The overview below shows a waveform or the whole spectrogram. const SHOW_WAVEFORM_KEY = 'signavis.labeling.show-waveform.v1'; + const OVERVIEW_MODE_KEY = 'signavis.labeling.overview-mode.v1'; const SHOW_CONF_KEY = 'signavis.labeling.show-confidence.v1'; const AUTO_FREQ_VIEW_KEY = 'signavis.labeling.auto-freq-view.v1'; const NORMALIZE_KEY = 'signavis.labeling.normalize.v1'; @@ -834,6 +838,7 @@

Tags

showFileOpen: false, enableProgressiveSpectrogram: true, showOverview: true, + overviewMode: (() => { try { return localStorage.getItem(OVERVIEW_MODE_KEY) === 'spectrogram' ? 'spectrogram' : 'waveform'; } catch { return 'waveform'; } })(), viewMode: showMainWaveform ? 'both' : 'spectrogram', showTransport: true, // BirdNET detections (no frequency bounds) as strips along the top edge @@ -1383,6 +1388,23 @@

Tags

}); specPanel.el.insertBefore(wf, sec); + const ov = document.createElement('div'); + ov.className = 'settings-section'; + ov.style.cssText = 'padding:8px 14px;border-top:1px solid var(--color-border)'; + ov.innerHTML = ` + + `; + const ovSel = ov.querySelector('select'); + ovSel.value = player.overviewMode; + ovSel.addEventListener('change', () => { + try { localStorage.setItem(OVERVIEW_MODE_KEY, ovSel.value); } catch { /* ignore */ } + player.setOverviewMode(ovSel.value); + }); + specPanel.el.insertBefore(ov, sec); + // ── Automatic behaviour: what the app does by itself, each one switchable ── const auto = document.createElement('div'); auto.className = 'settings-section'; @@ -2212,7 +2234,7 @@

Tags

function downloadRavenTable() { const entry = recordingManager.getActive(); const name = entry?.filename || fileLabel.textContent || 'labels'; - const text = buildRavenSelectionTable(state.labels, { sampleRate: entry?.sampleRate || state.sampleRate, setName: ravenSetName }); + const text = buildRavenSelectionTable(state.labels, { sampleRate: entry?.sampleRate || state.sampleRate, setName: ravenSetName, recordingStart: entry?.recordingStart || player.recordingStart }); saveFile(text, ravenTablePath(name.split('/').pop()), 'text/plain') .catch((e) => { selectionMeta.textContent = `Export failed: ${e?.message || e}`; }); } @@ -2225,7 +2247,7 @@

Tags

const labels = entryLabels(entry); if (!labels.length) continue; const target = entry.path ? ravenTablePath(entry.path) : `exports/raven/${ravenTablePath(entry.filename)}`; - await projectStore.writeText(target, buildRavenSelectionTable(labels, { sampleRate: entry.sampleRate, setName: ravenSetName })); + await projectStore.writeText(target, buildRavenSelectionTable(labels, { sampleRate: entry.sampleRate, setName: ravenSetName, recordingStart: entry.recordingStart || null })); written++; } selectionMeta.textContent = written @@ -2597,7 +2619,13 @@

Tags

// Blob URL (local file) — same origin, no CORS issue _lastAudioUrl = entry.audioUrl?.startsWith('blob:') ? entry.audioUrl : null; player.showLoadingOverlay('Loading…'); - await player.loadUrl(entry.audioUrl); + if (_lastAudioUrl) { + // As a File with its name, so a start time in the file name is found again. + const blob = await (await fetch(entry.audioUrl)).blob(); + await player.loadFile(new File([blob], (entry.filename || 'audio').split('/').pop(), { type: blob.type })); + } else { + await player.loadUrl(entry.audioUrl); + } } } catch (e) { _audioFailed = true; @@ -3236,6 +3264,7 @@

Open another recording

const out = buildLabelExport(format, labels, base, { sampleRate: entry?.sampleRate || state.sampleRate, setName: ravenSetName, + recordingStart: entry?.recordingStart || player.recordingStart, columns, displayName: (l) => resolveName(l).display, }); @@ -4276,6 +4305,31 @@

Open another recording

catch (e) { showLoadError(e?.message || e, 'Cannot open'); return; } if (picked) await loadLocalAudioFile(picked.file, { filePath: picked.path }); }); + // ── Microphone recording: live spectrogram, then added like a local file ── + const recordBtn = document.getElementById('recordBtn'); + if (recordBtn && BirdNETPlayer.recordingSupported) { + recordBtn.hidden = false; + const recordText = recordBtn.querySelector('.record-btn-text'); + const setRecording = (on) => { + recordBtn.classList.toggle('is-recording', on); + recordText.textContent = on ? 'Stop recording' : 'Record from microphone'; + }; + recordBtn.addEventListener('click', async () => { + if (player.isRecording) { await player.stopRecording(); return; } + try { + await player.startRecording({ maxSeconds: 300, loadWhenDone: false }); + setRecording(true); + } catch (e) { + showLoadError(e?.name === 'NotAllowedError' ? 'Microphone access was denied.' : (e?.message || e), 'Cannot record'); + } + }); + player.on('recordingstop', () => setRecording(false)); + player.on('recordingcomplete', async (e) => { + setRecording(false); + const file = e.detail?.file; + if (file) await loadLocalAudioFile(file); + }); + } fileInput.addEventListener('change', async () => { const file = fileInput.files?.[0]; if (!file) return; @@ -4331,6 +4385,7 @@

Open another recording

audioUrl: _blobUrl, isBlobUrl: true, ...extra, + recordingStart: player.recordingStart || null, }); recordingManager.setActive(_fileEntry.id); state._labelsEntryId = _fileEntry.id; // empty labels now belong to the new entry @@ -4414,7 +4469,11 @@

Open another recording

setTimeout(() => requestMissingSnr(), 500); if (e.detail?.sampleRate) { state.sampleRate = e.detail.sampleRate; - if (recordingManager?.activeId) recordingManager.update(recordingManager.activeId, { sampleRate: e.detail.sampleRate }); + if (recordingManager?.activeId) { + // Keep a known start when this load could not tell (e.g. metadata stripped). + const known = recordingManager.getById(recordingManager.activeId)?.recordingStart || null; + recordingManager.update(recordingManager.activeId, { sampleRate: e.detail.sampleRate, recordingStart: player.recordingStart || known }); + } } propsPanel.setRecordingMeta(state.recordingMeta, { sampleRate: state.sampleRate, diff --git a/docs/usage/integration.md b/docs/usage/integration.md index 30311259..702aa767 100644 --- a/docs/usage/integration.md +++ b/docs/usage/integration.md @@ -28,6 +28,34 @@ input.addEventListener('change', async () => { }) ``` +## Record from the microphone + +While recording, the spectrogram pane shows the sound as it arrives (it flies +in from the right, 20 s across). When it stops — Stop button, `stopRecording()` +or after `maxSeconds` — the recording is loaded like a file. The WAV carries +its start time, so `player.recordingStart` knows the clock time (the labeling app's +Raven selection tables use it; `player.exportAnnotationsRaven()` has no clock columns). + +```js +if (BirdNETPlayer.recordingSupported) { + player.on('recordingcomplete', ({ detail }) => { + console.log(detail.file.name, detail.duration) // Recording_20261115_143012.wav + }) + await player.startRecording({ maxSeconds: 300 }) // asks for microphone access + // … later + await player.stopRecording() +} +``` + +Pass `loadWhenDone: false` to handle `detail.file` yourself. + +## Recording start time + +`player.recordingStart` is read from AudioMoth, GUANO, BWF `bext` or WAV INFO +metadata, or from a date and time in the file name (`20230512_190000.wav`); +`null` when unknown. `player.clockTimeAt(seconds)` gives the wall-clock time at +a position, e.g. `"2023-05-12T19:00:03.000+01:00"`. + ## React ```jsx diff --git a/docs/usage/player-options.md b/docs/usage/player-options.md index 85285f79..7da799a5 100644 --- a/docs/usage/player-options.md +++ b/docs/usage/player-options.md @@ -18,6 +18,7 @@ | `showDisplayGain` | boolean | `true` | Show floor/ceiling sliders, auto contrast | | `showStatusbar` | boolean | `true` | Show bottom status bar | | `showOverview` | boolean | `true` | Show overview navigator | +| `overviewMode` | string | `'waveform'` | `'waveform'` or `'spectrogram'` — the whole recording as a max-pooled spectrogram (short calls stay visible); switch at runtime with `player.setOverviewMode(mode)` | | `showWaveformTimeline` | boolean | `true` | Show bottom timeline in waveform view | | `compactToolbar` | string | `'auto'` | `'auto'`, `'on'`, `'off'` — responsive toolbar compaction | | `labelTaxonomy` | array | see docs | Custom label presets (name, color, shortcut) | diff --git a/python-wrapper/README.md b/python-wrapper/README.md index c960abad..656e5d01 100644 --- a/python-wrapper/README.md +++ b/python-wrapper/README.md @@ -114,6 +114,7 @@ HTML(html) | `viewMode` | `'spectrogram'` | Show only the spectrogram | | `transportStyle` | `'hero'` | Large centered play button | | `showOverview` | `False` | Hide overview navigator | +| `overviewMode` | `'spectrogram'` | Overview shows the whole spectrogram instead of the waveform | | `showFileOpen` | `False` | Hide file open button | | `showStatusbar` | `False` | Hide status bar | | `labelTaxonomy` | `[{'name': 'Species', 'color': '#0ea5e9'}]` | Custom label presets | diff --git a/src/app/BirdNETPlayer.ts b/src/app/BirdNETPlayer.ts index f9be08cc..c710db94 100644 --- a/src/app/BirdNETPlayer.ts +++ b/src/app/BirdNETPlayer.ts @@ -91,6 +91,18 @@ export { importXenoCantoSpectrogramLabels, } from '../infrastructure/xeno-canto/xenoCantoRecordingsApi.ts'; export { TaxonomyResolver } from '../infrastructure/taxonomyResolver.ts'; +import { isoClockTime, type RecordingStart } from '../domain/recordingTime.ts'; +import { LiveRecorder, type LiveRecorderOptions, type LiveRecording } from '../infrastructure/audio/LiveRecorder.ts'; +import { LiveSpectrogramView } from '../ui/components/live-view/live-spectrogram-view.ts'; +export { LiveRecorder, encodeWav, recordingFileName, type LiveRecorderOptions, type LiveRecording } from '../infrastructure/audio/LiveRecorder.ts'; + +export interface StartRecordingOptions extends LiveRecorderOptions { + /** Load the recording into the player when it stops (default true). */ + loadWhenDone?: boolean; + /** Seconds across the live view (default 20). */ + spanSeconds?: number; +} +export { readRecordingStart, isoClockTime, ravenClock, utcLabel, formatRecordingStart, type RecordingStart } from '../domain/recordingTime.ts'; export { BirdNETInference, BIRDNET_MODEL_URL } from '../infrastructure/birdnetInference.ts'; export class BirdNETPlayer { @@ -714,6 +726,8 @@ export class BirdNETPlayer { /** Tear down the player and free resources */ destroy() { + this._liveRecorder?.cancel(); + this._liveView?.detach(); this._labelGrid?.destroy(); this._labelGrid = null; if (this._globalKeyHandler) { @@ -962,6 +976,87 @@ export class BirdNETPlayer { get normalize() { return !!this._state?.normalize; } + /** + * When the loaded recording started, from AudioMoth / GUANO / BWF / WAV INFO + * metadata or a date and time in the file name; null when unknown. + * `ms` is the recorder's wall-clock time written as UTC, `tz` its offset in minutes. + */ + get recordingStart(): RecordingStart | null { return this._state?.recordingStart ?? null; } + + /** Wall-clock time at `seconds` into the recording ("2023-05-12T19:00:03.000+01:00"), null when the start is unknown. */ + clockTimeAt(seconds: number): string | null { + const start = this.recordingStart; + return start ? isoClockTime(start, seconds) : null; + } + + // ── Live recording ────────────────────────────────────────────── + + _liveRecorder: LiveRecorder | null = null; + _liveView: LiveSpectrogramView | null = null; + + /** True when this browser can record from a microphone. */ + static get recordingSupported(): boolean { return LiveRecorder.supported; } + + get isRecording(): boolean { return !!this._liveRecorder?.recording; } + + /** + * Records from the microphone; the spectrogram pane shows the sound as it + * arrives. Emits 'recordingstart', then 'recordingcomplete' { file, + * startedAt, duration, sampleRate, reason } (or 'recordingstop' { reason } + * without a recording). Stops by itself after `maxSeconds` (default 300). + * The WAV carries the start time, so `recordingStart` knows it after loading. + */ + async startRecording(opts: StartRecordingOptions = {}): Promise { + if (this._liveRecorder) throw new Error('Already recording'); + const s = this._state; + if (!s) throw new Error('Player not ready'); + this.pause(); + const rec = new LiveRecorder(opts); + this._liveRecorder = rec; + const host = (s.d.spectrogramContainer && s._showSpectrogram ? s.d.spectrogramContainer : this.container) as HTMLElement; + const view = new LiveSpectrogramView(host, { + spanSeconds: opts.spanSeconds, + getColorScheme: () => s.d.colorSchemeSelect?.value || 'inferno', + getMaxFreq: () => parseFloat(s.d.maxFreqSelect?.value || '0') || 0, + }); + view.addEventListener('stoprequest', () => { void this.stopRecording(); }); + let reason: 'user' | 'limit' | 'ended' = 'user'; + rec.addEventListener('stop', (e: Event) => { reason = (e as CustomEvent).detail?.reason ?? 'user'; }); + const finished = new Promise((resolve) => { + rec.addEventListener('stop', () => resolve(rec.stop()), { once: true }); + }); + try { + await rec.start(); + } catch (err) { + this._liveRecorder = null; + throw err; + } + this._liveView = view; + view.attach(rec); + this._emit('recordingstart', { sampleRate: rec.sampleRate, maxSeconds: rec.maxSeconds }); + void finished.then(async (recording) => { + view.detach(); + if (this._liveView === view) this._liveView = null; + if (this._liveRecorder === rec) this._liveRecorder = null; + if (!recording) { this._emit('recordingstop', { reason }); return; } + this._emit('recordingcomplete', { ...recording, reason }); + if (opts.loadWhenDone !== false) await this.loadFile(recording.file).catch(() => {}); + }); + } + + /** Stops the recording; resolves with it (null when nothing was recorded or none was running). */ + async stopRecording(): Promise { + return this._liveRecorder ? this._liveRecorder.stop('user') : null; + } + + /** Stops the recording and throws it away. */ + cancelRecording(): void { this._liveRecorder?.cancel(); } + + /** Overview strip below the player: 'waveform' or a whole-file 'spectrogram'. */ + setOverviewMode(mode: 'waveform' | 'spectrogram') { this._state?._setOverviewMode(mode); } + + get overviewMode(): 'waveform' | 'spectrogram' { return this._state?._overviewMode ?? 'waveform'; } + // ── Built-in label table ──────────────────────────────────────── _labelGrid: LabelGrid | null = null; diff --git a/src/app/PlayerState.ts b/src/app/PlayerState.ts index 2a5e2aa1..4ba1d612 100644 --- a/src/app/PlayerState.ts +++ b/src/app/PlayerState.ts @@ -30,7 +30,8 @@ import { CrosshairController } from './player/CrosshairController.ts'; import { PerfOverlay } from './player/PerfOverlay.ts'; import { ViewResizeController } from './player/ViewResizeController.ts'; import { FrequencyZoomController } from './player/FrequencyZoomController.ts'; -import { WaveformRenderer } from './player/WaveformRenderer.ts'; +import { WaveformRenderer, type OverviewMode } from './player/WaveformRenderer.ts'; +import { formatRecordingStart, type RecordingStart } from '../domain/recordingTime.ts'; import { ToolbarController } from './player/ToolbarController.ts'; import { ToolbarFitController } from './player/ToolbarFitController.ts'; import { CarrierLineController } from './player/CarrierLineController.ts'; @@ -43,6 +44,7 @@ import { computeBandProfile, autoViewMaxHz, type BandProfile } from '../domain/b * @typedef {Object} PlayerOptions * @property {string} [viewMode] * @property {boolean} [showOverview] + * @property {'waveform'|'spectrogram'} [overviewMode] Overview strip content (default 'waveform') * @property {boolean} [transportOverlay] * @property {string} [compactToolbar] * @property {boolean} [showWaveformTimeline] @@ -137,6 +139,8 @@ export class PlayerState { _showWaveform: any; _showSpectrogram: any; _showOverview: any; + _overviewMode: OverviewMode = 'waveform'; + _overviewRedrawTimer: ReturnType | null = null; _transportOverlay: any; _keyboardEnabled = true; _compactToolbarMode: any; @@ -243,6 +247,7 @@ export class PlayerState { this._showWaveform = this._viewMode !== 'spectrogram'; this._showSpectrogram = this._viewMode !== 'waveform'; this._showOverview = this.options.showOverview !== false; + this._overviewMode = this.options.overviewMode === 'spectrogram' ? 'spectrogram' : 'waveform'; this._transportOverlay = this.options.transportOverlay === true; this._keyboardEnabled = this.options.keyboardShortcuts !== false; this._compactToolbarMode = this.options.compactToolbar && ['auto', 'on', 'off'].includes(this.options.compactToolbar) @@ -275,6 +280,7 @@ export class PlayerState { this._createFrequencyLabels(); this._drawSpectrogram(); this._syncOverviewWindowToViewport(); + this._scheduleOverviewSpectrogram(); }); // ── Audio / analysis state ── @@ -417,6 +423,8 @@ export class PlayerState { scheduleUiUpdate: () => this._scheduleUiUpdate({ time: this._getCurrentTime(), fromPlayback: false, immediate: true }), getScrollLeft: () => this._getPrimaryScrollLeft(), getViewportWidth: () => this._getViewportWidth(), + getOverviewMode: () => this._overviewMode, + getOverviewImage: (cols: number) => this._spectro.overviewImage(cols), }); // ── ToolbarController ── @@ -861,7 +869,11 @@ export class PlayerState { } this._spectro.updateMaxFreqOptions(sampleRate); - this.d.fileInfo!.innerHTML = `${escapeHtml(displayName)} ${formatTime(duration)}`; + const start = this._engine.recordingStart ?? null; + const startHtml = start + ? ` · ${escapeHtml(formatRecordingStart(start))}` + : ''; + this.d.fileInfo!.innerHTML = `${escapeHtml(displayName)} ${formatTime(duration)}${startHtml}`; if (this.d.sampleRateInfo) this.d.sampleRateInfo.textContent = `${sampleRate} Hz`; if (this.d.totalTimeDisplay) this.d.totalTimeDisplay.textContent = formatTime(duration); if (this.d.currentTimeDisplay) this.d.currentTimeDisplay.textContent = formatTime(0); @@ -876,6 +888,7 @@ export class PlayerState { // spectrogram (a pyramid for long files) is still being computed. this._syncTrackWidth(); this._drawMainWaveform(); + this._waveformRenderer.resetOverview(); this._drawOverviewWaveform(); this._syncOverviewWindowToViewport(); @@ -1054,6 +1067,8 @@ export class PlayerState { // second update picks up the canvas height draw() may have changed. this._updateCoords(); this._spectro.draw(this._getSpectrogramDrawParams()); + // Display changes (colors, contrast, noise reduction) end here too; the image is cached. + this._scheduleOverviewSpectrogram(); this._updateCoords(); this._carrierLine.update(); this._scheduleUiUpdate({ time: this._getCurrentTime(), fromPlayback: false, immediate: true }); @@ -1264,6 +1279,29 @@ export class PlayerState { _drawMainWaveform() { this._waveformRenderer.drawMainWaveform(); } _drawOverviewWaveform() { this._waveformRenderer.drawOverviewWaveform(); } + + /** When the loaded recording started (file metadata or name), null when unknown. */ + get recordingStart(): RecordingStart | null { return this._engine.recordingStart ?? null; } + + /** Overview strip: 'waveform' or a whole-file 'spectrogram' (max-pooled). */ + _setOverviewMode(mode: OverviewMode) { + const next: OverviewMode = mode === 'spectrogram' ? 'spectrogram' : 'waveform'; + if (next === this._overviewMode) return; + this._overviewMode = next; + this.options.overviewMode = next; + this._waveformRenderer.resetOverview(); + this._drawOverviewWaveform(); + this._emit('overviewmodechange', { mode: next }); + } + + /** Spectrogram tiles/colors changed: redraw the spectrogram overview, at most a few times a second. */ + _scheduleOverviewSpectrogram() { + if (this._overviewMode !== 'spectrogram' || this._overviewRedrawTimer) return; + this._overviewRedrawTimer = setTimeout(() => { + this._overviewRedrawTimer = null; + this._drawOverviewWaveform(); + }, 200); + } _createFrequencyLabels() { this._waveformRenderer.createFrequencyLabels(); } _updateAmplitudeLabels() { this._waveformRenderer.updateAmplitudeLabels(); } diff --git a/src/app/SpectrogramController.ts b/src/app/SpectrogramController.ts index ae518208..1d7590e2 100644 --- a/src/app/SpectrogramController.ts +++ b/src/app/SpectrogramController.ts @@ -35,6 +35,7 @@ import { visibleContrastPoints, noiseFloorStats, detectMaxFrequency, + buildOverviewImage, buildSpectrogramGrayscale, colorizeSpectrogram, windowGrayscale, @@ -523,6 +524,42 @@ export class SpectrogramController extends EventTarget { this._emit('needsredraw'); } + /** + * Whole-file spectrogram image for the overview strip, `cols` columns wide + * (max-pooled, current colors). Null when there is nothing to show yet + * (pyramid still building, external image mode): callers fall back to the waveform. + */ + async overviewImage(cols: number): Promise { + if (this._externalMode || cols <= 0) return null; + const tm = this._tileManager; + if (tm) return tm.overviewImage(cols); + if (!this._data || this._nFrames <= 0 || this._nMels <= 0) return null; + const o = this._getCurrentColorOptions(); + const colourScale = this._colourScale || o.colourScale; + const win = colourScale === 'phase' ? null : this._dbWindow(); + const [lo, hi] = win ?? (() => { + // Phase: fixed −π…π (windowed by floor01/ceil01 when colorizing), as in the main view. + if (colourScale === 'phase') return [-Math.PI, Math.PI]; + const a = this._absLogMin, b = this._absLogMax; + return [a + o.floor01 * (b - a), a + o.ceil01 * (b - a)]; + })(); + const key = `${cols}|${lo}|${hi}|${this._activeScale}|${colourScale}|${JSON.stringify(o)}`; + const cache = this._overviewCache; + if (cache && cache.data === this._data && cache.key === key) return cache.image; + const image = buildOverviewImage({ + data: this._data, nFrames: this._nFrames, bins: this._nMels, cols, + sampleRateHz: this._sampleRateHz, maxFreq: o.maxFreq, lo, hi, + scale: this._activeScale || o.scale, binFreqs: this.binFreqs, + colourScale, + noiseReduction: o.noiseReduction, bandFloor: this._bandFloor, + colorScheme: o.colorScheme, + floor01: o.floor01, ceil01: o.ceil01, + }); + this._overviewCache = { key, data: this._data, image }; + return image; + } + _overviewCache: { key: string; data: Float32Array; image: HTMLCanvasElement | OffscreenCanvas | null } | null = null; + private _getCurrentColorOptions(): TileColorOptions { const d = this._d; this._syncGainSliders(); diff --git a/src/app/WasmTileManager.ts b/src/app/WasmTileManager.ts index e7b22c08..b64999c3 100644 --- a/src/app/WasmTileManager.ts +++ b/src/app/WasmTileManager.ts @@ -14,6 +14,7 @@ // ═══════════════════════════════════════════════════════════════════════ import { + buildOverviewImage, buildSpectrogramGrayscale, colorizeSpectrogram, drawTimeGrid, @@ -609,6 +610,47 @@ export class WasmTileManager extends EventTarget { }); } + private overviewLevel: { level: number; meta: PyramidMeta; data: Promise } | null = null; + private overviewCache: { key: string; data: Float32Array; image: HTMLCanvasElement | OffscreenCanvas | null } | null = null; + + /** + * Whole-file image for the overview strip: the coarsest pyramid level with + * at least `cols` frames, max-pooled to `cols` columns and colored with the + * current options. Null while only the preview exists (no pyramid yet). + */ + async overviewImage(cols: number): Promise { + if (this.disposed || this.previewMode || cols <= 0) return null; + const levels = this.meta.levels; + let level = 0; + levels.forEach((l, i) => { if (l.nFrames >= cols) level = i; }); + if (!this.overviewLevel || this.overviewLevel.meta !== this.meta || this.overviewLevel.level !== level) { + this.overviewLevel = { level, meta: this.meta, data: this.svc.tile(level, 0, levels[level].nFrames) }; + } + const entry = this.overviewLevel; + let data: Float32Array; + try { data = await entry.data; } catch (err) { + if (this.overviewLevel === entry) this.overviewLevel = null; + throw err; + } + if (this.disposed) return null; + const opts = this.colorOptions; + const key = `${cols}|${level}|${JSON.stringify(opts)}`; + if (this.overviewCache && this.overviewCache.data === data && this.overviewCache.key === key) return this.overviewCache.image; + const [lo, hi] = this.valueWindow(level); + const image = buildOverviewImage({ + data, nFrames: levels[level].nFrames, bins: this.nMels, cols, + sampleRateHz: this.meta.params.sampleRate, + maxFreq: opts.maxFreq, lo, hi, + scale: opts.scale, binFreqs: this.binFreqs, colourScale: this.phase ? 'phase' : opts.colourScale, + noiseReduction: opts.noiseReduction, + bandFloor: levels[level]?.stats.bandFloor ?? null, + colorScheme: opts.colorScheme, + floor01: opts.floor01, ceil01: opts.ceil01, + }); + this.overviewCache = { key, data, image }; + return image; + } + invalidate(): void { this.tiles.clear(); this.queue = []; diff --git a/src/app/player/WaveformRenderer.ts b/src/app/player/WaveformRenderer.ts index 09efe8d1..e9640ff4 100644 --- a/src/app/player/WaveformRenderer.ts +++ b/src/app/player/WaveformRenderer.ts @@ -16,6 +16,7 @@ import { clamp } from '../../shared/utils.ts'; import { renderMainWaveform, renderOverviewWaveform, + renderOverviewImage, renderFrequencyLabels, } from '../../ui/components/waveform/waveform.ts'; import type { CoordinateSystem } from '../../domain/coordinateSystem.ts'; @@ -42,8 +43,14 @@ export interface WaveformRendererDeps { scheduleUiUpdate: () => void; getScrollLeft?: () => number; getViewportWidth?: () => number; + /** 'waveform' (default) or 'spectrogram' for the overview strip. */ + getOverviewMode?: () => OverviewMode; + /** Whole-file spectrogram image `cols` wide; null → the waveform is shown instead. */ + getOverviewImage?: (cols: number) => Promise; } +export type OverviewMode = 'waveform' | 'spectrogram'; + export class WaveformRenderer { #d: WaveformRendererDeps['d']; #getAudioBuffer: () => AudioBuffer | null; @@ -58,6 +65,14 @@ export class WaveformRenderer { #scheduleUiUpdate: () => void; #getScrollLeft: () => number; #getViewportWidth: () => number; + #getOverviewMode: () => OverviewMode; + #getOverviewImage: (cols: number) => Promise; + #overviewToken = 0; + /** True while the strip shows a spectrogram image (kept on redraws until a new one is ready). */ + #overviewHasImage = false; + /** What the strip's waveform was drawn for; skips rescanning every sample when nothing changed. */ + #overviewWaveKey: { audioBuffer: AudioBuffer; w: number; h: number; peak: number } | null = null; + #overviewImage: CanvasImageSource | null = null; constructor(deps: WaveformRendererDeps) { this.#d = deps.d; @@ -73,6 +88,8 @@ export class WaveformRenderer { this.#scheduleUiUpdate = deps.scheduleUiUpdate; this.#getScrollLeft = deps.getScrollLeft ?? (() => 0); this.#getViewportWidth = deps.getViewportWidth ?? (() => 0); + this.#getOverviewMode = deps.getOverviewMode ?? (() => 'waveform'); + this.#getOverviewImage = deps.getOverviewImage ?? (async () => null); } // ── Public rendering API ───────────────────────────────────────── @@ -102,15 +119,61 @@ export class WaveformRenderer { if (!this.#getShowOverview()) return; const { overviewCanvas, overviewContainer } = this.#d; if (!overviewCanvas || !overviewContainer) return; - renderOverviewWaveform({ - audioBuffer: this.#getAudioBuffer()!, - overviewCanvas, - overviewContainer, - amplitudePeakAbs: this.#getAmplitudePeakAbs(), + const audioBuffer = this.#getAudioBuffer(); + if (!audioBuffer) return; + const token = ++this.#overviewToken; + const drawWaveform = (force = true) => { + const rect = overviewContainer.getBoundingClientRect(); + const key = { audioBuffer, w: Math.floor(rect.width), h: Math.floor(rect.height), peak: this.#getAmplitudePeakAbs() }; + const k = this.#overviewWaveKey; + const same = !this.#overviewHasImage && k && k.audioBuffer === key.audioBuffer && k.w === key.w && k.h === key.h && k.peak === key.peak; + this.#overviewHasImage = false; + this.#overviewImage = null; + if (same && !force) return; + this.#overviewWaveKey = key; + renderOverviewWaveform({ + audioBuffer, + overviewCanvas, + overviewContainer, + amplitudePeakAbs: this.#getAmplitudePeakAbs(), + }); + }; + if (this.#getOverviewMode() !== 'spectrogram') { + drawWaveform(); + this.#scheduleUiUpdate(); + return; + } + // Spectrogram: keep what is shown until the new image is ready; the + // waveform stands in while the spectrogram is still being computed. + if (!this.#overviewHasImage) drawWaveform(false); + const cols = Math.max(1, Math.floor(overviewContainer.getBoundingClientRect().width)); + this.#getOverviewImage(cols).then((image) => { + if (token !== this.#overviewToken || this.#getAudioBuffer() !== audioBuffer) return; + if (image) { + // Cached images come back as the same object: redraw only on change or resize. + const sized = overviewCanvas.width === Math.max(1, cols); + if (image !== this.#overviewImage || !this.#overviewHasImage || !sized) { + renderOverviewImage({ image, overviewCanvas, overviewContainer }); + } + this.#overviewHasImage = true; + this.#overviewImage = image; + } else if (this.#overviewHasImage) { + drawWaveform(); + } + }).catch((err) => { + if (token === this.#overviewToken) console.warn('Overview spectrogram failed:', err); }); this.#scheduleUiUpdate(); } + /** A new file is loading: the old spectrogram image must not stay on screen. */ + resetOverview(): void { + this.#overviewHasImage = false; + this.#overviewImage = null; + this.#overviewWaveKey = null; + this.#overviewToken++; + } + createFrequencyLabels(): void { if (this.#d.freqLabels) { renderFrequencyLabels({ labelsElement: this.#d.freqLabels, coords: this.#getCoords() }); diff --git a/src/domain/recordingTime.ts b/src/domain/recordingTime.ts new file mode 100644 index 00000000..f89f082a --- /dev/null +++ b/src/domain/recordingTime.ts @@ -0,0 +1,188 @@ +// ═══════════════════════════════════════════════════════════════════════ +// domain/recordingTime.ts — when a recording started, from its metadata +// +// Adapted from recording-time.js in the BirdNET website demo +// (https://github.com/birdnet-team/birdnet-website, MIT License, +// Copyright (c) 2025 BirdNET-Team). +// +// Read in this order: +// - AudioMoth comment in the WAV INFO chunk ("Recorded at 19:00:00 +// 12/05/2023 (UTC+1) by AudioMoth …"), also when it survived conversion +// to FLAC or another format (searched in the file's first bytes) +// - GUANO "Timestamp" (Wildlife Acoustics, Titley, Pettersson and others) +// - BWF bext origination date and time (Zoom, Tascam, Sound Devices, …) +// - WAV INFO creation date, when it includes a time of day +// - a date and time in the file name ("20230512_190000.WAV", +// "SMA01234_20230512_190000.wav", "2023-05-12 19-00-00.flac") +// +// `ms` is the recorder's wall-clock time written as if it were UTC, so +// formatting never shifts it into the viewer's time zone; `tz` is the offset +// from UTC in minutes, or null when the metadata does not say. +// ═══════════════════════════════════════════════════════════════════════ + +export type RecordingStartSource = 'AudioMoth' | 'GUANO' | 'BWF' | 'WAV INFO' | 'fileName' | 'device'; + +export interface RecordingStart { + ms: number; + tz: number | null; + source: RecordingStartSource; +} + +const MAX_CHUNK = 1 << 16; +const HEAD_BYTES = 1 << 18; + +const ascii = (b: Uint8Array, p: number, n: number) => String.fromCharCode(...b.subarray(p, p + n)); +const pad = (n: number, w = 2) => String(n).padStart(w, '0'); +const latin1 = (b: Uint8Array) => new TextDecoder('latin1').decode(b); +const utf8 = (b: Uint8Array) => new TextDecoder('utf-8').decode(b).replace(/\0+$/, ''); + +type Num = string | number | undefined | null; + +function make(y: Num, mo: Num, d: Num, h: Num, mi: Num, s: Num, frac: Num, tz: number | null, source: RecordingStartSource): RecordingStart | null { + const [Y, MO, D, H, MI, S] = [y, mo, d, h, mi, s].map(Number); + if (!(Y >= 1990 && Y <= 2100 && MO >= 1 && MO <= 12 && D >= 1 && D <= 31 && H <= 23 && MI <= 59 && S <= 60)) return null; + const ms = Date.UTC(Y, MO - 1, D, H, MI, S) + (frac ? Math.round(Number(`0.${frac}`) * 1000) : 0); + // Date.UTC rolls 31 April into May; such dates are no dates. + if (new Date(ms).getUTCDate() !== D) return null; + return { ms, tz, source }; +} + +/** "+1", "-5", "+5:30", "+0530" → minutes; null when absent. */ +function tzMinutes(sign: string | undefined, hours: Num, minutes: Num): number | null { + if (sign == null) return null; + return (sign === '-' ? -1 : 1) * (Number(hours || 0) * 60 + Number(minutes || 0)); +} + +const AUDIOMOTH = /Recorded at (\d{2}):(\d{2}):(\d{2})(?:\.(\d+))? (\d{2})\/(\d{2})\/(\d{4}) \(UTC(?:([+-])(\d{1,2})(?::(\d{2}))?)?\)/; + +function fromAudioMoth(text: string): RecordingStart | null { + const m = text.match(AUDIOMOTH); + return m ? make(m[7], m[6], m[5], m[1], m[2], m[3], m[4], tzMinutes(m[8] || '+', m[9], m[10]), 'AudioMoth') : null; +} + +const ISO = /(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})(?:[.,](\d+))?\s*(?:(Z)|([+-])(\d{2}):?(\d{2})?)?/; + +function fromIso(text: string, source: RecordingStartSource): RecordingStart | null { + const m = text.match(ISO); + if (!m) return null; + const tz = m[8] ? 0 : tzMinutes(m[9], m[10], m[11]); + return make(m[1], m[2], m[3], m[4], m[5], m[6], m[7], tz, source); +} + +function fromGuano(text: string): RecordingStart | null { + const line = text.split(/\r?\n/).find((l) => /^Timestamp\s*:/.test(l)); + return line ? fromIso(line.replace(/^Timestamp\s*:\s*/, ''), 'GUANO') : null; +} + +/** bext: OriginationDate at 320 ("yyyy-mm-dd") and OriginationTime at 330 ("hh:mm:ss"), local time. */ +function fromBext(b: Uint8Array): RecordingStart | null { + if (b.length < 338) return null; + const d = ascii(b, 320, 10).match(/^(\d{4})\D(\d{2})\D(\d{2})$/); + const t = ascii(b, 330, 8).match(/^(\d{2})\D(\d{2})\D(\d{2})$/); + return d && t ? make(d[1], d[2], d[3], t[1], t[2], t[3], null, null, 'BWF') : null; +} + +/** Date and time in a file name, e.g. "SMA01234_20230512_190000.wav". */ +export function recordingStartFromFileName(name: string): RecordingStart | null { + const base = (name.split(/[\\/]/).pop() || '').replace(/\.[^.]+$/, ''); + const m = base.match(/(?> = {}; + for (let p = 12; p + 8 <= bytes.length;) { + const id = ascii(bytes, p, 4); + const size = view.getUint32(p + 4, true); + if (size === 0xffffffff) break; // RF64 audio data: its real size is elsewhere, and nothing we need follows + if (size <= MAX_CHUNK && ['LIST', 'guan', 'bext'].includes(id)) { + const b = bytes.subarray(p + 8, Math.min(bytes.length, p + 8 + size)); + if (id === 'guan') found.guano ??= fromGuano(utf8(b)); + else if (id === 'bext') found.bext ??= fromBext(b); + else if (b.length >= 4 && ascii(b, 0, 4) === 'INFO') { + const bv = new DataView(b.buffer, b.byteOffset, b.byteLength); + for (let q = 4; q + 8 <= b.length;) { + const sid = ascii(b, q, 4); + const n = bv.getUint32(q + 4, true); + const value = utf8(b.subarray(q + 8, q + 8 + n)); + if (sid === 'ICMT') found.audiomoth ??= fromAudioMoth(value); + else if (sid === 'ICRD') found.icrd ??= fromIso(value, 'WAV INFO'); + q += 8 + n + (n & 1); + } + } + } + p += 8 + size + (size & 1); + } + return found.audiomoth || found.guano || found.bext || found.icrd || null; +} + +/** Start of the recording in this file (bytes and/or name), or null when nothing says. */ +export function readRecordingStart(buffer: ArrayBuffer | Uint8Array | null, name = ''): RecordingStart | null { + try { + if (buffer) { + const bytes = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer); + const start = fromWav(bytes) || fromAudioMoth(latin1(bytes.subarray(0, HEAD_BYTES))); + if (start) return start; + } + } catch (err) { + console.warn('Could not read the recording start time', err); + } + return recordingStartFromFileName(name); +} + +/** Start of a recording made on this device, in its local time. */ +export function localRecordingStart(date: Date): RecordingStart { + const tz = -date.getTimezoneOffset(); + return { ms: date.getTime() + tz * 60000, tz, source: 'device' }; +} + +// ── Formatting ────────────────────────────────────────────────────── + +function parts(start: RecordingStart, offsetSec: number) { + const d = new Date(start.ms + Math.round(offsetSec * 1000)); + return { + date: `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())}`, + time: `${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}.${pad(d.getUTCMilliseconds(), 3)}`, + }; +} + +/** "+01:00", "Z", or "" when the offset is unknown. */ +function isoOffset(tz: number | null): string { + if (tz == null) return ''; + if (tz === 0) return 'Z'; + const a = Math.abs(tz); + return `${tz < 0 ? '-' : '+'}${pad(Math.floor(a / 60))}:${pad(a % 60)}`; +} + +/** Raven selection table columns for the recorder's wall-clock time. */ +export const RAVEN_CLOCK_COLUMNS = ['Begin Date', 'Begin Clock Time', 'End Clock Time']; + +/** Values for RAVEN_CLOCK_COLUMNS: "2023-05-12", "19:00:03.000", "19:00:06.000". */ +export function ravenClock(start: RecordingStart, beginSec: number, endSec: number): string[] { + const b = parts(start, beginSec); + return [b.date, b.time, parts(start, endSec).time]; +} + +/** "2023-05-12T19:00:03.000+01:00" (no offset when unknown). */ +export function isoClockTime(start: RecordingStart, offsetSec = 0): string { + const p = parts(start, offsetSec); + return `${p.date}T${p.time}${isoOffset(start.tz)}`; +} + +/** "UTC+1", "UTC+5:30", "UTC", or "" when unknown. */ +export function utcLabel(tz: number | null): string { + if (tz == null) return ''; + if (tz === 0) return 'UTC'; + const a = Math.abs(tz); + return `UTC${tz < 0 ? '-' : '+'}${Math.floor(a / 60)}${a % 60 ? `:${pad(a % 60)}` : ''}`; +} + +/** "2023-05-12 19:00:03 UTC+1" for display. */ +export function formatRecordingStart(start: RecordingStart, offsetSec = 0): string { + const p = parts(start, offsetSec); + return [`${p.date} ${p.time.slice(0, 8)}`, utcLabel(start.tz)].filter(Boolean).join(' '); +} diff --git a/src/domain/spectrogram.ts b/src/domain/spectrogram.ts index 7c60abd9..f6b4e417 100644 --- a/src/domain/spectrogram.ts +++ b/src/domain/spectrogram.ts @@ -1179,3 +1179,90 @@ export function createSpectralFeaturesProcessor() { return { computeFeatures, computeRidges: computeRidgesWorker, dispose }; } + +// ─── Overview (whole-file thumbnail) ──────────────────────────────── + +/** + * Pools `nFrames` frame-major frames (`bins` values each) into `cols` + * columns, keeping each bin's maximum, so short calls stay visible in a + * whole-file overview. Returns the data unchanged when it has no more + * frames than columns. + */ +export function poolFramesMax(data: ArrayLike, nFrames: number, bins: number, cols: number, + mode: 'max' | 'sample' = 'max'): { data: Float32Array; frames: number } { + const n = Math.max(0, Math.min(nFrames, Math.floor(data.length / Math.max(1, bins)))); + if (n <= cols || cols <= 0) { + return { data: data instanceof Float32Array ? data.subarray(0, n * bins) : Float32Array.from({ length: n * bins }, (_, i) => data[i]), frames: n }; + } + if (mode === 'sample') { + // Phase: the maximum angle means nothing; take the frame at each column's centre. + const out = new Float32Array(cols * bins); + for (let c = 0; c < cols; c++) { + const f = Math.min(n - 1, Math.floor((c + 0.5) * n / cols)); + for (let b = 0; b < bins; b++) out[c * bins + b] = data[f * bins + b]; + } + return { data: out, frames: cols }; + } + const out = new Float32Array(cols * bins).fill(-Infinity); + for (let c = 0; c < cols; c++) { + const f0 = Math.floor(c * n / cols); + const f1 = Math.max(f0 + 1, Math.floor((c + 1) * n / cols)); + const o = c * bins; + for (let f = f0; f < f1; f++) { + const s = f * bins; + for (let b = 0; b < bins; b++) { + const v = data[s + b]; + if (v > out[o + b]) out[o + b] = v; + } + } + } + return { data: out, frames: cols }; +} + +export interface OverviewImageOpts { + data: ArrayLike; + nFrames: number; + bins: number; + cols: number; + sampleRateHz: number; + maxFreq: number; + /** Value window [black, white] of `data`. */ + lo: number; + hi: number; + scale: string; + binFreqs: ArrayLike | null; + colourScale: string; + noiseReduction: boolean; + bandFloor: ArrayLike | null; + colorScheme: string; + /** Phase values: sampled instead of max-pooled; the window is [floor01, ceil01] of −π…π. */ + floor01?: number; + ceil01?: number; +} + +/** Colored whole-file spectrogram image (one column per overview pixel, max-pooled). */ +export function buildOverviewImage(o: OverviewImageOpts) { + const phase = o.colourScale === 'phase'; + const pooled = poolFramesMax(o.data, o.nFrames, o.bins, o.cols, phase ? 'sample' : 'max'); + if (pooled.frames <= 0) return null; + const grayInfo = buildSpectrogramGrayscale({ + spectrogramData: pooled.data, + spectrogramFrames: pooled.frames, + spectrogramMels: o.bins, + sampleRateHz: o.sampleRateHz, + maxFreq: o.maxFreq, + spectrogramAbsLogMin: o.lo, + spectrogramAbsLogMax: o.hi, + scale: o.scale, + binFreqs: o.binFreqs, + colourScale: o.colourScale, + noiseReduction: o.noiseReduction, + bandFloor: o.bandFloor, + clahe: false, + }); + if (!grayInfo) return null; + // Phase maps −π…π to 0…1 in the grayscale stage; the sliders window that range. + return phase + ? colorizeSpectrogram(grayInfo, o.floor01 ?? 0, o.ceil01 ?? 1, o.colorScheme) + : colorizeSpectrogram(grayInfo, 0, 1, o.colorScheme); +} diff --git a/src/infrastructure/audio/AudioEngine.ts b/src/infrastructure/audio/AudioEngine.ts index 2121271d..3e22d277 100644 --- a/src/infrastructure/audio/AudioEngine.ts +++ b/src/infrastructure/audio/AudioEngine.ts @@ -22,6 +22,7 @@ import { clamp, channelData, parseNativeSampleRate } from '../../shared/utils.ts import { AudioEngineBase } from './AudioEngineBase.ts'; import { isMp3, decodeMp3Chunked, CHUNKED_DECODE_MIN_BYTES } from './chunkedMp3Decode.ts'; import { parseGuano } from '../../domain/ultrasound.ts'; +import { readRecordingStart, type RecordingStart } from '../../domain/recordingTime.ts'; import { computeAmplitudePeak } from '../../domain/spectrogram.ts'; import { listenModeKey, type ListenMode, type DetectorChunk } from './detectorRender.ts'; import type { DetectorRenderer } from './detectorWorkerClient.ts'; @@ -153,6 +154,8 @@ export class AudioEngine extends AudioEngineBase { _peakFor: AudioBuffer | null; _peak: number; guano: Record | null; + /** When the loaded recording started (metadata or file name), null when unknown. */ + recordingStart: RecordingStart | null = null; ctx: any; bandpass: any; emitEnd: any; @@ -230,8 +233,10 @@ export class AudioEngine extends AudioEngineBase { */ async loadFromArrayBuffer(arrayBuffer: ArrayBuffer, name?: string) { this.guano = parseGuano(arrayBuffer); + const recordingStart = readRecordingStart(arrayBuffer, name || ''); const audioBuffer = await decodeArrayBuffer(arrayBuffer); this.audioBuffer = audioBuffer; + this.recordingStart = recordingStart; const displayName = name || 'audio'; this._setupWaveSurfer(null, displayName); @@ -251,13 +256,15 @@ export class AudioEngine extends AudioEngineBase { if (!response.ok) throw new Error(`HTTP ${response.status}`); const arrayBuffer = await readWithProgress(response, (loaded, total) => onProgress?.('download', loaded, total)); this.guano = parseGuano(arrayBuffer); - onProgress?.('decode'); - const audioBuffer = await decodeArrayBuffer(arrayBuffer); - this.audioBuffer = audioBuffer; - const name = decodeURIComponent( new URL(url, location.href).pathname.split('/').pop() || 'audio', ); + // Before decoding: decodeAudioData detaches the buffer. + const recordingStart = readRecordingStart(arrayBuffer, name); + onProgress?.('decode'); + const audioBuffer = await decodeArrayBuffer(arrayBuffer); + this.audioBuffer = audioBuffer; + this.recordingStart = recordingStart; await this._setupWaveSurfer(url, name); return { duration: audioBuffer.duration, sampleRate: audioBuffer.sampleRate }; } @@ -270,8 +277,10 @@ export class AudioEngine extends AudioEngineBase { async loadFromFile(file: File) { const arrayBuffer = await file.arrayBuffer(); this.guano = parseGuano(arrayBuffer); + const recordingStart = readRecordingStart(arrayBuffer, file.name); const audioBuffer = await decodeArrayBuffer(arrayBuffer); this.audioBuffer = audioBuffer; + this.recordingStart = recordingStart; await this._setupWaveSurfer(file, file.name); return { duration: audioBuffer.duration, sampleRate: audioBuffer.sampleRate }; } diff --git a/src/infrastructure/audio/LiveRecorder.ts b/src/infrastructure/audio/LiveRecorder.ts new file mode 100644 index 00000000..71eae2d9 --- /dev/null +++ b/src/infrastructure/audio/LiveRecorder.ts @@ -0,0 +1,252 @@ +// ═══════════════════════════════════════════════════════════════════════ +// LiveRecorder.ts — records the microphone as mono PCM +// +// The samples are kept as they arrive (AudioWorklet, ScriptProcessor as a +// fallback); an AnalyserNode on the same source feeds the live spectrogram. +// stop() returns a 16-bit WAV whose INFO chunk carries the start time with +// the device's UTC offset (ICRD), so readRecordingStart() finds it again +// after export or reload. +// +// Events: +// 'start' { sampleRate } +// 'stop' { reason: 'user' | 'limit' | 'ended' } (the track ended, e.g. the device was unplugged) +// ═══════════════════════════════════════════════════════════════════════ + +import { isoClockTime, localRecordingStart } from '../../domain/recordingTime.ts'; + +export interface LiveRecorderOptions { + /** Recording stops by itself after this many seconds (default 300). */ + maxSeconds?: number; + /** Specific input device (MediaDeviceInfo.deviceId). */ + deviceId?: string; +} + +export interface LiveRecording { + file: File; + startedAt: Date; + duration: number; + sampleRate: number; +} + +const WORKLET = ` +class SvCapture extends AudioWorkletProcessor { + process(inputs) { + const ch = inputs[0] && inputs[0][0]; + if (ch) this.port.postMessage(ch.slice(0)); + return true; + } +} +registerProcessor('sv-capture', SvCapture); +`; + +const pad = (n: number) => String(n).padStart(2, '0'); + +/** "Recording_20261115_143012.wav" in the device's local time. */ +export function recordingFileName(d: Date): string { + return `Recording_${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}_${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}.wav`; +} + +/** Mono 16-bit PCM WAV with a LIST/INFO chunk (ICRD start time, ISFT software). */ +export function encodeWav(chunks: Float32Array[], sampleRate: number, info: Record = {}): Blob { + const n = chunks.reduce((s, c) => s + c.length, 0); + const enc = new TextEncoder(); + const subs = Object.entries(info).map(([id, value]) => { + const v = enc.encode(value + '\0'); + const b = new Uint8Array(8 + v.length + (v.length & 1)); + for (let i = 0; i < 4; i++) b[i] = id.charCodeAt(i); + new DataView(b.buffer).setUint32(4, v.length, true); + b.set(v, 8); + return b; + }); + const infoBytes = subs.length ? 4 + subs.reduce((s, b) => s + b.length, 0) : 0; + const listBytes = infoBytes ? 8 + infoBytes : 0; + const dataBytes = n * 2; + const head = new DataView(new ArrayBuffer(36 + listBytes + 8)); + const tag = (at: number, s: string) => { for (let i = 0; i < 4; i++) head.setUint8(at + i, s.charCodeAt(i)); }; + tag(0, 'RIFF'); head.setUint32(4, 36 + listBytes + 8 + dataBytes - 8, true); tag(8, 'WAVE'); + tag(12, 'fmt '); head.setUint32(16, 16, true); + head.setUint16(20, 1, true); // PCM + head.setUint16(22, 1, true); // mono + head.setUint32(24, sampleRate, true); + head.setUint32(28, sampleRate * 2, true); + head.setUint16(32, 2, true); + head.setUint16(34, 16, true); + let o = 36; + if (listBytes) { + tag(o, 'LIST'); head.setUint32(o + 4, infoBytes, true); tag(o + 8, 'INFO'); + o += 12; + for (const b of subs) { for (let i = 0; i < b.length; i++) head.setUint8(o + i, b[i]); o += b.length; } + } + tag(o, 'data'); head.setUint32(o + 4, dataBytes, true); + const pcm = new Int16Array(n); + let k = 0; + for (const c of chunks) { + for (let i = 0; i < c.length; i++) { + const v = Math.max(-1, Math.min(1, c[i])); + pcm[k++] = v < 0 ? v * 0x8000 : v * 0x7fff; + } + } + return new Blob([head.buffer, pcm.buffer], { type: 'audio/wav' }); +} + +export class LiveRecorder extends EventTarget { + readonly maxSeconds: number; + private readonly deviceId: string | undefined; + private ctx: AudioContext | null = null; + private stream: MediaStream | null = null; + private source: MediaStreamAudioSourceNode | null = null; + private capture: AudioWorkletNode | ScriptProcessorNode | null = null; + private mute: GainNode | null = null; + private chunks: Float32Array[] = []; + private samples = 0; + private startedAt: Date | null = null; + private stopping: Promise | null = null; + analyser: AnalyserNode | null = null; + sampleRate = 0; + + constructor(opts: LiveRecorderOptions = {}) { + super(); + this.maxSeconds = opts.maxSeconds && opts.maxSeconds > 0 ? opts.maxSeconds : 300; + this.deviceId = opts.deviceId; + } + + static get supported(): boolean { + return typeof navigator !== 'undefined' && !!navigator.mediaDevices?.getUserMedia + && typeof AudioContext !== 'undefined'; + } + + get recording(): boolean { return !!this.startedAt && !this.stopping; } + + /** Seconds recorded so far. */ + get elapsed(): number { return this.sampleRate ? this.samples / this.sampleRate : 0; } + + get startTime(): Date | null { return this.startedAt; } + + /** Asks for the microphone and starts recording. Rejects when access is denied. */ + async start(): Promise { + if (this.startedAt) throw new Error('Already recording'); + if (!LiveRecorder.supported) throw new Error('Recording is not supported in this browser'); + if (this.stopping) throw new Error('Recording was cancelled'); + // stop()/cancel() while waiting (e.g. for microphone permission) cancels the start. + const cancelled = () => !!this.stopping; + // Raw signal: the browser's voice processing would remove quiet calls. + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + ...(this.deviceId ? { deviceId: { exact: this.deviceId } } : {}), + echoCancellation: false, noiseSuppression: false, autoGainControl: false, + channelCount: 1, + }, + }); + if (cancelled()) { + stream.getTracks().forEach((t) => t.stop()); + throw new Error('Recording was cancelled'); + } + this.stream = stream; + try { + const ctx = new AudioContext(); + this.ctx = ctx; + if (ctx.state === 'suspended') await ctx.resume(); + if (cancelled()) throw new Error('Recording was cancelled'); + this.sampleRate = ctx.sampleRate; + this.source = ctx.createMediaStreamSource(this.stream); + this.analyser = ctx.createAnalyser(); + this.analyser.fftSize = 2048; + this.analyser.smoothingTimeConstant = 0; + this.source.connect(this.analyser); + // The capture node must reach the destination to run; silence it. + this.mute = ctx.createGain(); + this.mute.gain.value = 0; + this.mute.connect(ctx.destination); + this.capture = await this.createCapture(ctx); + if (cancelled()) throw new Error('Recording was cancelled'); + this.source.connect(this.capture); + this.capture.connect(this.mute); + this.stream.getAudioTracks()[0]?.addEventListener('ended', () => { void this.stop('ended'); }); + this.startedAt = new Date(); + this.dispatchEvent(new CustomEvent('start', { detail: { sampleRate: this.sampleRate } })); + } catch (err) { + this.release(); + throw err; + } + } + + private async createCapture(ctx: AudioContext): Promise { + if (ctx.audioWorklet && typeof AudioWorkletNode !== 'undefined') { + const url = URL.createObjectURL(new Blob([WORKLET], { type: 'application/javascript' })); + try { + await ctx.audioWorklet.addModule(url); + const node = new AudioWorkletNode(ctx, 'sv-capture', { numberOfInputs: 1, numberOfOutputs: 1, channelCount: 1 }); + node.port.onmessage = (e: MessageEvent) => this.push(e.data); + return node; + } catch (err) { + console.warn('AudioWorklet unavailable, using ScriptProcessor:', err); + } finally { + URL.revokeObjectURL(url); + } + } + const node = ctx.createScriptProcessor(4096, 1, 1); + node.onaudioprocess = (e) => this.push(e.inputBuffer.getChannelData(0).slice(0)); + return node; + } + + private push(chunk: Float32Array): void { + if (!this.startedAt || this.stopping) return; + const limit = Math.round(this.maxSeconds * this.sampleRate); + const room = limit - this.samples; + if (room > 0) { + const c = chunk.length > room ? chunk.subarray(0, room) : chunk; + this.chunks.push(c); + this.samples += c.length; + } + if (this.samples >= limit) void this.stop('limit'); + } + + /** Stops and returns the recording (null when nothing was recorded). */ + stop(reason: 'user' | 'limit' | 'ended' = 'user'): Promise { + if (this.stopping) return this.stopping; + let resolve!: (r: LiveRecording | null) => void; + this.stopping = new Promise((r) => { resolve = r; }); + const startedAt = this.startedAt; + const sampleRate = this.sampleRate; + const chunks = this.chunks; + const samples = this.samples; + this.release(); + let result: LiveRecording | null = null; + if (startedAt && samples > 0) { + const start = localRecordingStart(startedAt); + const blob = encodeWav(chunks, sampleRate, { + ICRD: isoClockTime(start).replace(/\.\d{3}/, ''), + ISFT: 'SignaVis', + }); + const file = new File([blob], recordingFileName(startedAt), { type: 'audio/wav', lastModified: Date.now() }); + result = { file, startedAt, duration: samples / sampleRate, sampleRate }; + } + resolve(result); + // After `stopping` is set: listeners may call stop() again and get the same result. + this.dispatchEvent(new CustomEvent('stop', { detail: { reason } })); + return this.stopping; + } + + /** Stops without keeping anything. */ + cancel(): void { + this.chunks = []; + this.samples = 0; + void this.stop('user'); + } + + private release(): void { + try { this.capture?.disconnect(); } catch { /* already gone */ } + try { this.source?.disconnect(); } catch { /* already gone */ } + if (this.capture && 'port' in this.capture) this.capture.port.onmessage = null; + this.stream?.getTracks().forEach((t) => t.stop()); + void this.ctx?.close().catch(() => {}); + this.capture = null; + this.source = null; + this.mute = null; + this.stream = null; + this.ctx = null; + this.chunks = []; + this.samples = 0; + // The analyser stays readable (silent) until the view lets go of it. + } +} diff --git a/src/infrastructure/audio/MockAudioEngine.ts b/src/infrastructure/audio/MockAudioEngine.ts index 6fe9e889..cb79584b 100644 --- a/src/infrastructure/audio/MockAudioEngine.ts +++ b/src/infrastructure/audio/MockAudioEngine.ts @@ -163,6 +163,7 @@ export class MockAudioEngine extends AudioEngineBase { override setNormalize(on: boolean): number { this.normalize = !!on; return 1; } timeExpansion = 1; guano: Record | null = null; + recordingStart: import('../../domain/recordingTime.ts').RecordingStart | null = null; listenMode: any = { kind: 'expansion', factor: 1 }; override setTimeExpansion(factor: number): void { this.setListenMode({ kind: 'expansion', factor }); } override setListenMode(mode: any): void { diff --git a/src/infrastructure/export/labelTableFormats.ts b/src/infrastructure/export/labelTableFormats.ts index 3d1fcaa3..d0c53852 100644 --- a/src/infrastructure/export/labelTableFormats.ts +++ b/src/infrastructure/export/labelTableFormats.ts @@ -23,6 +23,8 @@ export interface LabelExportColumn { export interface LabelExportOptions { sampleRate?: number | null; + /** Recording start (Raven clock-time columns). */ + recordingStart?: import('../../domain/recordingTime.ts').RecordingStart | null; setName?: (setId: string) => string; /** Columns for CSV (the table's visible columns). */ columns?: LabelExportColumn[]; diff --git a/src/infrastructure/export/ravenSelectionTable.ts b/src/infrastructure/export/ravenSelectionTable.ts index 2b5d256a..bfb021fa 100644 --- a/src/infrastructure/export/ravenSelectionTable.ts +++ b/src/infrastructure/export/ravenSelectionTable.ts @@ -5,8 +5,12 @@ //