From bea783afa0be843d51b7b6fe7a9fc1273adc73e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 08:00:49 +0000 Subject: [PATCH 1/4] feat(overview): choose waveform or whole-file spectrogram for the overview strip The spectrogram overview pools the coarsest pyramid level (or the in-memory spectrogram) to one column per pixel, keeping each band's maximum so short calls stay visible. Option overviewMode, player.setOverviewMode(), and a Display setting in the labeling app. --- CHANGELOG.md | 8 +++ demo/labeling-app.html | 21 +++++++- docs/usage/player-options.md | 1 + python-wrapper/README.md | 1 + src/app/BirdNETPlayer.ts | 5 ++ src/app/PlayerState.ts | 30 ++++++++++- src/app/SpectrogramController.ts | 27 ++++++++++ src/app/WasmTileManager.ts | 36 +++++++++++++ src/app/player/WaveformRenderer.ts | 56 +++++++++++++++++++-- src/domain/spectrogram.ts | 70 ++++++++++++++++++++++++++ src/ui/components/waveform/waveform.ts | 24 +++++++++ src/ui/template.ts | 1 + tests/overview-spectrogram.test.mjs | 28 +++++++++++ 13 files changed, 301 insertions(+), 7 deletions(-) create mode 100644 tests/overview-spectrogram.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 58081e59..54662943 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. +## [Unreleased] + +### Added +- `overviewMode: 'spectrogram'` (and `player.setOverviewMode()`): the overview + strip shows the whole recording as a spectrogram, max-pooled so short calls + stay visible; the waveform stands in while the spectrogram is computed. + The labeling app offers the choice under Display → Overview. + ## [1.1.3] - 2026-11-14 ### Added diff --git a/demo/labeling-app.html b/demo/labeling-app.html index 5d4c0371..d8bf9851 100644 --- a/demo/labeling-app.html +++ b/demo/labeling-app.html @@ -820,8 +820,9 @@

Tags

} // ── Player ──────────────────────────────────────────────────────── - // Waveform above the spectrogram is optional (the overview below always shows one); applied on load + // Waveform above the spectrogram is optional; applied on load. The overview below shows a waveform or the whole spectrogram. const SHOW_WAVEFORM_KEY = 'signavis.labeling.show-waveform.v1'; + const OVERVIEW_MODE_KEY = 'signavis.labeling.overview-mode.v1'; const SHOW_CONF_KEY = 'signavis.labeling.show-confidence.v1'; const AUTO_FREQ_VIEW_KEY = 'signavis.labeling.auto-freq-view.v1'; const NORMALIZE_KEY = 'signavis.labeling.normalize.v1'; @@ -834,6 +835,7 @@

Tags

showFileOpen: false, enableProgressiveSpectrogram: true, showOverview: true, + overviewMode: (() => { try { return localStorage.getItem(OVERVIEW_MODE_KEY) === 'spectrogram' ? 'spectrogram' : 'waveform'; } catch { return 'waveform'; } })(), viewMode: showMainWaveform ? 'both' : 'spectrogram', showTransport: true, // BirdNET detections (no frequency bounds) as strips along the top edge @@ -1383,6 +1385,23 @@

Tags

}); specPanel.el.insertBefore(wf, sec); + const ov = document.createElement('div'); + ov.className = 'settings-section'; + ov.style.cssText = 'padding:8px 14px;border-top:1px solid var(--color-border)'; + ov.innerHTML = ` + + `; + const ovSel = ov.querySelector('select'); + ovSel.value = player.overviewMode; + ovSel.addEventListener('change', () => { + try { localStorage.setItem(OVERVIEW_MODE_KEY, ovSel.value); } catch { /* ignore */ } + player.setOverviewMode(ovSel.value); + }); + specPanel.el.insertBefore(ov, sec); + // ── Automatic behaviour: what the app does by itself, each one switchable ── const auto = document.createElement('div'); auto.className = 'settings-section'; diff --git a/docs/usage/player-options.md b/docs/usage/player-options.md index 85285f79..7da799a5 100644 --- a/docs/usage/player-options.md +++ b/docs/usage/player-options.md @@ -18,6 +18,7 @@ | `showDisplayGain` | boolean | `true` | Show floor/ceiling sliders, auto contrast | | `showStatusbar` | boolean | `true` | Show bottom status bar | | `showOverview` | boolean | `true` | Show overview navigator | +| `overviewMode` | string | `'waveform'` | `'waveform'` or `'spectrogram'` — the whole recording as a max-pooled spectrogram (short calls stay visible); switch at runtime with `player.setOverviewMode(mode)` | | `showWaveformTimeline` | boolean | `true` | Show bottom timeline in waveform view | | `compactToolbar` | string | `'auto'` | `'auto'`, `'on'`, `'off'` — responsive toolbar compaction | | `labelTaxonomy` | array | see docs | Custom label presets (name, color, shortcut) | diff --git a/python-wrapper/README.md b/python-wrapper/README.md index c960abad..656e5d01 100644 --- a/python-wrapper/README.md +++ b/python-wrapper/README.md @@ -114,6 +114,7 @@ HTML(html) | `viewMode` | `'spectrogram'` | Show only the spectrogram | | `transportStyle` | `'hero'` | Large centered play button | | `showOverview` | `False` | Hide overview navigator | +| `overviewMode` | `'spectrogram'` | Overview shows the whole spectrogram instead of the waveform | | `showFileOpen` | `False` | Hide file open button | | `showStatusbar` | `False` | Hide status bar | | `labelTaxonomy` | `[{'name': 'Species', 'color': '#0ea5e9'}]` | Custom label presets | diff --git a/src/app/BirdNETPlayer.ts b/src/app/BirdNETPlayer.ts index f9be08cc..4e708d9c 100644 --- a/src/app/BirdNETPlayer.ts +++ b/src/app/BirdNETPlayer.ts @@ -962,6 +962,11 @@ export class BirdNETPlayer { get normalize() { return !!this._state?.normalize; } + /** Overview strip below the player: 'waveform' or a whole-file 'spectrogram'. */ + setOverviewMode(mode: 'waveform' | 'spectrogram') { this._state?._setOverviewMode(mode); } + + get overviewMode(): 'waveform' | 'spectrogram' { return this._state?._overviewMode ?? 'waveform'; } + // ── Built-in label table ──────────────────────────────────────── _labelGrid: LabelGrid | null = null; diff --git a/src/app/PlayerState.ts b/src/app/PlayerState.ts index 2a5e2aa1..b0245bd5 100644 --- a/src/app/PlayerState.ts +++ b/src/app/PlayerState.ts @@ -30,7 +30,7 @@ import { CrosshairController } from './player/CrosshairController.ts'; import { PerfOverlay } from './player/PerfOverlay.ts'; import { ViewResizeController } from './player/ViewResizeController.ts'; import { FrequencyZoomController } from './player/FrequencyZoomController.ts'; -import { WaveformRenderer } from './player/WaveformRenderer.ts'; +import { WaveformRenderer, type OverviewMode } from './player/WaveformRenderer.ts'; import { ToolbarController } from './player/ToolbarController.ts'; import { ToolbarFitController } from './player/ToolbarFitController.ts'; import { CarrierLineController } from './player/CarrierLineController.ts'; @@ -43,6 +43,7 @@ import { computeBandProfile, autoViewMaxHz, type BandProfile } from '../domain/b * @typedef {Object} PlayerOptions * @property {string} [viewMode] * @property {boolean} [showOverview] + * @property {'waveform'|'spectrogram'} [overviewMode] Overview strip content (default 'waveform') * @property {boolean} [transportOverlay] * @property {string} [compactToolbar] * @property {boolean} [showWaveformTimeline] @@ -137,6 +138,8 @@ export class PlayerState { _showWaveform: any; _showSpectrogram: any; _showOverview: any; + _overviewMode: OverviewMode = 'waveform'; + _overviewRedrawTimer: ReturnType | null = null; _transportOverlay: any; _keyboardEnabled = true; _compactToolbarMode: any; @@ -243,6 +246,7 @@ export class PlayerState { this._showWaveform = this._viewMode !== 'spectrogram'; this._showSpectrogram = this._viewMode !== 'waveform'; this._showOverview = this.options.showOverview !== false; + this._overviewMode = this.options.overviewMode === 'spectrogram' ? 'spectrogram' : 'waveform'; this._transportOverlay = this.options.transportOverlay === true; this._keyboardEnabled = this.options.keyboardShortcuts !== false; this._compactToolbarMode = this.options.compactToolbar && ['auto', 'on', 'off'].includes(this.options.compactToolbar) @@ -275,6 +279,7 @@ export class PlayerState { this._createFrequencyLabels(); this._drawSpectrogram(); this._syncOverviewWindowToViewport(); + this._scheduleOverviewSpectrogram(); }); // ── Audio / analysis state ── @@ -417,6 +422,8 @@ export class PlayerState { scheduleUiUpdate: () => this._scheduleUiUpdate({ time: this._getCurrentTime(), fromPlayback: false, immediate: true }), getScrollLeft: () => this._getPrimaryScrollLeft(), getViewportWidth: () => this._getViewportWidth(), + getOverviewMode: () => this._overviewMode, + getOverviewImage: (cols: number) => this._spectro.overviewImage(cols), }); // ── ToolbarController ── @@ -876,6 +883,7 @@ export class PlayerState { // spectrogram (a pyramid for long files) is still being computed. this._syncTrackWidth(); this._drawMainWaveform(); + this._waveformRenderer.resetOverview(); this._drawOverviewWaveform(); this._syncOverviewWindowToViewport(); @@ -1264,6 +1272,26 @@ export class PlayerState { _drawMainWaveform() { this._waveformRenderer.drawMainWaveform(); } _drawOverviewWaveform() { this._waveformRenderer.drawOverviewWaveform(); } + + /** Overview strip: 'waveform' or a whole-file 'spectrogram' (max-pooled). */ + _setOverviewMode(mode: OverviewMode) { + const next: OverviewMode = mode === 'spectrogram' ? 'spectrogram' : 'waveform'; + if (next === this._overviewMode) return; + this._overviewMode = next; + this.options.overviewMode = next; + this._waveformRenderer.resetOverview(); + this._drawOverviewWaveform(); + this._emit('overviewmodechange', { mode: next }); + } + + /** Spectrogram tiles/colors changed: redraw the spectrogram overview, at most a few times a second. */ + _scheduleOverviewSpectrogram() { + if (this._overviewMode !== 'spectrogram' || this._overviewRedrawTimer) return; + this._overviewRedrawTimer = setTimeout(() => { + this._overviewRedrawTimer = null; + this._drawOverviewWaveform(); + }, 200); + } _createFrequencyLabels() { this._waveformRenderer.createFrequencyLabels(); } _updateAmplitudeLabels() { this._waveformRenderer.updateAmplitudeLabels(); } diff --git a/src/app/SpectrogramController.ts b/src/app/SpectrogramController.ts index ae518208..e6074544 100644 --- a/src/app/SpectrogramController.ts +++ b/src/app/SpectrogramController.ts @@ -35,6 +35,7 @@ import { visibleContrastPoints, noiseFloorStats, detectMaxFrequency, + buildOverviewImage, buildSpectrogramGrayscale, colorizeSpectrogram, windowGrayscale, @@ -523,6 +524,32 @@ export class SpectrogramController extends EventTarget { this._emit('needsredraw'); } + /** + * Whole-file spectrogram image for the overview strip, `cols` columns wide + * (max-pooled, current colors). Null when there is nothing to show yet + * (pyramid still building, external image mode): callers fall back to the waveform. + */ + async overviewImage(cols: number): Promise { + if (this._externalMode || cols <= 0) return null; + const tm = this._tileManager; + if (tm) return tm.overviewImage(cols); + if (!this._data || this._nFrames <= 0 || this._nMels <= 0) return null; + const o = this._getCurrentColorOptions(); + const win = this._dbWindow(); + const [lo, hi] = win ?? (() => { + const a = this._absLogMin, b = this._absLogMax; + return [a + o.floor01 * (b - a), a + o.ceil01 * (b - a)]; + })(); + return buildOverviewImage({ + data: this._data, nFrames: this._nFrames, bins: this._nMels, cols, + sampleRateHz: this._sampleRateHz, maxFreq: o.maxFreq, lo, hi, + scale: this._activeScale || o.scale, binFreqs: this.binFreqs, + colourScale: this._colourScale || o.colourScale, + noiseReduction: o.noiseReduction, bandFloor: this._bandFloor, + colorScheme: o.colorScheme, + }); + } + private _getCurrentColorOptions(): TileColorOptions { const d = this._d; this._syncGainSliders(); diff --git a/src/app/WasmTileManager.ts b/src/app/WasmTileManager.ts index e7b22c08..a54d9373 100644 --- a/src/app/WasmTileManager.ts +++ b/src/app/WasmTileManager.ts @@ -14,6 +14,7 @@ // ═══════════════════════════════════════════════════════════════════════ import { + buildOverviewImage, buildSpectrogramGrayscale, colorizeSpectrogram, drawTimeGrid, @@ -609,6 +610,41 @@ export class WasmTileManager extends EventTarget { }); } + private overviewLevel: { level: number; meta: PyramidMeta; data: Promise } | null = null; + + /** + * Whole-file image for the overview strip: the coarsest pyramid level with + * at least `cols` frames, max-pooled to `cols` columns and colored with the + * current options. Null while only the preview exists (no pyramid yet). + */ + async overviewImage(cols: number): Promise { + if (this.disposed || this.previewMode || cols <= 0) return null; + const levels = this.meta.levels; + let level = 0; + levels.forEach((l, i) => { if (l.nFrames >= cols) level = i; }); + if (!this.overviewLevel || this.overviewLevel.meta !== this.meta || this.overviewLevel.level !== level) { + this.overviewLevel = { level, meta: this.meta, data: this.svc.tile(level, 0, levels[level].nFrames) }; + } + const entry = this.overviewLevel; + let data: Float32Array; + try { data = await entry.data; } catch (err) { + if (this.overviewLevel === entry) this.overviewLevel = null; + throw err; + } + if (this.disposed) return null; + const opts = this.colorOptions; + const [lo, hi] = this.valueWindow(level); + return buildOverviewImage({ + data, nFrames: levels[level].nFrames, bins: this.nMels, cols, + sampleRateHz: this.meta.params.sampleRate, + maxFreq: opts.maxFreq, lo, hi, + scale: opts.scale, binFreqs: this.binFreqs, colourScale: opts.colourScale, + noiseReduction: opts.noiseReduction, + bandFloor: levels[level]?.stats.bandFloor ?? null, + colorScheme: opts.colorScheme, + }); + } + invalidate(): void { this.tiles.clear(); this.queue = []; diff --git a/src/app/player/WaveformRenderer.ts b/src/app/player/WaveformRenderer.ts index 09efe8d1..4e06cffa 100644 --- a/src/app/player/WaveformRenderer.ts +++ b/src/app/player/WaveformRenderer.ts @@ -16,6 +16,7 @@ import { clamp } from '../../shared/utils.ts'; import { renderMainWaveform, renderOverviewWaveform, + renderOverviewImage, renderFrequencyLabels, } from '../../ui/components/waveform/waveform.ts'; import type { CoordinateSystem } from '../../domain/coordinateSystem.ts'; @@ -42,8 +43,14 @@ export interface WaveformRendererDeps { scheduleUiUpdate: () => void; getScrollLeft?: () => number; getViewportWidth?: () => number; + /** 'waveform' (default) or 'spectrogram' for the overview strip. */ + getOverviewMode?: () => OverviewMode; + /** Whole-file spectrogram image `cols` wide; null → the waveform is shown instead. */ + getOverviewImage?: (cols: number) => Promise; } +export type OverviewMode = 'waveform' | 'spectrogram'; + export class WaveformRenderer { #d: WaveformRendererDeps['d']; #getAudioBuffer: () => AudioBuffer | null; @@ -58,6 +65,11 @@ export class WaveformRenderer { #scheduleUiUpdate: () => void; #getScrollLeft: () => number; #getViewportWidth: () => number; + #getOverviewMode: () => OverviewMode; + #getOverviewImage: (cols: number) => Promise; + #overviewToken = 0; + /** True while the strip shows a spectrogram image (kept on redraws until a new one is ready). */ + #overviewHasImage = false; constructor(deps: WaveformRendererDeps) { this.#d = deps.d; @@ -73,6 +85,8 @@ export class WaveformRenderer { this.#scheduleUiUpdate = deps.scheduleUiUpdate; this.#getScrollLeft = deps.getScrollLeft ?? (() => 0); this.#getViewportWidth = deps.getViewportWidth ?? (() => 0); + this.#getOverviewMode = deps.getOverviewMode ?? (() => 'waveform'); + this.#getOverviewImage = deps.getOverviewImage ?? (async () => null); } // ── Public rendering API ───────────────────────────────────────── @@ -102,15 +116,47 @@ export class WaveformRenderer { if (!this.#getShowOverview()) return; const { overviewCanvas, overviewContainer } = this.#d; if (!overviewCanvas || !overviewContainer) return; - renderOverviewWaveform({ - audioBuffer: this.#getAudioBuffer()!, - overviewCanvas, - overviewContainer, - amplitudePeakAbs: this.#getAmplitudePeakAbs(), + const audioBuffer = this.#getAudioBuffer(); + if (!audioBuffer) return; + const token = ++this.#overviewToken; + const drawWaveform = () => { + this.#overviewHasImage = false; + renderOverviewWaveform({ + audioBuffer, + overviewCanvas, + overviewContainer, + amplitudePeakAbs: this.#getAmplitudePeakAbs(), + }); + }; + if (this.#getOverviewMode() !== 'spectrogram') { + drawWaveform(); + this.#scheduleUiUpdate(); + return; + } + // Spectrogram: keep what is shown until the new image is ready; the + // waveform stands in while the spectrogram is still being computed. + if (!this.#overviewHasImage) drawWaveform(); + const cols = Math.max(1, Math.floor(overviewContainer.getBoundingClientRect().width)); + this.#getOverviewImage(cols).then((image) => { + if (token !== this.#overviewToken || this.#getAudioBuffer() !== audioBuffer) return; + if (image) { + renderOverviewImage({ image, overviewCanvas, overviewContainer }); + this.#overviewHasImage = true; + } else if (this.#overviewHasImage) { + drawWaveform(); + } + }).catch((err) => { + if (token === this.#overviewToken) console.warn('Overview spectrogram failed:', err); }); this.#scheduleUiUpdate(); } + /** A new file is loading: the old spectrogram image must not stay on screen. */ + resetOverview(): void { + this.#overviewHasImage = false; + this.#overviewToken++; + } + createFrequencyLabels(): void { if (this.#d.freqLabels) { renderFrequencyLabels({ labelsElement: this.#d.freqLabels, coords: this.#getCoords() }); diff --git a/src/domain/spectrogram.ts b/src/domain/spectrogram.ts index 7c60abd9..08405000 100644 --- a/src/domain/spectrogram.ts +++ b/src/domain/spectrogram.ts @@ -1179,3 +1179,73 @@ export function createSpectralFeaturesProcessor() { return { computeFeatures, computeRidges: computeRidgesWorker, dispose }; } + +// ─── Overview (whole-file thumbnail) ──────────────────────────────── + +/** + * Pools `nFrames` frame-major frames (`bins` values each) into `cols` + * columns, keeping each bin's maximum, so short calls stay visible in a + * whole-file overview. Returns the data unchanged when it has no more + * frames than columns. + */ +export function poolFramesMax(data: ArrayLike, nFrames: number, bins: number, cols: number): { data: Float32Array; frames: number } { + const n = Math.max(0, Math.min(nFrames, Math.floor(data.length / Math.max(1, bins)))); + if (n <= cols || cols <= 0) { + return { data: data instanceof Float32Array ? data.subarray(0, n * bins) : Float32Array.from({ length: n * bins }, (_, i) => data[i]), frames: n }; + } + const out = new Float32Array(cols * bins).fill(-Infinity); + for (let c = 0; c < cols; c++) { + const f0 = Math.floor(c * n / cols); + const f1 = Math.max(f0 + 1, Math.floor((c + 1) * n / cols)); + const o = c * bins; + for (let f = f0; f < f1; f++) { + const s = f * bins; + for (let b = 0; b < bins; b++) { + const v = data[s + b]; + if (v > out[o + b]) out[o + b] = v; + } + } + } + return { data: out, frames: cols }; +} + +export interface OverviewImageOpts { + data: ArrayLike; + nFrames: number; + bins: number; + cols: number; + sampleRateHz: number; + maxFreq: number; + /** Value window [black, white] of `data`. */ + lo: number; + hi: number; + scale: string; + binFreqs: ArrayLike | null; + colourScale: string; + noiseReduction: boolean; + bandFloor: ArrayLike | null; + colorScheme: string; +} + +/** Colored whole-file spectrogram image (one column per overview pixel, max-pooled). */ +export function buildOverviewImage(o: OverviewImageOpts) { + const pooled = poolFramesMax(o.data, o.nFrames, o.bins, o.cols); + if (pooled.frames <= 0) return null; + const grayInfo = buildSpectrogramGrayscale({ + spectrogramData: pooled.data, + spectrogramFrames: pooled.frames, + spectrogramMels: o.bins, + sampleRateHz: o.sampleRateHz, + maxFreq: o.maxFreq, + spectrogramAbsLogMin: o.lo, + spectrogramAbsLogMax: o.hi, + scale: o.scale, + binFreqs: o.binFreqs, + colourScale: o.colourScale, + noiseReduction: o.noiseReduction, + bandFloor: o.bandFloor, + clahe: false, + }); + if (!grayInfo) return null; + return colorizeSpectrogram(grayInfo, 0, 1, o.colorScheme); +} diff --git a/src/ui/components/waveform/waveform.ts b/src/ui/components/waveform/waveform.ts index 0db714e3..b9641630 100644 --- a/src/ui/components/waveform/waveform.ts +++ b/src/ui/components/waveform/waveform.ts @@ -311,6 +311,30 @@ export function renderOverviewWaveform({ }); } +/** + * Draws a whole-file spectrogram image into the overview strip (overview + * mode 'spectrogram'). Clears the waveform's theme re-render so a theme + * change does not paint the waveform over it. + */ +export function renderOverviewImage({ + image, + overviewCanvas, + overviewContainer, +}: { + image: CanvasImageSource; + overviewCanvas: HTMLCanvasElement; + overviewContainer: HTMLElement; +}) { + const ctx = overviewCanvas.getContext('2d'); + if (!ctx) return; + (overviewCanvas as any)._aw_lastRender = null; + const rect = overviewContainer.getBoundingClientRect(); + overviewCanvas.width = Math.max(1, Math.floor(rect.width)); + overviewCanvas.height = Math.max(1, Math.floor(rect.height)); + ctx.imageSmoothingEnabled = true; + ctx.drawImage(image, 0, 0, overviewCanvas.width, overviewCanvas.height); +} + // ─── Frequency Labels ─────────────────────────────────────────────── export function renderFrequencyLabels({ labelsElement, coords }: { labelsElement: HTMLElement; coords: any }) { diff --git a/src/ui/template.ts b/src/ui/template.ts index ed763028..464e6ff1 100644 --- a/src/ui/template.ts +++ b/src/ui/template.ts @@ -17,6 +17,7 @@ export const DEFAULT_OPTIONS = { showDisplayGain: true, // Floor / Ceil / AC sliders showStatusbar: true, // Bottom status bar showOverview: true, // Bottom overview navigator + overviewMode: 'waveform', // waveform | spectrogram (whole file, max-pooled) viewMode: 'both', // both | waveform | spectrogram transportStyle: 'default', // default | hero transportOverlay: false, // Overlay mode: centered play button without toolbar height diff --git a/tests/overview-spectrogram.test.mjs b/tests/overview-spectrogram.test.mjs new file mode 100644 index 00000000..1a99574d --- /dev/null +++ b/tests/overview-spectrogram.test.mjs @@ -0,0 +1,28 @@ +// overview-spectrogram.test.mjs — max-pooled whole-file overview +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { poolFramesMax } from '../src/domain/spectrogram.ts'; + +test('poolFramesMax keeps each bin maximum per column', () => { + // 6 frames × 2 bins, frame-major; a short "call" in frame 4, bin 1 + const data = Float32Array.from([ + 0, 1, 0, 1, 0, 1, + 0, 1, 0, 9, 2, 1, + ]); + const { data: out, frames } = poolFramesMax(data, 6, 2, 3); + assert.equal(frames, 3); + assert.deepEqual([...out], [0, 1, 0, 1, 2, 9]); +}); + +test('poolFramesMax passes short data through', () => { + const data = Float32Array.from([1, 2, 3, 4]); + const { data: out, frames } = poolFramesMax(data, 2, 2, 10); + assert.equal(frames, 2); + assert.deepEqual([...out], [1, 2, 3, 4]); +}); + +test('poolFramesMax covers uneven splits without gaps', () => { + const data = Float32Array.from([5, 1, 1, 1, 7]); // 5 frames × 1 bin + const { data: out } = poolFramesMax(data, 5, 1, 2); + assert.deepEqual([...out], [5, 7]); +}); From b0bdb7ba71e07d92045830f4961a78b7cf366cd0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 08:02:40 +0000 Subject: [PATCH 2/4] feat(metadata): read the recording start time; clock times in Raven tables Adapted from recording-time.js of the BirdNET website demo (MIT): AudioMoth comment, GUANO Timestamp, BWF bext, WAV INFO ICRD, or the file name. Shown in the status bar, exposed as player.recordingStart / clockTimeAt(), and Raven selection tables gain Begin Date / Begin Clock Time / End Clock Time. --- CHANGELOG.md | 5 + demo/labeling-app.html | 7 +- src/app/BirdNETPlayer.ts | 15 ++ src/app/PlayerState.ts | 10 +- src/domain/recordingTime.ts | 188 ++++++++++++++++++ src/infrastructure/audio/AudioEngine.ts | 6 + src/infrastructure/audio/MockAudioEngine.ts | 1 + .../export/labelTableFormats.ts | 2 + .../export/ravenSelectionTable.ts | 11 +- tests/recording-time.test.mjs | 84 ++++++++ 10 files changed, 324 insertions(+), 5 deletions(-) create mode 100644 src/domain/recordingTime.ts create mode 100644 tests/recording-time.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 54662943..4496d9e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ All notable changes to this project will be documented in this file. strip shows the whole recording as a spectrogram, max-pooled so short calls stay visible; the waveform stands in while the spectrogram is computed. The labeling app offers the choice under Display → Overview. +- Recording start time from AudioMoth, GUANO, BWF `bext` and WAV INFO metadata + or a date and time in the file name (adapted from the BirdNET website demo, + MIT): shown in the status bar, `player.recordingStart`, + `player.clockTimeAt(seconds)`, and Raven selection tables get + Begin Date / Begin Clock Time / End Clock Time. ## [1.1.3] - 2026-11-14 diff --git a/demo/labeling-app.html b/demo/labeling-app.html index d8bf9851..71db3b2a 100644 --- a/demo/labeling-app.html +++ b/demo/labeling-app.html @@ -2231,7 +2231,7 @@

Tags

function downloadRavenTable() { const entry = recordingManager.getActive(); const name = entry?.filename || fileLabel.textContent || 'labels'; - const text = buildRavenSelectionTable(state.labels, { sampleRate: entry?.sampleRate || state.sampleRate, setName: ravenSetName }); + const text = buildRavenSelectionTable(state.labels, { sampleRate: entry?.sampleRate || state.sampleRate, setName: ravenSetName, recordingStart: entry?.recordingStart || player.recordingStart }); saveFile(text, ravenTablePath(name.split('/').pop()), 'text/plain') .catch((e) => { selectionMeta.textContent = `Export failed: ${e?.message || e}`; }); } @@ -2244,7 +2244,7 @@

Tags

const labels = entryLabels(entry); if (!labels.length) continue; const target = entry.path ? ravenTablePath(entry.path) : `exports/raven/${ravenTablePath(entry.filename)}`; - await projectStore.writeText(target, buildRavenSelectionTable(labels, { sampleRate: entry.sampleRate, setName: ravenSetName })); + await projectStore.writeText(target, buildRavenSelectionTable(labels, { sampleRate: entry.sampleRate, setName: ravenSetName, recordingStart: entry.recordingStart || null })); written++; } selectionMeta.textContent = written @@ -3255,6 +3255,7 @@

Open another recording

const out = buildLabelExport(format, labels, base, { sampleRate: entry?.sampleRate || state.sampleRate, setName: ravenSetName, + recordingStart: entry?.recordingStart || player.recordingStart, columns, displayName: (l) => resolveName(l).display, }); @@ -4433,7 +4434,7 @@

Open another recording

setTimeout(() => requestMissingSnr(), 500); if (e.detail?.sampleRate) { state.sampleRate = e.detail.sampleRate; - if (recordingManager?.activeId) recordingManager.update(recordingManager.activeId, { sampleRate: e.detail.sampleRate }); + if (recordingManager?.activeId) recordingManager.update(recordingManager.activeId, { sampleRate: e.detail.sampleRate, recordingStart: player.recordingStart || null }); } propsPanel.setRecordingMeta(state.recordingMeta, { sampleRate: state.sampleRate, diff --git a/src/app/BirdNETPlayer.ts b/src/app/BirdNETPlayer.ts index 4e708d9c..680bf2e2 100644 --- a/src/app/BirdNETPlayer.ts +++ b/src/app/BirdNETPlayer.ts @@ -91,6 +91,8 @@ export { importXenoCantoSpectrogramLabels, } from '../infrastructure/xeno-canto/xenoCantoRecordingsApi.ts'; export { TaxonomyResolver } from '../infrastructure/taxonomyResolver.ts'; +import { isoClockTime, type RecordingStart } from '../domain/recordingTime.ts'; +export { readRecordingStart, isoClockTime, ravenClock, utcLabel, formatRecordingStart, type RecordingStart } from '../domain/recordingTime.ts'; export { BirdNETInference, BIRDNET_MODEL_URL } from '../infrastructure/birdnetInference.ts'; export class BirdNETPlayer { @@ -962,6 +964,19 @@ export class BirdNETPlayer { get normalize() { return !!this._state?.normalize; } + /** + * When the loaded recording started, from AudioMoth / GUANO / BWF / WAV INFO + * metadata or a date and time in the file name; null when unknown. + * `ms` is the recorder's wall-clock time written as UTC, `tz` its offset in minutes. + */ + get recordingStart(): RecordingStart | null { return this._state?.recordingStart ?? null; } + + /** Wall-clock time at `seconds` into the recording ("2023-05-12T19:00:03.000+01:00"), null when the start is unknown. */ + clockTimeAt(seconds: number): string | null { + const start = this.recordingStart; + return start ? isoClockTime(start, seconds) : null; + } + /** Overview strip below the player: 'waveform' or a whole-file 'spectrogram'. */ setOverviewMode(mode: 'waveform' | 'spectrogram') { this._state?._setOverviewMode(mode); } diff --git a/src/app/PlayerState.ts b/src/app/PlayerState.ts index b0245bd5..ea9c7eb6 100644 --- a/src/app/PlayerState.ts +++ b/src/app/PlayerState.ts @@ -31,6 +31,7 @@ import { PerfOverlay } from './player/PerfOverlay.ts'; import { ViewResizeController } from './player/ViewResizeController.ts'; import { FrequencyZoomController } from './player/FrequencyZoomController.ts'; import { WaveformRenderer, type OverviewMode } from './player/WaveformRenderer.ts'; +import { formatRecordingStart, type RecordingStart } from '../domain/recordingTime.ts'; import { ToolbarController } from './player/ToolbarController.ts'; import { ToolbarFitController } from './player/ToolbarFitController.ts'; import { CarrierLineController } from './player/CarrierLineController.ts'; @@ -868,7 +869,11 @@ export class PlayerState { } this._spectro.updateMaxFreqOptions(sampleRate); - this.d.fileInfo!.innerHTML = `${escapeHtml(displayName)} ${formatTime(duration)}`; + const start = this._engine.recordingStart ?? null; + const startHtml = start + ? ` · ${escapeHtml(formatRecordingStart(start))}` + : ''; + this.d.fileInfo!.innerHTML = `${escapeHtml(displayName)} ${formatTime(duration)}${startHtml}`; if (this.d.sampleRateInfo) this.d.sampleRateInfo.textContent = `${sampleRate} Hz`; if (this.d.totalTimeDisplay) this.d.totalTimeDisplay.textContent = formatTime(duration); if (this.d.currentTimeDisplay) this.d.currentTimeDisplay.textContent = formatTime(0); @@ -1273,6 +1278,9 @@ export class PlayerState { _drawMainWaveform() { this._waveformRenderer.drawMainWaveform(); } _drawOverviewWaveform() { this._waveformRenderer.drawOverviewWaveform(); } + /** When the loaded recording started (file metadata or name), null when unknown. */ + get recordingStart(): RecordingStart | null { return this._engine.recordingStart ?? null; } + /** Overview strip: 'waveform' or a whole-file 'spectrogram' (max-pooled). */ _setOverviewMode(mode: OverviewMode) { const next: OverviewMode = mode === 'spectrogram' ? 'spectrogram' : 'waveform'; diff --git a/src/domain/recordingTime.ts b/src/domain/recordingTime.ts new file mode 100644 index 00000000..f89f082a --- /dev/null +++ b/src/domain/recordingTime.ts @@ -0,0 +1,188 @@ +// ═══════════════════════════════════════════════════════════════════════ +// domain/recordingTime.ts — when a recording started, from its metadata +// +// Adapted from recording-time.js in the BirdNET website demo +// (https://github.com/birdnet-team/birdnet-website, MIT License, +// Copyright (c) 2025 BirdNET-Team). +// +// Read in this order: +// - AudioMoth comment in the WAV INFO chunk ("Recorded at 19:00:00 +// 12/05/2023 (UTC+1) by AudioMoth …"), also when it survived conversion +// to FLAC or another format (searched in the file's first bytes) +// - GUANO "Timestamp" (Wildlife Acoustics, Titley, Pettersson and others) +// - BWF bext origination date and time (Zoom, Tascam, Sound Devices, …) +// - WAV INFO creation date, when it includes a time of day +// - a date and time in the file name ("20230512_190000.WAV", +// "SMA01234_20230512_190000.wav", "2023-05-12 19-00-00.flac") +// +// `ms` is the recorder's wall-clock time written as if it were UTC, so +// formatting never shifts it into the viewer's time zone; `tz` is the offset +// from UTC in minutes, or null when the metadata does not say. +// ═══════════════════════════════════════════════════════════════════════ + +export type RecordingStartSource = 'AudioMoth' | 'GUANO' | 'BWF' | 'WAV INFO' | 'fileName' | 'device'; + +export interface RecordingStart { + ms: number; + tz: number | null; + source: RecordingStartSource; +} + +const MAX_CHUNK = 1 << 16; +const HEAD_BYTES = 1 << 18; + +const ascii = (b: Uint8Array, p: number, n: number) => String.fromCharCode(...b.subarray(p, p + n)); +const pad = (n: number, w = 2) => String(n).padStart(w, '0'); +const latin1 = (b: Uint8Array) => new TextDecoder('latin1').decode(b); +const utf8 = (b: Uint8Array) => new TextDecoder('utf-8').decode(b).replace(/\0+$/, ''); + +type Num = string | number | undefined | null; + +function make(y: Num, mo: Num, d: Num, h: Num, mi: Num, s: Num, frac: Num, tz: number | null, source: RecordingStartSource): RecordingStart | null { + const [Y, MO, D, H, MI, S] = [y, mo, d, h, mi, s].map(Number); + if (!(Y >= 1990 && Y <= 2100 && MO >= 1 && MO <= 12 && D >= 1 && D <= 31 && H <= 23 && MI <= 59 && S <= 60)) return null; + const ms = Date.UTC(Y, MO - 1, D, H, MI, S) + (frac ? Math.round(Number(`0.${frac}`) * 1000) : 0); + // Date.UTC rolls 31 April into May; such dates are no dates. + if (new Date(ms).getUTCDate() !== D) return null; + return { ms, tz, source }; +} + +/** "+1", "-5", "+5:30", "+0530" → minutes; null when absent. */ +function tzMinutes(sign: string | undefined, hours: Num, minutes: Num): number | null { + if (sign == null) return null; + return (sign === '-' ? -1 : 1) * (Number(hours || 0) * 60 + Number(minutes || 0)); +} + +const AUDIOMOTH = /Recorded at (\d{2}):(\d{2}):(\d{2})(?:\.(\d+))? (\d{2})\/(\d{2})\/(\d{4}) \(UTC(?:([+-])(\d{1,2})(?::(\d{2}))?)?\)/; + +function fromAudioMoth(text: string): RecordingStart | null { + const m = text.match(AUDIOMOTH); + return m ? make(m[7], m[6], m[5], m[1], m[2], m[3], m[4], tzMinutes(m[8] || '+', m[9], m[10]), 'AudioMoth') : null; +} + +const ISO = /(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2}):(\d{2})(?:[.,](\d+))?\s*(?:(Z)|([+-])(\d{2}):?(\d{2})?)?/; + +function fromIso(text: string, source: RecordingStartSource): RecordingStart | null { + const m = text.match(ISO); + if (!m) return null; + const tz = m[8] ? 0 : tzMinutes(m[9], m[10], m[11]); + return make(m[1], m[2], m[3], m[4], m[5], m[6], m[7], tz, source); +} + +function fromGuano(text: string): RecordingStart | null { + const line = text.split(/\r?\n/).find((l) => /^Timestamp\s*:/.test(l)); + return line ? fromIso(line.replace(/^Timestamp\s*:\s*/, ''), 'GUANO') : null; +} + +/** bext: OriginationDate at 320 ("yyyy-mm-dd") and OriginationTime at 330 ("hh:mm:ss"), local time. */ +function fromBext(b: Uint8Array): RecordingStart | null { + if (b.length < 338) return null; + const d = ascii(b, 320, 10).match(/^(\d{4})\D(\d{2})\D(\d{2})$/); + const t = ascii(b, 330, 8).match(/^(\d{2})\D(\d{2})\D(\d{2})$/); + return d && t ? make(d[1], d[2], d[3], t[1], t[2], t[3], null, null, 'BWF') : null; +} + +/** Date and time in a file name, e.g. "SMA01234_20230512_190000.wav". */ +export function recordingStartFromFileName(name: string): RecordingStart | null { + const base = (name.split(/[\\/]/).pop() || '').replace(/\.[^.]+$/, ''); + const m = base.match(/(?> = {}; + for (let p = 12; p + 8 <= bytes.length;) { + const id = ascii(bytes, p, 4); + const size = view.getUint32(p + 4, true); + if (size === 0xffffffff) break; // RF64 audio data: its real size is elsewhere, and nothing we need follows + if (size <= MAX_CHUNK && ['LIST', 'guan', 'bext'].includes(id)) { + const b = bytes.subarray(p + 8, Math.min(bytes.length, p + 8 + size)); + if (id === 'guan') found.guano ??= fromGuano(utf8(b)); + else if (id === 'bext') found.bext ??= fromBext(b); + else if (b.length >= 4 && ascii(b, 0, 4) === 'INFO') { + const bv = new DataView(b.buffer, b.byteOffset, b.byteLength); + for (let q = 4; q + 8 <= b.length;) { + const sid = ascii(b, q, 4); + const n = bv.getUint32(q + 4, true); + const value = utf8(b.subarray(q + 8, q + 8 + n)); + if (sid === 'ICMT') found.audiomoth ??= fromAudioMoth(value); + else if (sid === 'ICRD') found.icrd ??= fromIso(value, 'WAV INFO'); + q += 8 + n + (n & 1); + } + } + } + p += 8 + size + (size & 1); + } + return found.audiomoth || found.guano || found.bext || found.icrd || null; +} + +/** Start of the recording in this file (bytes and/or name), or null when nothing says. */ +export function readRecordingStart(buffer: ArrayBuffer | Uint8Array | null, name = ''): RecordingStart | null { + try { + if (buffer) { + const bytes = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer); + const start = fromWav(bytes) || fromAudioMoth(latin1(bytes.subarray(0, HEAD_BYTES))); + if (start) return start; + } + } catch (err) { + console.warn('Could not read the recording start time', err); + } + return recordingStartFromFileName(name); +} + +/** Start of a recording made on this device, in its local time. */ +export function localRecordingStart(date: Date): RecordingStart { + const tz = -date.getTimezoneOffset(); + return { ms: date.getTime() + tz * 60000, tz, source: 'device' }; +} + +// ── Formatting ────────────────────────────────────────────────────── + +function parts(start: RecordingStart, offsetSec: number) { + const d = new Date(start.ms + Math.round(offsetSec * 1000)); + return { + date: `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())}`, + time: `${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}.${pad(d.getUTCMilliseconds(), 3)}`, + }; +} + +/** "+01:00", "Z", or "" when the offset is unknown. */ +function isoOffset(tz: number | null): string { + if (tz == null) return ''; + if (tz === 0) return 'Z'; + const a = Math.abs(tz); + return `${tz < 0 ? '-' : '+'}${pad(Math.floor(a / 60))}:${pad(a % 60)}`; +} + +/** Raven selection table columns for the recorder's wall-clock time. */ +export const RAVEN_CLOCK_COLUMNS = ['Begin Date', 'Begin Clock Time', 'End Clock Time']; + +/** Values for RAVEN_CLOCK_COLUMNS: "2023-05-12", "19:00:03.000", "19:00:06.000". */ +export function ravenClock(start: RecordingStart, beginSec: number, endSec: number): string[] { + const b = parts(start, beginSec); + return [b.date, b.time, parts(start, endSec).time]; +} + +/** "2023-05-12T19:00:03.000+01:00" (no offset when unknown). */ +export function isoClockTime(start: RecordingStart, offsetSec = 0): string { + const p = parts(start, offsetSec); + return `${p.date}T${p.time}${isoOffset(start.tz)}`; +} + +/** "UTC+1", "UTC+5:30", "UTC", or "" when unknown. */ +export function utcLabel(tz: number | null): string { + if (tz == null) return ''; + if (tz === 0) return 'UTC'; + const a = Math.abs(tz); + return `UTC${tz < 0 ? '-' : '+'}${Math.floor(a / 60)}${a % 60 ? `:${pad(a % 60)}` : ''}`; +} + +/** "2023-05-12 19:00:03 UTC+1" for display. */ +export function formatRecordingStart(start: RecordingStart, offsetSec = 0): string { + const p = parts(start, offsetSec); + return [`${p.date} ${p.time.slice(0, 8)}`, utcLabel(start.tz)].filter(Boolean).join(' '); +} diff --git a/src/infrastructure/audio/AudioEngine.ts b/src/infrastructure/audio/AudioEngine.ts index 2121271d..888f32b2 100644 --- a/src/infrastructure/audio/AudioEngine.ts +++ b/src/infrastructure/audio/AudioEngine.ts @@ -22,6 +22,7 @@ import { clamp, channelData, parseNativeSampleRate } from '../../shared/utils.ts import { AudioEngineBase } from './AudioEngineBase.ts'; import { isMp3, decodeMp3Chunked, CHUNKED_DECODE_MIN_BYTES } from './chunkedMp3Decode.ts'; import { parseGuano } from '../../domain/ultrasound.ts'; +import { readRecordingStart, type RecordingStart } from '../../domain/recordingTime.ts'; import { computeAmplitudePeak } from '../../domain/spectrogram.ts'; import { listenModeKey, type ListenMode, type DetectorChunk } from './detectorRender.ts'; import type { DetectorRenderer } from './detectorWorkerClient.ts'; @@ -153,6 +154,8 @@ export class AudioEngine extends AudioEngineBase { _peakFor: AudioBuffer | null; _peak: number; guano: Record | null; + /** When the loaded recording started (metadata or file name), null when unknown. */ + recordingStart: RecordingStart | null = null; ctx: any; bandpass: any; emitEnd: any; @@ -230,6 +233,7 @@ export class AudioEngine extends AudioEngineBase { */ async loadFromArrayBuffer(arrayBuffer: ArrayBuffer, name?: string) { this.guano = parseGuano(arrayBuffer); + this.recordingStart = readRecordingStart(arrayBuffer, name || ''); const audioBuffer = await decodeArrayBuffer(arrayBuffer); this.audioBuffer = audioBuffer; @@ -258,6 +262,7 @@ export class AudioEngine extends AudioEngineBase { const name = decodeURIComponent( new URL(url, location.href).pathname.split('/').pop() || 'audio', ); + this.recordingStart = readRecordingStart(arrayBuffer, name); await this._setupWaveSurfer(url, name); return { duration: audioBuffer.duration, sampleRate: audioBuffer.sampleRate }; } @@ -270,6 +275,7 @@ export class AudioEngine extends AudioEngineBase { async loadFromFile(file: File) { const arrayBuffer = await file.arrayBuffer(); this.guano = parseGuano(arrayBuffer); + this.recordingStart = readRecordingStart(arrayBuffer, file.name); const audioBuffer = await decodeArrayBuffer(arrayBuffer); this.audioBuffer = audioBuffer; await this._setupWaveSurfer(file, file.name); diff --git a/src/infrastructure/audio/MockAudioEngine.ts b/src/infrastructure/audio/MockAudioEngine.ts index 6fe9e889..cb79584b 100644 --- a/src/infrastructure/audio/MockAudioEngine.ts +++ b/src/infrastructure/audio/MockAudioEngine.ts @@ -163,6 +163,7 @@ export class MockAudioEngine extends AudioEngineBase { override setNormalize(on: boolean): number { this.normalize = !!on; return 1; } timeExpansion = 1; guano: Record | null = null; + recordingStart: import('../../domain/recordingTime.ts').RecordingStart | null = null; listenMode: any = { kind: 'expansion', factor: 1 }; override setTimeExpansion(factor: number): void { this.setListenMode({ kind: 'expansion', factor }); } override setListenMode(mode: any): void { diff --git a/src/infrastructure/export/labelTableFormats.ts b/src/infrastructure/export/labelTableFormats.ts index 3d1fcaa3..d0c53852 100644 --- a/src/infrastructure/export/labelTableFormats.ts +++ b/src/infrastructure/export/labelTableFormats.ts @@ -23,6 +23,8 @@ export interface LabelExportColumn { export interface LabelExportOptions { sampleRate?: number | null; + /** Recording start (Raven clock-time columns). */ + recordingStart?: import('../../domain/recordingTime.ts').RecordingStart | null; setName?: (setId: string) => string; /** Columns for CSV (the table's visible columns). */ columns?: LabelExportColumn[]; diff --git a/src/infrastructure/export/ravenSelectionTable.ts b/src/infrastructure/export/ravenSelectionTable.ts index 2b5d256a..bfb021fa 100644 --- a/src/infrastructure/export/ravenSelectionTable.ts +++ b/src/infrastructure/export/ravenSelectionTable.ts @@ -5,8 +5,12 @@ //