diff --git a/README.md b/README.md index d921626..1569b78 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,8 @@ On **Slopsmith Desktop**, when the native audio engine has the Spotify **Basic P This path is fully feature-detected and fail-soft: a desktop build without `detectNotes` still scores single notes via `getPitchDetection`; one without the `scoreChord` IPC skips chord scoring; and a build without the ML model falls back to YIN/`ChordScorer`. The browser build is unaffected and keeps the JS YIN/HPS/CREPE + constraint path. See `slopsmith-desktop/src/audio/MlNoteDetector.*`. +> **Retuned / transformed charts.** Expected pitches follow the chart the player actually sees. When a core [`chart-transform`](https://github.com/got-feedback/feedback/blob/main/docs/capability-recipes.md#chart-transform-provider) provider rewrites the chart (e.g. a retune, transposition or "convert to E standard" view), the plugin scores against the host's *effective* `highway.getTuning()`, `getCapo()` and `getCentOffset()` instead of the original `getSongInfo()` metadata, and re-syncs within a frame when a provider restages mid-song. The effective `centOffset` is applied too (minus whole octaves, which octave-folded matching and the bass base tables already cover — an extended-range bass's `-1200` changes nothing), so a pitch-shifted backing track (say `-100` cents for a half-step-down retune, or `+11.8` for an A443 recording) scores against where the audio really sits. A plugin that pitch-shifts the song should publish that shift as a chart-transform provider returning `tuning` and/or `centOffset`; note detection follows with no extra wiring. On the desktop engine path offsets are rounded to whole semitones (the engine reads integers), so a sub-semitone residue is a browser-path refinement only. + ## Requirements - Browser with `getUserMedia` support (all modern browsers) diff --git a/package.json b/package.json index 98485f7..c04f63e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "slopsmith-plugin-notedetect", - "version": "1.32.0", + "version": "1.33.0", "private": true, "description": "Slopsmith note detection plugin — test harness for detection core", "scripts": { diff --git a/plugin.json b/plugin.json index bb332a3..b5f4e05 100644 --- a/plugin.json +++ b/plugin.json @@ -1,7 +1,7 @@ { "id": "note_detect", "name": "Note Detection", - "version": "1.32.0", + "version": "1.33.0", "private": false, "script": "screen.js", "styles": "assets/plugin.css", diff --git a/screen.js b/screen.js index bd9025c..fb7a4e0 100644 --- a/screen.js +++ b/screen.js @@ -316,7 +316,7 @@ const _ND_AUTO_ENABLE_RETRY_MS = 1500; // exact build that produced it. The script tag has no `import`/`fetch` // hook to read package.json at load time, so this is the single // hand-maintained constant the diagnostic path keys off of. -const _ND_VERSION = '1.32.0'; +const _ND_VERSION = '1.33.0'; // Bleed-rescue tuning for the low-bass blind spot. A bass DI's fundamental is // weaker than its 2nd harmonic and an open string / neighbour often rings the @@ -613,6 +613,34 @@ function _ndMidiFromStringFret(string, fret, arrangement, stringCount, offsets, return base[string] + offset + (capo || 0) + fret; } +// Whole-chart reference-pitch shift (cents) → the part note detection applies. +// The host's effective `centOffset` (`highway.getCentOffset()`: a chart- +// transform provider's value, else the chart's own ``) says the +// audio sits N cents off the chart's nominal pitch — a retune/pitch-shift +// transform, a true-tuned A443 recording (+11.8), a song shifted to E standard. +// Whole octaves are dropped: matching is already octave-folded and the +// per-arrangement base tables already place bass in its own octave, so the +// common extended-range-bass `-1200` must keep scoring exactly as before. The +// remainder lands in (-600, 600] and is folded into the per-string offsets +// (as fractional semitones) by _syncChartStateFromHw, so it reaches every +// expected MIDI / band range through _ndMidiFromStringFret unchanged. +function _ndRefShiftCents(centOffset) { + const c = Number(centOffset); + if (!Number.isFinite(c)) return 0; + const r = c - 1200 * Math.round(c / 1200); + return Math.abs(r) < 1e-6 ? 0 : r; +} + +// The desktop engine's chart/scoreChord contexts read tuning offsets as +// Int32 (truncating), so a fractional reference shift is rounded to the +// nearest semitone at the bridge boundary — a whole-semitone retune still +// reaches the engine exactly; a sub-semitone residue (e.g. A443) stays a +// JS-path refinement that sits inside the engine's pitch tolerance anyway. +function _ndEngineOffsets(offsets) { + if (!Array.isArray(offsets)) return offsets; + return offsets.map(o => Math.round(Number(o) || 0)); +} + function _ndClassifyTiming(timingErrorMs, timingThresholdMs, lateGraceMs) { if (!Number.isFinite(timingErrorMs)) return null; const grace = Number.isFinite(lateGraceMs) && lateGraceMs > 0 ? lateGraceMs : 0; @@ -3373,6 +3401,9 @@ function createNoteDetector(options = {}) { let tuningOffsets = [0, 0, 0, 0, 0, 0]; let capo = 0; let currentStringCount = 6; // kept in sync with tuningOffsets.length + // Effective-tuning inputs the fields above were last synced from (see + // _ndResyncIfTuningChanged); null = never synced. + let _ndSyncedTuningKey = null; // Audio buffers let accumBuffer = new Float32Array(0); @@ -3526,6 +3557,7 @@ function createNoteDetector(options = {}) { } function _ndBridgeScoreChord(ctx) { const a = _ndBridgeAudio(); if (!a) return Promise.resolve(null); + if (ctx && Array.isArray(ctx.offsets)) ctx = { ...ctx, offsets: _ndEngineOffsets(ctx.offsets) }; if (sourceId != null && typeof a.scoreSourceChord === 'function') return a.scoreSourceChord(sourceId, ctx); if (sourceId == null && typeof a.scoreChord === 'function') @@ -3540,6 +3572,9 @@ function createNoteDetector(options = {}) { } function _ndBridgeSetChart(chart) { const a = _ndBridgeAudio(); if (!a) return Promise.resolve(null); + if (chart && Array.isArray(chart.tuningOffsets)) { + chart = { ...chart, tuningOffsets: _ndEngineOffsets(chart.tuningOffsets) }; + } if (sourceId != null && typeof a.setSourceChart === 'function') return a.setSourceChart(sourceId, chart); if (sourceId == null && typeof a.setChart === 'function') @@ -4181,6 +4216,10 @@ function createNoteDetector(options = {}) { // song:loaded (and can switch mid-session); // re-push whenever the live chart diverges // from what the engine currently holds. + // A live retune restage changes the + // effective tuning, which is in the + // signature, so it re-pushes here too. + _ndResyncIfTuningChanged(); if (_ndChartSignature() !== _ndVerifierChartSig) { await _ndPushChartToBridge(); if (!enabled || gen !== sessionGen) return; @@ -5059,6 +5098,9 @@ function createNoteDetector(options = {}) { // ── Frame processing ────────────────────────────────────────────── async function processFrame(buffer) { + // Follow a mid-song chart-transform restage (live retune) before + // mapping any (string, fret) to an expected pitch this frame. + _ndResyncIfTuningChanged(); // Append to the rolling raw-audio buffer for the bass miss rescue // (checkMisses). Bass only — guitar's higher fundamentals resolve in // the short window, so it doesn't need (or pay for) this. @@ -13626,10 +13668,26 @@ function createNoteDetector(options = {}) { tuningOffsets = [0, 0, 0, 0, 0, 0]; capo = 0; const info = (hw && hw.getSongInfo) ? hw.getSongInfo() : null; + const eff = _ndReadEffectiveTuning(); + _ndSyncedTuningKey = eff; if (!info) return; if (info.arrangement) currentArrangement = _ndArrangementKindFromName(info.arrangement); - if (Array.isArray(info.tuning)) tuningOffsets = info.tuning; - if (Number.isFinite(info.capo)) capo = info.capo; + // Tuning / capo / reference pitch come from the host's EFFECTIVE + // getters, not getSongInfo(): getSongInfo() keeps the original chart + // metadata, while getNotes()/getChords() (what we score) already carry + // any chart-transform provider's output. Reading the original tuning + // against transformed frets made a retuned / converted-to-E-standard + // chart score against its old pitches. Hosts without the getters + // (pre-chart-transform) fall back to song_info. + const tuning = Array.isArray(eff.tuning) ? eff.tuning : info.tuning; + if (Array.isArray(tuning)) { + const shift = _ndRefShiftCents(eff.centOffset !== undefined ? eff.centOffset : info.centOffset); + tuningOffsets = shift + ? tuning.map(o => (Number(o) || 0) + shift / 100) + : tuning; + } + const effCapo = eff.capo !== undefined ? eff.capo : info.capo; + if (Number.isFinite(effCapo)) capo = effCapo; // String-count resolution, in precedence order: // 1. hw.getStringCount() — host's authoritative count. Asked @@ -13654,7 +13712,7 @@ function createNoteDetector(options = {}) { if (Number.isFinite(hostStringCount)) { currentStringCount = hostStringCount; } else if (info.arrangement) { - const tuneLen = Array.isArray(info.tuning) ? info.tuning.length : null; + const tuneLen = Array.isArray(tuning) ? tuning.length : null; const consistent = currentArrangement === 'bass' ? (tuneLen === 4 || tuneLen === 5) : (tuneLen === 6 || tuneLen === 7 || tuneLen === 8); @@ -13663,11 +13721,47 @@ function createNoteDetector(options = {}) { } else { currentStringCount = currentArrangement === 'bass' ? 4 : 6; } - } else if (Array.isArray(info.tuning)) { - currentStringCount = info.tuning.length; + } else if (Array.isArray(tuning)) { + currentStringCount = tuning.length; } } + // The host's effective tuning inputs, read raw (undefined when the host + // predates the chart-transform getters). Compared by identity in + // _ndResyncIfTuningChanged: the highway hands back the same array/number + // until a transform restages, so this is a few getter calls per tick. + function _ndReadEffectiveTuning() { + const read = (name) => { + if (!hw || typeof hw[name] !== 'function') return undefined; + try { return hw[name](); } catch (_) { return undefined; } + }; + return { + tuning: read('getTuning'), + capo: read('getCapo'), + centOffset: read('getCentOffset'), + }; + } + // A chart-transform provider can restage mid-song (select / clear / + // refresh — e.g. a live retune) without any song or arrangement event. + // Called from the detect tick so scoring follows the effective tuning + // within a frame; returns true when it resynced. + function _ndResyncIfTuningChanged() { + // Pre-chart-transform host: nothing can restage mid-song, and the + // song/arrangement events already cover every change. + if (!hw || (typeof hw.getTuning !== 'function' + && typeof hw.getCapo !== 'function' + && typeof hw.getCentOffset !== 'function')) return false; + const k = _ndSyncedTuningKey; + const now = _ndReadEffectiveTuning(); + if (k && k.tuning === now.tuning && k.capo === now.capo + && (k.centOffset === now.centOffset + || (Number.isNaN(k.centOffset) && Number.isNaN(now.centOffset)))) { + return false; + } + _syncChartStateFromHw(); + return true; + } + // ── Engine-side chart verification (desktop bridge) ─────────────────── // Build the song's note chart from the host and push it to the desktop // engine, which scores it continuously on a background thread. Replaces @@ -13723,7 +13817,8 @@ function createNoteDetector(options = {}) { // value up live — without this the popup sliders are inert on the // engine-verifier path until the next song/arrangement change. return n + ':' + c + ':' + currentArrangement + ':' + currentStringCount - + ':' + capo + ':' + timingTolerance + ':' + pitchTolerance + + ':' + capo + ':' + tuningOffsets.slice(0, currentStringCount).join(',') + + ':' + timingTolerance + ':' + pitchTolerance + ':' + (fnv >>> 0); } @@ -18541,6 +18636,7 @@ function createNoteDetector(options = {}) { _bindChartStateEvents: _chartStateBindEvents, _unbindChartStateEvents: _chartStateUnbindEvents, _syncChartStateFromHw: _syncChartStateFromHw, + _resyncIfTuningChanged: _ndResyncIfTuningChanged, // Auto-record test hooks — same rationale. Production binds from // enableImpl() / unbinds from destroy(); tests drive them without // the audio pipeline to assert the song:loaded auto-arm. @@ -18585,6 +18681,9 @@ function createNoteDetector(options = {}) { if (Number.isFinite(ctx.stringCount)) currentStringCount = ctx.stringCount; if (Array.isArray(ctx.tuningOffsets)) tuningOffsets = ctx.tuningOffsets.slice(); if (Number.isFinite(ctx.capo)) capo = ctx.capo; + // An injected context is authoritative — don't let the + // per-tick effective-tuning resync overwrite it. + _ndSyncedTuningKey = _ndReadEffectiveTuning(); }, setSettings: (s) => { s = s || {}; diff --git a/test/_loader.js b/test/_loader.js index 660c9f3..78e903c 100644 --- a/test/_loader.js +++ b/test/_loader.js @@ -259,6 +259,13 @@ function loadDetectionCore({ sandboxBeforeRun } = {}) { midiToStringFret: midiToStringFretWrapped, resolveDisplayFingering: resolveDisplayFingeringWrapped, nearestOctaveCents: sandbox._ndNearestOctaveCents, + // Effective-tuning helpers (chart-transform / centOffset follow). + rawMidiFromStringFret: sandbox._ndMidiFromStringFret, + refShiftCents: sandbox._ndRefShiftCents, + engineOffsets: (offsets) => { + const r = sandbox._ndEngineOffsets(offsets); + return Array.isArray(r) ? Array.from(r) : r; + }, stringBandHz: sandbox._ndStringBandHz, bandEnergy: sandbox._ndBandEnergy, constraintCheckString: sandbox._ndConstraintCheckString, diff --git a/test/effective_tuning.test.js b/test/effective_tuning.test.js new file mode 100644 index 0000000..cef80e0 --- /dev/null +++ b/test/effective_tuning.test.js @@ -0,0 +1,184 @@ +// Effective-tuning follow — scoring maps (string, fret) → pitch through the +// host's EFFECTIVE tuning / capo / reference pitch, not the original song_info. +// +// Regression target (notedetect#24): when the chart is retuned / converted to +// E standard, the highway shows the new notes but detection kept expecting the +// old pitches. Core's chart-transform domain (feedBack#952) is the official +// signal for "the chart the player sees differs from the original": a provider +// may replace tuning, capo and centOffset, exposed through +// `highway.getTuning()` / `getCapo()` / `getCentOffset()` while +// `getSongInfo()` keeps the original metadata. These tests pin: +// 1. _ndRefShiftCents keeps only the sub-octave part of a centOffset. +// 2. _syncChartStateFromHw prefers the effective getters over song_info. +// 3. A whole-semitone centOffset shifts every expected MIDI; the common +// extended-range-bass -1200 changes nothing. +// 4. A mid-song transform restage (no song/arrangement event) is picked up +// by the per-tick resync. +// 5. The desktop engine receives integer offsets (it reads Int32). +// 6. A pre-chart-transform host (no getters) keeps the song_info path. + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { loadDetectionCore } = require('./_loader'); + +function mountWith(core, { info, tuning, capo, centOffset, stringCount = 6 }) { + core.highway.getSongInfo = () => info; + core.highway.getStringCount = () => stringCount; + if (tuning !== undefined) core.highway.getTuning = () => tuning; + if (capo !== undefined) core.highway.getCapo = () => capo; + if (centOffset !== undefined) core.highway.getCentOffset = () => centOffset; + return core.createNoteDetector(); +} + +test('_ndRefShiftCents drops whole octaves, keeps the remainder', () => { + const core = loadDetectionCore(); + const f = core.refShiftCents; + assert.equal(f(0), 0); + assert.equal(f(undefined), 0); + assert.equal(f(NaN), 0); + assert.equal(f('x'), 0); + assert.equal(f(-1200), 0, 'extended-range bass octave is not a retune'); + assert.equal(f(2400), 0); + assert.equal(f(-100), -100, 'a half-step-down retune'); + assert.equal(f(-1300), -100, 'octave + semitone keeps the semitone'); + assert.ok(Math.abs(f(11.8) - 11.8) < 1e-9, 'A443 true-tuning'); +}); + +test('_ndEngineOffsets rounds to whole semitones for the Int32 engine', () => { + const core = loadDetectionCore(); + assert.deepEqual(core.engineOffsets([-1, -1.3, 0.118, -2.6]), [-1, -1, 0, -3]); + assert.equal(core.engineOffsets(null), null); +}); + +test('effective tuning/capo from a chart transform beat original song_info', () => { + const core = loadDetectionCore(); + // Original chart: Eb standard, capo 0. Transform: converted to E standard + // with a capo-2 view. + const det = mountWith(core, { + info: { arrangement: 'Lead', tuning: [-1, -1, -1, -1, -1, -1], capo: 0, centOffset: 0 }, + tuning: [0, 0, 0, 0, 0, 0], + capo: 2, + centOffset: 0, + }); + det._syncChartStateFromHw(); + const st = det._getChartState(); + assert.equal(st.tuningOffsets.join(','), '0,0,0,0,0,0', 'transformed tuning, not the original'); + assert.equal(st.capo, 2, 'transformed capo'); + assert.equal(st.stringCount, 6); + det.destroy(); +}); + +test('a semitone centOffset shifts every expected pitch; -1200 bass does not', () => { + const core = loadDetectionCore(); + const det = mountWith(core, { + info: { arrangement: 'Lead', tuning: [0, 0, 0, 0, 0, 0], centOffset: 0 }, + tuning: [0, 0, 0, 0, 0, 0], + capo: 0, + centOffset: -100, + }); + det._syncChartStateFromHw(); + const st = det._getChartState(); + assert.equal(st.tuningOffsets.join(','), '-1,-1,-1,-1,-1,-1'); + // Low E open → Eb2 (39) instead of E2 (40); the chokepoint is unchanged. + assert.equal(core.rawMidiFromStringFret(0, 0, 'guitar', 6, st.tuningOffsets, st.capo), 39); + det.destroy(); + + const core2 = loadDetectionCore(); + const bass = mountWith(core2, { + info: { arrangement: 'Bass', tuning: [0, 0, 0, 0], centOffset: -1200 }, + tuning: [0, 0, 0, 0], + capo: 0, + centOffset: -1200, + stringCount: 4, + }); + bass._syncChartStateFromHw(); + const bs = bass._getChartState(); + assert.equal(bs.tuningOffsets.join(','), '0,0,0,0', 'octave offset leaves bass scoring as before'); + assert.equal(bs.arrangement, 'bass'); + bass.destroy(); +}); + +test('a mid-song transform restage is picked up by the per-tick resync', () => { + const core = loadDetectionCore(); + let tuning = [0, 0, 0, 0, 0, 0]; + let cents = 0; + core.highway.getSongInfo = () => ({ arrangement: 'Lead', tuning: [0, 0, 0, 0, 0, 0] }); + core.highway.getStringCount = () => 6; + core.highway.getTuning = () => tuning; + core.highway.getCapo = () => 0; + core.highway.getCentOffset = () => cents; + const det = core.createNoteDetector(); + det._syncChartStateFromHw(); + assert.equal(det._resyncIfTuningChanged(), false, 'unchanged inputs → no resync'); + + // Live retune provider restages: whole-step down via tuning… + tuning = [-2, -2, -2, -2, -2, -2]; + assert.equal(det._resyncIfTuningChanged(), true); + assert.equal(det._getChartState().tuningOffsets.join(','), '-2,-2,-2,-2,-2,-2'); + assert.equal(det._resyncIfTuningChanged(), false); + + // …then a reference-pitch nudge via centOffset. + cents = 50; + assert.equal(det._resyncIfTuningChanged(), true); + assert.equal(det._getChartState().tuningOffsets.join(','), '-1.5,-1.5,-1.5,-1.5,-1.5,-1.5'); + det.destroy(); +}); + +test('pre-chart-transform host (no getters) keeps the song_info path', () => { + const core = loadDetectionCore(); + delete core.highway.getTuning; + delete core.highway.getCapo; + delete core.highway.getCentOffset; + core.highway.getSongInfo = () => ({ arrangement: 'Lead', tuning: [-2, 0, 0, 0, 0, 0], capo: 3 }); + core.highway.getStringCount = () => 6; + const det = core.createNoteDetector(); + det._syncChartStateFromHw(); + const st = det._getChartState(); + assert.equal(st.tuningOffsets.join(','), '-2,0,0,0,0,0'); + assert.equal(st.capo, 3); + assert.equal(det._resyncIfTuningChanged(), false, 'nothing to poll without the getters'); + det.destroy(); +}); + +test('the desktop engine chart gets integer offsets under a fractional shift', async () => { + const pushed = []; + const audio = { + isAvailable: async () => true, + isAudioRunning: async () => true, + startAudio: async () => {}, + getLevels: async () => ({ inputLevel: 0, inputPeak: 0, outputLevel: 0, outputPeak: 0 }), + getSampleRate: async () => 48000, + getPitchDetection: async () => ({ midiNote: -1, confidence: 0, frequency: -1, cents: 0, noteName: '' }), + getRawPitch: async () => ({ midiNote: -1, confidence: 0, frequency: -1, cents: 0, noteName: '' }), + setChart: async (chart) => { pushed.push(chart); return true; }, + getNoteVerdicts: async () => [], + }; + const { createNoteDetector } = loadDetectionCore({ + sandboxBeforeRun(sandbox) { + sandbox.setInterval = () => 1; + sandbox.dispatchEvent = () => true; + sandbox.highway.getNotes = () => ([{ s: 1, f: 2, t: 3.0, sus: 0 }]); + sandbox.highway.getChords = () => ([]); + sandbox.highway.getTime = () => 0; + sandbox.highway.getAvOffset = () => 0; + sandbox.highway.getSongInfo = () => ({ arrangement: 'Lead', tuning: [0, 0, 0, 0, 0, 0] }); + sandbox.highway.getStringCount = () => 6; + sandbox.highway.getTuning = () => [-1, -1, -1, -1, -1, -1]; + sandbox.highway.getCapo = () => 0; + sandbox.highway.getCentOffset = () => -30; // -1.3 semitones + sandbox.window.feedBackDesktop = { isDesktop: true, platform: 'linux', audio }; + }, + }); + const det = createNoteDetector({ isDefault: false }); + await det.enable(); + for (let i = 0; i < 6; i++) await new Promise((r) => setImmediate(r)); + // Host live tuning (no ctx) → the funnel rounds before the engine sees it. + const armed = await det.setContainedChart([{ id: 'x', t: 0.5, s: 0, f: 0 }]); + assert.equal(armed, true); + const last = pushed[pushed.length - 1]; + assert.equal(last.tuningOffsets.join(','), '-1,-1,-1,-1,-1,-1'); + assert.equal(det._getChartState().tuningOffsets.join(','), '-1.3,-1.3,-1.3,-1.3,-1.3,-1.3', + 'the JS path keeps the sub-semitone shift'); + det.destroy(); + for (let i = 0; i < 6; i++) await new Promise((r) => setImmediate(r)); +});