diff --git a/main/Analyser.cpp b/main/Analyser.cpp index 957bfe63..634845d6 100644 --- a/main/Analyser.cpp +++ b/main/Analyser.cpp @@ -5,7 +5,7 @@ An intonation analysis and annotation tool Centre for Digital Music, Queen Mary, University of London. This file copyright 2006-2012 Chris Cannam and QMUL. - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the @@ -14,6 +14,9 @@ */ #include "Analyser.h" +#include "RealtimeAnalyser.h" + +#include #include "transform/TransformFactory.h" #include "transform/ModelTransformer.h" @@ -44,9 +47,9 @@ using std::endl; using namespace sv; Analyser::Analyser() : - m_document(0), - m_paneStack(0), - m_pane(0), + m_document(nullptr), + m_paneStack(nullptr), + m_pane(nullptr), m_currentCandidate(-1), m_candidatesVisible(false), m_currentAsyncHandle(0) @@ -65,10 +68,18 @@ Analyser::Analyser() : QString("") .arg(int(FlexiNoteLayer::AutoAlignScale))); settings.endGroup(); + + m_realtimeAnalyser = new RealtimeAnalyser(this); + connect(m_realtimeAnalyser, &RealtimeAnalyser::layersChanged, + this, &Analyser::layersChanged, + Qt::QueuedConnection); } Analyser::~Analyser() { + if (m_realtimeAnalyser) { + m_realtimeAnalyser->cleanup(); + } } std::map @@ -83,19 +94,43 @@ Analyser::getAnalysisSettings() QString Analyser::newFileLoaded(Document *doc, ModelId model, - PaneStack *paneStack, Pane *pane) + PaneStack *paneStack, Pane *pane) { + if (m_document && m_document != doc) { + disconnect(m_document, nullptr, this, nullptr); + } + + // Clean up any in-flight realtime analysis layers/callbacks tied to the previous document + if (m_realtimeAnalyser) { + m_realtimeAnalyser->cleanup(); + } + m_document = doc; m_fileModel = model; m_paneStack = paneStack; m_pane = pane; + // Realtime analysis progress is relative to this model. Recording a + // second time reaches us through here rather than through + // fileClosed(), so without this reset the first chunk of the new + // recording would be analysed from the *previous* recording's end + // frame -- which Selection silently turns into a large backwards + // range, wiping the new pitch track. + m_analysedFrames = 0; + + if (m_realtimeAnalyser) { + // Targets are (re)created later in addAnalyses(); set minimal context now + m_realtimeAnalyser->setContext(m_document, m_fileModel, m_pane, nullptr, nullptr); + m_realtimeAnalyser->invalidateGeneration(); + } + if (!ModelById::isa(m_fileModel)) { return "Internal error: Analyser::newFileLoaded() called with no model, or a non-WaveFileModel"; } - - connect(doc, SIGNAL(layerAboutToBeDeleted(Layer *)), - this, SLOT(layerAboutToBeDeleted(Layer *))); + + connect(doc, &Document::layerAboutToBeDeleted, + this, &Analyser::layerAboutToBeDeleted, + Qt::UniqueConnection); QSettings settings; settings.beginGroup("Analyser"); @@ -113,19 +148,49 @@ Analyser::analyseExistingFile() if (!m_pane) return "Internal error: Analyser::analyseExistingFile() called with no pane present"; if (m_fileModel.isNone()) return "Internal error: Analyser::analyseExistingFile() called with no model present"; - + if (m_layers[PitchTrack]) { m_document->removeLayerFromView(m_pane, m_layers[PitchTrack]); - m_layers[PitchTrack] = 0; + m_layers[PitchTrack] = nullptr; } if (m_layers[Notes]) { m_document->removeLayerFromView(m_pane, m_layers[Notes]); - m_layers[Notes] = 0; + m_layers[Notes] = nullptr; } return doAllAnalyses(true); } +QString +Analyser::analyseRecordingToEnd(sv_frame_t record_duration) +{ + if (!m_document) return "Internal error: Analyser::analyseRecordingToEnd() called with no document present"; + + if (!m_pane) return "Internal error: Analyser::analyseRecordingToEnd() called with no pane present"; + + if (m_fileModel.isNone()) return "Internal error: Analyser::analyseRecordingToEnd() called with no model present"; + + // We start with a 2500-frame overlap to ensure we capture instrument attacks in time (~56ms) + const sv_frame_t overlap = 2500; + const sv_frame_t startPosition = std::max(m_analysedFrames - overlap, + sv_frame_t(0)); + const sv_frame_t endPosition = record_duration; + + if (endPosition <= startPosition) { + // Nothing new to analyse. Selection would silently swap the two + // frames and hand us a large backwards range, so bail out + // instead of leaving that to be discovered downstream. + return ""; + } + + QString error = analyseRecording(Selection(startPosition, endPosition)); + if (error != "") return error; + + m_analysedFrames = endPosition; + + return ""; +} + QString Analyser::doAllAnalyses(bool withPitchTrack) { @@ -170,10 +235,31 @@ void Analyser::fileClosed() { cerr << "Analyser::fileClosed" << endl; + + if (m_currentAsyncHandle && m_document) { + m_document->cancelAsyncLayerCreation(m_currentAsyncHandle); + } + m_currentAsyncHandle = 0; + + if (m_realtimeAnalyser) { + // cleanup() already abandons outstanding work and bumps the + // generation, so it is safe to drop the context straight after + m_realtimeAnalyser->cleanup(); + m_realtimeAnalyser->clearContext(); + } + m_layers.clear(); m_reAnalysisCandidates.clear(); m_currentCandidate = -1; m_reAnalysingSelection = Selection(); + m_reAnalysingRange = FrequencyRange(); + m_candidatesVisible = false; + m_analysedFrames = 0; + + m_document = nullptr; + m_paneStack = nullptr; + m_pane = nullptr; + m_fileModel = ModelId(); } bool @@ -204,7 +290,7 @@ Analyser::getInitialAnalysisCompletion() int c = m_layers[Notes]->getCompletion(m_pane); if (c < completion) completion = c; } - + return completion; } @@ -227,7 +313,7 @@ Analyser::layerCompletionChanged(ModelId) auto audioModel = ModelById::get(m_layers[Audio]->getModel()); sv_frame_t endFrame = audioModel->getEndFrame(); - + if (m_layers[PitchTrack]) { auto model = ModelById::getAs (m_layers[PitchTrack]->getModel()); @@ -255,7 +341,7 @@ Analyser::addVisualisations() /* This is roughly what we'd do for a constant-Q spectrogram, but it currently has issues with y-axis alignment - + TransformFactory *tf = TransformFactory::getInstance(); QString name = "Constant-Q"; @@ -264,7 +350,7 @@ Analyser::addVisualisations() QString notFound = tr("Transform \"%1\" not found, spectrogram will not be enabled.

Is the %2 Vamp plugin correctly installed?"); if (!tf->haveTransform(base + out)) { - return notFound.arg(base + out).arg(name); + return notFound.arg(base + out).arg(name); } Transform transform = tf->getDefaultTransformFor @@ -275,7 +361,7 @@ Analyser::addVisualisations() (m_document->createDerivedLayer(transform, m_fileModel)); if (!spectrogram) return tr("Transform \"%1\" did not run correctly (no layer or wrong layer type returned)").arg(base + out); -*/ +*/ // As with all the visualisation layers, if we already have one in // the pane we do not create another, just record its @@ -338,21 +424,96 @@ Analyser::addWaveform() params->setPlayPan(-1); params->setPlayGain(1); } - + m_document->addLayerToView(m_pane, waveform); m_layers[Audio] = waveform; return ""; } +std::map getAnalysisSettingsFromSettings() +{ + std::map analysisSettings; + + QSettings settings; + settings.beginGroup("Analyser"); + + analysisSettings["precision-analysis"] = settings.value("precision-analysis", false).toBool(); + analysisSettings["lowamp-analysis"] = settings.value("lowamp-analysis", true).toBool(); + analysisSettings["onset-analysis"] = settings.value("onset-analysis", true).toBool(); + analysisSettings["prune-analysis"] = settings.value("prune-analysis", true).toBool(); + + settings.endGroup(); + + return analysisSettings; +} + +static void setAnalysisSettings(Transform& transform) +{ + const auto analysisSettings = getAnalysisSettingsFromSettings(); + + if (analysisSettings.count("precision-analysis") > 0) { + bool precise = analysisSettings.at("precision-analysis"); + if (precise) { + cerr << "setting parameters for precise mode" << endl; + transform.setParameter("precisetime", 1); + } + else { + cerr << "setting parameters for vague mode" << endl; + transform.setParameter("precisetime", 0); + } + } + + if (analysisSettings.count("lowamp-analysis") > 0) { + bool lowamp = analysisSettings.at("lowamp-analysis"); + if (lowamp) { + cerr << "setting parameters for lowamp suppression" << endl; + transform.setParameter("lowampsuppression", 0.2f); + } + else { + cerr << "setting parameters for no lowamp suppression" << endl; + transform.setParameter("lowampsuppression", 0.0f); + } + } + + if (analysisSettings.count("onset-analysis") > 0) { + bool onset = analysisSettings.at("onset-analysis"); + if (onset) { + cerr << "setting parameters for increased onset sensitivity" << endl; + transform.setParameter("onsetsensitivity", 0.7f); + } + else { + cerr << "setting parameters for non-increased onset sensitivity" << endl; + transform.setParameter("onsetsensitivity", 0.0f); + } + } + + if (analysisSettings.count("prune-analysis") > 0) { + bool prune = analysisSettings.at("prune-analysis"); + if (prune) { + cerr << "setting parameters for duration pruning" << endl; + transform.setParameter("prunethresh", 0.1f); + } + else { + cerr << "setting parameters for no duration pruning" << endl; + transform.setParameter("prunethresh", 0.0f); + } + } +} + QString Analyser::addAnalyses() { + if (m_realtimeAnalyser) { + // Prevent stale callbacks touching soon-to-be-replaced pitch/note layers + m_realtimeAnalyser->invalidateGeneration(); + m_realtimeAnalyser->cleanup(); + } auto waveFileModel = ModelById::getAs(m_fileModel); if (!waveFileModel) { return "Internal error: Analyser::addAnalyses() called with no model present"; } - + // As with the spectrogram above, if these layers exist we use // them TimeValueLayer *existingPitch = 0; @@ -373,20 +534,15 @@ Analyser::addAnalyses() } else { if (existingPitch) { m_document->removeLayerFromView(m_pane, existingPitch); - m_layers[PitchTrack] = 0; + m_layers[PitchTrack] = nullptr; } if (existingNotes) { m_document->removeLayerFromView(m_pane, existingNotes); - m_layers[Notes] = 0; + m_layers[Notes] = nullptr; } } TransformFactory *tf = TransformFactory::getInstance(); - - QString plugname = "pYIN"; - QString base = "vamp:pyin:pyin:"; - QString f0out = "smoothedpitchtrack"; - QString noteout = "notes"; Transforms transforms; @@ -401,81 +557,28 @@ Analyser::addAnalyses() m_document->addLayerToView(m_pane, lx); } */ + auto f0_transform = QString(PYIN_TRANSFORM_BASE) + QString(PYIN_F0_OUT); + auto note_transform = QString(PYIN_TRANSFORM_BASE) + QString(PYIN_NOTE_OUT); QString notFound = tr("Transform \"%1\" not found. Unable to analyse audio file.

Is the %2 Vamp plugin correctly installed?"); - if (!tf->haveTransform(base + f0out)) { - return notFound.arg(base + f0out).arg(plugname); + if (!tf->haveTransform(f0_transform)) { + return notFound.arg(f0_transform).arg(PYIN_PLUGIN_NAME); } - if (!tf->haveTransform(base + noteout)) { - return notFound.arg(base + noteout).arg(plugname); + if (!tf->haveTransform(note_transform)) { + return notFound.arg(note_transform).arg(PYIN_PLUGIN_NAME); } - QSettings settings; - settings.beginGroup("Analyser"); - - bool precise = false, lowamp = true, onset = true, prune = true; - - std::map flags { - { "precision-analysis", precise }, - { "lowamp-analysis", lowamp }, - { "onset-analysis", onset }, - { "prune-analysis", prune } - }; - - auto keyMap = getAnalysisSettings(); - - for (auto p: flags) { - auto ki = keyMap.find(p.first); - if (ki != keyMap.end()) { - p.second = settings.value(ki->first, ki->second).toBool(); - } else { - throw std::logic_error("Internal error: One or more analysis settings keys not found in map: check addAnalyses and getAnalysisSettings"); - } - } - - settings.endGroup(); - Transform t = tf->getDefaultTransformFor - (base + f0out, waveFileModel->getSampleRate()); + (f0_transform, waveFileModel->getSampleRate()); t.setStepSize(256); t.setBlockSize(2048); - if (precise) { - cerr << "setting parameters for precise mode" << endl; - t.setParameter("precisetime", 1); - } else { - cerr << "setting parameters for vague mode" << endl; - t.setParameter("precisetime", 0); - } - - if (lowamp) { - cerr << "setting parameters for lowamp suppression" << endl; - t.setParameter("lowampsuppression", 0.2f); - } else { - cerr << "setting parameters for no lowamp suppression" << endl; - t.setParameter("lowampsuppression", 0.0f); - } - - if (onset) { - cerr << "setting parameters for increased onset sensitivity" << endl; - t.setParameter("onsetsensitivity", 0.7f); - } else { - cerr << "setting parameters for non-increased onset sensitivity" << endl; - t.setParameter("onsetsensitivity", 0.0f); - } - - if (prune) { - cerr << "setting parameters for duration pruning" << endl; - t.setParameter("prunethresh", 0.1f); - } else { - cerr << "setting parameters for no duration pruning" << endl; - t.setParameter("prunethresh", 0.0f); - } + setAnalysisSettings(t); transforms.push_back(t); - t.setOutput(noteout); - + t.setOutput(PYIN_NOTE_OUT); + transforms.push_back(t); std::vector layers = @@ -485,16 +588,20 @@ Analyser::addAnalyses() FlexiNoteLayer *f = qobject_cast(layers[i]); TimeValueLayer *t = qobject_cast(layers[i]); - - if (f) m_layers[Notes] = f; - if (t) m_layers[PitchTrack] = t; - + + if (f) { + m_layers[Notes] = f; + } + else if (t) { + m_layers[PitchTrack] = t; + } + m_document->addLayerToView(m_pane, layers[i]); } - + ColourDatabase *cdb = ColourDatabase::getInstance(); - - TimeValueLayer *pitchLayer = + + TimeValueLayer *pitchLayer = qobject_cast(m_layers[PitchTrack]); if (pitchLayer) { pitchLayer->setBaseColour(cdb->getColourIndex(tr("Black"))); @@ -506,8 +613,8 @@ Analyser::addAnalyses() connect(pitchLayer, SIGNAL(modelCompletionChanged(ModelId)), this, SLOT(layerCompletionChanged(ModelId))); } - - FlexiNoteLayer *flexiNoteLayer = + + FlexiNoteLayer *flexiNoteLayer = qobject_cast(m_layers[Notes]); if (flexiNoteLayer) { flexiNoteLayer->setBaseColour(cdb->getColourIndex(tr("Bright Blue"))); @@ -523,7 +630,13 @@ Analyser::addAnalyses() connect(flexiNoteLayer, SIGNAL(materialiseReAnalysis()), this, SLOT(materialiseReAnalysis())); } - + + if (m_realtimeAnalyser) { + auto *pitchLayer = qobject_cast(m_layers[PitchTrack]); + auto *noteLayer = qobject_cast(m_layers[Notes]); + m_realtimeAnalyser->setContext(m_document, m_fileModel, m_pane, pitchLayer, noteLayer); + } + return ""; } @@ -544,6 +657,35 @@ Analyser::materialiseReAnalysis() switchPitchCandidate(m_reAnalysingSelection, true); // or false, doesn't matter } +void +Analyser::updatePitchTrack(sv::ModelId) +{ + // Implementation for updatePitchTrack slot + emit layersChanged(); +} + +void +Analyser::updateNoteLayer(sv::ModelId) +{ + // Implementation for updateNoteLayer slot + emit layersChanged(); +} + +QString +Analyser::analyseRecording(Selection sel) +{ + if (!m_realtimeAnalyser) { + return "Internal error: Analyser::analyseRecording() called with no realtime analyser present"; + } + + auto *pitchLayer = qobject_cast(m_layers[PitchTrack]); + auto *noteLayer = qobject_cast(m_layers[Notes]); + + m_realtimeAnalyser->setContext(m_document, m_fileModel, m_pane, pitchLayer, noteLayer); + return m_realtimeAnalyser->analyseChunk(sel); +} + + QString Analyser::reAnalyseSelection(Selection sel, FrequencyRange range) { @@ -553,7 +695,7 @@ Analyser::reAnalyseSelection(Selection sel, FrequencyRange range) if (!waveFileModel) { return "Internal error: Analyser::reAnalyseSelection() called with no model present"; } - + if (!m_reAnalysingSelection.isEmpty()) { if (sel == m_reAnalysingSelection && range == m_reAnalysingRange) { cerr << "selection & range are same as current analysis, ignoring" << endl; @@ -584,13 +726,13 @@ Analyser::reAnalyseSelection(Selection sel, FrequencyRange range) } TransformFactory *tf = TransformFactory::getInstance(); - - QString plugname1 = "pYIN"; + QString plugname2 = "CHP"; QString base = "vamp:pyin:localcandidatepyin:"; QString out = "pitchtrackcandidates"; + if (range.isConstrained()) { base = "vamp:chp:constrainedharmonicpeak:"; out = "peak"; @@ -600,9 +742,9 @@ Analyser::reAnalyseSelection(Selection sel, FrequencyRange range) QString notFound = tr("Transform \"%1\" not found. Unable to perform interactive analysis.

Are the %2 and %3 Vamp plugins correctly installed?"); if (!tf->haveTransform(base + out)) { - return notFound.arg(base + out).arg(plugname1).arg(plugname2); + return notFound.arg(base + out).arg(PYIN_PLUGIN_NAME).arg(plugname2); } - + Transform t = tf->getDefaultTransformFor (base + out, waveFileModel->getSampleRate()); t.setStepSize(256); @@ -626,7 +768,7 @@ Analyser::reAnalyseSelection(Selection sel, FrequencyRange range) } else { endSample -= 9*grid; // MM says: not sure what the CHP plugin does there } - RealTime start = RealTime::frame2RealTime(startSample, waveFileModel->getSampleRate()); + RealTime start = RealTime::frame2RealTime(startSample, waveFileModel->getSampleRate()); RealTime end = RealTime::frame2RealTime(endSample, waveFileModel->getSampleRate()); RealTime duration; @@ -641,12 +783,12 @@ Analyser::reAnalyseSelection(Selection sel, FrequencyRange range) cerr << "Analyser::reAnalyseSelection: duration <= 0, not analysing" << endl; return ""; } - + t.setStartTime(start); t.setDuration(duration); transforms.push_back(t); - + m_currentAsyncHandle = m_document->createDerivedLayersAsync(transforms, m_fileModel, this); @@ -660,7 +802,7 @@ Analyser::arePitchCandidatesShown() const } void -Analyser::showPitchCandidates(bool shown) +Analyser::showPitchCandidates(bool shown) { if (m_candidatesVisible == shown) return; @@ -687,7 +829,7 @@ Analyser::layersCreated(Document::LayerCreationAsyncHandle handle, { QMutexLocker locker(&m_asyncMutex); - if (handle != m_currentAsyncHandle || + if (handle != m_currentAsyncHandle || m_reAnalysingSelection == Selection()) { // We don't want these! for (int i = 0; i < (int)primary.size(); ++i) { @@ -750,14 +892,14 @@ Analyser::haveHigherPitchCandidate() const if (m_reAnalysisCandidates.empty()) return false; return (m_currentCandidate < 0 || (m_currentCandidate + 1 < (int)m_reAnalysisCandidates.size())); -} +} bool Analyser::haveLowerPitchCandidate() const { if (m_reAnalysisCandidates.empty()) return false; return (m_currentCandidate < 0 || m_currentCandidate >= 1); -} +} void Analyser::switchPitchCandidate(Selection sel, bool up) @@ -803,14 +945,14 @@ void Analyser::shiftOctave(Selection sel, bool up) { float factor = (up ? 2.f : 0.5f); - + vector actOn; Layer *pitchTrack = m_layers[PitchTrack]; if (pitchTrack) actOn.push_back(pitchTrack); foreach (Layer *layer, actOn) { - + Clipboard clip; layer->copy(m_pane, sel, clip); layer->deleteSelection(sel); @@ -824,7 +966,7 @@ Analyser::shiftOctave(Selection sel, bool up) shifted.addPoint(e); } } - + layer->paste(m_pane, shifted, 0, false); } } @@ -849,7 +991,7 @@ Analyser::abandonReAnalysis(Selection sel) if (!myLayer) return; myLayer->deleteSelection(sel); myLayer->paste(m_pane, m_preAnalysis, 0, false); -} +} void Analyser::clearReAnalysis() @@ -880,7 +1022,7 @@ void Analyser::layerAboutToBeDeleted(Layer *doomed) { cerr << "Analyser::layerAboutToBeDeleted(" << doomed << ")" << endl; - + vector notDoomed; foreach (Layer *layer, m_reAnalysisCandidates) { @@ -903,7 +1045,7 @@ Analyser::takePitchTrackFrom(Layer *otherLayer) if (!myModel || !otherModel) return; Clipboard clip; - + Selection sel = Selection(myModel->getStartFrame(), myModel->getEndFrame()); myLayer->deleteSelection(sel); @@ -932,7 +1074,7 @@ Analyser::takePitchTrackFrom(Layer *otherLayer) void Analyser::getEnclosingSelectionScope(sv_frame_t f, sv_frame_t &f0, sv_frame_t &f1) { - FlexiNoteLayer *flexiNoteLayer = + FlexiNoteLayer *flexiNoteLayer = qobject_cast(m_layers[Notes]); sv_frame_t f0i = f, f1i = f; @@ -942,7 +1084,7 @@ Analyser::getEnclosingSelectionScope(sv_frame_t f, sv_frame_t &f0, sv_frame_t &f f0 = f1 = f; return; } - + flexiNoteLayer->snapToFeatureFrame(m_pane, f0i, res, Layer::SnapLeft, -1); flexiNoteLayer->snapToFeatureFrame(m_pane, f1i, res, Layer::SnapRight, -1); @@ -976,11 +1118,11 @@ Analyser::loadState(Component c) } void -Analyser::setIntelligentActions(bool on) +Analyser::setIntelligentActions(bool on) { std::cerr << "toggle setIntelligentActions " << on << std::endl; - FlexiNoteLayer *flexiNoteLayer = + FlexiNoteLayer *flexiNoteLayer = qobject_cast(m_layers[Notes]); if (flexiNoteLayer) { flexiNoteLayer->setIntelligentActions(on); @@ -1055,7 +1197,7 @@ Analyser::getGain(Component c) const return 1.f; } } - + void Analyser::setGain(Component c, float gain) { @@ -1078,7 +1220,7 @@ Analyser::getPan(Component c) const return 1.f; } } - + void Analyser::setPan(Component c, float pan) { @@ -1089,6 +1231,3 @@ Analyser::setPan(Component c, float pan) saveState(c); } } - - - diff --git a/main/Analyser.h b/main/Analyser.h index 918d239c..0364daa1 100644 --- a/main/Analyser.h +++ b/main/Analyser.h @@ -5,7 +5,7 @@ An intonation analysis and annotation tool Centre for Digital Music, Queen Mary, University of London. This file copyright 2006-2012 Chris Cannam and QMUL. - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -36,6 +37,8 @@ class TimeValueLayer; class Layer; } +class RealtimeAnalyser; + class Analyser : public QObject, public sv::Document::LayerCreationHandler { @@ -56,9 +59,12 @@ class Analyser : public QObject, // layers; return "" on success or error string on failure QString analyseExistingFile(); + // Completes analysis from the last position to the end + QString analyseRecordingToEnd(sv::sv_frame_t record_duration); + // Discard any layers etc associated with the current document void fileClosed(); - + void setIntelligentActions(bool); bool getDisplayFrequencyExtents(double &min, double &max); @@ -128,7 +134,14 @@ class Analyser : public QObject, * group in QSettings. */ static std::map getAnalysisSettings(); - + + /** + * Analyse the selection and schedule asynchronous adds of + * candidate layers for the region it contains. Returns "" on + * success or a user-readable error string on failure. + */ + QString analyseRecording(sv::Selection sel); + /** * Analyse the selection and schedule asynchronous adds of * candidate layers for the region it contains. Returns "" on @@ -232,6 +245,8 @@ class Analyser : public QObject, void initialAnalysisCompleted(); protected slots: + void updatePitchTrack(sv::ModelId); + void updateNoteLayer(sv::ModelId); void layerAboutToBeDeleted(sv::Layer *); void layerCompletionChanged(sv::ModelId); void reAnalyseRegion(sv::sv_frame_t, sv::sv_frame_t, float, float); @@ -247,12 +262,16 @@ protected slots: sv::Clipboard m_preAnalysis; sv::Selection m_reAnalysingSelection; + sv::sv_frame_t m_analysedFrames = 0; FrequencyRange m_reAnalysingRange; std::vector m_reAnalysisCandidates; + + RealtimeAnalyser *m_realtimeAnalyser = nullptr; + int m_currentCandidate; bool m_candidatesVisible; sv::Document::LayerCreationAsyncHandle m_currentAsyncHandle; - QMutex m_asyncMutex; + mutable QMutex m_asyncMutex; QString doAllAnalyses(bool withPitchTrack); @@ -263,13 +282,20 @@ protected slots: void discardPitchCandidates(); void stackLayers(); - + // Document::LayerCreationHandler method void layersCreated(sv::Document::LayerCreationAsyncHandle, std::vector, std::vector); void saveState(Component c) const; void loadState(Component c); + + static constexpr const char* PYIN_PLUGIN_NAME = "pYIN"; + static constexpr const char* PYIN_TRANSFORM_BASE = "vamp:pyin:pyin:"; + static constexpr const char* PYIN_F0_OUT = "smoothedpitchtrack"; + static constexpr const char* PYIN_NOTE_OUT = "notes"; + + }; #endif diff --git a/main/MainWindow.cpp b/main/MainWindow.cpp index cd9fa4af..bea4780d 100644 --- a/main/MainWindow.cpp +++ b/main/MainWindow.cpp @@ -5,7 +5,7 @@ An intonation analysis and annotation tool Centre for Digital Music, Queen Mary, University of London. This file copyright 2006-2012 Chris Cannam and QMUL. - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the @@ -102,7 +102,7 @@ using std::endl; using namespace sv; MainWindow::MainWindow(AudioMode audioMode, - bool withSonification, + bool withSonification, bool withSpectrogram) : MainWindowBase(audioMode, MainWindowBase::MIDI_NONE, @@ -111,13 +111,16 @@ MainWindow::MainWindow(AudioMode audioMode, m_overview(0), m_mainMenusCreated(false), m_playbackMenu(0), - m_recentFilesMenu(0), + m_recentFilesMenu(0), m_rightButtonMenu(0), m_rightButtonPlaybackMenu(0), m_deleteSelectedAction(0), m_ffwdAction(0), m_rwdAction(0), m_intelligentActionOn(true), //GF: !!! temporary + m_autoAnalyse(nullptr), + m_analyseDuringRecord(nullptr), + m_analyseAfterRecord(nullptr), m_activityLog(new ActivityLog()), m_keyReference(new KeyReference()), m_selectionAnchor(0), @@ -186,13 +189,13 @@ MainWindow::MainWindow(AudioMode audioMode, m_viewManager->setOverlayMode(ViewManager::GlobalOverlays); connect(m_viewManager, SIGNAL(selectionChangedByUser()), - this, SLOT(selectionChangedByUser())); + this, SLOT(selectionChangedByUser())); QFrame *frame = new QFrame; setCentralWidget(frame); QGridLayout *layout = new QGridLayout; - + QScrollArea *scroll = new QScrollArea(frame); scroll->setWidgetResizable(true); scroll->setHorizontalScrollBarPolicy(Qt::ScrollBarAlwaysOff); @@ -230,7 +233,7 @@ MainWindow::MainWindow(AudioMode audioMode, } else { m_panLayer->setBaseColour (ColourDatabase::getInstance()->getColourIndex(tr("Blue"))); - } + } m_fader = new Fader(frame, false); connect(m_fader, SIGNAL(mouseEntered()), this, SLOT(mouseEnteredWidget())); @@ -311,7 +314,7 @@ MainWindow::MainWindow(AudioMode audioMode, m_activityLog->hide(); setAudioRecordMode(RecordReplaceSession); - + newSession(); settings.beginGroup("MainWindow"); @@ -350,7 +353,7 @@ MainWindow::setupMenus() // workaround, to remove the appmenu-qt5 package, but that is // awkward and the problem is so severe that it merits disabling // the system menubar integration altogether. Like this: - menuBar()->setNativeMenuBar(false); + menuBar()->setNativeMenuBar(false); #endif m_rightButtonMenu = new QMenu(); @@ -417,7 +420,7 @@ MainWindow::setupFileMenu() m_keyReference->registerShortcut(action); menu->addAction(action); toolbar->addAction(action); - + icon = il.load("filesaveas"); action = new QAction(icon, tr("Save Session &As..."), this); action->setShortcut(tr("Ctrl+Shift+S")); @@ -455,7 +458,7 @@ MainWindow::setupFileMenu() menu->addAction(action); menu->addSeparator(); - + action = new QAction(tr("Browse Recorded Audio"), this); action->setStatusTip(tr("Open the Recorded Audio folder in the system file browser")); connect(action, SIGNAL(triggered()), this, SLOT(browseRecordedAudio())); @@ -484,17 +487,17 @@ MainWindow::setupEditMenu() m_keyReference->setCategory (tr("Selection Strip Mouse Actions")); m_keyReference->registerShortcut - (tr("Jump"), tr("Left"), + (tr("Jump"), tr("Left"), tr("Click left button to move the playback position to a time")); m_keyReference->registerShortcut - (tr("Select"), tr("Left"), + (tr("Select"), tr("Left"), tr("Click left button and drag to select a region of time")); m_keyReference->registerShortcut - (tr("Select Note Duration"), tr("Double-Click Left"), + (tr("Select Note Duration"), tr("Double-Click Left"), tr("Double-click left button to select the region of time corresponding to a note")); QToolBar *toolbar = addToolBar(tr("Tools Toolbar")); - + CommandHistory::getInstance()->registerToolbar(toolbar); QActionGroup *group = new QActionGroup(this); @@ -517,18 +520,18 @@ MainWindow::setupEditMenu() m_keyReference->setCategory (tr("Navigate Tool Mouse Actions")); m_keyReference->registerShortcut - (tr("Navigate"), tr("Left"), + (tr("Navigate"), tr("Left"), tr("Click left button and drag to move around")); m_keyReference->registerShortcut - (tr("Re-Analyse Area"), tr("Shift+Left"), + (tr("Re-Analyse Area"), tr("Shift+Left"), tr("Shift-click left button and drag to define a specific pitch and time range to re-analyse")); m_keyReference->registerShortcut - (tr("Edit"), tr("Double-Click Left"), + (tr("Edit"), tr("Double-Click Left"), tr("Double-click left button on an item to edit it")); m_keyReference->setCategory(tr("Tool Selection")); action = toolbar->addAction(il.load("move"), - tr("Edit")); + tr("Edit")); action->setCheckable(true); action->setShortcut(tr("2")); action->setStatusTip(tr("Edit with Note Intelligence")); @@ -540,16 +543,16 @@ MainWindow::setupEditMenu() m_keyReference->setCategory (tr("Note Edit Tool Mouse Actions")); m_keyReference->registerShortcut - (tr("Adjust Pitch"), tr("Left"), + (tr("Adjust Pitch"), tr("Left"), tr("Click left button on the main part of a note and drag to move it up or down")); m_keyReference->registerShortcut - (tr("Split"), tr("Left"), + (tr("Split"), tr("Left"), tr("Click left button on the bottom edge of a note to split it at the click point")); m_keyReference->registerShortcut - (tr("Resize"), tr("Left"), + (tr("Resize"), tr("Left"), tr("Click left button on the left or right edge of a note and drag to change the time or duration of the note")); m_keyReference->registerShortcut - (tr("Erase"), tr("Shift+Left"), + (tr("Erase"), tr("Shift+Left"), tr("Shift-click left button on a note to remove it")); @@ -557,7 +560,7 @@ MainWindow::setupEditMenu() m_keyReference->setCategory(tr("Tool Selection")); action = toolbar->addAction(il.load("notes"), - tr("Free Edit")); + tr("Free Edit")); action->setCheckable(true); action->setShortcut(tr("3")); action->setStatusTip(tr("Free Edit")); @@ -567,7 +570,7 @@ MainWindow::setupEditMenu() */ menu->addSeparator(); - + m_keyReference->setCategory(tr("Selection")); action = new QAction(tr("Select &All"), this); @@ -593,9 +596,9 @@ MainWindow::setupEditMenu() menu->addSeparator(); m_rightButtonMenu->addSeparator(); - + m_keyReference->setCategory(tr("Pitch Track")); - + action = new QAction(tr("Choose Higher Pitch"), this); action->setShortcut(tr("Ctrl+Up")); action->setStatusTip(tr("Move pitches up an octave, or to the next higher pitch candidate")); @@ -604,7 +607,7 @@ MainWindow::setupEditMenu() connect(this, SIGNAL(canClearSelection(bool)), action, SLOT(setEnabled(bool))); menu->addAction(action); m_rightButtonMenu->addAction(action); - + action = new QAction(tr("Choose Lower Pitch"), this); action->setShortcut(tr("Ctrl+Down")); action->setStatusTip(tr("Move pitches down an octave, or to the next lower pitch candidate")); @@ -622,7 +625,7 @@ MainWindow::setupEditMenu() connect(this, SIGNAL(canClearSelection(bool)), m_showCandidatesAction, SLOT(setEnabled(bool))); menu->addAction(m_showCandidatesAction); m_rightButtonMenu->addAction(m_showCandidatesAction); - + action = new QAction(tr("Remove Pitches"), this); action->setShortcut(tr("Ctrl+Backspace")); action->setStatusTip(tr("Remove all pitch estimates within the selected region, making it unvoiced")); @@ -634,7 +637,7 @@ MainWindow::setupEditMenu() menu->addSeparator(); m_rightButtonMenu->addSeparator(); - + m_keyReference->setCategory(tr("Note Track")); action = new QAction(tr("Split Note"), this); @@ -663,7 +666,7 @@ MainWindow::setupEditMenu() connect(this, SIGNAL(canSnapNotes(bool)), action, SLOT(setEnabled(bool))); menu->addAction(action); m_rightButtonMenu->addAction(action); - + action = new QAction(tr("Form Note from Selection"), this); action->setShortcut(tr("=")); action->setStatusTip(tr("Form a note spanning the selected region, splitting any existing notes at its boundaries")); @@ -702,7 +705,7 @@ MainWindow::setupViewMenu() connect(this, SIGNAL(canScroll(bool)), action, SLOT(setEnabled(bool))); m_keyReference->registerShortcut(action); menu->addAction(action); - + action = new QAction(tr("Peek &Right"), this); action->setShortcut(tr("Alt+Right")); action->setStatusTip(tr("Scroll the current pane to the right without changing the play position")); @@ -723,7 +726,7 @@ MainWindow::setupViewMenu() connect(this, SIGNAL(canZoom(bool)), action, SLOT(setEnabled(bool))); m_keyReference->registerShortcut(action); menu->addAction(action); - + action = new QAction(il.load("zoom-out"), tr("Zoom &Out"), this); action->setShortcut(tr("Down")); @@ -732,7 +735,7 @@ MainWindow::setupViewMenu() connect(this, SIGNAL(canZoom(bool)), action, SLOT(setEnabled(bool))); m_keyReference->registerShortcut(action); menu->addAction(action); - + action = new QAction(tr("Restore &Default Zoom"), this); action->setStatusTip(tr("Restore the zoom level to the default")); connect(action, SIGNAL(triggered()), this, SLOT(zoomDefault())); @@ -749,7 +752,7 @@ MainWindow::setupViewMenu() menu->addAction(action); menu->addSeparator(); - + action = new QAction(tr("Set Displayed Fre&quency Range..."), this); action->setStatusTip(tr("Set the minimum and maximum frequencies in the visible display")); connect(action, SIGNAL(triggered()), this, SLOT(editDisplayExtents())); @@ -768,11 +771,32 @@ MainWindow::setupAnalysisMenu() QMenu *menu = menuBar()->addMenu(tr("&Analysis")); menu->setTearOffEnabled(true); + // Create a submenu for analysis mode options + QMenu *modeMenu = menu->addMenu(tr("Analysis &Mode")); + modeMenu->setTearOffEnabled(true); + + // Create a radio button group for recording analysis mode + QActionGroup *analysisGroup = new QActionGroup(this); + + m_analyseAfterRecord = new QAction(tr("Analyse &After Recording"), this); + m_analyseAfterRecord->setStatusTip(tr("Automatically trigger analysis after recording is complete.")); + m_analyseAfterRecord->setCheckable(true); + m_analyseAfterRecord->setActionGroup(analysisGroup); + connect(m_analyseAfterRecord, SIGNAL(triggered()), this, SLOT(analyseAfterRecordToggled())); + modeMenu->addAction(m_analyseAfterRecord); + + m_analyseDuringRecord = new QAction(tr("Analyse &During Recording"), this); + m_analyseDuringRecord->setStatusTip(tr("Automatically trigger analysis during recording.")); + m_analyseDuringRecord->setCheckable(true); + m_analyseDuringRecord->setActionGroup(analysisGroup); + connect(m_analyseDuringRecord, SIGNAL(triggered()), this, SLOT(recordAnalysisToggled())); + modeMenu->addAction(m_analyseDuringRecord); + m_autoAnalyse = new QAction(tr("Auto-Analyse &New Audio"), this); m_autoAnalyse->setStatusTip(tr("Automatically trigger analysis upon opening of a new audio file.")); m_autoAnalyse->setCheckable(true); connect(m_autoAnalyse, SIGNAL(triggered()), this, SLOT(autoAnalysisToggled())); - menu->addAction(m_autoAnalyse); + modeMenu->addAction(m_autoAnalyse); action = new QAction(tr("&Analyse Now!"), this); action->setStatusTip(tr("Trigger analysis of pitches and notes. (This will delete all existing pitches and notes.)")); @@ -823,7 +847,9 @@ MainWindow::resetAnalyseOptions() settings.beginGroup("Analyser"); settings.setValue("auto-analysis", true); - + settings.setValue("recording-analysis-mode", + static_cast(RecordingAnalysisMode::AfterRecording)); + auto keyMap = Analyser::getAnalysisSettings(); for (auto p: keyMap) { settings.setValue(p.first, p.second); @@ -833,14 +859,52 @@ MainWindow::resetAnalyseOptions() updateAnalyseStates(); } +MainWindow::RecordingAnalysisMode +MainWindow::getRecordingAnalysisMode() const +{ + QSettings settings; + settings.beginGroup("Analyser"); + int mode = settings.value("recording-analysis-mode", + static_cast(RecordingAnalysisMode::AfterRecording)).toInt(); + settings.endGroup(); + return static_cast(mode); +} + +void +MainWindow::setRecordingAnalysisMode(RecordingAnalysisMode mode) +{ + QSettings settings; + settings.beginGroup("Analyser"); + settings.setValue("recording-analysis-mode", static_cast(mode)); + settings.endGroup(); + updateAnalyseStates(); +} + void MainWindow::updateAnalyseStates() { QSettings settings; settings.beginGroup("Analyser"); - bool autoAnalyse = settings.value("auto-analysis", true).toBool(); - m_autoAnalyse->setChecked(autoAnalyse); + // Handle recording analysis mode with enum-based approach + RecordingAnalysisMode mode = getRecordingAnalysisMode(); + + switch (mode) { + case RecordingAnalysisMode::AfterRecording: + m_analyseAfterRecord->setChecked(true); + m_analyseDuringRecord->setChecked(false); + break; + case RecordingAnalysisMode::DuringRecording: + m_analyseDuringRecord->setChecked(true); + m_analyseAfterRecord->setChecked(false); + break; + } + + // Auto-analysis of newly opened files is independent of the + // recording analysis mode: newFileLoaded() reads this key directly, + // so deriving the checkbox from the mode instead would leave the two + // permanently out of step. + m_autoAnalyse->setChecked(settings.value("auto-analysis", true).toBool()); std::map actions { { "precision-analysis", m_precise }, @@ -850,7 +914,7 @@ MainWindow::updateAnalyseStates() }; auto keyMap = Analyser::getAnalysisSettings(); - + for (auto p: actions) { auto ki = keyMap.find(p.first); if (ki != keyMap.end()) { @@ -864,6 +928,30 @@ MainWindow::updateAnalyseStates() settings.endGroup(); } +void +MainWindow::recordAnalysisToggled() +{ + QAction *a = qobject_cast(sender()); + if (!a) return; + + bool set = a->isChecked(); + if (set) { + setRecordingAnalysisMode(RecordingAnalysisMode::DuringRecording); + } +} + +void +MainWindow::analyseAfterRecordToggled() +{ + QAction *a = qobject_cast(sender()); + if (!a) return; + + bool set = a->isChecked(); + if (set) { + setRecordingAnalysisMode(RecordingAnalysisMode::AfterRecording); + } +} + void MainWindow::autoAnalysisToggled() { @@ -962,7 +1050,7 @@ MainWindow::setupHelpMenu() { QMenu *menu = menuBar()->addMenu(tr("&Help")); menu->setTearOffEnabled(true); - + m_keyReference->setCategory(tr("Help")); IconLoader il; @@ -971,9 +1059,9 @@ MainWindow::setupHelpMenu() QAction *action; action = new QAction(il.load("help"), - tr("&Help Reference"), this); + tr("&Help Reference"), this); action->setShortcut(tr("F1")); - action->setStatusTip(tr("Open the %1 reference manual").arg(name)); + action->setStatusTip(tr("Open the %1 reference manual").arg(name)); connect(action, SIGNAL(triggered()), this, SLOT(help())); m_keyReference->registerShortcut(action); menu->addAction(action); @@ -984,14 +1072,14 @@ MainWindow::setupHelpMenu() connect(action, SIGNAL(triggered()), this, SLOT(keyReference())); m_keyReference->registerShortcut(action); menu->addAction(action); - - action = new QAction(tr("What's &New In This Release?"), this); - action->setStatusTip(tr("List the changes in this release (and every previous release) of %1").arg(name)); + + action = new QAction(tr("What's &New In This Release?"), this); + action->setStatusTip(tr("List the changes in this release (and every previous release) of %1").arg(name)); connect(action, SIGNAL(triggered()), this, SLOT(whatsNew())); menu->addAction(action); - - action = new QAction(tr("&About %1").arg(name), this); - action->setStatusTip(tr("Show information about %1").arg(name)); + + action = new QAction(tr("&About %1").arg(name), this); + action->setStatusTip(tr("Show information about %1").arg(name)); connect(action, SIGNAL(triggered()), this, SLOT(about())); menu->addAction(action); } @@ -1076,10 +1164,14 @@ MainWindow::setupToolbars() recordAction->setShortcut(tr("Ctrl+Space")); recordAction->setStatusTip(tr("Record a new audio file")); connect(recordAction, SIGNAL(triggered()), this, SLOT(record())); + + connect(recordAction, SIGNAL(triggered()), + this, SLOT(analyseDuringRecordingRunner())); + connect(m_recordTarget, SIGNAL(recordStatusChanged(bool)), - recordAction, SLOT(setChecked(bool))); + recordAction, SLOT(setChecked(bool))); connect(m_recordTarget, SIGNAL(recordCompleted()), - this, SLOT(analyseNow())); + this, SLOT(analyseAfterRecording())); connect(this, SIGNAL(canRecord(bool)), recordAction, SLOT(setEnabled(bool))); @@ -1112,7 +1204,7 @@ MainWindow::setupToolbars() oneLeftAction->setStatusTip(tr("Move cursor to the preceding note (or silence) onset.")); connect(oneLeftAction, SIGNAL(triggered()), this, SLOT(moveOneNoteLeft())); connect(this, SIGNAL(canScroll(bool)), oneLeftAction, SLOT(setEnabled(bool))); - + QAction *oneRightAction = new QAction(tr("O&ne Note Right"), this); oneRightAction->setShortcut(tr("Ctrl+Right")); oneRightAction->setStatusTip(tr("Move cursor to the succeeding note (or silence).")); @@ -1124,7 +1216,7 @@ MainWindow::setupToolbars() selectOneLeftAction->setStatusTip(tr("Select to the preceding note (or silence) onset.")); connect(selectOneLeftAction, SIGNAL(triggered()), this, SLOT(selectOneNoteLeft())); connect(this, SIGNAL(canScroll(bool)), selectOneLeftAction, SLOT(setEnabled(bool))); - + QAction *selectOneRightAction = new QAction(tr("S&elect One Note Right"), this); selectOneRightAction->setShortcut(tr("Ctrl+Shift+Right")); selectOneRightAction->setStatusTip(tr("Select to the succeeding note (or silence).")); @@ -1186,7 +1278,7 @@ MainWindow::setupToolbars() fastAction->setStatusTip(tr("Time-stretch playback to speed it up without changing pitch")); connect(fastAction, SIGNAL(triggered()), this, SLOT(speedUpPlayback())); connect(this, SIGNAL(canSpeedUpPlayback(bool)), fastAction, SLOT(setEnabled(bool))); - + QAction *slowAction = menu->addAction(tr("Slow Down")); slowAction->setShortcut(tr("Ctrl+PgDown")); slowAction->setStatusTip(tr("Time-stretch playback to slow it down without changing pitch")); @@ -1234,7 +1326,7 @@ MainWindow::setupToolbars() lpwSize = m_viewManager->scalePixelSize(26); bigLpwSize = int(lpwSize * 2.8); #endif - + m_audioLPW->setImageSize(lpwSize); m_audioLPW->setBigImageSize(bigLpwSize); toolbar->addWidget(m_audioLPW); @@ -1303,7 +1395,7 @@ MainWindow::setupToolbars() Pane::registerShortcuts(*m_keyReference); updateLayerStatuses(); - + // QTimer::singleShot(500, this, SLOT(betaReleaseWarning())); } @@ -1340,7 +1432,7 @@ MainWindow::moveByOneNote(bool right, bool doSelect) { sv_frame_t frame = m_viewManager->getPlaybackFrame(); cerr << "MainWindow::moveByOneNote startframe: " << frame << endl; - + bool isAtSelectionBoundary = false; MultiSelection::SelectionList selections = m_viewManager->getSelections(); if (!selections.empty()) { @@ -1360,7 +1452,7 @@ MainWindow::moveByOneNote(bool right, bool doSelect) //!!! This seems like a strange and inefficient way to do this - //!!! there is almost certainly a better way making use of //!!! EventSeries api - + EventVector points = model->getAllEvents(); if (points.empty()) return; @@ -1434,24 +1526,24 @@ MainWindow::updateMenuStates() if (currentPane) currentLayer = currentPane->getSelectedLayer(); bool haveMainModel = - (getMainModel() != 0); + (getMainModel() != 0); bool havePlayTarget = - (m_playTarget != 0 || m_audioIO != 0); + (m_playTarget != 0 || m_audioIO != 0); bool haveCurrentPane = (currentPane != 0); bool haveCurrentLayer = (haveCurrentPane && (currentLayer != 0)); - bool haveSelection = + bool haveSelection = (m_viewManager && !m_viewManager->getSelections().empty()); - bool haveCurrentTimeInstantsLayer = + bool haveCurrentTimeInstantsLayer = (haveCurrentLayer && qobject_cast(currentLayer)); - bool haveCurrentTimeValueLayer = + bool haveCurrentTimeValueLayer = (haveCurrentLayer && qobject_cast(currentLayer)); - bool pitchCandidatesVisible = + bool pitchCandidatesVisible = m_analyser->arePitchCandidatesShown(); emit canChangePlaybackSpeed(true); @@ -1463,11 +1555,11 @@ MainWindow::updateMenuStates() m_analyser->isVisible(Analyser::Audio) && m_analyser->getLayer(Analyser::Audio); - bool havePitchTrack = + bool havePitchTrack = m_analyser->isVisible(Analyser::PitchTrack) && m_analyser->getLayer(Analyser::PitchTrack); - bool haveNotes = + bool haveNotes = m_analyser->isVisible(Analyser::Notes) && m_analyser->getLayer(Analyser::Notes); @@ -1611,7 +1703,7 @@ MainWindow::updateLayerStatuses() m_audioLPW->setEnabled(m_analyser->isAudible(Analyser::Audio)); m_audioLPW->setLevel(m_analyser->getGain(Analyser::Audio)); m_audioLPW->setPan(m_analyser->getPan(Analyser::Audio)); - + m_showPitch->setChecked(m_analyser->isVisible(Analyser::PitchTrack)); m_playPitch->setChecked(m_analyser->isAudible(Analyser::PitchTrack)); m_pitchLPW->setEnabled(m_analyser->isAudible(Analyser::PitchTrack)); @@ -1633,7 +1725,7 @@ MainWindow::editDisplayExtents() double min, max; double vmin = 0; double vmax = getMainModel()->getSampleRate() /2; - + if (!m_analyser->getDisplayFrequencyExtents(min, max)) { //!!! return; @@ -1691,7 +1783,7 @@ MainWindow::newSession() m_viewManager->setGlobalCentreFrame (pane->getFrameForX(width() / 2)); - + connect(pane, SIGNAL(contextHelpChanged(const QString &)), this, SLOT(contextHelpChanged(const QString &))); @@ -1730,7 +1822,7 @@ MainWindow::closeSession() m_document->removeLayerFromView (pane, pane->getLayer(pane->getLayerCount() - 1)); } - + m_overview->unregisterView(pane); m_paneStack->deletePane(pane); } @@ -1739,12 +1831,12 @@ MainWindow::closeSession() Pane *pane = m_paneStack->getHiddenPane (m_paneStack->getHiddenPaneCount() - 1); - + while (pane->getLayerCount() > 0) { m_document->removeLayerFromView (pane, pane->getLayer(pane->getLayerCount() - 1)); } - + m_overview->unregisterView(pane); m_paneStack->deletePane(pane); } @@ -1818,7 +1910,7 @@ MainWindow::openRecentFile() { QObject *obj = sender(); QAction *action = qobject_cast(obj); - + if (!action) { cerr << "WARNING: MainWindow::openRecentFile: sender is not an action" << endl; @@ -1845,19 +1937,19 @@ MainWindow::paneAdded(Pane *pane) pane->setPlaybackFollow(PlaybackScrollPage); m_paneStack->sizePanesEqually(); if (m_overview) m_overview->registerView(pane); -} +} void MainWindow::paneHidden(Pane *pane) { - if (m_overview) m_overview->unregisterView(pane); -} + if (m_overview) m_overview->unregisterView(pane); +} void MainWindow::paneAboutToBeDeleted(Pane *pane) { - if (m_overview) m_overview->unregisterView(pane); -} + if (m_overview) m_overview->unregisterView(pane); +} void MainWindow::paneDropAccepted(Pane *pane, QStringList uriList) @@ -1884,8 +1976,8 @@ MainWindow::paneDropAccepted(Pane *pane, QString text) if (pane) m_paneStack->setCurrentPane(pane); QUrl testUrl(text); - if (testUrl.scheme() == "file" || - testUrl.scheme() == "http" || + if (testUrl.scheme() == "file" || + testUrl.scheme() == "http" || testUrl.scheme() == "ftp") { QStringList list; list.push_back(text); @@ -1950,7 +2042,7 @@ MainWindow::commitData(bool mayAskUser) } else { if (!QFileInfo(svDir).isDir()) return false; } - + // This name doesn't have to be unguessable #ifndef _WIN32 QString fname = QString("tmp-%1-%2.sv") @@ -1979,7 +2071,7 @@ MainWindow::checkSaveModified() if (!m_documentModified) return true; - int button = + int button = QMessageBox::warning(this, tr("Session modified"), tr("The current session has been modified.\nDo you want to save it?"), @@ -2009,7 +2101,7 @@ MainWindow::waitForInitialAnalysis() // initial analysis is happening, because then we end up with an // incomplete session on reload. There are certainly theoretically // better ways to handle this... - + QSettings settings; settings.beginGroup("Analyser"); bool autoAnalyse = settings.value("auto-analysis", true).toBool(); @@ -2029,7 +2121,7 @@ MainWindow::waitForInitialAnalysis() QMessageBox::Cancel, this); - connect(m_analyser, SIGNAL(initialAnalysisCompleted()), + connect(m_analyser, SIGNAL(initialAnalysisCompleted()), &mb, SLOT(accept())); if (mb.exec() == QDialog::Accepted) { @@ -2164,14 +2256,14 @@ MainWindow::exportToSVL(QString path, Layer *layer) << "\n" << "\n" << " \n"; - + model->toXml(out, " "); - + out << " \n" << " \n"; - + layer->toXml(out, " "); - + out << " \n" << "\n"; @@ -2206,10 +2298,10 @@ MainWindow::importPitchLayer(FileSource source) source.waitForData(); if (!waitForInitialAnalysis()) return FileOpenCancelled; - + QString path = source.getLocalFilename(); - RDFImporter::RDFDocumentType rdfType = + RDFImporter::RDFDocumentType rdfType = RDFImporter::identifyDocumentType(QUrl::fromLocalFile(path).toString()); if (rdfType != RDFImporter::NotRDF) { @@ -2221,12 +2313,12 @@ MainWindow::importPitchLayer(FileSource source) (source.getExtension().toLower() == "xml" && (SVFileReader::identifyXmlFile(source.getLocalFilename()) == SVFileReader::SVLayerFile))) { - + //!!! return FileOpenFailed; } else { - + try { CSVFormat format(path); @@ -2246,7 +2338,7 @@ MainWindow::importPitchLayer(FileSource source) ModelId modelId = ModelById::add (std::shared_ptr(model)); - + CommandHistory::getInstance()->startCompoundOperation (tr("Import Pitch Track"), true); @@ -2272,7 +2364,7 @@ MainWindow::importPitchLayer(FileSource source) } } } - + return FileOpenFailed; } @@ -2292,7 +2384,7 @@ MainWindow::exportPitchLayer() if (path == "") return; if (!waitForInitialAnalysis()) return; - + if (QFileInfo(path).suffix() == "") path += ".svl"; QString suffix = QFileInfo(path).suffix().toLower(); @@ -2314,7 +2406,7 @@ MainWindow::exportPitchLayer() } else { DataExportOptions options = DataExportFillGaps; - + CSVFileWriter writer(path, model.get(), ((suffix == "csv") ? "," : "\t"), options); @@ -2358,7 +2450,7 @@ MainWindow::exportNoteLayer() error = exportToSVL(path, layer); } else if (suffix == "mid" || suffix == "midi") { - + MIDIFileWriter writer(path, model.get(), model->getSampleRate()); writer.write(); if (!writer.isOK()) { @@ -2376,7 +2468,7 @@ MainWindow::exportNoteLayer() } else { DataExportOptions options = DataExportOmitLevel; - + CSVFileWriter writer(path, model.get(), ((suffix == "csv") ? "," : "\t"), options); @@ -2411,7 +2503,7 @@ MainWindow::doubleClickSelectInvoked(sv_frame_t frame) { sv_frame_t f0, f1; m_analyser->getEnclosingSelectionScope(frame, f0, f1); - + cerr << "MainWindow::doubleClickSelectInvoked(" << frame << "): [" << f0 << "," << f1 << "]" << endl; Selection sel(f0, f1); @@ -2496,11 +2588,11 @@ MainWindow::regionOutlined(QRect r) sv_frame_t f0 = pane->getFrameForX(r.x()); sv_frame_t f1 = pane->getFrameForX(r.x() + r.width()); - + double v0 = spectrogram->getFrequencyForY(pane, r.y() + r.height()); double v1 = spectrogram->getFrequencyForY(pane, r.y()); - cerr << "MainWindow::regionOutlined: frame " << f0 << " -> " << f1 + cerr << "MainWindow::regionOutlined: frame " << f0 << " -> " << f1 << ", frequency " << v0 << " -> " << v1 << endl; m_pendingConstraint = Analyser::FrequencyRange(v0, v1); @@ -2589,7 +2681,7 @@ MainWindow::switchPitchDown() (tr("Choose Lower Pitch Candidate"), true); MultiSelection::SelectionList selections = m_viewManager->getSelections(); - + for (MultiSelection::SelectionList::iterator k = selections.begin(); k != selections.end(); ++k) { m_analyser->switchPitchCandidate(*k, false); @@ -2613,12 +2705,12 @@ MainWindow::snapNotesToPitches() CommandHistory::getInstance()->startCompoundOperation (tr("Snap Notes to Pitches"), true); - + for (MultiSelection::SelectionList::iterator k = selections.begin(); k != selections.end(); ++k) { auxSnapNotes(*k); } - + CommandHistory::getInstance()->endCompoundOperation(); } } @@ -2632,7 +2724,7 @@ MainWindow::auxSnapNotes(Selection s) if (!layer) return; layer->snapSelectedNotesToPitchTrack(m_analyser->getPane(), s); -} +} void MainWindow::splitNote() @@ -2657,12 +2749,12 @@ MainWindow::mergeNotes() CommandHistory::getInstance()->startCompoundOperation (tr("Merge Notes"), true); - + for (MultiSelection::SelectionList::iterator k = selections.begin(); k != selections.end(); ++k) { layer->mergeNotes(m_analyser->getPane(), *k, true); } - + CommandHistory::getInstance()->endCompoundOperation(); } } @@ -2680,12 +2772,12 @@ MainWindow::deleteNotes() CommandHistory::getInstance()->startCompoundOperation (tr("Delete Notes"), true); - + for (MultiSelection::SelectionList::iterator k = selections.begin(); k != selections.end(); ++k) { layer->deleteSelectionInclusive(*k); } - + CommandHistory::getInstance()->endCompoundOperation(); } } @@ -2703,7 +2795,7 @@ MainWindow::formNoteFromSelection() MultiSelection::SelectionList selections = m_viewManager->getSelections(); if (!selections.empty()) { - + CommandHistory::getInstance()->startCompoundOperation (tr("Form Note from Selection"), true); @@ -2717,7 +2809,7 @@ MainWindow::formNoteFromSelection() // existing pitch track if possible. This way we should // handle all the possible cases of existing notes that // may or may not overlap the start or end times - + sv_frame_t start = k->getStartFrame(); sv_frame_t end = k->getEndFrame(); @@ -2728,18 +2820,18 @@ MainWindow::formNoteFromSelection() if (!existing.empty()) { defaultPitch = int(roundf(existing.begin()->getValue())); } - + layer->splitNotesAt(pane, start); layer->splitNotesAt(pane, end); layer->deleteSelection(*k); - + layer->addNoteOn(start, defaultPitch, 100); layer->addNoteOff(end, defaultPitch); - + layer->mergeNotes(pane, *k, false); } - CommandHistory::getInstance()->endCompoundOperation(); + CommandHistory::getInstance()->endCompoundOperation(); } } @@ -2758,7 +2850,7 @@ MainWindow::playSpeedChanged(int position) char pcbuf[30]; char facbuf[30]; - + if (position == centre) { contextHelpChanged(tr("Playback speed: Normal")); } else if (position < centre) { @@ -2802,7 +2894,7 @@ MainWindow::playMonoToggled() playSpeedChanged(m_playSpeed->value()); // TODO: pitch gain? -} +} void MainWindow::speedUpPlayback() @@ -2841,7 +2933,7 @@ MainWindow::audioGainChanged(float gain) m_analyser->setGain(Analyser::Audio, gain); } updateMenuStates(); -} +} void MainWindow::pitchGainChanged(float gain) @@ -2856,7 +2948,7 @@ MainWindow::pitchGainChanged(float gain) m_analyser->setGain(Analyser::PitchTrack, gain); } updateMenuStates(); -} +} void MainWindow::notesGainChanged(float gain) @@ -2871,7 +2963,7 @@ MainWindow::notesGainChanged(float gain) m_analyser->setGain(Analyser::Notes, gain); } updateMenuStates(); -} +} void MainWindow::audioPanChanged(float pan) @@ -2879,7 +2971,7 @@ MainWindow::audioPanChanged(float pan) contextHelpChanged(tr("Audio Pan: %1").arg(pan)); m_analyser->setPan(Analyser::Audio, pan); updateMenuStates(); -} +} void MainWindow::pitchPanChanged(float pan) @@ -2887,7 +2979,7 @@ MainWindow::pitchPanChanged(float pan) contextHelpChanged(tr("Pitch Pan: %1").arg(pan)); m_analyser->setPan(Analyser::PitchTrack, pan); updateMenuStates(); -} +} void MainWindow::notesPanChanged(float pan) @@ -2895,7 +2987,7 @@ MainWindow::notesPanChanged(float pan) contextHelpChanged(tr("Notes Pan: %1").arg(pan)); m_analyser->setPan(Analyser::Notes, pan); updateMenuStates(); -} +} void MainWindow::updateVisibleRangeDisplay(Pane *p) const @@ -2944,7 +3036,7 @@ MainWindow::updateVisibleRangeDisplay(Pane *p) const m_myStatusMessage = tr("Visible: %1 to %2 (duration %3)") .arg(startStr).arg(endStr).arg(durationStr); } - + getStatusLabel()->setText(m_myStatusMessage); } @@ -3045,6 +3137,87 @@ MainWindow::analyseNow() } } +void +MainWindow::disconnectRecordingAnalysis() +{ + if (m_recordDurationConnection) { + disconnect(m_recordDurationConnection); + m_recordDurationConnection = QMetaObject::Connection(); + } + if (m_recordCompletedConnection) { + disconnect(m_recordCompletedConnection); + m_recordCompletedConnection = QMetaObject::Connection(); + } +} + +void +MainWindow::analyseDuringRecordingRunner() +{ + if (!m_recordTarget) return; + if (!m_recordTarget->isRecording()) return; + + if (getRecordingAnalysisMode() != RecordingAnalysisMode::DuringRecording) { + return; + } + + // Defensive: the record action can only be triggered once per + // recording, but re-running would otherwise stack up duplicate + // connections + disconnectRecordingAnalysis(); + + m_recordDurationConnection = + connect(m_recordTarget, &AudioCallbackRecordTarget::recordDurationChanged, + this, &MainWindow::analyseDuringRecording); + + m_recordCompletedConnection = + connect(m_recordTarget, &AudioCallbackRecordTarget::recordCompleted, + this, [this]() { + // Only our own two connections, by handle. Passing + // nullptr receivers to disconnect() here used to take + // out MainWindowBase's record-duration display and + // the recordCompleted -> analyseAfterRecording() + // connection along with them, for the rest of the + // session. + disconnectRecordingAnalysis(); + analyseDuringRecording(); + }); +} + +void +MainWindow::analyseDuringRecording() +{ + if (!m_recordTarget || !m_analyser) return; + + QString error = m_analyser->analyseRecordingToEnd + (m_recordTarget->getRecordDuration()); + + // This runs several times a second while recording, so report + // failures to the log rather than to a modal dialog + if (error != "") { + SVCERR << "MainWindow::analyseDuringRecording: " << error << endl; + } +} + +void +MainWindow::analyseAfterRecording() +{ + // If realtime analysis is running, the final chunk is analysed by + // the recordCompleted handler installed in + // analyseDuringRecordingRunner(). Running a full re-analysis as well + // would replace the pitch and note layers out from under the chunk + // being scheduled at the same moment. + // + // This tests the live connection rather than re-reading the mode, + // so that switching mode part-way through a recording can't leave + // both paths active. We run before that handler does (we were + // connected first), so the connection is still valid here. + if (m_recordCompletedConnection) { + return; + } + + analyseNow(); +} + void MainWindow::analyseNewMainModel() { @@ -3053,7 +3226,7 @@ MainWindow::analyseNewMainModel() SVDEBUG << "MainWindow::analyseNewMainModel: main model is " << model << endl; SVDEBUG << "(document is " << m_document << ", it says main model is " << m_document->getMainModel() << ")" << endl; - + if (!model) { cerr << "no main model!" << endl; return; @@ -3117,7 +3290,7 @@ MainWindow::analyseNewMainModel() m_analyser->setAudible(Analyser::PitchTrack, false); m_analyser->setAudible(Analyser::Notes, false); } - + updateLayerStatuses(); documentRestored(); } @@ -3265,16 +3438,16 @@ MainWindow::whatsNew() QDialog *d = new QDialog(this); d->setWindowTitle(tr("What's New")); - + QGridLayout *layout = new QGridLayout; d->setLayout(layout); int row = 0; - + QLabel *iconLabel = new QLabel; iconLabel->setPixmap(QApplication::windowIcon().pixmap(64, 64)); layout->addWidget(iconLabel, row, 0); - + layout->addWidget (new QLabel(tr("

What's New in %1

") .arg(QApplication::applicationName())), @@ -3287,7 +3460,7 @@ MainWindow::whatsNew() if (m_newerVersionIs != "") { layout->addWidget(new QLabel(tr("Note: A newer version of %1 is available.
(Version %2 is available; you are using version %3)").arg(QApplication::applicationName()).arg(m_newerVersionIs).arg(TONY_VERSION)), row++, 1, 1, 2); } - + QDialogButtonBox *bb = new QDialogButtonBox(QDialogButtonBox::Ok); layout->addWidget(bb, row++, 0, 1, 3); connect(bb, SIGNAL(accepted()), d, SLOT(accept())); @@ -3316,7 +3489,7 @@ MainWindow::whatsNew() d->setMinimumSize(m_viewManager->scalePixelSize(520), m_viewManager->scalePixelSize(450)); - + d->exec(); delete d; @@ -3360,7 +3533,7 @@ MainWindow::about() aboutText += tr("

Using Qt framework version %1.

") .arg(QT_VERSION_STR); - aboutText += + aboutText += "

Copyright © 2005–2019 Chris Cannam, Queen Mary University of London, and the Tony project authors: Matthias Mauch, George Fazekas, Justin Salamon, and Rachel Bittner.

" "

pYIN analysis plugin written by Matthias Mauch.

" "

This program is free software; you can redistribute it and/or " @@ -3368,18 +3541,18 @@ MainWindow::about() "published by the Free Software Foundation; either version 2 of the " "License, or (at your option) any later version.
See the file " "COPYING included with this distribution for more information.

"; - + // use our own dialog so we can influence the size QDialog *d = new QDialog(this); d->setWindowTitle(tr("About %1").arg(QApplication::applicationName())); - + QGridLayout *layout = new QGridLayout; d->setLayout(layout); int row = 0; - + QLabel *iconLabel = new QLabel; iconLabel->setPixmap(QApplication::windowIcon().pixmap(64, 64)); layout->addWidget(iconLabel, row, 0, Qt::AlignTop); @@ -3402,7 +3575,7 @@ MainWindow::about() d->setMinimumSize(m_viewManager->scalePixelSize(420), m_viewManager->scalePixelSize(200)); - + d->exec(); delete d; @@ -3418,7 +3591,7 @@ void MainWindow::newerVersionAvailable(QString version) { m_newerVersionIs = version; - + //!!! nicer URL would be nicer QSettings settings; settings.beginGroup("NewerVersionWarning"); @@ -3442,9 +3615,9 @@ MainWindow::ffwd() sv_samplerate_t sr = getMainModel()->getSampleRate(); - // The step is supposed to scale and be as wide as a step of + // The step is supposed to scale and be as wide as a step of // m_defaultFfwdRwdStep seconds at zoom level 720 and sr = 44100 - + ZoomLevel zoom = m_viewManager->getGlobalZoom(); double framesPerPixel = 1.0; if (zoom.zone == ZoomLevel::FramesPerPixel) { @@ -3455,20 +3628,20 @@ MainWindow::ffwd() double defaultFramesPerPixel = (720 * 44100) / sr; double scaler = framesPerPixel / defaultFramesPerPixel; RealTime step = m_defaultFfwdRwdStep * scaler; - + frame = RealTime::realTime2Frame (RealTime::frame2RealTime(frame, sr) + step, sr); if (frame > getMainModel()->getEndFrame()) { frame = getMainModel()->getEndFrame(); } - + if (frame < 0) frame = 0; if (m_viewManager->getPlaySelectionMode()) { frame = m_viewManager->constrainFrameToSelection(frame); } - + m_viewManager->setPlaybackFrame(frame); if (frame == getMainModel()->getEndFrame() && @@ -3489,7 +3662,7 @@ MainWindow::rewind() sv_samplerate_t sr = getMainModel()->getSampleRate(); - // The step is supposed to scale and be as wide as a step of + // The step is supposed to scale and be as wide as a step of // m_defaultFfwdRwdStep seconds at zoom level 720 and sr = 44100 ZoomLevel zoom = m_viewManager->getGlobalZoom(); @@ -3505,7 +3678,7 @@ MainWindow::rewind() frame = RealTime::realTime2Frame (RealTime::frame2RealTime(frame, sr) - step, sr); - + if (frame < getMainModel()->getStartFrame()) { frame = getMainModel()->getStartFrame(); } diff --git a/main/MainWindow.h b/main/MainWindow.h index f14d5ca9..82bc43b9 100644 --- a/main/MainWindow.h +++ b/main/MainWindow.h @@ -1,3 +1,4 @@ +#pragma once /* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ /* @@ -5,7 +6,7 @@ An intonation analysis and annotation tool Centre for Digital Music, Queen Mary, University of London. This file copyright 2006-2012 Chris Cannam and QMUL. - + This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the @@ -30,8 +31,13 @@ class MainWindow : public sv::MainWindowBase Q_OBJECT public: + enum class RecordingAnalysisMode { + AfterRecording = 0, // Default + DuringRecording = 1 + }; + MainWindow(AudioMode audioMode, - bool withSonification = true, + bool withSonification = true, bool withSpectrogram = true); virtual ~MainWindow(); @@ -87,7 +93,14 @@ protected slots: virtual void editDisplayExtents(); virtual void analyseNow(); + + virtual void analyseDuringRecording(); + virtual void analyseDuringRecordingRunner(); + virtual void analyseAfterRecording(); virtual void resetAnalyseOptions(); + virtual void recordAnalysisToggled(); + virtual void analyseAfterRecordToggled(); + virtual void autoAnalysisToggled(); virtual void precisionAnalysisToggled(); virtual void lowampAnalysisToggled(); @@ -164,7 +177,7 @@ protected slots: virtual void whatsNew(); virtual void betaReleaseWarning(); - + virtual void newerVersionAvailable(QString); virtual void selectionChangedByUser(); @@ -205,11 +218,13 @@ protected slots: bool m_intelligentActionOn; // GF: !!! temporary QAction *m_autoAnalyse; + QAction *m_analyseDuringRecord; + QAction *m_analyseAfterRecord; QAction *m_precise; QAction *m_lowamp; QAction *m_onset; QAction *m_prune; - + QAction *m_showAudio; QAction *m_showSpect; QAction *m_showPitch; @@ -224,6 +239,7 @@ protected slots: sv::ActivityLog *m_activityLog; sv::KeyReference *m_keyReference; sv::VersionTester *m_versionTester; + QString m_newerVersionIs; sv::sv_frame_t m_selectionAnchor; @@ -246,6 +262,18 @@ protected slots: virtual void setupHelpMenu(); virtual void setupToolbars(); + RecordingAnalysisMode getRecordingAnalysisMode() const; + void setRecordingAnalysisMode(RecordingAnalysisMode mode); + + // Tear down only the connections analyseDuringRecordingRunner() + // made. A blanket disconnect on m_recordTarget's signals would also + // remove MainWindowBase's record-duration handler and our own + // recordCompleted handler, permanently. + void disconnectRecordingAnalysis(); + + QMetaObject::Connection m_recordDurationConnection; + QMetaObject::Connection m_recordCompletedConnection; + virtual void octaveShift(bool up); virtual void auxSnapNotes(sv::Selection s); diff --git a/main/OverlapProcessor.cpp b/main/OverlapProcessor.cpp new file mode 100644 index 00000000..35f29e4e --- /dev/null +++ b/main/OverlapProcessor.cpp @@ -0,0 +1,415 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Tony + An intonation analysis and annotation tool + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2006-2012 Chris Cannam and QMUL. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#include "OverlapProcessor.h" + +#include +#include +#include + +using namespace sv; + +namespace { + +bool lessEventForDedup(const Event &a, const Event &b) +{ + if (a.getFrame() != b.getFrame()) return a.getFrame() < b.getFrame(); + if (a.getDuration() != b.getDuration()) return a.getDuration() < b.getDuration(); + + if (a.hasValue() != b.hasValue()) return a.hasValue() < b.hasValue(); + if (a.hasValue() && a.getValue() != b.getValue()) return a.getValue() < b.getValue(); + + if (a.hasLabel() != b.hasLabel()) return a.hasLabel() < b.hasLabel(); + if (a.hasLabel() && a.getLabel() != b.getLabel()) return a.getLabel() < b.getLabel(); + + if (a.hasLevel() != b.hasLevel()) return a.hasLevel() < b.hasLevel(); + if (a.hasLevel() && a.getLevel() != b.getLevel()) return a.getLevel() < b.getLevel(); + + return false; +} + +bool equalEventForDedup(const Event &a, const Event &b) +{ + return a.getFrame() == b.getFrame() && + a.getDuration() == b.getDuration() && + a.hasValue() == b.hasValue() && + (!a.hasValue() || a.getValue() == b.getValue()) && + a.hasLabel() == b.hasLabel() && + (!a.hasLabel() || a.getLabel() == b.getLabel()) && + a.hasLevel() == b.hasLevel() && + (!a.hasLevel() || a.getLevel() == b.getLevel()); +} + +void sortAndDedupeEvents(EventVector &events) +{ + std::sort(events.begin(), events.end(), lessEventForDedup); + events.erase(std::unique(events.begin(), events.end(), equalEventForDedup), + events.end()); +} + +EventVector shiftedBy(const EventVector &events, sv_frame_t offset) +{ + EventVector shifted; + shifted.reserve(events.size()); + for (const auto &event : events) { + shifted.push_back(event.withFrame(event.getFrame() + offset)); + } + return shifted; +} + +} + +// OverlapGroup implementation +OverlapGroup::OverlapGroup() : startFrame(0), endFrame(0) +{ +} + +OverlapGroup::OverlapGroup(size_t index, const Event &event) : + indices{index}, + startFrame(event.getFrame()), + endFrame(event.getFrame() + event.getDuration()) +{ +} + +void OverlapGroup::addEvent(size_t index, const Event &event) +{ + if (indices.empty()) { + startFrame = event.getFrame(); + endFrame = event.getFrame() + event.getDuration(); + } else { + startFrame = std::min(startFrame, event.getFrame()); + endFrame = std::max(endFrame, event.getFrame() + event.getDuration()); + } + + indices.push_back(index); +} + +// OverlapProcessor implementation +OverlapProcessor::OverlapProcessor(const OverlapConfig &config) : + m_config(config) +{ +} + +std::vector +OverlapProcessor::findOverlapGroups(const EventVector &events) const +{ + std::vector groups; + if (events.size() < 2) return groups; + + // The connected components of an interval overlap graph are + // contiguous once the intervals are sorted by start frame, so a + // single sweep tracking the running maximum end frame finds them + // all. (The previous implementation rescanned the whole event set + // after every match, which is cubic in the worst case and was being + // run over the entire note track once per realtime chunk.) + + std::vector order(events.size()); + std::iota(order.begin(), order.end(), size_t(0)); + + std::sort(order.begin(), order.end(), + [&events](size_t a, size_t b) { + if (events[a].getFrame() != events[b].getFrame()) { + return events[a].getFrame() < events[b].getFrame(); + } + return a < b; + }); + + OverlapGroup current(order[0], events[order[0]]); + + for (size_t i = 1; i < order.size(); ++i) { + + const size_t index = order[i]; + const Event &event = events[index]; + + // Equivalent to eventsOverlap() against some group member, + // given that no later event can start before this one does. + if (event.getFrame() < current.endFrame) { + current.addEvent(index, event); + } else { + if (current.size() > 1) groups.push_back(current); + current = OverlapGroup(index, event); + } + } + + if (current.size() > 1) groups.push_back(current); + + return groups; +} + +float OverlapProcessor::calculateWeightedFrequency(const EventVector& overlappingEvents, + sv_frame_t overlapStart, + sv_frame_t overlapDuration) const { + if (overlappingEvents.empty()) return 0.0f; + if (overlappingEvents.size() == 1) { + return overlappingEvents[0].hasValue() ? overlappingEvents[0].getValue() : 0.0f; + } + + std::vector frequencies; + std::vector weights; + frequencies.reserve(overlappingEvents.size()); + weights.reserve(overlappingEvents.size()); + + const auto overlapEnd = overlapStart + overlapDuration; + + for (const auto& event : overlappingEvents) { + const float freq = event.hasValue() ? event.getValue() : 0.0f; + if (freq <= 0.0f) continue; + + const auto eventStart = event.getFrame(); + const auto eventEnd = event.getFrame() + event.getDuration(); + + const auto eventOverlapStart = std::max(eventStart, overlapStart); + const auto eventOverlapEnd = std::min(eventEnd, overlapEnd); + const auto eventOverlapContrib = std::max(0, eventOverlapEnd - eventOverlapStart); + + if (eventOverlapContrib > 0) { + frequencies.push_back(freq); + weights.push_back(static_cast(eventOverlapContrib)); + } + } + + if (frequencies.empty()) return 0.0f; + if (frequencies.size() == 1) return frequencies[0]; + + double totalWeight = 0.0; + for (double weight : weights) totalWeight += weight; + if (totalWeight <= 0.0) { + double logSum = 0.0; + for (float freq : frequencies) { + logSum += std::log(freq); + } + return float(std::exp(logSum / double(frequencies.size()))); + } + + for (double& weight : weights) weight /= totalWeight; + + double weightedLogSum = 0.0; + for (size_t i = 0; i < frequencies.size(); ++i) { + weightedLogSum += std::log(frequencies[i]) * weights[i]; + } + + return float(std::exp(weightedLogSum)); +} + +std::optional OverlapProcessor::mergeOverlapGroup(const OverlapGroup& group, const EventVector& events) const { + if (group.isEmpty()) { + return std::nullopt; + } + + if (group.size() == 1) { + return events[group.indices[0]]; + } + + const auto mergedStart = group.startFrame; + const auto mergedEnd = group.endFrame; + const auto mergedDuration = mergedEnd - mergedStart; + + EventVector overlappingEvents; + overlappingEvents.reserve(group.indices.size()); + for (size_t index : group.indices) { + overlappingEvents.push_back(events[index]); + } + + const float weightedFreq = + calculateWeightedFrequency(overlappingEvents, mergedStart, mergedDuration); + + const Event* longestEvent = findLongestEvent(group, events); + if (!longestEvent) { + return std::nullopt; + } + + Event mergedEvent = longestEvent->withFrame(mergedStart) + .withDuration(mergedDuration); + + if (weightedFreq > 0.0f) { + mergedEvent = mergedEvent.withValue(weightedFreq); + } + + if (longestEvent->hasLabel()) { + mergedEvent = mergedEvent.withLabel(longestEvent->getLabel()); + } + + if (longestEvent->hasLevel()) { + mergedEvent = mergedEvent.withLevel(longestEvent->getLevel()); + } + + return mergedEvent; +} + +OverlapProcessor::EventPatch OverlapProcessor::processPitchEvents(sv_frame_t contextStart, + const EventVector& incomingEvents, + const EventVector& existingEvents) const { + EventPatch patch; + + // Incoming pitch frames are NOT shifted, unlike note frames. + // + // pYIN's smoothedpitchtrack is a FixedSampleRate output, and its + // features carry the host-supplied block timestamp verbatim + // (PYinVamp.cpp: f.timestamp = m_timestamp[iFrame]). The host feeds + // absolute block timestamps, and FeatureExtractionModelTransformer's + // FixedSampleRate branch reconstructs the frame from that timestamp, + // so these frames are already absolute. + // + // Its notes output behaves differently -- it counts frames from zero + // and ignores the host timestamps -- so processNoteEvents() does have + // to shift. Adding contextStart here as well was what put the live + // pitch track at roughly twice its correct frame. + const EventVector &incoming = incomingEvents; + + // Everything from contextStart onwards is superseded by the fresh + // analysis. We need the latest-ending event before contextStart in + // order to bridge the seam, but not a copy of the whole preceding + // track -- this runs once per realtime chunk over the entire pitch + // track, which grows for the length of the recording. + const Event *lastBefore = nullptr; + + patch.remove.reserve(existingEvents.size()); + + for (const auto& event : existingEvents) { + const auto eventStart = event.getFrame(); + const auto eventEnd = eventStart + event.getDuration(); + + if (eventStart >= contextStart || eventEnd > contextStart) { + patch.remove.push_back(event); + } else if (!lastBefore || + eventEnd > lastBefore->getFrame() + lastBefore->getDuration()) { + lastBefore = &event; + } + } + + sortAndDedupeEvents(patch.remove); + + patch.add = incoming; + + if (lastBefore && !incoming.empty()) { + + const auto& firstNewEvent = incoming.front(); + + const auto lastExistingEnd = + lastBefore->getFrame() + lastBefore->getDuration(); + const auto gapFrames = firstNewEvent.getFrame() - lastExistingEnd; + + if (gapFrames > 0 && gapFrames <= m_config.interpolationThreshold) { + if (lastBefore->hasValue() && firstNewEvent.hasValue()) { + const auto lastValue = lastBefore->getValue(); + const auto firstValue = firstNewEvent.getValue(); + + if (lastValue > 0.0f && firstValue > 0.0f) { + const auto valueDiff = std::abs(lastValue - firstValue) / lastValue; + + if (valueDiff <= m_config.pitchSimilarityThreshold) { + const auto midFrame = lastExistingEnd + gapFrames / 2; + const auto midValue = (lastValue + firstValue) / 2.0f; + patch.add.push_back(Event(midFrame, midValue, "interpolated")); + } + } + } + } + } + + sortAndDedupeEvents(patch.add); + + return patch; +} + +OverlapProcessor::EventPatch OverlapProcessor::processNoteEvents(sv_frame_t contextStart, + const EventVector& incomingEvents, + const EventVector& existingEvents) const { + EventPatch patch; + + const EventVector shiftedIncoming = shiftedBy(incomingEvents, contextStart); + if (shiftedIncoming.empty()) return patch; + + // Single index space: [0, existingCount) are existing events, the + // rest are incoming ones. Overlap grouping is transitive, so an + // existing note can be pulled into a merge by way of another + // existing note even when it does not itself overlap anything + // incoming. It is still superseded by the merged event and must be + // removed, or it would be left behind as a duplicate underneath it. + const size_t existingCount = existingEvents.size(); + + EventVector combined; + combined.reserve(existingCount + shiftedIncoming.size()); + combined.insert(combined.end(), existingEvents.begin(), existingEvents.end()); + combined.insert(combined.end(), shiftedIncoming.begin(), shiftedIncoming.end()); + + const auto groups = findOverlapGroups(combined); + + std::vector grouped(combined.size(), 0); + + for (const auto& group : groups) { + + bool touchesIncoming = false; + for (size_t index : group.indices) { + grouped[index] = 1; + if (index >= existingCount) touchesIncoming = true; + } + + if (!touchesIncoming) { + // Pre-existing notes that overlap only each other, outside + // the region we just analysed: leave them exactly as they + // are. + continue; + } + + for (size_t index : group.indices) { + if (index < existingCount) { + patch.remove.push_back(combined[index]); + } + } + + if (auto merged = mergeOverlapGroup(group, combined)) { + patch.add.push_back(*merged); + } + } + + // Incoming notes that overlapped nothing at all + for (size_t i = existingCount; i < combined.size(); ++i) { + if (!grouped[i]) patch.add.push_back(combined[i]); + } + + // Groups are disjoint, so remove holds each existing event at most + // once already; sort for determinism but don't dedupe, so that a + // model containing genuine duplicates has both instances removed. + std::sort(patch.remove.begin(), patch.remove.end(), lessEventForDedup); + sortAndDedupeEvents(patch.add); + + return patch; +} + +// Private helper methods +bool OverlapProcessor::eventsOverlap(const Event& a, const Event& b) const { + const auto aStart = a.getFrame(); + const auto aEnd = a.getFrame() + a.getDuration(); + const auto bStart = b.getFrame(); + const auto bEnd = b.getFrame() + b.getDuration(); + + // Intentionally strict overlap test: both grouping and patch inclusion + // only consider events that truly overlap in time. + return aStart < bEnd && aEnd > bStart; +} + +const Event* OverlapProcessor::findLongestEvent(const OverlapGroup& group, const EventVector& events) const { + if (group.indices.empty()) return nullptr; + + const Event* longestEvent = &events[group.indices[0]]; + for (size_t index : group.indices) { + const Event &event = events[index]; + if (event.getDuration() > longestEvent->getDuration()) { + longestEvent = &event; + } + } + return longestEvent; +} diff --git a/main/OverlapProcessor.h b/main/OverlapProcessor.h new file mode 100644 index 00000000..e36c8bdf --- /dev/null +++ b/main/OverlapProcessor.h @@ -0,0 +1,141 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Tony + An intonation analysis and annotation tool + Centre for Digital Music, Queen Mary, University of London. + This file copyright 2006-2012 Chris Cannam and QMUL. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef OVERLAP_PROCESSOR_H +#define OVERLAP_PROCESSOR_H + +#include +#include +#include + +#include "base/BaseTypes.h" +#include "base/Event.h" + +/** + * Configuration parameters for overlap processing algorithms + */ +struct OverlapConfig { + sv::sv_frame_t interpolationThreshold = 512; // ~11ms at 44.1kHz + double pitchSimilarityThreshold = 0.1; // 10% pitch difference threshold + + OverlapConfig() = default; + OverlapConfig(sv::sv_frame_t interpThresh, double pitchThresh) + : interpolationThreshold(interpThresh) + , pitchSimilarityThreshold(pitchThresh) {} +}; + +/** + * Structure to represent a group of overlapping events. The indices + * refer to positions within the event vector the group was derived + * from; startFrame and endFrame span the whole group. + */ +struct OverlapGroup { + std::vector indices; + sv::sv_frame_t startFrame; + sv::sv_frame_t endFrame; + + OverlapGroup(); + explicit OverlapGroup(size_t index, const sv::Event &event); + bool isEmpty() const { return indices.empty(); } + size_t size() const { return indices.size(); } + void addEvent(size_t index, const sv::Event &event); +}; + +/** + * Main overlap processing class that handles detection and merging of + * overlapping events. + * + * The process*Events methods are pure functions of their inputs: they + * return the set of events to remove from, and add to, the target + * model, without touching any model themselves. + */ +class OverlapProcessor { +public: + explicit OverlapProcessor(const OverlapConfig& config = OverlapConfig()); + + struct EventPatch { + sv::EventVector remove; + sv::EventVector add; + }; + + /** + * Return the connected components of the overlap graph over the + * given events, omitting any component with only one member. Two + * events overlap if their half-open frame ranges intersect; + * grouping is transitive, so A-B and B-C yields one group {A,B,C}. + */ + std::vector findOverlapGroups(const sv::EventVector& events) const; + + /** + * Collapse a group into a single event spanning the whole group, + * taking its label and level from the longest member and its value + * from the duration-weighted geometric mean of the members. + */ + std::optional mergeOverlapGroup(const OverlapGroup& group, + const sv::EventVector& events) const; + + float calculateWeightedFrequency(const sv::EventVector& overlappingEvents, + sv::sv_frame_t overlapStart, + sv::sv_frame_t overlapDuration) const; + + /** + * Produce the patch that replaces the pitch track from + * contextStart onwards with incomingEvents. + * + * incomingEvents are expected to carry ABSOLUTE frames, as pYIN's + * smoothedpitchtrack output does: it is a FixedSampleRate output + * whose features carry the host's block timestamp verbatim, and the + * transformer reconstructs the frame from that timestamp. Here + * contextStart is only the boundary of the region being replaced, + * not an offset to add. + */ + EventPatch processPitchEvents(sv::sv_frame_t contextStart, + const sv::EventVector& incomingEvents, + const sv::EventVector& existingEvents) const; + + /** + * Produce the patch that merges incomingEvents into existingEvents. + * Existing notes absorbed into a merge are included in the remove + * list, and existing notes untouched by the incoming ones are left + * alone. + * + * Unlike processPitchEvents(), incomingEvents are expected to carry + * frames RELATIVE to contextStart, as pYIN's notes output does: it + * derives its timestamps from a frame index counting from zero and + * ignores the host's block timestamps. contextStart is added to them. + */ + EventPatch processNoteEvents(sv::sv_frame_t contextStart, + const sv::EventVector& incomingEvents, + const sv::EventVector& existingEvents) const; + + /** + * The pairwise overlap predicate that grouping is the transitive + * closure of: true if the half-open ranges [frame, frame+duration) + * of a and b intersect. Events that merely touch end-to-end do not + * overlap. + */ + bool eventsOverlap(const sv::Event& a, const sv::Event& b) const; + + const OverlapConfig& getConfig() const { return m_config; } + void setConfig(const OverlapConfig& config) { m_config = config; } + +private: + OverlapConfig m_config; + + const sv::Event* findLongestEvent(const OverlapGroup& group, + const sv::EventVector& events) const; +}; + +#endif // OVERLAP_PROCESSOR_H diff --git a/main/RealtimeAnalyser.cpp b/main/RealtimeAnalyser.cpp new file mode 100644 index 00000000..36626173 --- /dev/null +++ b/main/RealtimeAnalyser.cpp @@ -0,0 +1,694 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Tony + Realtime analysis helper (extracted from Analyser) +*/ + +#include "RealtimeAnalyser.h" + +#include "OverlapProcessor.h" + +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "transform/TransformFactory.h" +#include "framework/Document.h" +#include "data/model/WaveFileModel.h" +#include "data/model/SparseTimeValueModel.h" +#include "data/model/NoteModel.h" + +#include "view/Pane.h" +#include "layer/Layer.h" +#include "layer/TimeValueLayer.h" +#include "layer/FlexiNoteLayer.h" +#include "layer/ColourDatabase.h" + +using std::cerr; +using std::endl; + +using namespace sv; + +// Global overlap processor instance for efficient processing (same as previously in Analyser.cpp) +static OverlapProcessor s_overlapProcessor; + +static void logPatchFrameRange(const char *label, + const EventVector &events) +{ + if (events.empty()) { + cerr << label << ": empty" << endl; + return; + } + + auto minFrame = events.front().getFrame(); + auto maxFrame = events.front().getFrame() + events.front().getDuration(); + for (const auto &event : events) { + minFrame = std::min(minFrame, event.getFrame()); + maxFrame = std::max(maxFrame, event.getFrame() + event.getDuration()); + } + + cerr << label << ": count=" << events.size() + << " start=" << minFrame + << " end=" << maxFrame << endl; +} + +// Wrapper functions +static OverlapProcessor::EventPatch processPitchEvents(sv_frame_t contextStart, + const std::shared_ptr &fromModel, + const std::shared_ptr &toModel) +{ + return s_overlapProcessor.processPitchEvents(contextStart, + fromModel->getAllEvents(), + toModel->getAllEvents()); +} + +static OverlapProcessor::EventPatch processNoteEvents(sv_frame_t contextStart, + const std::shared_ptr &fromModel, + const std::shared_ptr &toModel) +{ + return s_overlapProcessor.processNoteEvents(contextStart, + fromModel->getAllEvents(), + toModel->getAllEvents()); +} + +static std::map getAnalysisSettingsFromSettings() +{ + std::map analysisSettings; + + QSettings settings; + settings.beginGroup("Analyser"); + + analysisSettings["precision-analysis"] = settings.value("precision-analysis", false).toBool(); + analysisSettings["lowamp-analysis"] = settings.value("lowamp-analysis", true).toBool(); + analysisSettings["onset-analysis"] = settings.value("onset-analysis", true).toBool(); + analysisSettings["prune-analysis"] = settings.value("prune-analysis", true).toBool(); + + settings.endGroup(); + + return analysisSettings; +} + +static void setAnalysisSettings(Transform &transform) +{ + const auto analysisSettings = getAnalysisSettingsFromSettings(); + + if (analysisSettings.count("precision-analysis") > 0) { + bool precise = analysisSettings.at("precision-analysis"); + if (precise) { + cerr << "setting parameters for precise mode" << endl; + transform.setParameter("precisetime", 1); + } else { + cerr << "setting parameters for vague mode" << endl; + transform.setParameter("precisetime", 0); + } + } + + if (analysisSettings.count("lowamp-analysis") > 0) { + bool lowamp = analysisSettings.at("lowamp-analysis"); + if (lowamp) { + cerr << "setting parameters for lowamp suppression" << endl; + transform.setParameter("lowampsuppression", 0.2f); + } else { + cerr << "setting parameters for no lowamp suppression" << endl; + transform.setParameter("lowampsuppression", 0.0f); + } + } + + if (analysisSettings.count("onset-analysis") > 0) { + bool onset = analysisSettings.at("onset-analysis"); + if (onset) { + cerr << "setting parameters for increased onset sensitivity" << endl; + transform.setParameter("onsetsensitivity", 0.7f); + } else { + cerr << "setting parameters for non-increased onset sensitivity" << endl; + transform.setParameter("onsetsensitivity", 0.0f); + } + } + + if (analysisSettings.count("prune-analysis") > 0) { + bool prune = analysisSettings.at("prune-analysis"); + if (prune) { + cerr << "setting parameters for duration pruning" << endl; + transform.setParameter("prunethresh", 0.1f); + } else { + cerr << "setting parameters for no duration pruning" << endl; + transform.setParameter("prunethresh", 0.0f); + } + } +} + +RealtimeAnalyser::RealtimeAnalyser(QObject *parent) : + QObject(parent) +{ +} + +RealtimeAnalyser::~RealtimeAnalyser() +{ + cleanup(); +} + +void +RealtimeAnalyser::setContext(Document *document, + ModelId fileModel, + Pane *pane, + TimeValueLayer *targetPitchLayer, + FlexiNoteLayer *targetNoteLayer) +{ + QMutexLocker locker(&m_mutex); + + m_ctx.document = document; + m_ctx.fileModel = fileModel; + m_ctx.pane = pane; + m_ctx.targetPitchLayer = targetPitchLayer; + m_ctx.targetNoteLayer = targetNoteLayer; +} + +void +RealtimeAnalyser::clearContext() +{ + QMutexLocker locker(&m_mutex); + + m_ctx.document = nullptr; + m_ctx.fileModel = ModelId(); + m_ctx.pane = nullptr; + m_ctx.targetPitchLayer = nullptr; + m_ctx.targetNoteLayer = nullptr; +} + +void +RealtimeAnalyser::cleanup() +{ + std::vector> layersToClean; + QPointer doc; + quint64 newGeneration = 0; + bool wasInFlight = false; + bool hadPending = false; + + { + QMutexLocker locker(&m_mutex); + + layersToClean.swap(m_tempLayers); + + wasInFlight = m_inFlight; + hadPending = m_pendingSelection.has_value(); + + // Every caller of cleanup() is reacting to the ground moving -- + // the document closing, a new file loading, or the target layers + // being replaced. Restarting the queued chunk here would schedule + // work against state that is being torn down (and, from the + // destructor, would connect signals to a half-destroyed object), + // so all outstanding work is simply abandoned. Bumping the + // generation makes any callback already queued for delivery a + // no-op. + m_inFlight = false; + m_pendingSelection = std::nullopt; + + newGeneration = ++m_generation; + + doc = m_ctx.document; + + cerr << "RealtimeAnalyser::cleanup: generation=" << newGeneration + << " wasInFlight=" << wasInFlight + << " hadPending=" << hadPending + << " tempLayers=" << layersToClean.size() << endl; + } + + deleteTempLayers(std::move(layersToClean), doc); +} + +void +RealtimeAnalyser::invalidateGeneration() +{ + QMutexLocker locker(&m_mutex); + + // If we invalidate generation, any existing callbacks become stale. + // Do not clear m_inFlight here: the owning run must retire itself via + // completion so we don't break the in-flight/pending state machine. + ++m_generation; + m_pendingSelection = std::nullopt; + + cerr << "RealtimeAnalyser::invalidateGeneration: generation=" << m_generation + << " inFlight=" << m_inFlight << endl; +} + +QString +RealtimeAnalyser::analyseChunk(Selection sel) +{ + bool startedChunk = false; + + if (sel.isEmpty()) return ""; + + quint64 generation = 0; + QPointer safeDocument; + QPointer safePane; + QPointer safeTargetPitchLayer; + QPointer safeTargetNoteLayer; + ModelId fileModel; + std::shared_ptr waveFileModel; + + { + QMutexLocker locker(&m_mutex); + + if (m_inFlight) { + + // Merge rather than replace: recordDurationChanged fires + // several times during a single chunk, and simply keeping + // the newest selection would leave the frames between the + // running chunk's end and the newest chunk's start + // permanently unanalysed. + if (m_pendingSelection) { + m_pendingSelection = Selection + (std::min(m_pendingSelection->getStartFrame(), + sel.getStartFrame()), + std::max(m_pendingSelection->getEndFrame(), + sel.getEndFrame())); + } else { + m_pendingSelection = sel; + } + + cerr << "RealtimeAnalyser::analyseChunk: already in flight, merged into pending selection" + << " generation=" << m_generation + << " pendingStart=" << m_pendingSelection->getStartFrame() + << " pendingEnd=" << m_pendingSelection->getEndFrame() << endl; + return ""; + } + + if (!m_ctx.document || !m_ctx.pane) { + return "Internal error: RealtimeAnalyser::analyseChunk() called with no document or pane present"; + } + + if (m_ctx.fileModel.isNone()) { + return "Internal error: RealtimeAnalyser::analyseChunk() called with no model present"; + } + + if (!m_ctx.targetPitchLayer || !m_ctx.targetNoteLayer) { + return "Internal error: RealtimeAnalyser::analyseChunk() called with no target pitch/note layers present"; + } + + m_inFlight = true; + startedChunk = true; + generation = m_generation; + + safeDocument = m_ctx.document; + safePane = m_ctx.pane; + safeTargetPitchLayer = m_ctx.targetPitchLayer; + safeTargetNoteLayer = m_ctx.targetNoteLayer; + fileModel = m_ctx.fileModel; + + cerr << "RealtimeAnalyser::analyseChunk: acquired in-flight slot" + << " generation=" << generation + << " start=" << sel.getStartFrame() + << " end=" << sel.getEndFrame() << endl; + } + + auto finishIfStarted = [this, startedChunk, generation]() { + if (startedChunk) { + finishChunk(generation); + } + }; + + waveFileModel = ModelById::getAs(fileModel); + if (!waveFileModel) { + finishIfStarted(); + return "Internal error: RealtimeAnalyser::analyseChunk() called with no WaveFileModel"; + } + + auto cleanupTempLayer = [this](QPointer safeTempLayer, + QPointer doc) { + Layer *layerToDelete = safeTempLayer.data(); + if (!layerToDelete) return; + + { + QMutexLocker locker(&m_mutex); + untrackTempLayerLocked(layerToDelete); + } + + // Temp layers are never added to a view, so removeLayerFromView() + // must not be used here: it would push an undoable + // RemoveLayerCommand holding a raw pointer to a layer we are + // about to delete, leaving the undo stack full of commands that + // dereference freed memory on undo. Delete directly instead. + if (doc) { + doc->deleteLayer(layerToDelete, true); + } + }; + + auto state = std::make_shared(); + + auto completePart = [this, state, generation](bool canFinish) { + --state->remainingParts; + cerr << "RealtimeAnalyser::completePart: generation=" << generation + << " canFinish=" << canFinish + << " remainingParts=" << state->remainingParts << endl; + if (state->remainingParts == 0) { + cerr << "RealtimeAnalyser::completePart: retiring chunk for generation=" << generation + << " finishAllowed=" << canFinish << endl; + finishChunk(generation); + } + }; + + // Registers a completion handler for one temp layer. The handler + // runs at most once, whether it is reached through the layer's + // signal or through the immediate check below -- if the transform + // has already finished by the time we get here the signal has been + // and gone, and without the check nothing would ever retire the + // chunk, leaving m_inFlight stuck true and realtime analysis + // silently dead for the rest of the session. + auto registerPart = [this, state](Layer *layer, + std::function handler) { + + ++state->remainingParts; + + auto done = std::make_shared(false); + auto once = [done, handler](ModelId modelId) { + if (*done) return; + *done = true; + handler(modelId); + }; + + QObject::connect(layer, &Layer::modelCompletionChanged, this, + [once](ModelId modelId) { + auto model = ModelById::get(modelId); + if (!model || model->getCompletion() != 100) return; + once(modelId); + }, + Qt::QueuedConnection); + + const ModelId modelId = layer->getModel(); + auto model = ModelById::get(modelId); + if (model && model->getCompletion() == 100) { + QMetaObject::invokeMethod(this, + [once, modelId]() { once(modelId); }, + Qt::QueuedConnection); + } + }; + + TransformFactory *tf = TransformFactory::getInstance(); + + const auto f0_transform = QString(PYIN_TRANSFORM_BASE) + QString(PYIN_F0_OUT); + const auto note_transform = QString(PYIN_TRANSFORM_BASE) + QString(PYIN_NOTE_OUT); + + QString notFound = + tr("Transform \"%1\" not found. Unable to perform interactive analysis." + "

Are the %2 and %3 Vamp plugins correctly installed?"); + + if (!tf->haveTransform(f0_transform)) { + finishIfStarted(); + return notFound.arg(f0_transform).arg(PYIN_PLUGIN_NAME); + } + + if (!tf->haveTransform(note_transform)) { + finishIfStarted(); + return notFound.arg(note_transform).arg(PYIN_PLUGIN_NAME); + } + + Transform t = tf->getDefaultTransformFor(f0_transform, waveFileModel->getSampleRate()); + t.setStepSize(256); + t.setBlockSize(2048); + + setAnalysisSettings(t); + + const RealTime start = + RealTime::frame2RealTime(sel.getStartFrame(), waveFileModel->getSampleRate()); + const RealTime end = + RealTime::frame2RealTime(sel.getEndFrame(), waveFileModel->getSampleRate()); + + RealTime duration; + if (sel.getEndFrame() > sel.getStartFrame()) { + duration = end - start; + } + + cerr << "RealtimeAnalyser::analyseChunk: start " << start + << " end " << end + << " original selection start " << sel.getStartFrame() + << " end " << sel.getEndFrame() + << " duration " << duration << endl; + + if (duration <= RealTime::zeroTime) { + cerr << "RealtimeAnalyser::analyseChunk: duration <= 0, not analysing" << endl; + finishIfStarted(); + return ""; + } + + t.setStartTime(start); + t.setDuration(duration); + + Transforms transforms; + transforms.push_back(t); + + t.setOutput(PYIN_NOTE_OUT); + transforms.push_back(t); + + if (!safeDocument) { + finishIfStarted(); + return "Internal error: RealtimeAnalyser::analyseChunk() document deleted during scheduling"; + } + + const std::vector layers = safeDocument->createDerivedLayers(transforms, fileModel); + + if (layers.empty()) { + cerr << "WARNING: RealtimeAnalyser::analyseChunk: no layers returned from createDerivedLayers" << endl; + finishIfStarted(); + return ""; + } + + { + QMutexLocker locker(&m_mutex); + for (auto *layer : layers) { + m_tempLayers.push_back(QPointer(layer)); + } + } + + ColourDatabase *cdb = ColourDatabase::getInstance(); + + std::vector unrecognised; + + for (auto *layer : layers) { + + if (auto *tempPitchLayer = qobject_cast(layer)) { + + tempPitchLayer->setBaseColour(cdb->getColourIndex(tr("Black"))); + + QPointer safeTempLayer(tempPitchLayer); + + registerPart + (tempPitchLayer, + [this, safeTempLayer, safeTargetPitchLayer, safeDocument, safePane, + sel, generation, cleanupTempLayer, completePart](ModelId modelId) { + + const auto fromModel = ModelById::getAs(modelId); + if (!fromModel) { + cleanupTempLayer(safeTempLayer, safeDocument); + completePart(false); + return; + } + + cerr << "RealtimeAnalyser::analyseChunk: Processing pitch track completion" << endl; + + bool stale = false; + { + QMutexLocker locker(&m_mutex); + stale = (generation != m_generation); + } + + if (stale) { + cerr << "RealtimeAnalyser::analyseChunk: Ignoring stale pitch callback from old generation" + << " callbackGeneration=" << generation << endl; + cleanupTempLayer(safeTempLayer, safeDocument); + completePart(false); + return; + } + + if (safeTargetPitchLayer) { + const auto toModel = + ModelById::getAs(safeTargetPitchLayer->getModel()); + + if (toModel) { + const auto patch = + processPitchEvents(sel.getStartFrame(), fromModel, toModel); + + cerr << "RealtimeAnalyser::pitchPatch: selectionStart=" << sel.getStartFrame() + << " selectionEnd=" << sel.getEndFrame() << endl; + logPatchFrameRange("RealtimeAnalyser::pitchPatch.remove", patch.remove); + logPatchFrameRange("RealtimeAnalyser::pitchPatch.add", patch.add); + + for (const Event &p : patch.remove) { + toModel->remove(p); + } + for (const Event &p : patch.add) { + toModel->add(p); + } + } else { + cerr << "ERROR: RealtimeAnalyser pitch callback - target model is null" << endl; + } + } else { + cerr << "WARNING: RealtimeAnalyser pitch callback - target layer deleted" << endl; + } + + cleanupTempLayer(safeTempLayer, safeDocument); + + if (safeTargetPitchLayer && safePane) { + safeTargetPitchLayer->layerParametersChanged(); + safePane->layerParametersChanged(); + } + emit layersChanged(); + + completePart(true); + }); + + } else if (auto *tempNoteLayer = qobject_cast(layer)) { + + tempNoteLayer->setBaseColour(cdb->getColourIndex(tr("Bright Blue"))); + + QPointer safeTempLayer(tempNoteLayer); + + registerPart + (tempNoteLayer, + [this, safeTempLayer, safeTargetNoteLayer, safeDocument, + sel, generation, cleanupTempLayer, completePart](ModelId modelId) { + + const auto fromModel = ModelById::getAs(modelId); + if (!fromModel) { + cleanupTempLayer(safeTempLayer, safeDocument); + completePart(false); + return; + } + + cerr << "RealtimeAnalyser::analyseChunk: Processing note layer completion" << endl; + + bool stale = false; + { + QMutexLocker locker(&m_mutex); + stale = (generation != m_generation); + } + + if (stale) { + cerr << "RealtimeAnalyser::analyseChunk: Ignoring stale note callback from old generation" + << " callbackGeneration=" << generation << endl; + cleanupTempLayer(safeTempLayer, safeDocument); + completePart(false); + return; + } + + if (safeTargetNoteLayer) { + const auto toModel = + ModelById::getAs(safeTargetNoteLayer->getModel()); + + if (toModel) { + const auto patch = + processNoteEvents(sel.getStartFrame(), fromModel, toModel); + + logPatchFrameRange("RealtimeAnalyser::notePatch.remove", patch.remove); + logPatchFrameRange("RealtimeAnalyser::notePatch.add", patch.add); + + for (const Event &p : patch.remove) { + toModel->remove(p); + } + for (const Event &p : patch.add) { + toModel->add(p); + } + } else { + cerr << "ERROR: RealtimeAnalyser note callback - target model is null" << endl; + } + } else { + cerr << "WARNING: RealtimeAnalyser note callback - target layer deleted" << endl; + } + + cleanupTempLayer(safeTempLayer, safeDocument); + + emit layersChanged(); + + completePart(true); + }); + + } else { + unrecognised.push_back(layer); + } + } + + // Anything we can't drive to completion would otherwise sit in + // m_tempLayers until the next cleanup() + for (auto *layer : unrecognised) { + cerr << "WARNING: RealtimeAnalyser::analyseChunk: discarding unrecognised temp layer" << endl; + cleanupTempLayer(QPointer(layer), safeDocument); + } + + if (state->remainingParts == 0) { + cerr << "WARNING: RealtimeAnalyser::analyseChunk: no recognised temp layers created" << endl; + finishIfStarted(); + } + + return ""; +} + +void +RealtimeAnalyser::untrackTempLayerLocked(Layer *layer) +{ + m_tempLayers.erase( + std::remove_if(m_tempLayers.begin(), + m_tempLayers.end(), + [layer](const QPointer &p) { + return p.isNull() || p.data() == layer; + }), + m_tempLayers.end()); +} + +void +RealtimeAnalyser::deleteTempLayers(std::vector> layersToClean, + const QPointer &doc) +{ + // If the document has already gone it owns and destroys its layers, + // so there is nothing left for us to release. + if (!doc) return; + + for (const auto &layerPtr : layersToClean) { + Layer *layer = layerPtr.data(); + if (!layer) continue; + + // See cleanupTempLayer(): these layers were never added to a + // view, so they must not go through removeLayerFromView(). + doc->deleteLayer(layer, true); + } +} + +void +RealtimeAnalyser::finishChunk(quint64 generation) +{ + std::optional pending; + + { + QMutexLocker locker(&m_mutex); + + if (generation != m_generation) { + // A cleanup() or invalidateGeneration() has superseded this + // chunk, so it no longer owns the in-flight slot. Clearing + // m_inFlight here would release a slot that a newer chunk + // may already have taken, letting two chunks write to the + // same models at once. + cerr << "RealtimeAnalyser::finishChunk: ignoring retirement of stale generation=" + << generation << " current=" << m_generation << endl; + return; + } + + m_inFlight = false; + pending = std::exchange(m_pendingSelection, std::nullopt); + + cerr << "RealtimeAnalyser::finishChunk: generation=" << generation + << " pending=" << (pending.has_value() ? "yes" : "no") << endl; + } + + if (pending) { + cerr << "RealtimeAnalyser::finishChunk: starting pending realtime selection" + << " start=" << pending->getStartFrame() + << " end=" << pending->getEndFrame() << endl; + (void)analyseChunk(*pending); + } +} diff --git a/main/RealtimeAnalyser.h b/main/RealtimeAnalyser.h new file mode 100644 index 00000000..a75b1a98 --- /dev/null +++ b/main/RealtimeAnalyser.h @@ -0,0 +1,108 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + + /* + Tony + Realtime analysis helper (extracted from Analyser) + */ + +#ifndef REALTIMEANALYSER_H +#define REALTIMEANALYSER_H + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "framework/Document.h" +#include "base/Selection.h" + +namespace sv { +class Pane; +class Layer; +class TimeValueLayer; +class FlexiNoteLayer; +} + +class RealtimeAnalyser : public QObject +{ + Q_OBJECT + +public: + explicit RealtimeAnalyser(QObject *parent = nullptr); + ~RealtimeAnalyser() override; + + void setContext(sv::Document *document, + sv::ModelId fileModel, + sv::Pane *pane, + sv::TimeValueLayer *targetPitchLayer, + sv::FlexiNoteLayer *targetNoteLayer); + + void clearContext(); + + /** + * Abandon all in-flight and pending work, delete any temporary + * layers, and bump the generation so that callbacks still queued + * for delivery are ignored. Never starts new work, so it is safe to + * call from a destructor or while tearing down a document. + */ + void cleanup(); + + /** + * Ignore stale callbacks after external state changes, and drop any + * pending follow-up chunk. Does not delete layers, and does not + * retire the in-flight chunk: that chunk still owns the in-flight + * slot and must retire itself through completion. + */ + void invalidateGeneration(); + + /** + * One-in-flight realtime analysis. If a chunk is already running, + * the selection is merged into the pending one -- rather than + * replacing it, which would drop the region between them -- and is + * analysed when the running chunk retires. + */ + QString analyseChunk(sv::Selection sel); + +signals: + void layersChanged(); + +private: + struct Context { + QPointer document; + sv::ModelId fileModel; + QPointer pane; + QPointer targetPitchLayer; + QPointer targetNoteLayer; + }; + + struct RealtimeChunkState { + int remainingParts = 0; + }; + + static constexpr const char* PYIN_PLUGIN_NAME = "pYIN"; + static constexpr const char* PYIN_TRANSFORM_BASE = "vamp:pyin:pyin:"; + static constexpr const char* PYIN_F0_OUT = "smoothedpitchtrack"; + static constexpr const char* PYIN_NOTE_OUT = "notes"; + + void untrackTempLayerLocked(sv::Layer *layer); + void deleteTempLayers(std::vector> layersToClean, + const QPointer &doc); + + void finishChunk(quint64 generation); + +private: + mutable QMutex m_mutex; + Context m_ctx; + std::vector> m_tempLayers; + + bool m_inFlight = false; + std::optional m_pendingSelection; + quint64 m_generation = 0; +}; + +#endif diff --git a/main/test/TestOverlapProcessor.h b/main/test/TestOverlapProcessor.h new file mode 100644 index 00000000..2db3bc87 --- /dev/null +++ b/main/test/TestOverlapProcessor.h @@ -0,0 +1,665 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Tony + An intonation analysis and annotation tool + Centre for Digital Music, Queen Mary, University of London. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#ifndef TEST_OVERLAP_PROCESSOR_H +#define TEST_OVERLAP_PROCESSOR_H + +#include "../OverlapProcessor.h" + +#include +#include + +#include +#include +#include +#include + +using namespace sv; + +class TestOverlapProcessor : public QObject +{ + Q_OBJECT + +private: + static Event note(sv_frame_t frame, sv_frame_t duration, float value) { + return Event(frame, value, duration, QString()); + } + + static Event pitch(sv_frame_t frame, float value) { + return Event(frame, value, QString()); + } + + /** Apply a patch the way RealtimeAnalyser does: remove first, then + * add. Removing an event drops a single matching instance, as + * EventSeries::remove does. */ + static EventVector applyPatch(EventVector model, + const OverlapProcessor::EventPatch &patch) { + for (const auto &e: patch.remove) { + auto itr = std::find(model.begin(), model.end(), e); + if (itr != model.end()) model.erase(itr); + } + for (const auto &e: patch.add) { + model.push_back(e); + } + std::sort(model.begin(), model.end()); + return model; + } + + /** Brute-force reference implementation of grouping: the transitive + * closure of the pairwise overlap predicate. */ + static std::vector> + referenceGroups(const OverlapProcessor &p, const EventVector &events) { + + const size_t n = events.size(); + std::vector component(n); + for (size_t i = 0; i < n; ++i) component[i] = i; + + bool changed = true; + while (changed) { + changed = false; + for (size_t i = 0; i < n; ++i) { + for (size_t j = i + 1; j < n; ++j) { + if (!p.eventsOverlap(events[i], events[j])) continue; + const size_t a = std::min(component[i], component[j]); + const size_t b = std::max(component[i], component[j]); + if (a == b) continue; + for (size_t k = 0; k < n; ++k) { + if (component[k] == b) component[k] = a; + } + changed = true; + } + } + } + + std::vector> groups; + for (size_t root = 0; root < n; ++root) { + std::set members; + for (size_t k = 0; k < n; ++k) { + if (component[k] == root) members.insert(k); + } + if (members.size() > 1) groups.push_back(members); + } + return groups; + } + + static std::vector> + actualGroups(const OverlapProcessor &p, const EventVector &events) { + std::vector> groups; + for (const auto &g: p.findOverlapGroups(events)) { + groups.push_back(std::set(g.indices.begin(), + g.indices.end())); + } + std::sort(groups.begin(), groups.end()); + return groups; + } + + /** True if any two distinct events in the vector overlap. */ + static bool hasAnyOverlap(const OverlapProcessor &p, + const EventVector &events) { + for (size_t i = 0; i < events.size(); ++i) { + for (size_t j = i + 1; j < events.size(); ++j) { + if (p.eventsOverlap(events[i], events[j])) return true; + } + } + return false; + } + +private slots: + + // ---- findOverlapGroups ------------------------------------------ + + void groupsEmptyAndSingle() { + + OverlapProcessor p; + + QCOMPARE(int(p.findOverlapGroups(EventVector()).size()), 0); + + EventVector one { note(0, 100, 440.f) }; + QCOMPARE(int(p.findOverlapGroups(one).size()), 0); + } + + void groupsIgnoreNonOverlapping() { + + OverlapProcessor p; + + EventVector events { + note(0, 100, 440.f), + note(200, 100, 450.f), + note(400, 100, 460.f) + }; + + QCOMPARE(int(p.findOverlapGroups(events).size()), 0); + } + + void groupsIgnoreTouchingEndToEnd() { + + // [0,100) and [100,200) share a boundary but do not overlap + OverlapProcessor p; + + EventVector events { + note(0, 100, 440.f), + note(100, 100, 450.f) + }; + + QCOMPARE(int(p.findOverlapGroups(events).size()), 0); + QCOMPARE(p.eventsOverlap(events[0], events[1]), false); + } + + void groupsAreTransitive() { + + // A and C do not overlap each other, but both overlap B + OverlapProcessor p; + + EventVector events { + note(0, 100, 440.f), // A + note(90, 100, 450.f), // B + note(180, 100, 460.f) // C + }; + + auto groups = p.findOverlapGroups(events); + QCOMPARE(int(groups.size()), 1); + QCOMPARE(int(groups[0].size()), 3); + QCOMPARE(groups[0].startFrame, sv_frame_t(0)); + QCOMPARE(groups[0].endFrame, sv_frame_t(280)); + QCOMPARE(p.eventsOverlap(events[0], events[2]), false); + } + + void groupsSpanIsNotBrokenByAShortMember() { + + // The short event in the middle must not end the run: the + // running span is the maximum end frame of the group so far, + // not the end frame of the previous event. + OverlapProcessor p; + + EventVector events { + note(0, 1000, 440.f), // long + note(10, 5, 450.f), // short, entirely inside the long one + note(900, 200, 460.f) // overlaps the long one only + }; + + auto groups = p.findOverlapGroups(events); + QCOMPARE(int(groups.size()), 1); + QCOMPARE(int(groups[0].size()), 3); + QCOMPARE(groups[0].endFrame, sv_frame_t(1100)); + } + + void groupsSeparateRuns() { + + OverlapProcessor p; + + EventVector events { + note(0, 100, 440.f), + note(50, 100, 450.f), + note(1000, 100, 460.f), + note(1050, 100, 470.f) + }; + + auto groups = p.findOverlapGroups(events); + QCOMPARE(int(groups.size()), 2); + QCOMPARE(int(groups[0].size()), 2); + QCOMPARE(int(groups[1].size()), 2); + } + + void groupsHandleZeroDurationEvents() { + + // Pitch-style events have no duration and so never overlap each + // other, not even at the same frame + OverlapProcessor p; + + EventVector events { + pitch(100, 440.f), + pitch(100, 441.f), + pitch(200, 442.f) + }; + + QCOMPARE(int(p.findOverlapGroups(events).size()), 0); + + // ...but a durationed event does overlap the point events it + // spans, and only those: [50,150) covers the two at frame 100 + // but not the one at frame 200. + events.push_back(note(50, 100, 443.f)); + auto groups = p.findOverlapGroups(events); + QCOMPARE(int(groups.size()), 1); + QCOMPARE(groups[0].indices, std::vector({ 3, 0, 1 })); + } + + void groupsMatchTransitiveClosureReference() { + + // The sweep implementation must agree with a brute-force + // transitive closure of eventsOverlap() on unsorted, nested, + // duplicated and zero-duration input. + OverlapProcessor p; + + EventVector events { + note(500, 100, 440.f), + note(0, 50, 441.f), + note(40, 5, 442.f), + note(490, 400, 443.f), + pitch(45, 444.f), + note(1000, 0, 445.f), + note(880, 130, 446.f), + note(0, 50, 441.f), + note(2000, 10, 447.f), + note(300, 100, 448.f) + }; + + auto expected = referenceGroups(p, events); + std::sort(expected.begin(), expected.end()); + + QCOMPARE(actualGroups(p, events), expected); + } + + void groupsScaleToLongTracks() { + + // Regression guard for the old rescan-on-every-match loop, which + // was cubic in the worst case and ran over the whole note track + // once per realtime chunk. 20000 chained notes would not finish + // in any reasonable time under that implementation. + OverlapProcessor p; + + const int count = 20000; + EventVector events; + events.reserve(count); + for (int i = 0; i < count; ++i) { + events.push_back(note(i * 100, 150, 440.f)); // each overlaps the next + } + + QElapsedTimer timer; + timer.start(); + auto groups = p.findOverlapGroups(events); + const qint64 elapsed = timer.elapsed(); + + QCOMPARE(int(groups.size()), 1); + QCOMPARE(int(groups[0].size()), count); + QVERIFY2(elapsed < 5000, + QString("findOverlapGroups took %1 ms for %2 events") + .arg(elapsed).arg(count).toLocal8Bit().data()); + } + + // ---- mergeOverlapGroup / weighted frequency --------------------- + + void weightedFrequencyIsDurationWeightedGeometricMean() { + + OverlapProcessor p; + + EventVector events { + note(0, 300, 400.f), + note(0, 100, 800.f) + }; + + // weights 3/4 and 1/4 => exp(0.75*ln400 + 0.25*ln800) + const double expected = + std::exp(0.75 * std::log(400.0) + 0.25 * std::log(800.0)); + + const float actual = p.calculateWeightedFrequency(events, 0, 300); + QVERIFY(std::fabs(double(actual) - expected) < 0.01); + } + + void weightedFrequencyIgnoresUnvoicedEvents() { + + OverlapProcessor p; + + EventVector events { + note(0, 100, 440.f), + note(0, 100, 0.f) // unvoiced, must not drag the mean to zero + }; + + QCOMPARE(p.calculateWeightedFrequency(events, 0, 100), 440.f); + } + + void mergeSpansWholeGroupAndTakesLabelFromLongest() { + + OverlapProcessor p; + + EventVector events { + Event(0, 440.f, 100, "short"), + Event(50, 450.f, 400, "long") + }; + + OverlapGroup group(0, events[0]); + group.addEvent(1, events[1]); + + auto merged = p.mergeOverlapGroup(group, events); + QVERIFY(merged.has_value()); + QCOMPARE(merged->getFrame(), sv_frame_t(0)); + QCOMPARE(merged->getDuration(), sv_frame_t(450)); + QCOMPARE(merged->getLabel(), QString("long")); + } + + // ---- processNoteEvents ------------------------------------------ + + void notesEmptyIncomingIsANoOp() { + + OverlapProcessor p; + + EventVector existing { note(0, 100, 440.f) }; + + auto patch = p.processNoteEvents(1000, EventVector(), existing); + QCOMPARE(int(patch.remove.size()), 0); + QCOMPARE(int(patch.add.size()), 0); + } + + void notesIncomingIsShiftedByContextStart() { + + OverlapProcessor p; + + EventVector incoming { note(0, 100, 440.f), note(200, 100, 450.f) }; + + auto patch = p.processNoteEvents(5000, incoming, EventVector()); + + QCOMPARE(int(patch.remove.size()), 0); + QCOMPARE(int(patch.add.size()), 2); + QCOMPARE(patch.add[0].getFrame(), sv_frame_t(5000)); + QCOMPARE(patch.add[1].getFrame(), sv_frame_t(5200)); + } + + void notesLeaveUntouchedExistingAlone() { + + // An existing group far from the analysed region must be left + // exactly as it is -- neither removed nor re-added. + OverlapProcessor p; + + EventVector existing { + note(0, 100, 440.f), + note(50, 100, 441.f) // overlaps its neighbour, but not the incoming + }; + + EventVector incoming { note(0, 100, 460.f) }; + + auto patch = p.processNoteEvents(5000, incoming, existing); + + QCOMPARE(int(patch.remove.size()), 0); + QCOMPARE(int(patch.add.size()), 1); + QCOMPARE(patch.add[0].getFrame(), sv_frame_t(5000)); + + auto result = applyPatch(existing, patch); + QCOMPARE(int(result.size()), 3); + } + + void notesOverlappingExistingIsRemovedAndMerged() { + + OverlapProcessor p; + + EventVector existing { note(100, 150, 440.f) }; // [100,250) + EventVector incoming { note(0, 200, 445.f) }; // shifted to [200,400) + + auto patch = p.processNoteEvents(200, incoming, existing); + + QCOMPARE(int(patch.remove.size()), 1); + QCOMPARE(patch.remove[0].getFrame(), sv_frame_t(100)); + + QCOMPARE(int(patch.add.size()), 1); + QCOMPARE(patch.add[0].getFrame(), sv_frame_t(100)); + QCOMPARE(patch.add[0].getFrame() + patch.add[0].getDuration(), + sv_frame_t(400)); + + auto result = applyPatch(existing, patch); + QCOMPARE(int(result.size()), 1); + } + + void notesChainedExistingIsAlsoRemoved() { + + // Regression test. + // + // R overlaps D, and D overlaps the incoming note N, but R does + // not overlap N. Grouping is transitive, so {N,D,R} merges into + // a single event -- which means R is superseded too. Removing + // only D (the one that directly overlapped N) left R behind + // underneath the merged note, producing two overlapping notes + // covering the same span. + OverlapProcessor p; + + EventVector existing { + note(150, 100, 440.f), // D: [150,250) + note(240, 60, 441.f) // R: [240,300) + }; + EventVector incoming { note(0, 100, 450.f) }; // N: [100,200) + + QCOMPARE(p.eventsOverlap(existing[1], + incoming[0].withFrame(100)), false); + + auto patch = p.processNoteEvents(100, incoming, existing); + + QCOMPARE(int(patch.remove.size()), 2); + QCOMPARE(int(patch.add.size()), 1); + + auto result = applyPatch(existing, patch); + QCOMPARE(int(result.size()), 1); + QCOMPARE(result[0].getFrame(), sv_frame_t(100)); + QCOMPARE(result[0].getFrame() + result[0].getDuration(), + sv_frame_t(300)); + QCOMPARE(hasAnyOverlap(p, result), false); + } + + void notesLeaveNoOverlapsBehindInAChain() { + + // A longer version of the same shape: a chain of existing notes + // reached only through their neighbours. Whatever the merge + // decides to produce, the resulting model must not contain two + // notes overlapping each other. + OverlapProcessor p; + + EventVector existing; + for (int i = 0; i < 8; ++i) { + existing.push_back(note(300 + i * 90, 100, 440.f + i)); + } + + EventVector incoming { note(0, 100, 500.f) }; // -> [350,450) + + auto patch = p.processNoteEvents(350, incoming, existing); + auto result = applyPatch(existing, patch); + + QCOMPARE(hasAnyOverlap(p, result), false); + } + + void notesRepeatedChunksDoNotAccumulate() { + + // Realtime chunks overlap each other by design, so the same + // audio is analysed more than once. Re-applying an identical + // chunk must not grow the model. + OverlapProcessor p; + + EventVector incoming { + note(0, 100, 440.f), + note(200, 100, 450.f) + }; + + EventVector model; + model = applyPatch(model, p.processNoteEvents(1000, incoming, model)); + const int afterFirst = int(model.size()); + + for (int i = 0; i < 5; ++i) { + model = applyPatch(model, + p.processNoteEvents(1000, incoming, model)); + } + + QCOMPARE(int(model.size()), afterFirst); + QCOMPARE(hasAnyOverlap(p, model), false); + } + + // ---- processPitchEvents ----------------------------------------- + // + // Note the asymmetry with processNoteEvents throughout this section: + // incoming PITCH frames are absolute and are not shifted, because + // pYIN's smoothedpitchtrack echoes the host's absolute block + // timestamp, whereas its notes output counts frames from zero. See + // pitchAndNoteFrameOriginsDiffer() below. + + void pitchIncomingIsNotShiftedByContextStart() { + + // Regression test. Adding contextStart to the incoming pitch + // frames -- correct for notes, wrong for pitch -- put the live + // pitch track at roughly twice its true frame, so during + // recording it drifted off past the record head. + OverlapProcessor p; + + EventVector incoming { pitch(5000, 440.f), pitch(5100, 441.f) }; + + auto patch = p.processPitchEvents(5000, incoming, EventVector()); + + QCOMPARE(int(patch.add.size()), 2); + QCOMPARE(patch.add[0].getFrame(), sv_frame_t(5000)); + QCOMPARE(patch.add[1].getFrame(), sv_frame_t(5100)); + } + + void pitchAndNoteFrameOriginsDiffer() { + + // Pins the contract that the two entry points disagree on + // purpose. Same contextStart, same nominal event position, two + // different frame conventions. + OverlapProcessor p; + + const sv_frame_t contextStart = 5000; + + auto pitchPatch = p.processPitchEvents + (contextStart, EventVector { pitch(5000, 440.f) }, EventVector()); + + auto notePatch = p.processNoteEvents + (contextStart, EventVector { note(0, 100, 440.f) }, EventVector()); + + QCOMPARE(int(pitchPatch.add.size()), 1); + QCOMPARE(int(notePatch.add.size()), 1); + + // Absolute in, absolute out -- unchanged + QCOMPARE(pitchPatch.add[0].getFrame(), sv_frame_t(5000)); + // Relative in, shifted to absolute out + QCOMPARE(notePatch.add[0].getFrame(), sv_frame_t(5000)); + } + + void pitchReplacesFromContextStartOnly() { + + OverlapProcessor p; + + EventVector existing { + pitch(0, 400.f), + pitch(100, 401.f), + pitch(1000, 402.f), + pitch(1100, 403.f) + }; + + EventVector incoming { pitch(1000, 500.f), pitch(1100, 501.f) }; + + auto patch = p.processPitchEvents(1000, incoming, existing); + + QCOMPARE(int(patch.remove.size()), 2); + QCOMPARE(patch.remove[0].getFrame(), sv_frame_t(1000)); + QCOMPARE(patch.remove[1].getFrame(), sv_frame_t(1100)); + + auto result = applyPatch(existing, patch); + QCOMPARE(int(result.size()), 4); + QCOMPARE(result[0].getValue(), 400.f); + QCOMPARE(result[1].getValue(), 401.f); + QCOMPARE(result[2].getFrame(), sv_frame_t(1000)); + QCOMPARE(result[2].getValue(), 500.f); + QCOMPARE(result[3].getFrame(), sv_frame_t(1100)); + QCOMPARE(result[3].getValue(), 501.f); + } + + void pitchInterpolatesAcrossASmallSimilarGap() { + + OverlapProcessor p; // threshold 512 frames, 10% pitch difference + + EventVector existing { pitch(900, 440.f) }; + EventVector incoming { pitch(1100, 445.f) }; // 200-frame gap + + auto patch = p.processPitchEvents(1000, incoming, existing); + + QCOMPARE(int(patch.add.size()), 2); + QCOMPARE(patch.add[0].getFrame(), sv_frame_t(1000)); // midpoint + QCOMPARE(patch.add[0].getLabel(), QString("interpolated")); + QCOMPARE(patch.add[0].getValue(), (440.f + 445.f) / 2.f); + QCOMPARE(patch.add[1].getFrame(), sv_frame_t(1100)); + } + + void pitchDoesNotInterpolateAcrossALargeGap() { + + OverlapProcessor p; + + EventVector existing { pitch(0, 440.f) }; + EventVector incoming { pitch(1100, 445.f) }; // gap of 1100 frames + + auto patch = p.processPitchEvents(1000, incoming, existing); + + QCOMPARE(int(patch.add.size()), 1); + QCOMPARE(patch.add[0].getFrame(), sv_frame_t(1100)); + } + + void pitchDoesNotInterpolateAcrossADissimilarPitch() { + + OverlapProcessor p; + + EventVector existing { pitch(900, 440.f) }; + EventVector incoming { pitch(1100, 880.f) }; // an octave up + + auto patch = p.processPitchEvents(1000, incoming, existing); + + QCOMPARE(int(patch.add.size()), 1); + QCOMPARE(patch.add[0].getFrame(), sv_frame_t(1100)); + } + + void pitchSeamUsesLatestEventBeforeContextStart() { + + // The seam must be measured from the last event before the + // analysed region, whichever order the existing events arrive in + EventVector existing { + pitch(100, 300.f), + pitch(900, 440.f), + pitch(500, 350.f) + }; + + OverlapProcessor p; + EventVector incoming { pitch(1100, 445.f) }; + + auto patch = p.processPitchEvents(1000, incoming, existing); + + QCOMPARE(int(patch.add.size()), 2); + QCOMPARE(patch.add[0].getFrame(), sv_frame_t(1000)); + QCOMPARE(patch.add[0].getValue(), (440.f + 445.f) / 2.f); + } + + void pitchEmptyIncomingStillClearsTheRegion() { + + // An analysis that found no pitch at all should clear whatever + // the previous chunk left behind in that region + OverlapProcessor p; + + EventVector existing { pitch(0, 400.f), pitch(1000, 401.f) }; + + auto patch = p.processPitchEvents(1000, EventVector(), existing); + + QCOMPARE(int(patch.remove.size()), 1); + QCOMPARE(patch.remove[0].getFrame(), sv_frame_t(1000)); + QCOMPARE(int(patch.add.size()), 0); + } + + void pitchRepeatedChunksDoNotAccumulate() { + + OverlapProcessor p; + + EventVector incoming { pitch(1000, 440.f), pitch(1100, 441.f) }; + + EventVector model; + model = applyPatch(model, p.processPitchEvents(1000, incoming, model)); + const int afterFirst = int(model.size()); + + for (int i = 0; i < 5; ++i) { + model = applyPatch(model, + p.processPitchEvents(1000, incoming, model)); + } + + QCOMPARE(int(model.size()), afterFirst); + } +}; + +#endif diff --git a/main/test/tony-main-test.cpp b/main/test/tony-main-test.cpp new file mode 100644 index 00000000..2b57fce9 --- /dev/null +++ b/main/test/tony-main-test.cpp @@ -0,0 +1,46 @@ +/* -*- c-basic-offset: 4 indent-tabs-mode: nil -*- vi:set ts=8 sts=4 sw=4: */ + +/* + Tony + An intonation analysis and annotation tool + Centre for Digital Music, Queen Mary, University of London. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. See the file + COPYING included with this distribution for more information. +*/ + +#include "TestOverlapProcessor.h" + +#include "system/Init.h" + +#include + +int main(int argc, char *argv[]) +{ + int good = 0, bad = 0; + + svSystemSpecificInitialisation(); + + QCoreApplication app(argc, argv); + app.setOrganizationName("sonic-visualiser"); + app.setApplicationName("test-tony-main"); + + { + TestOverlapProcessor t; + if (QTest::qExec(&t, argc, argv) == 0) ++good; + else ++bad; + } + + (void)good; + + if (bad > 0) { + SVCERR << "\n********* " << bad << " test suite(s) failed!\n" << endl; + return 1; + } else { + SVCERR << "All tests passed" << endl; + return 0; + } +} diff --git a/meson.build b/meson.build index 41c9392d..0a73137a 100644 --- a/meson.build +++ b/meson.build @@ -757,6 +757,11 @@ svcore_data_fileio_test_moc_files = qt.preprocess( 'svcore/data/fileio/test/CSVStreamWriterTest.h', ]) +tony_main_test_moc_files = qt.preprocess( + moc_headers: [ + 'main/test/TestOverlapProcessor.h', +]) + svgui_files = [ 'svgui/layer/Colour3DPlotExporter.cpp', 'svgui/layer/Colour3DPlotLayer.cpp', @@ -1021,6 +1026,8 @@ chp_plugin = shared_library( tony_main_files = [ 'main/main.cpp', 'main/Analyser.cpp', + 'main/RealtimeAnalyser.cpp', + 'main/OverlapProcessor.cpp', 'main/MainWindow.cpp', 'main/NetworkPermissionTester.cpp', ] @@ -1029,6 +1036,7 @@ tony_main_moc_files = qt.preprocess( moc_headers: [ 'main/MainWindow.h', 'main/Analyser.h', + 'main/RealtimeAnalyser.h', ]) qt_resource_files = qt.preprocess( @@ -1199,6 +1207,31 @@ svcore_data_fileio_test_exe = executable( win_subsystem: 'console' ) +tony_main_test_exe = executable( + 'test-tony-main', + tony_main_test_moc_files, + 'main/OverlapProcessor.cpp', + 'main/test/tony-main-test.cpp', + include_directories: [ + 'main', + ], + dependencies: [ + svcore_dep, + qt_dep, + feature_dependencies, + dl_dep, + ], + cpp_args: [ + feature_defines, + general_defines, + ], + link_args: [ + feature_additional_libs, + general_link_args, + ], + win_subsystem: 'console' +) + test('svcore-base', svcore_base_test_exe) test('svcore-system', svcore_system_test_exe) test('svcore-data-model', svcore_data_model_test_exe) @@ -1206,6 +1239,7 @@ test('svcore-data-fileio', svcore_data_fileio_test_exe, args: [ '--testdir', meson.current_source_dir() / 'svcore/data/fileio/test' ]) +test('tony-main', tony_main_test_exe) summary({'prefix': get_option('prefix'), 'bindir': get_option('bindir'),