diff --git a/src/audiocapture.cpp b/src/audiocapture.cpp index 8886f3d..87653c7 100644 --- a/src/audiocapture.cpp +++ b/src/audiocapture.cpp @@ -14,6 +14,7 @@ AudioCapture::~AudioCapture() { stop(); } void AudioCapture::start(const QByteArray &device, const QAudioFormat &format) { m_device = device; m_format = format; + m_timeline = AudioTimeline(format.sampleRate()); m_loop = pa_mainloop_new(); if (!m_loop) { fail(tr("Could not create the audio capture loop.")); return; } m_context = pa_context_new(pa_mainloop_get_api(m_loop), "Monologue"); @@ -115,7 +116,13 @@ void AudioCapture::readAvailable() { QByteArray pcm = data ? QByteArray(static_cast(data), qsizetype(bytes)) : QByteArray(qsizetype(bytes), m_format.sampleFormat() == QAudioFormat::UInt8 ? char(128) : char(0)); if (pa_stream_drop(m_stream) < 0) { fail(tr("Could not advance microphone capture.")); break; } - emit samples(pcm, capturedAt); + const int frameBytes = m_format.bytesPerFrame(); + const auto placed = m_timeline.place(capturedAt, qint64(bytes) / frameBytes); + if (placed.resynced) qCDebug(audioTiming) << "Microphone timeline restarted at" << capturedAt; + if (placed.slip) qCDebug(audioTiming) << "Following microphone clock drift by" << placed.slip << "frame"; + if (placed.slip > 0) pcm.append(pcm.right(frameBytes)); + else if (placed.slip < 0) pcm.chop(frameBytes); + emit samples(pcm, placed.capturedAt); } m_reading = false; } diff --git a/src/audiocapture.h b/src/audiocapture.h index 7a6e785..c1dda21 100644 --- a/src/audiocapture.h +++ b/src/audiocapture.h @@ -3,8 +3,70 @@ #include #include #include +#include +#include #include +// Places capture buffers on one continuous timeline. A buffer's measured time +// (clock minus reported latency) is noisy: it jitters between buffers, wanders +// by a few milliseconds over tens of seconds, and takes about half a minute to +// settle after the stream opens. A writer that follows it trims or zero-pads +// at buffer boundaries, which is heard as crackle. The sample count is steady, +// so between takes the timeline follows the smoothed measurement, and from +// lock() on it only counts samples. An offset that outgrows the tolerance, from +// clock drift or a few lost samples, is then walked back one frame at a time. +class AudioTimeline { +public: + struct Placement { + qint64 capturedAt; // Time of the buffer's first sample. + int slip; // +1: repeat one frame, -1: drop one, to follow drift. + bool resynced; // The measurement left the timeline; it starts over here. + }; + explicit AudioTimeline(int sampleRate = 48000) : m_rate(sampleRate) {} + void lock() { m_locked = true; m_error = 0; m_following = false; } + Placement place(qint64 measured, qint64 frames) { + const double period = 1000000.0 / m_rate; + const double error = measured - (m_origin + m_frames * period); + Placement placed{0, 0, false}; + // One far reading is noise. Several in a row are a real discontinuity, + // such as dropped samples or a stalled device. + m_far = std::abs(error) > Resync ? m_far + 1 : 0; + if (!m_started || m_far >= Confirm) { + placed.resynced = m_started; + m_started = true; + m_origin = double(measured) - m_frames * period; + m_buffers = 1; m_error = 0; m_far = 0; m_following = false; + } else if (m_far) { + // Not confirmed yet: place it by count and wait for the next reading. + } else if (!m_locked) { + // A plain mean at first, so one early reading does not set the origin. + m_origin += error * std::max(frames * period / Smoothing, 1.0 / ++m_buffers); + } else { + m_error += (error - m_error) * std::min(1.0, frames * period / Smoothing); + // Follow all the way back, so the offset does not rest at the tolerance. + m_following = std::abs(m_error) > (m_following ? period : Tolerance); + if (frames > 1 && m_following) { + placed.slip = m_error > 0 ? 1 : -1; + m_error -= placed.slip * period; + } + } + placed.capturedAt = qRound64(m_origin + m_frames * period); + m_frames += frames + placed.slip; + return placed; + } +private: + // Microseconds. The tolerance sits above the settling and wander of the + // reported latency, and below an offset anyone would notice. A restart + // takes an offset no one should have to hear while it is walked back. + static constexpr double Resync = 50000, Smoothing = 5000000, Tolerance = 20000; + static constexpr int Confirm = 3; + int m_rate; + bool m_started = false, m_locked = false, m_following = false; + qint64 m_frames = 0, m_buffers = 0; + int m_far = 0; + double m_origin = 0, m_error = 0; +}; + // PulseAudio's capture latency includes unread/server/device buffering. Qt's // QAudioSource only exposes samples and delivery time, losing that information. // PipeWire's PulseAudio server implements the same timestamped capture API. @@ -16,6 +78,8 @@ class AudioCapture : public QObject { void start(const QByteArray &device, const QAudioFormat &format); void stop(); void readAvailable(); + // A take starts: from here on buffer times are contiguous by sample count. + void lockTimeline() { m_timeline.lock(); } static qint64 captureTime(qint64 deliveredAt, qint64 latency, bool negative) { return deliveredAt + (negative ? latency : -latency); } @@ -30,6 +94,7 @@ class AudioCapture : public QObject { std::function m_clock; QTimer m_pump; QAudioFormat m_format; + AudioTimeline m_timeline; QByteArray m_device; pa_mainloop *m_loop = nullptr; pa_context *m_context = nullptr; diff --git a/src/backend.cpp b/src/backend.cpp index cc3170a..4946b7e 100644 --- a/src/backend.cpp +++ b/src/backend.cpp @@ -285,6 +285,7 @@ void Backend::startTake() { m_clipPath=directory+"/"+m_clipFileName; m_interruption.clear(); m_duration=0; readAudio(); + if(m_audio) m_audio->lockTimeline(); m_writer=new Writer(this); connect(m_writer,&Writer::finished,this,&Backend::writerFinished,Qt::QueuedConnection); connect(m_writer,&Writer::failed,this,[this](const QString &error) { m_interruption=error; m_message=error; m_state="finalizing"; emit changed(); }); diff --git a/src/writer.cpp b/src/writer.cpp index 6fa6d6c..c765db9 100644 --- a/src/writer.cpp +++ b/src/writer.cpp @@ -336,10 +336,12 @@ void Writer::audio(QByteArray data, qint64 capturedAt) { // Qt counts audio samples rather than honoring PTS. Materialize capture // gaps as silence and trim overlaps, keeping the original event times. const qint64 desiredFrame = qRound64(position * m_audioFormat.sampleRate() / 1000000.0); - if (desiredFrame < m_audioFramesWritten) { + // Capture times are whole microseconds, so a contiguous buffer can round + // one frame to either side of the running count. That is neither. + if (desiredFrame < m_audioFramesWritten - 1) { const auto overlap = (m_audioFramesWritten - desiredFrame) * frameBytes; part.remove(0, std::min(qsizetype(part.size()), qsizetype(overlap))); - } else padAudioTo(position); + } else if (desiredFrame > m_audioFramesWritten + 1) padAudioTo(position); if (!part.isEmpty()) appendAudio(part); } } diff --git a/tests/backend_tests.cpp b/tests/backend_tests.cpp index 78f145e..63bc95d 100644 --- a/tests/backend_tests.cpp +++ b/tests/backend_tests.cpp @@ -13,12 +13,14 @@ #include #include #include +#include #include #include #include #include "mediautils.h" #include "theme.h" #include "writer.h" +#include "audiocapture.h" #include "backend.h" #include "translator.h" @@ -572,6 +574,82 @@ private slots: QVERIFY(probe.waitForFinished()); QCOMPARE(QString::fromUtf8(probe.readAllStandardOutput()).trimmed(),QString("yuv420p,bt709")); } + void microphoneTimelineCountsSamplesDuringATake() { + const double period=1000000.0/48000; + QRandomGenerator random(7); + const auto jitter=[&random] { return qint64(random.bounded(601))-300; }; + // Before the take the origin is a plain mean of the readings, the first + // one included: after 0, +3 ms and +3 ms it sits at their mean. + { + AudioTimeline timeline(48000); + QCOMPARE(timeline.place(0,960).capturedAt,0LL); + QCOMPARE(timeline.place(20000+3000,960).capturedAt,20000LL+1500); + QCOMPARE(timeline.place(40000+3000,960).capturedAt,40000LL+2000); + } + // Before the take the timeline settles on the measurements, even when + // the first reading is 15 ms off. During it, jitter and a few + // milliseconds of wander move nothing: times are contiguous by count. + { + AudioTimeline timeline(48000); + timeline.place(-15000,960); + qint64 last=0; + for(int i=1;i<500;++i) last=timeline.place(i*20000LL+jitter(),960).capturedAt; + QVERIFY(std::abs(last-499*20000LL)<1000); + timeline.lock(); + qint64 first=0; + for(int i=0;i<3000;++i) { + const qint64 wander=qRound64(4000*std::sin(i/500.0)); + const auto placed=timeline.place((500+i)*20000LL+wander+jitter(),960); + if(i==0) first=placed.capturedAt; + QCOMPARE(placed.slip,0); QVERIFY(!placed.resynced); + QCOMPARE(placed.capturedAt,first+i*20000LL); + } + // One far reading is noise and moves nothing. Three in a row are a + // real gap, and restart the timeline at the measurement. + QCOMPARE(timeline.place(3500*20000LL+70000,960).capturedAt,first+3000*20000LL); + QCOMPARE(timeline.place(3501*20000LL,960).capturedAt,first+3001*20000LL); + for(int i=3502;i<3504;++i) QVERIFY(!timeline.place(i*20000LL+300000,960).resynced); + const auto gap=timeline.place(3504*20000LL+300000,960); + QVERIFY(gap.resynced); QCOMPARE(gap.capturedAt,3504*20000LL+300000); + } + // Thirty milliseconds of lost samples, too few for a restart, are walked + // back by single frames instead of being left as an offset. + { + AudioTimeline timeline(48000); + timeline.lock(); + qint64 last=0; + for(int i=0;i<3000;++i) { + const auto placed=timeline.place(i*20000LL+(i<50?0:30000)+jitter(),960); + QVERIFY(!placed.resynced); QVERIFY(placed.slip>=0); + last=placed.capturedAt; + } + QVERIFY2(std::abs(last-(2999*20000LL+30000))<1000,qPrintable(QString::number(last))); + } + // Thirty minutes from a microphone running 100 ppm slow, then from one + // running 100 ppm fast: each time the drift outgrows the 20 ms tolerance + // it is walked back by single frames, repeated for the slow microphone + // and dropped for the fast one. + for(const int direction:{1,-1}) { + AudioTimeline timeline(48000); + timeline.lock(); + qint64 origin=0, written=0, slips=0; + bool following=false; + for(int i=0;i<90000;++i) { + const double trueTime=1000000+i*960*period*(1+direction*0.0001); + const auto placed=timeline.place(qRound64(trueTime)+jitter(),960); + if(i==0) origin=placed.capturedAt; + QVERIFY(!placed.resynced); + QVERIFY(std::abs(placed.capturedAt-(origin+written*period))<1); // Rounding only. + QVERIFY(std::abs(placed.capturedAt-trueTime)<22000); + QVERIFY(placed.slip*direction>=0); + if(following && !placed.slip) QVERIFY(std::abs(placed.capturedAt-trueTime)<2000); // All the way back. + following=placed.slip; + slips+=placed.slip*direction; written+=960+placed.slip; + } + // 180 ms of drift, less whatever has accumulated since the last walk back. + QVERIFY2(slips>8640-960-150 && slips<=8640,qPrintable(QString::number(slips*direction))); + } + } void delayedCaptureStaysInSync() { QTemporaryDir directory; const auto path=directory.filePath("sync.mp4");