From ed0c18ab9a3b98abc4fd70d30af301c1dcf51ee2 Mon Sep 17 00:00:00 2001 From: Michal Dojtrowski <220743396+michaldojtrowski@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:21:04 +0100 Subject: [PATCH 1/3] Stop microphone crackle by counting samples during a take The writer trimmed or zero-padded each capture buffer to follow its measured time, clock minus the reported latency. That measurement moves by 10 to 24 ms in the seconds after the stream opens and keeps wandering by a few milliseconds, so the writer edited a few samples at buffer boundaries many times a second, which is heard as crackle. AudioTimeline places buffers on one continuous timeline. Between takes it follows the smoothed measurement; once a take starts it counts samples. An offset above 20 ms is walked back one frame at a time, and three readings more than 50 ms off restart the timeline. The writer ignores a one-frame rounding difference. Co-Authored-By: Claude Opus 5.5 --- src/audiocapture.cpp | 9 +++++- src/audiocapture.h | 65 ++++++++++++++++++++++++++++++++++++++++ src/backend.cpp | 1 + src/writer.cpp | 6 ++-- tests/backend_tests.cpp | 66 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 144 insertions(+), 3 deletions(-) diff --git a/src/audiocapture.cpp b/src/audiocapture.cpp index 8886f3d..87653c7 100644 --- a/src/audiocapture.cpp +++ b/src/audiocapture.cpp @@ -14,6 +14,7 @@ AudioCapture::~AudioCapture() { stop(); } void AudioCapture::start(const QByteArray &device, const QAudioFormat &format) { m_device = device; m_format = format; + m_timeline = AudioTimeline(format.sampleRate()); m_loop = pa_mainloop_new(); if (!m_loop) { fail(tr("Could not create the audio capture loop.")); return; } m_context = pa_context_new(pa_mainloop_get_api(m_loop), "Monologue"); @@ -115,7 +116,13 @@ void AudioCapture::readAvailable() { QByteArray pcm = data ? QByteArray(static_cast(data), qsizetype(bytes)) : QByteArray(qsizetype(bytes), m_format.sampleFormat() == QAudioFormat::UInt8 ? char(128) : char(0)); if (pa_stream_drop(m_stream) < 0) { fail(tr("Could not advance microphone capture.")); break; } - emit samples(pcm, capturedAt); + const int frameBytes = m_format.bytesPerFrame(); + const auto placed = m_timeline.place(capturedAt, qint64(bytes) / frameBytes); + if (placed.resynced) qCDebug(audioTiming) << "Microphone timeline restarted at" << capturedAt; + if (placed.slip) qCDebug(audioTiming) << "Following microphone clock drift by" << placed.slip << "frame"; + if (placed.slip > 0) pcm.append(pcm.right(frameBytes)); + else if (placed.slip < 0) pcm.chop(frameBytes); + emit samples(pcm, placed.capturedAt); } m_reading = false; } diff --git a/src/audiocapture.h b/src/audiocapture.h index 7a6e785..e8581ae 100644 --- a/src/audiocapture.h +++ b/src/audiocapture.h @@ -3,8 +3,70 @@ #include #include #include +#include +#include #include +// Places capture buffers on one continuous timeline. A buffer's measured time +// (clock minus reported latency) is noisy: it jitters between buffers, wanders +// by a few milliseconds over tens of seconds, and takes about half a minute to +// settle after the stream opens. A writer that follows it trims or zero-pads +// at buffer boundaries, which is heard as crackle. The sample count is steady, +// so between takes the timeline follows the smoothed measurement, and from +// lock() on it only counts samples. An offset that outgrows the tolerance, from +// clock drift or a few lost samples, is then walked back one frame at a time. +class AudioTimeline { +public: + struct Placement { + qint64 capturedAt; // Time of the buffer's first sample. + int slip; // +1: repeat one frame, -1: drop one, to follow drift. + bool resynced; // The measurement left the timeline; it starts over here. + }; + explicit AudioTimeline(int sampleRate = 48000) : m_rate(sampleRate) {} + void lock() { m_locked = true; m_error = 0; m_following = false; } + Placement place(qint64 measured, qint64 frames) { + const double period = 1000000.0 / m_rate; + const double error = measured - (m_origin + m_frames * period); + Placement placed{0, 0, false}; + // One far reading is noise. Several in a row are a real discontinuity, + // such as dropped samples or a stalled device. + m_far = std::abs(error) > Resync ? m_far + 1 : 0; + if (!m_started || m_far >= Confirm) { + placed.resynced = m_started; + m_started = true; + m_origin = double(measured) - m_frames * period; + m_buffers = 0; m_error = 0; m_far = 0; m_following = false; + } else if (m_far) { + // Not confirmed yet: place it by count and wait for the next reading. + } else if (!m_locked) { + // A plain mean at first, so one early reading does not set the origin. + m_origin += error * std::max(frames * period / Smoothing, 1.0 / ++m_buffers); + } else { + m_error += (error - m_error) * std::min(1.0, frames * period / Smoothing); + // Follow all the way back, so the offset does not rest at the tolerance. + m_following = std::abs(m_error) > (m_following ? period : Tolerance); + if (frames > 1 && m_following) { + placed.slip = m_error > 0 ? 1 : -1; + m_error -= placed.slip * period; + } + } + placed.capturedAt = qRound64(m_origin + m_frames * period); + m_frames += frames + placed.slip; + return placed; + } +private: + // Microseconds. The tolerance sits above the settling and wander of the + // reported latency, and below an offset anyone would notice. A restart + // takes an offset no one should have to hear while it is walked back. + static constexpr double Resync = 50000, Smoothing = 5000000, Tolerance = 20000; + static constexpr int Confirm = 3; + int m_rate; + bool m_started = false, m_locked = false, m_following = false; + qint64 m_frames = 0, m_buffers = 0; + int m_far = 0; + double m_origin = 0, m_error = 0; +}; + // PulseAudio's capture latency includes unread/server/device buffering. Qt's // QAudioSource only exposes samples and delivery time, losing that information. // PipeWire's PulseAudio server implements the same timestamped capture API. @@ -16,6 +78,8 @@ class AudioCapture : public QObject { void start(const QByteArray &device, const QAudioFormat &format); void stop(); void readAvailable(); + // A take starts: from here on buffer times are contiguous by sample count. + void lockTimeline() { m_timeline.lock(); } static qint64 captureTime(qint64 deliveredAt, qint64 latency, bool negative) { return deliveredAt + (negative ? latency : -latency); } @@ -30,6 +94,7 @@ class AudioCapture : public QObject { std::function m_clock; QTimer m_pump; QAudioFormat m_format; + AudioTimeline m_timeline; QByteArray m_device; pa_mainloop *m_loop = nullptr; pa_context *m_context = nullptr; diff --git a/src/backend.cpp b/src/backend.cpp index cc3170a..4946b7e 100644 --- a/src/backend.cpp +++ b/src/backend.cpp @@ -285,6 +285,7 @@ void Backend::startTake() { m_clipPath=directory+"/"+m_clipFileName; m_interruption.clear(); m_duration=0; readAudio(); + if(m_audio) m_audio->lockTimeline(); m_writer=new Writer(this); connect(m_writer,&Writer::finished,this,&Backend::writerFinished,Qt::QueuedConnection); connect(m_writer,&Writer::failed,this,[this](const QString &error) { m_interruption=error; m_message=error; m_state="finalizing"; emit changed(); }); diff --git a/src/writer.cpp b/src/writer.cpp index 6fa6d6c..c765db9 100644 --- a/src/writer.cpp +++ b/src/writer.cpp @@ -336,10 +336,12 @@ void Writer::audio(QByteArray data, qint64 capturedAt) { // Qt counts audio samples rather than honoring PTS. Materialize capture // gaps as silence and trim overlaps, keeping the original event times. const qint64 desiredFrame = qRound64(position * m_audioFormat.sampleRate() / 1000000.0); - if (desiredFrame < m_audioFramesWritten) { + // Capture times are whole microseconds, so a contiguous buffer can round + // one frame to either side of the running count. That is neither. + if (desiredFrame < m_audioFramesWritten - 1) { const auto overlap = (m_audioFramesWritten - desiredFrame) * frameBytes; part.remove(0, std::min(qsizetype(part.size()), qsizetype(overlap))); - } else padAudioTo(position); + } else if (desiredFrame > m_audioFramesWritten + 1) padAudioTo(position); if (!part.isEmpty()) appendAudio(part); } } diff --git a/tests/backend_tests.cpp b/tests/backend_tests.cpp index 78f145e..36d8b08 100644 --- a/tests/backend_tests.cpp +++ b/tests/backend_tests.cpp @@ -13,12 +13,14 @@ #include #include #include +#include #include #include #include #include "mediautils.h" #include "theme.h" #include "writer.h" +#include "audiocapture.h" #include "backend.h" #include "translator.h" @@ -572,6 +574,70 @@ private slots: QVERIFY(probe.waitForFinished()); QCOMPARE(QString::fromUtf8(probe.readAllStandardOutput()).trimmed(),QString("yuv420p,bt709")); } + void microphoneTimelineCountsSamplesDuringATake() { + const double period=1000000.0/48000; + QRandomGenerator random(7); + const auto jitter=[&random] { return qint64(random.bounded(601))-300; }; + // Before the take the timeline settles on the measurements, even when + // the first reading is 15 ms off. During it, jitter and a few + // milliseconds of wander move nothing: times are contiguous by count. + { + AudioTimeline timeline(48000); + timeline.place(-15000,960); + qint64 last=0; + for(int i=1;i<500;++i) last=timeline.place(i*20000LL+jitter(),960).capturedAt; + QVERIFY(std::abs(last-499*20000LL)<1000); + timeline.lock(); + qint64 first=0; + for(int i=0;i<3000;++i) { + const qint64 wander=qRound64(4000*std::sin(i/500.0)); + const auto placed=timeline.place((500+i)*20000LL+wander+jitter(),960); + if(i==0) first=placed.capturedAt; + QCOMPARE(placed.slip,0); QVERIFY(!placed.resynced); + QCOMPARE(placed.capturedAt,first+i*20000LL); + } + // One far reading is noise and moves nothing. Three in a row are a + // real gap, and restart the timeline at the measurement. + QCOMPARE(timeline.place(3500*20000LL+70000,960).capturedAt,first+3000*20000LL); + QCOMPARE(timeline.place(3501*20000LL,960).capturedAt,first+3001*20000LL); + for(int i=3502;i<3504;++i) QVERIFY(!timeline.place(i*20000LL+300000,960).resynced); + const auto gap=timeline.place(3504*20000LL+300000,960); + QVERIFY(gap.resynced); QCOMPARE(gap.capturedAt,3504*20000LL+300000); + } + // Thirty milliseconds of lost samples, too few for a restart, are walked + // back by single frames instead of being left as an offset. + { + AudioTimeline timeline(48000); + timeline.lock(); + qint64 last=0; + for(int i=0;i<3000;++i) { + const auto placed=timeline.place(i*20000LL+(i<50?0:30000)+jitter(),960); + QVERIFY(!placed.resynced); QVERIFY(placed.slip>=0); + last=placed.capturedAt; + } + QVERIFY2(std::abs(last-(2999*20000LL+30000))<1000,qPrintable(QString::number(last))); + } + // Thirty minutes from a microphone running 100 ppm slow: each time the + // drift outgrows the 20 ms tolerance it is walked back by single frames. + AudioTimeline timeline(48000); + timeline.lock(); + qint64 origin=0, written=0, slips=0; + bool following=false; + for(int i=0;i<90000;++i) { + const double trueTime=1000000+i*960*period*1.0001; + const auto placed=timeline.place(qRound64(trueTime)+jitter(),960); + if(i==0) origin=placed.capturedAt; + QVERIFY(!placed.resynced); + QVERIFY(std::abs(placed.capturedAt-(origin+written*period))<1); // Rounding only. + QVERIFY(std::abs(placed.capturedAt-trueTime)<22000); + QVERIFY(placed.slip>=0); + if(following && !placed.slip) QVERIFY(std::abs(placed.capturedAt-trueTime)<2000); // All the way back. + following=placed.slip; + slips+=placed.slip; written+=960+placed.slip; + } + // 180 ms of drift, less whatever has accumulated since the last walk back. + QVERIFY2(slips>8640-960-150 && slips<=8640,qPrintable(QString::number(slips))); + } void delayedCaptureStaysInSync() { QTemporaryDir directory; const auto path=directory.filePath("sync.mp4"); From b8666d100bc2a8ed0020a7f379d4fac31c3692a4 Mon Sep 17 00:00:00 2001 From: Michal Dojtrowski <220743396+michaldojtrowski@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:56:10 +0100 Subject: [PATCH 2/3] Test the timeline against a fast microphone too The drift test only covered a microphone running slow, where frames are repeated. Run it in both directions, so the path that drops frames for a fast microphone is checked the same way. Co-Authored-By: Claude Fable 5.2 --- tests/backend_tests.cpp | 42 ++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/tests/backend_tests.cpp b/tests/backend_tests.cpp index 36d8b08..269831c 100644 --- a/tests/backend_tests.cpp +++ b/tests/backend_tests.cpp @@ -617,26 +617,30 @@ private slots: } QVERIFY2(std::abs(last-(2999*20000LL+30000))<1000,qPrintable(QString::number(last))); } - // Thirty minutes from a microphone running 100 ppm slow: each time the - // drift outgrows the 20 ms tolerance it is walked back by single frames. - AudioTimeline timeline(48000); - timeline.lock(); - qint64 origin=0, written=0, slips=0; - bool following=false; - for(int i=0;i<90000;++i) { - const double trueTime=1000000+i*960*period*1.0001; - const auto placed=timeline.place(qRound64(trueTime)+jitter(),960); - if(i==0) origin=placed.capturedAt; - QVERIFY(!placed.resynced); - QVERIFY(std::abs(placed.capturedAt-(origin+written*period))<1); // Rounding only. - QVERIFY(std::abs(placed.capturedAt-trueTime)<22000); - QVERIFY(placed.slip>=0); - if(following && !placed.slip) QVERIFY(std::abs(placed.capturedAt-trueTime)<2000); // All the way back. - following=placed.slip; - slips+=placed.slip; written+=960+placed.slip; + // Thirty minutes from a microphone running 100 ppm slow, then from one + // running 100 ppm fast: each time the drift outgrows the 20 ms tolerance + // it is walked back by single frames, repeated for the slow microphone + // and dropped for the fast one. + for(const int direction:{1,-1}) { + AudioTimeline timeline(48000); + timeline.lock(); + qint64 origin=0, written=0, slips=0; + bool following=false; + for(int i=0;i<90000;++i) { + const double trueTime=1000000+i*960*period*(1+direction*0.0001); + const auto placed=timeline.place(qRound64(trueTime)+jitter(),960); + if(i==0) origin=placed.capturedAt; + QVERIFY(!placed.resynced); + QVERIFY(std::abs(placed.capturedAt-(origin+written*period))<1); // Rounding only. + QVERIFY(std::abs(placed.capturedAt-trueTime)<22000); + QVERIFY(placed.slip*direction>=0); + if(following && !placed.slip) QVERIFY(std::abs(placed.capturedAt-trueTime)<2000); // All the way back. + following=placed.slip; + slips+=placed.slip*direction; written+=960+placed.slip; + } + // 180 ms of drift, less whatever has accumulated since the last walk back. + QVERIFY2(slips>8640-960-150 && slips<=8640,qPrintable(QString::number(slips*direction))); } - // 180 ms of drift, less whatever has accumulated since the last walk back. - QVERIFY2(slips>8640-960-150 && slips<=8640,qPrintable(QString::number(slips))); } void delayedCaptureStaysInSync() { QTemporaryDir directory; From e2f26b0df95cb4b8314026a101e19dd047e8d82f Mon Sep 17 00:00:00 2001 From: Michal Dojtrowski <220743396+michaldojtrowski@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:46:50 +0100 Subject: [PATCH 3/3] Count the first reading in the timeline's starting mean The buffer count started at zero after the first reading, so the second reading replaced the origin outright instead of being averaged with the first. Start it at one, so the origin is the plain mean the comment describes. Co-Authored-By: Claude Fable 5.2 --- src/audiocapture.h | 2 +- tests/backend_tests.cpp | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/audiocapture.h b/src/audiocapture.h index e8581ae..c1dda21 100644 --- a/src/audiocapture.h +++ b/src/audiocapture.h @@ -35,7 +35,7 @@ class AudioTimeline { placed.resynced = m_started; m_started = true; m_origin = double(measured) - m_frames * period; - m_buffers = 0; m_error = 0; m_far = 0; m_following = false; + m_buffers = 1; m_error = 0; m_far = 0; m_following = false; } else if (m_far) { // Not confirmed yet: place it by count and wait for the next reading. } else if (!m_locked) { diff --git a/tests/backend_tests.cpp b/tests/backend_tests.cpp index 269831c..63bc95d 100644 --- a/tests/backend_tests.cpp +++ b/tests/backend_tests.cpp @@ -578,6 +578,14 @@ private slots: const double period=1000000.0/48000; QRandomGenerator random(7); const auto jitter=[&random] { return qint64(random.bounded(601))-300; }; + // Before the take the origin is a plain mean of the readings, the first + // one included: after 0, +3 ms and +3 ms it sits at their mean. + { + AudioTimeline timeline(48000); + QCOMPARE(timeline.place(0,960).capturedAt,0LL); + QCOMPARE(timeline.place(20000+3000,960).capturedAt,20000LL+1500); + QCOMPARE(timeline.place(40000+3000,960).capturedAt,40000LL+2000); + } // Before the take the timeline settles on the measurements, even when // the first reading is 15 ms off. During it, jitter and a few // milliseconds of wander move nothing: times are contiguous by count.