Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion src/audiocapture.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ AudioCapture::~AudioCapture() { stop(); }
void AudioCapture::start(const QByteArray &device, const QAudioFormat &format) {
m_device = device;
m_format = format;
m_timeline = AudioTimeline(format.sampleRate());
m_loop = pa_mainloop_new();
if (!m_loop) { fail(tr("Could not create the audio capture loop.")); return; }
m_context = pa_context_new(pa_mainloop_get_api(m_loop), "Monologue");
Expand Down Expand Up @@ -115,7 +116,13 @@ void AudioCapture::readAvailable() {
QByteArray pcm = data ? QByteArray(static_cast<const char*>(data), qsizetype(bytes))
: QByteArray(qsizetype(bytes), m_format.sampleFormat() == QAudioFormat::UInt8 ? char(128) : char(0));
if (pa_stream_drop(m_stream) < 0) { fail(tr("Could not advance microphone capture.")); break; }
emit samples(pcm, capturedAt);
const int frameBytes = m_format.bytesPerFrame();
const auto placed = m_timeline.place(capturedAt, qint64(bytes) / frameBytes);
if (placed.resynced) qCDebug(audioTiming) << "Microphone timeline restarted at" << capturedAt;
if (placed.slip) qCDebug(audioTiming) << "Following microphone clock drift by" << placed.slip << "frame";
if (placed.slip > 0) pcm.append(pcm.right(frameBytes));
else if (placed.slip < 0) pcm.chop(frameBytes);
emit samples(pcm, placed.capturedAt);
}
m_reading = false;
}
Expand Down
65 changes: 65 additions & 0 deletions src/audiocapture.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,70 @@
#include <QAudioFormat>
#include <QTimer>
#include <functional>
#include <algorithm>
#include <cmath>
#include <pulse/pulseaudio.h>

// Places capture buffers on one continuous timeline. A buffer's measured time
// (clock minus reported latency) is noisy: it jitters between buffers, wanders
// by a few milliseconds over tens of seconds, and takes about half a minute to
// settle after the stream opens. A writer that follows it trims or zero-pads
// at buffer boundaries, which is heard as crackle. The sample count is steady,
// so between takes the timeline follows the smoothed measurement, and from
// lock() on it only counts samples. An offset that outgrows the tolerance, from
// clock drift or a few lost samples, is then walked back one frame at a time.
class AudioTimeline {
public:
struct Placement {
qint64 capturedAt; // Time of the buffer's first sample.
int slip; // +1: repeat one frame, -1: drop one, to follow drift.
bool resynced; // The measurement left the timeline; it starts over here.
};
explicit AudioTimeline(int sampleRate = 48000) : m_rate(sampleRate) {}
void lock() { m_locked = true; m_error = 0; m_following = false; }
Placement place(qint64 measured, qint64 frames) {
const double period = 1000000.0 / m_rate;
const double error = measured - (m_origin + m_frames * period);
Placement placed{0, 0, false};
// One far reading is noise. Several in a row are a real discontinuity,
// such as dropped samples or a stalled device.
m_far = std::abs(error) > Resync ? m_far + 1 : 0;
if (!m_started || m_far >= Confirm) {
placed.resynced = m_started;
m_started = true;
m_origin = double(measured) - m_frames * period;
m_buffers = 1; m_error = 0; m_far = 0; m_following = false;
} else if (m_far) {
// Not confirmed yet: place it by count and wait for the next reading.
} else if (!m_locked) {
// A plain mean at first, so one early reading does not set the origin.
m_origin += error * std::max(frames * period / Smoothing, 1.0 / ++m_buffers);
} else {
m_error += (error - m_error) * std::min(1.0, frames * period / Smoothing);
// Follow all the way back, so the offset does not rest at the tolerance.
m_following = std::abs(m_error) > (m_following ? period : Tolerance);
if (frames > 1 && m_following) {
placed.slip = m_error > 0 ? 1 : -1;
m_error -= placed.slip * period;
}
}
placed.capturedAt = qRound64(m_origin + m_frames * period);
m_frames += frames + placed.slip;
return placed;
}
private:
// Microseconds. The tolerance sits above the settling and wander of the
// reported latency, and below an offset anyone would notice. A restart
// takes an offset no one should have to hear while it is walked back.
static constexpr double Resync = 50000, Smoothing = 5000000, Tolerance = 20000;
static constexpr int Confirm = 3;
int m_rate;
bool m_started = false, m_locked = false, m_following = false;
qint64 m_frames = 0, m_buffers = 0;
int m_far = 0;
double m_origin = 0, m_error = 0;
};

// PulseAudio's capture latency includes unread/server/device buffering. Qt's
// QAudioSource only exposes samples and delivery time, losing that information.
// PipeWire's PulseAudio server implements the same timestamped capture API.
Expand All @@ -16,6 +78,8 @@ class AudioCapture : public QObject {
void start(const QByteArray &device, const QAudioFormat &format);
void stop();
void readAvailable();
// A take starts: from here on buffer times are contiguous by sample count.
void lockTimeline() { m_timeline.lock(); }
static qint64 captureTime(qint64 deliveredAt, qint64 latency, bool negative) {
return deliveredAt + (negative ? latency : -latency);
}
Expand All @@ -30,6 +94,7 @@ class AudioCapture : public QObject {
std::function<qint64()> m_clock;
QTimer m_pump;
QAudioFormat m_format;
AudioTimeline m_timeline;
QByteArray m_device;
pa_mainloop *m_loop = nullptr;
pa_context *m_context = nullptr;
Expand Down
1 change: 1 addition & 0 deletions src/backend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,7 @@ void Backend::startTake() {
m_clipPath=directory+"/"+m_clipFileName;
m_interruption.clear(); m_duration=0;
readAudio();
if(m_audio) m_audio->lockTimeline();
m_writer=new Writer(this);
connect(m_writer,&Writer::finished,this,&Backend::writerFinished,Qt::QueuedConnection);
connect(m_writer,&Writer::failed,this,[this](const QString &error) { m_interruption=error; m_message=error; m_state="finalizing"; emit changed(); });
Expand Down
6 changes: 4 additions & 2 deletions src/writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -336,10 +336,12 @@ void Writer::audio(QByteArray data, qint64 capturedAt) {
// Qt counts audio samples rather than honoring PTS. Materialize capture
// gaps as silence and trim overlaps, keeping the original event times.
const qint64 desiredFrame = qRound64(position * m_audioFormat.sampleRate() / 1000000.0);
if (desiredFrame < m_audioFramesWritten) {
// Capture times are whole microseconds, so a contiguous buffer can round
// one frame to either side of the running count. That is neither.
if (desiredFrame < m_audioFramesWritten - 1) {
const auto overlap = (m_audioFramesWritten - desiredFrame) * frameBytes;
part.remove(0, std::min(qsizetype(part.size()), qsizetype(overlap)));
} else padAudioTo(position);
} else if (desiredFrame > m_audioFramesWritten + 1) padAudioTo(position);
if (!part.isEmpty()) appendAudio(part);
}
}
Expand Down
78 changes: 78 additions & 0 deletions tests/backend_tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,14 @@
#include <QJsonArray>
#include <QStandardPaths>
#include <QSettings>
#include <QRandomGenerator>
#include <QAbstractVideoBuffer>
#include <cmath>
#include <sys/stat.h>
#include "mediautils.h"
#include "theme.h"
#include "writer.h"
#include "audiocapture.h"
#include "backend.h"
#include "translator.h"

Expand Down Expand Up @@ -572,6 +574,82 @@ private slots:
QVERIFY(probe.waitForFinished());
QCOMPARE(QString::fromUtf8(probe.readAllStandardOutput()).trimmed(),QString("yuv420p,bt709"));
}
void microphoneTimelineCountsSamplesDuringATake() {
const double period=1000000.0/48000;
QRandomGenerator random(7);
const auto jitter=[&random] { return qint64(random.bounded(601))-300; };
// Before the take the origin is a plain mean of the readings, the first
// one included: after 0, +3 ms and +3 ms it sits at their mean.
{
AudioTimeline timeline(48000);
QCOMPARE(timeline.place(0,960).capturedAt,0LL);
QCOMPARE(timeline.place(20000+3000,960).capturedAt,20000LL+1500);
QCOMPARE(timeline.place(40000+3000,960).capturedAt,40000LL+2000);
}
// Before the take the timeline settles on the measurements, even when
// the first reading is 15 ms off. During it, jitter and a few
// milliseconds of wander move nothing: times are contiguous by count.
{
AudioTimeline timeline(48000);
timeline.place(-15000,960);
qint64 last=0;
for(int i=1;i<500;++i) last=timeline.place(i*20000LL+jitter(),960).capturedAt;
QVERIFY(std::abs(last-499*20000LL)<1000);
timeline.lock();
qint64 first=0;
for(int i=0;i<3000;++i) {
const qint64 wander=qRound64(4000*std::sin(i/500.0));
const auto placed=timeline.place((500+i)*20000LL+wander+jitter(),960);
if(i==0) first=placed.capturedAt;
QCOMPARE(placed.slip,0); QVERIFY(!placed.resynced);
QCOMPARE(placed.capturedAt,first+i*20000LL);
}
// One far reading is noise and moves nothing. Three in a row are a
// real gap, and restart the timeline at the measurement.
QCOMPARE(timeline.place(3500*20000LL+70000,960).capturedAt,first+3000*20000LL);
QCOMPARE(timeline.place(3501*20000LL,960).capturedAt,first+3001*20000LL);
for(int i=3502;i<3504;++i) QVERIFY(!timeline.place(i*20000LL+300000,960).resynced);
const auto gap=timeline.place(3504*20000LL+300000,960);
QVERIFY(gap.resynced); QCOMPARE(gap.capturedAt,3504*20000LL+300000);
}
// Thirty milliseconds of lost samples, too few for a restart, are walked
// back by single frames instead of being left as an offset.
{
AudioTimeline timeline(48000);
timeline.lock();
qint64 last=0;
for(int i=0;i<3000;++i) {
const auto placed=timeline.place(i*20000LL+(i<50?0:30000)+jitter(),960);
QVERIFY(!placed.resynced); QVERIFY(placed.slip>=0);
last=placed.capturedAt;
}
QVERIFY2(std::abs(last-(2999*20000LL+30000))<1000,qPrintable(QString::number(last)));
}
// Thirty minutes from a microphone running 100 ppm slow, then from one
// running 100 ppm fast: each time the drift outgrows the 20 ms tolerance
// it is walked back by single frames, repeated for the slow microphone
// and dropped for the fast one.
for(const int direction:{1,-1}) {
AudioTimeline timeline(48000);
timeline.lock();
qint64 origin=0, written=0, slips=0;
bool following=false;
for(int i=0;i<90000;++i) {
const double trueTime=1000000+i*960*period*(1+direction*0.0001);
const auto placed=timeline.place(qRound64(trueTime)+jitter(),960);
if(i==0) origin=placed.capturedAt;
QVERIFY(!placed.resynced);
QVERIFY(std::abs(placed.capturedAt-(origin+written*period))<1); // Rounding only.
QVERIFY(std::abs(placed.capturedAt-trueTime)<22000);
QVERIFY(placed.slip*direction>=0);
if(following && !placed.slip) QVERIFY(std::abs(placed.capturedAt-trueTime)<2000); // All the way back.
following=placed.slip;
slips+=placed.slip*direction; written+=960+placed.slip;
}
// 180 ms of drift, less whatever has accumulated since the last walk back.
QVERIFY2(slips>8640-960-150 && slips<=8640,qPrintable(QString::number(slips*direction)));
}
}
void delayedCaptureStaysInSync() {
QTemporaryDir directory;
const auto path=directory.filePath("sync.mp4");
Expand Down