fix: A/V desync' when using opus audio encoder

Uses an `eventfd` to notify when Pipewire has filled a complete frame of audio data. This reduces the amount of copying and lock contention when processing audio buffers.
gmbeard · Aug 3, 2024 · 4bd8470 · 4bd8470
1 parent 9aeb5e6
commit 4bd8470
Show file tree

Hide file tree

Showing 8 changed files with 201 additions and 182 deletions.
diff --git a/.versioning/changes/nBpZUnmkTX.patch.md b/.versioning/changes/nBpZUnmkTX.patch.md
@@ -0,0 +1 @@
+Fixes an A/V desync issue with the opus encoder
diff --git a/src/av/sample_format.cpp b/src/av/sample_format.cpp
@@ -46,4 +46,36 @@ auto is_sample_rate_supported(std::uint32_t requested,
     return false;
 }
 
+auto sample_format_name(SampleFormat fmt) noexcept -> char const*
+{
+    switch (fmt) {
+    case SampleFormat::u8_interleaved:
+        return "u8_interleaved";
+    case SampleFormat::s16_interleaved:
+        return "s16_interleaved";
+    case SampleFormat::s32_interleaved:
+        return "s32_interleaved";
+    case SampleFormat::float_interleaved:
+        return "float_interleaved";
+    case SampleFormat::double_interleaved:
+        return "double_interleaved";
+    case SampleFormat::u8_planar:
+        return "u8_planar";
+    case SampleFormat::s16_planar:
+        return "s16_planar";
+    case SampleFormat::s32_planar:
+        return "s32_planar";
+    case SampleFormat::float_planar:
+        return "float_planar";
+    case SampleFormat::double_planar:
+        return "double_planar";
+    case SampleFormat::s64_interleaved:
+        return "s64_interleaved";
+    case SampleFormat::s64_planar:
+        return "s64_planar";
+    default:
+        return "unknown";
+    }
+}
+
 } // namespace sc
diff --git a/src/av/sample_format.hpp b/src/av/sample_format.hpp
@@ -182,6 +182,8 @@ auto constexpr convert_to_pipewire_format(SampleFormat fmt) -> spa_audio_format
     throw std::runtime_error { "No viable Pipewire sample format conversion " };
 }
 
+auto sample_format_name(SampleFormat fmt) noexcept -> char const*;
+
 auto find_supported_formats(sc::BorrowedPtr<AVCodec const> codec)
     -> std::vector<SampleFormat>;
 

diff --git a/src/handlers/audio_chunk_writer.cpp b/src/handlers/audio_chunk_writer.cpp
@@ -14,20 +14,20 @@ namespace sc
 
 ChunkWriter::ChunkWriter(AVCodecContext* codec_context,
                          AVStream* stream,
-                         Encoder encoder) noexcept
+                         Encoder encoder,
+                         std::size_t frame_size) noexcept
     : codec_context_ { codec_context }
     , stream_ { stream }
     , encoder_ { encoder }
+    , frame_size_ { frame_size }
     , frame_ { av_frame_alloc() }
     , total_samples_written_ { 0 }
 {
 }
 
 auto ChunkWriter::operator()(MediaChunk const& chunk) -> void
 {
-    SC_EXPECT(!codec_context_->frame_size ||
-              static_cast<int>(chunk.sample_count) ==
-                  codec_context_->frame_size);
+    SC_EXPECT(chunk.sample_count >= frame_size_);
     sc::SampleFormat const sample_format =
         sc::convert_from_libav_format(codec_context_->sample_fmt);
 
@@ -38,7 +38,7 @@ auto ChunkWriter::operator()(MediaChunk const& chunk) -> void
         encoder_.prepare_frame(codec_context_.get(), stream_.get());
     auto* frame = encoder_frame->frame.get();
 
-    frame->nb_samples = chunk.sample_count;
+    frame->nb_samples = frame_size_;
     frame->format = codec_context_->sample_fmt;
     frame->sample_rate = codec_context_->sample_rate;
 #if LIBAVCODEC_VERSION_MAJOR < 60

diff --git a/src/handlers/audio_chunk_writer.hpp b/src/handlers/audio_chunk_writer.hpp
@@ -12,14 +12,16 @@ struct ChunkWriter
 {
     explicit ChunkWriter(AVCodecContext* codec_context,
                          AVStream* stream,
-                         Encoder encoder) noexcept;
+                         Encoder encoder,
+                         std::size_t frame_size) noexcept;
 
     auto operator()(MediaChunk const& chunk) -> void;
 
 private:
     BorrowedPtr<AVCodecContext> codec_context_;
     BorrowedPtr<AVStream> stream_;
     Encoder encoder_;
+    std::size_t frame_size_;
     FramePtr frame_;
     std::size_t total_samples_written_ { 0 };
 };

diff --git a/src/main.cpp b/src/main.cpp
@@ -1,4 +1,5 @@
 #include "./shadow_cast.hpp"
+#include "utils/contracts.hpp"
 #include "utils/frame_time.hpp"
 
 #include <X11/Xlib.h>
@@ -9,9 +10,11 @@
 #include <cstdio>
 #include <initializer_list>
 #include <iostream>
+#include <libavutil/dict.h>
 #include <libavutil/pixfmt.h>
 #include <memory>
 #include <signal.h>
+#include <string_view>
 #include <thread>
 #include <type_traits>
 #include <utility>
@@ -79,6 +82,15 @@ struct DestroyCaptureSessionGuard
     sc::NvFBC nvfbc;
 };
 
+auto apply_audio_codec_modifiers(AVCodec const& codec, AVDictionary*& output)
+    -> void
+{
+    if (std::string_view { codec.name } == "libopus") {
+        av_dict_set_int(&output, "frame_duration", 40, 0);
+        av_dict_set_int(&output, "vbr", 0, 0);
+    }
+}
+
 auto run_loop(sc::Context& main,
               sc::Context& media,
               sc::Context& audio,
@@ -213,8 +225,11 @@ auto run_wayland(sc::Parameters const& params, sc::wayland::DisplayPtr display)
     audio_encoder_context->time_base = AVRational { 1, params.sample_rate };
     audio_encoder_context->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
 
+    AVDictionary* options = nullptr;
+    apply_audio_codec_modifiers(*encoder, options);
+
     if (auto const ret =
-            avcodec_open2(audio_encoder_context.get(), encoder.get(), nullptr);
+            avcodec_open2(audio_encoder_context.get(), encoder.get(), &options);
         ret < 0) {
         throw sc::CodecError { "Failed to open audio codec: " +
                                sc::av_error_to_string(ret) };
@@ -292,6 +307,10 @@ auto run_wayland(sc::Parameters const& params, sc::wayland::DisplayPtr display)
     sc::Context audio_ctx { params.frame_time };
     sc::Context media_ctx { params.frame_time };
 
+    std::size_t const frame_size = audio_encoder_context->frame_size
+                                       ? audio_encoder_context->frame_size
+                                       : 2048;
+
     ctx.services().add<sc::SignalService>(sc::SignalService {});
     add_signal_handler(ctx, SIGINT, [&](std::uint32_t) {
         ctx.request_stop();
@@ -300,11 +319,7 @@ auto run_wayland(sc::Parameters const& params, sc::wayland::DisplayPtr display)
 
     audio_ctx.services().add_from_factory<sc::AudioService>([&] {
         return std::make_unique<sc::AudioService>(
-            supported_formats.front(),
-            params.sample_rate,
-            audio_encoder_context->frame_size
-                ? audio_encoder_context->frame_size
-                : 2048);
+            supported_formats.front(), params.sample_rate, frame_size);
     });
 
     ctx.services().add_from_factory<sc::DRMVideoService>([&] {
@@ -321,7 +336,8 @@ auto run_wayland(sc::Parameters const& params, sc::wayland::DisplayPtr display)
     set_audio_chunk_handler(audio_ctx,
                             sc::ChunkWriter { audio_encoder_context.get(),
                                               stream.get(),
-                                              media_writer });
+                                              media_writer,
+                                              frame_size });
     set_drm_video_frame_handler(
         ctx,
         sc::DRMVideoFrameWriter {
@@ -407,8 +423,11 @@ auto run(sc::Parameters const& params) -> void
     audio_encoder_context->time_base = AVRational { 1, params.sample_rate };
     audio_encoder_context->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
 
+    AVDictionary* options = nullptr;
+    apply_audio_codec_modifiers(*encoder, options);
+
     if (auto const ret =
-            avcodec_open2(audio_encoder_context.get(), encoder.get(), nullptr);
+            avcodec_open2(audio_encoder_context.get(), encoder.get(), &options);
         ret < 0) {
         throw sc::CodecError { "Failed to open audio codec: " +
                                sc::av_error_to_string(ret) };
@@ -496,6 +515,10 @@ auto run(sc::Parameters const& params) -> void
     sc::Context audio_ctx { params.frame_time };
     sc::Context media_ctx { params.frame_time };
 
+    std::size_t const frame_size = audio_encoder_context->frame_size
+                                       ? audio_encoder_context->frame_size
+                                       : 2048;
+
     ctx.services().add<sc::SignalService>(sc::SignalService {});
     add_signal_handler(ctx, SIGINT, [&](std::uint32_t) {
         ctx.request_stop();
@@ -504,11 +527,7 @@ auto run(sc::Parameters const& params) -> void
 
     audio_ctx.services().add_from_factory<sc::AudioService>([&] {
         return std::make_unique<sc::AudioService>(
-            supported_formats.front(),
-            params.sample_rate,
-            audio_encoder_context->frame_size
-                ? audio_encoder_context->frame_size
-                : 2048);
+            supported_formats.front(), params.sample_rate, frame_size);
     });
 
     ctx.services().add_from_factory<sc::VideoService>([&] {
@@ -525,7 +544,8 @@ auto run(sc::Parameters const& params) -> void
     set_audio_chunk_handler(audio_ctx,
                             sc::ChunkWriter { audio_encoder_context.get(),
                                               stream.get(),
-                                              media_writer });
+                                              media_writer,
+                                              frame_size });
     set_video_frame_handler(ctx,
                             sc::VideoFrameWriter { video_encoder_context.get(),
                                                    video_stream.get(),
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Fixes an A/V desync issue with the opus encoder