From fb25d513dfc4354886b7fb0ae894e9793241c0c0 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Mon, 29 Jun 2026 18:05:19 +0800 Subject: [PATCH 01/14] fix: improve initial video quality by setting x-google-start-bitrate for all video codecs - Apply x-google-start-bitrate SDP hint to all video codecs (VP8, VP9, AV1, H264, H265), not just SVC codecs - Use 90% of target bitrate as start bitrate to prevent initial blurriness - Default degradationPreference to MAINTAIN_RESOLUTION for video tracks to prefer frame drops over resolution reduction when bandwidth is constrained This addresses the issue where video starts blurry for several seconds before improving, by telling WebRTC's bandwidth estimator to start at a higher bitrate instead of ramping up from ~300kbps. Co-Authored-By: Claude Opus 4.5 --- .../android/room/PeerConnectionTransport.kt | 31 +- .../room/participant/LocalParticipant.kt | 11 +- munging.patch | 591 ++++++++++++++++++ protocol | 2 +- 4 files changed, 621 insertions(+), 14 deletions(-) create mode 100644 munging.patch diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt index efae279a..04f405bd 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt @@ -437,13 +437,17 @@ fun ensureVideoDDExtensionForSVC(mediaDesc: MediaDescription) { } } -/* The svc codec (av1/vp9) would use a very low bitrate at the beginning and -increase slowly by the bandwidth estimator until it reach the target bitrate. The -process commonly cost more than 10 seconds cause subscriber will get blur video at -the first few seconds. So we use a 70% of target bitrate here as the start bitrate to -eliminate this issue. -*/ -private const val startBitrateForSVC = 0.7 +/* + * Video codecs use a very low bitrate at the beginning and increase slowly by + * the bandwidth estimator until they reach the target bitrate. The process commonly + * costs more than 10 seconds causing subscribers to get blurry video at the first + * few seconds. We use x-google-start-bitrate to hint the BWE to start higher. + * + * Why 90%: Gives ~10% headroom for bandwidth estimation while starting close to target. + * Why same for all codecs: Target bitrate already accounts for codec efficiency + * (e.g., users set lower targets for VP9/AV1 knowing they're more efficient). + */ +private const val startBitrateMultiplier = 0.9 /** * @suppress @@ -476,7 +480,7 @@ fun ensureCodecBitrates( fmtpFound = true var newFmtpConfig = fmtp.config if (!fmtp.config.contains("x-google-start-bitrate")) { - newFmtpConfig = "$newFmtpConfig;x-google-start-bitrate=${(trackBr.maxBitrate * startBitrateForSVC).roundToLong()}" + newFmtpConfig = "$newFmtpConfig;x-google-start-bitrate=${(trackBr.maxBitrate * startBitrateMultiplier).roundToLong()}" } if (!fmtp.config.contains("x-google-max-bitrate")) { newFmtpConfig = "$newFmtpConfig;x-google-max-bitrate=${trackBr.maxBitrate}" @@ -492,7 +496,7 @@ fun ensureCodecBitrates( media.addAttribute( SdpFmtp( payload = codecPayload, - config = "x-google-start-bitrate=${trackBr.maxBitrate * startBitrateForSVC};" + + config = "x-google-start-bitrate=${trackBr.maxBitrate * startBitrateMultiplier};" + "x-google-max-bitrate=${trackBr.maxBitrate}", ).toAttributeField(), ) @@ -506,6 +510,15 @@ internal fun isSVCCodec(codec: String?): Boolean { "vp9".equals(codec, ignoreCase = true)) } +internal fun isVideoCodec(codec: String?): Boolean { + return codec != null && + ("vp8".equals(codec, ignoreCase = true) || + "vp9".equals(codec, ignoreCase = true) || + "av1".equals(codec, ignoreCase = true) || + "h264".equals(codec, ignoreCase = true) || + "h265".equals(codec, ignoreCase = true)) +} + /** * @suppress */ diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt index 75ac69c1..0a9829c4 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt @@ -39,6 +39,7 @@ import io.livekit.android.room.SenderTransceiverHandle import io.livekit.android.room.TrackBitrateInfo import io.livekit.android.room.datastream.outgoing.OutgoingDataStreamManager import io.livekit.android.room.isSVCCodec +import io.livekit.android.room.isVideoCodec import io.livekit.android.room.rpc.RpcClientManager import io.livekit.android.room.rpc.RpcManager import io.livekit.android.room.rpc.RpcServerManager @@ -750,9 +751,9 @@ internal constructor( track.statsGetter = engine.createStatsGetter(transceiver.sender) val finalOptions = options - // Handle trackBitrates + // Handle trackBitrates - apply start bitrate for all video codecs to prevent initial blurriness if (encodings.isNotEmpty()) { - if (finalOptions is VideoTrackPublishOptions && isSVCCodec(finalOptions.videoCodec) && encodings.firstOrNull()?.maxBitrateBps != null) { + if (finalOptions is VideoTrackPublishOptions && isVideoCodec(finalOptions.videoCodec) && encodings.firstOrNull()?.maxBitrateBps != null) { engine.registerTrackBitrateInfo( cid = cid, TrackBitrateInfo( @@ -1591,7 +1592,8 @@ data class VideoTrackPublishDefaults( override val videoCodec: String = VideoCodec.VP8.codecName, override val scalabilityMode: String? = null, override val backupCodec: BackupVideoCodec? = null, - override val degradationPreference: RtpParameters.DegradationPreference? = null, + // Default to MAINTAIN_RESOLUTION to prevent initial video blurriness + override val degradationPreference: RtpParameters.DegradationPreference? = RtpParameters.DegradationPreference.MAINTAIN_RESOLUTION, override val simulcastLayers: List? = null, ) : BaseVideoTrackPublishOptions() @@ -1604,7 +1606,8 @@ data class VideoTrackPublishOptions( override val backupCodec: BackupVideoCodec? = null, override val source: Track.Source? = null, override val stream: String? = null, - override val degradationPreference: RtpParameters.DegradationPreference? = null, + // Default to MAINTAIN_RESOLUTION to prevent initial video blurriness + override val degradationPreference: RtpParameters.DegradationPreference? = RtpParameters.DegradationPreference.MAINTAIN_RESOLUTION, override val simulcastLayers: List? = null, ) : BaseVideoTrackPublishOptions(), TrackPublishOptions { constructor( diff --git a/munging.patch b/munging.patch new file mode 100644 index 00000000..1fb38448 --- /dev/null +++ b/munging.patch @@ -0,0 +1,591 @@ +diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml +index 3501a8d..df624b7 100644 +--- a/gradle/libs.versions.toml ++++ b/gradle/libs.versions.toml +@@ -1,7 +1,6 @@ + [versions] + webrtc = "137.7151.05" + +-androidJainSipRi = "1.3.0-91" + androidx-activity = "1.9.0" + androidx-camera = "1.4.2" + androidx-core = "1.13.1" +@@ -31,7 +30,6 @@ noise = "2.0.0" + lifecycleProcess = "2.8.7" + + [libraries] +-android-jain-sip-ri = { module = "javax.sip:android-jain-sip-ri", version.ref = "androidJainSipRi" } + androidx-activity-ktx = { module = "androidx.activity:activity-ktx", version.ref = "androidx-activity" } + androidx-camera-core = { module = "androidx.camera:camera-core", version.ref = "androidx-camera" } + androidx-camera-camera2 = { module = "androidx.camera:camera-camera2", version.ref = "androidx-camera" } +@@ -111,4 +109,3 @@ androidx-material3 = { group = "androidx.compose.material3", name = "material3" + lifecycle-process = { group = "androidx.lifecycle", name = "lifecycle-process", version.ref = "lifecycleProcess" } + + [plugins] +- +diff --git a/livekit-android-sdk/build.gradle b/livekit-android-sdk/build.gradle +index 45a3f69..2c019e1 100644 +--- a/livekit-android-sdk/build.gradle ++++ b/livekit-android-sdk/build.gradle +@@ -154,8 +154,6 @@ dependencies { + implementation libs.androidx.core + implementation libs.protobuf.javalite + +- implementation libs.android.jain.sip.ri +- + implementation libs.dagger.lib + kapt libs.dagger.compiler + +diff --git a/livekit-android-sdk/consumer-rules.pro b/livekit-android-sdk/consumer-rules.pro +index a3d6542..6ae7318 100644 +--- a/livekit-android-sdk/consumer-rules.pro ++++ b/livekit-android-sdk/consumer-rules.pro +@@ -28,13 +28,6 @@ + @livekit.**.CalledByNativeUnchecked ; + } + +-# NIST sdp parser +-######################################### +-# Preserve reflection used for Parser registrations +--keep class android.gov.nist.javax.sdp.parser.*Parser { *; } +--keep class android.gov.nist.javax.sdp.parser.ParserFactory { *; } +--keep class android.gov.nist.javax.sdp.parser.SDPParser { *; } +- + # Protobuf + ######################################### + -keep class * extends com.google.protobuf.GeneratedMessageLite { *; } +diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt b/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt +index 9f6da92..075f139 100644 +--- a/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt ++++ b/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt +@@ -17,7 +17,6 @@ + package io.livekit.android.dagger + + import android.content.Context +-import android.javax.sdp.SdpFactory + import android.media.AudioAttributes + import android.media.MediaRecorder + import android.os.Build +@@ -408,8 +407,6 @@ internal object RTCModule { + @Named(InjectionNames.OPTIONS_VIDEO_HW_ACCEL) + fun videoHwAccel() = true + +- @Provides +- fun sdpFactory() = SdpFactory.getInstance() + } + + /** +diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt +index fb6f5a0..f1e533b 100644 +--- a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt ++++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt +@@ -16,8 +16,6 @@ + + package io.livekit.android.room + +-import android.javax.sdp.MediaDescription +-import android.javax.sdp.SdpFactory + import androidx.annotation.VisibleForTesting + import dagger.assisted.Assisted + import dagger.assisted.AssistedFactory +@@ -38,6 +36,8 @@ import io.livekit.android.webrtc.getFmtps + import io.livekit.android.webrtc.getMsid + import io.livekit.android.webrtc.getRtps + import io.livekit.android.webrtc.isConnected ++import io.livekit.android.webrtc.parseSdpSections ++import io.livekit.android.webrtc.SdpMediaSection + import io.livekit.android.webrtc.peerconnection.RTCThreadToken + import io.livekit.android.webrtc.peerconnection.executeBlockingOnRTCThread + import io.livekit.android.webrtc.peerconnection.launchBlockingOnRTCThread +@@ -76,7 +76,6 @@ constructor( + @Named(InjectionNames.DISPATCHER_IO) + private val ioDispatcher: CoroutineDispatcher, + connectionFactory: PeerConnectionFactory, +- private val sdpFactory: SdpFactory, + private val rtcThreadToken: RTCThreadToken, + ) { + private val coroutineScope = CoroutineScope(ioDispatcher + SupervisorJob()) +@@ -203,21 +202,16 @@ constructor( + return@launchRTCIfNotClosed + } + // munge sdp +- val sdpDescription = sdpFactory.createSessionDescription(sdpOffer.description) +- +- val mediaDescs = sdpDescription.getMediaDescriptions(true) +- for (mediaDesc in mediaDescs) { +- if (mediaDesc !is MediaDescription) { +- continue +- } +- if (mediaDesc.media.mediaType == "audio") { ++ val sdpSections = parseSdpSections(sdpOffer.description) ++ for (mediaDesc in sdpSections.mediaSections) { ++ if (mediaDesc.mediaType == "audio") { + // TODO +- } else if (mediaDesc.media.mediaType == "video") { ++ } else if (mediaDesc.mediaType == "video") { + ensureVideoDDExtensionForSVC(mediaDesc) + ensureCodecBitrates(mediaDesc, trackBitrates = trackBitrates) + } + } +- finalSdp = setMungedSdp(sdpOffer, sdpDescription.toString()) ++ finalSdp = setMungedSdp(sdpOffer, sdpSections.toSdpString()) + } + + finalSdp?.let { sdp -> +@@ -402,7 +396,7 @@ private const val DD_EXTENSION_URI = "https://aomediacodec.github.io/av1-rtp-spe + * @suppress + */ + @VisibleForTesting +-fun ensureVideoDDExtensionForSVC(mediaDesc: MediaDescription) { ++fun ensureVideoDDExtensionForSVC(mediaDesc: SdpMediaSection) { + val codec = mediaDesc.getRtps() + .firstOrNull() + ?.second +@@ -426,13 +420,14 @@ fun ensureVideoDDExtensionForSVC(mediaDesc: MediaDescription) { + // Not found, add manually + if (!ddFound) { + mediaDesc.addAttribute( ++ "extmap", + SdpExt( + value = maxId + 1, + uri = DD_EXTENSION_URI, + config = null, + direction = null, + encryptUri = null, +- ).toAttributeField(), ++ ).toAttributeValue(), + ) + } + } +@@ -450,7 +445,7 @@ private const val startBitrateForSVC = 0.7 + */ + @VisibleForTesting + fun ensureCodecBitrates( +- media: MediaDescription, ++ media: SdpMediaSection, + trackBitrates: Map, + ) { + val msid = media.getMsid()?.value ?: return +@@ -482,7 +477,7 @@ fun ensureCodecBitrates( + newFmtpConfig = "$newFmtpConfig;x-google-max-bitrate=${trackBr.maxBitrate}" + } + if (fmtp.config != newFmtpConfig) { +- attribute.value = "${fmtp.payload} $newFmtpConfig" ++ attribute.mutableValue = "${fmtp.payload} $newFmtpConfig" + break + } + } +@@ -490,11 +485,12 @@ fun ensureCodecBitrates( + + if (!fmtpFound) { + media.addAttribute( ++ "fmtp", + SdpFmtp( + payload = codecPayload, + config = "x-google-start-bitrate=${trackBr.maxBitrate * startBitrateForSVC};" + + "x-google-max-bitrate=${trackBr.maxBitrate}", +- ).toAttributeField(), ++ ).toAttributeValue(), + ) + } + } +diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/webrtc/JainSdpUtils.kt b/livekit-android-sdk/src/main/java/io/livekit/android/webrtc/JainSdpUtils.kt +index c6e91c1..371669e 100644 +--- a/livekit-android-sdk/src/main/java/io/livekit/android/webrtc/JainSdpUtils.kt ++++ b/livekit-android-sdk/src/main/java/io/livekit/android/webrtc/JainSdpUtils.kt +@@ -16,8 +16,6 @@ + + package io.livekit.android.webrtc + +-import android.gov.nist.javax.sdp.fields.AttributeField +-import android.javax.sdp.MediaDescription + import io.livekit.android.util.LKLog + + /** +@@ -25,17 +23,142 @@ import io.livekit.android.util.LKLog + */ + data class SdpRtp(val payload: Long, val codec: String, val rate: Long?, val encoding: String?) + ++data class SdpAttributeRef( ++ val name: String, ++ val value: String, ++ private val onSetValue: (String) -> Unit, ++) { ++ var mutableValue: String = value ++ set(newValue) { ++ field = newValue ++ onSetValue(newValue) ++ } ++} ++ ++/** ++ * Represents a single `m=` section in SDP. ++ * ++ * @suppress ++ */ ++class SdpMediaSection internal constructor( ++ private val lines: MutableList, ++) { ++ internal fun toLines(): List = lines.toList() ++ ++ val mediaType: String? ++ get() { ++ val mediaLine = lines.firstOrNull()?.trim() ?: return null ++ if (!mediaLine.startsWith("m=")) return null ++ return mediaLine.removePrefix("m=") ++ .substringBefore(' ') ++ .ifEmpty { null } ++ } ++ ++ fun addAttribute(name: String, value: String? = null) { ++ if (value == null) { ++ lines.add("a=$name") ++ } else { ++ lines.add("a=$name:$value") ++ } ++ } ++ ++ fun getAttribute(name: String): String? { ++ return getAttributes() ++ .firstOrNull { it.name == name } ++ ?.value ++ } ++ ++ fun getAttributes(): List { ++ val output = mutableListOf() ++ lines.forEachIndexed { index, rawLine -> ++ if (!rawLine.startsWith("a=")) { ++ return@forEachIndexed ++ } ++ ++ val content = rawLine.removePrefix("a=") ++ val separator = content.indexOf(':') ++ val (attributeName, attributeValue) = if (separator < 0) { ++ content to "" ++ } else { ++ content.substring(0, separator) to content.substring(separator + 1) ++ } ++ ++ output.add( ++ SdpAttributeRef( ++ name = attributeName, ++ value = attributeValue, ++ ) { newValue -> ++ lines[index] = "a=$attributeName:$newValue" ++ }, ++ ) ++ } ++ return output ++ } ++} ++ + /** + * @suppress + */ +-fun MediaDescription.getRtps(): List> { +- return getAttributes(true) +- .filterIsInstance() +- .filter { it.attribute.name == "rtpmap" } ++data class SdpSections( ++ val sessionLines: List, ++ val mediaSections: List, ++) { ++ fun toSdpString(): String { ++ return buildString { ++ (sessionLines + mediaSections.flatMap { it.toLines() }).forEachIndexed { index, line -> ++ if (index > 0) append("\r\n") ++ append(line) ++ } ++ append("\r\n") ++ } ++ } ++} ++ ++/** ++ * @suppress ++ */ ++fun parseSdpSections(description: String): SdpSections { ++ val normalizedLines = description ++ .replace("\r\n", "\n") ++ .replace('\r', '\n') ++ .split('\n') ++ .filter { it.isNotEmpty() } ++ ++ val sessionLines = mutableListOf() ++ val mediaSections = mutableListOf() ++ ++ var currentMediaSection: MutableList? = null ++ for (line in normalizedLines) { ++ if (line.startsWith("m=")) { ++ currentMediaSection?.let { mediaSections.add(SdpMediaSection(it)) } ++ currentMediaSection = mutableListOf(line) ++ continue ++ } ++ ++ if (currentMediaSection == null) { ++ sessionLines.add(line) ++ } else { ++ currentMediaSection.add(line) ++ } ++ } ++ currentMediaSection?.let { mediaSections.add(SdpMediaSection(it)) } ++ ++ return SdpSections( ++ sessionLines = sessionLines, ++ mediaSections = mediaSections, ++ ) ++} ++ ++/** ++ * @suppress ++ */ ++fun SdpMediaSection.getRtps(): List> { ++ return getAttributes() ++ .filter { it.name == "rtpmap" } + .mapNotNull { +- val rtp = tryParseRtp(it.value) ++ val rtp = tryParseRtp(it.mutableValue) + if (rtp == null) { +- LKLog.w { "could not parse rtpmap: ${it.encode()}" } ++ LKLog.w { "could not parse rtpmap: a=${it.name}:${it.mutableValue}" } + return@mapNotNull null + } + it to rtp +@@ -60,7 +183,7 @@ data class SdpMsid( + /** + * @suppress + */ +-fun MediaDescription.getMsid(): SdpMsid? { ++fun SdpMediaSection.getMsid(): SdpMsid? { + val attribute = getAttribute("msid") ?: return null + return SdpMsid(attribute) + } +@@ -69,25 +192,21 @@ fun MediaDescription.getMsid(): SdpMsid? { + * @suppress + */ + data class SdpFmtp(val payload: Long, val config: String) { +- fun toAttributeField(): AttributeField { +- return AttributeField().apply { +- name = "fmtp" +- value = "$payload $config" +- } ++ fun toAttributeValue(): String { ++ return "$payload $config" + } + } + + /** + * @suppress + */ +-fun MediaDescription.getFmtps(): List> { +- return getAttributes(true) +- .filterIsInstance() +- .filter { it.attribute.name == "fmtp" } ++fun SdpMediaSection.getFmtps(): List> { ++ return getAttributes() ++ .filter { it.name == "fmtp" } + .mapNotNull { +- val fmtp = tryParseFmtp(it.value) ++ val fmtp = tryParseFmtp(it.mutableValue) + if (fmtp == null) { +- LKLog.w { "could not parse fmtp: ${it.encode()}" } ++ LKLog.w { "could not parse fmtp: a=${it.name}:${it.mutableValue}" } + return@mapNotNull null + } + it to fmtp +@@ -105,21 +224,18 @@ internal fun tryParseFmtp(string: String): SdpFmtp? { + * @suppress + */ + data class SdpExt(val value: Long, val direction: String?, val encryptUri: String?, val uri: String, val config: String?) { +- fun toAttributeField(): AttributeField { +- return AttributeField().apply { +- name = "extmap" +- value = buildString { +- append(this@SdpExt.value) +- if (direction != null) { +- append(" $direction") +- } +- if (encryptUri != null) { +- append(" $encryptUri") +- } +- append(" $uri") +- if (config != null) { +- append(" $config") +- } ++ fun toAttributeValue(): String { ++ return buildString { ++ append(this@SdpExt.value) ++ if (direction != null) { ++ append(" $direction") ++ } ++ if (encryptUri != null) { ++ append(" $encryptUri") ++ } ++ append(" $uri") ++ if (config != null) { ++ append(" $config") + } + } + } +@@ -128,14 +244,13 @@ data class SdpExt(val value: Long, val direction: String?, val encryptUri: Strin + /** + * @suppress + */ +-fun MediaDescription.getExts(): List> { +- return getAttributes(true) +- .filterIsInstance() +- .filter { it.attribute.name == "extmap" } ++fun SdpMediaSection.getExts(): List> { ++ return getAttributes() ++ .filter { it.name == "extmap" } + .mapNotNull { +- val ext = tryParseExt(it.value) ++ val ext = tryParseExt(it.mutableValue) + if (ext == null) { +- LKLog.w { "could not parse extmap: ${it.encode()}" } ++ LKLog.w { "could not parse extmap: a=${it.name}:${it.mutableValue}" } + return@mapNotNull null + } + it to ext +diff --git a/livekit-android-test/build.gradle b/livekit-android-test/build.gradle +index 30d84c2..a117d63 100644 +--- a/livekit-android-test/build.gradle ++++ b/livekit-android-test/build.gradle +@@ -82,7 +82,6 @@ dependencies { + api libs.audioswitch + implementation libs.androidx.annotation + api libs.protobuf.javalite +- implementation libs.android.jain.sip.ri + implementation libs.junit + implementation libs.robolectric + implementation libs.mockito.core +diff --git a/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt b/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt +index 0ed771c..5eb2761 100644 +--- a/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt ++++ b/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt +@@ -17,7 +17,6 @@ + package io.livekit.android.test.mock.dagger + + import android.content.Context +-import android.javax.sdp.SdpFactory + import dagger.Module + import dagger.Provides + import dagger.Reusable +@@ -129,9 +128,6 @@ object TestRTCModule { + @Named(InjectionNames.OPTIONS_VIDEO_HW_ACCEL) + fun videoHwAccel() = true + +- @Provides +- fun sdpFactory() = SdpFactory.getInstance() +- + @Provides + fun dataPacketCryptorManagerFactory(): DataPacketCryptorManager.Factory = object : DataPacketCryptorManager.Factory { + override fun create(keyProvider: KeyProvider): DataPacketCryptorManager { +diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt +index bfecd60..bd9db73 100644 +--- a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt ++++ b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt +@@ -16,11 +16,10 @@ + + package io.livekit.android.room + +-import android.javax.sdp.MediaDescription +-import android.javax.sdp.SdpFactory + import io.livekit.android.webrtc.JainSdpUtilsTest + import io.livekit.android.webrtc.getExts + import io.livekit.android.webrtc.getFmtps ++import io.livekit.android.webrtc.parseSdpSections + import org.junit.Assert.assertEquals + import org.junit.Assert.assertNotNull + import org.junit.Assert.assertNull +@@ -30,8 +29,7 @@ class SdpMungingTest { + + @Test + fun ensureVideoDDExtensionForSVCTest() { +- val sdp = SdpFactory.getInstance().createSessionDescription(NO_DD_DESCRIPTION) +- val mediaDescription = sdp.getMediaDescriptions(true).filterIsInstance()[1] ++ val mediaDescription = parseSdpSections(NO_DD_DESCRIPTION).mediaSections[1] + + ensureVideoDDExtensionForSVC(mediaDescription) + +@@ -53,8 +51,7 @@ class SdpMungingTest { + + @Test + fun ensureCodecBitratesTest() { +- val sdp = SdpFactory.getInstance().createSessionDescription(JainSdpUtilsTest.DESCRIPTION) +- val mediaDescription = sdp.getMediaDescriptions(true).filterIsInstance()[1] ++ val mediaDescription = parseSdpSections(JainSdpUtilsTest.DESCRIPTION).mediaSections[1] + + ensureCodecBitrates( + mediaDescription, +diff --git a/livekit-android-test/src/test/java/io/livekit/android/webrtc/JainSdpUtilsTest.kt b/livekit-android-test/src/test/java/io/livekit/android/webrtc/JainSdpUtilsTest.kt +index 18870af..f9f9a77 100644 +--- a/livekit-android-test/src/test/java/io/livekit/android/webrtc/JainSdpUtilsTest.kt ++++ b/livekit-android-test/src/test/java/io/livekit/android/webrtc/JainSdpUtilsTest.kt +@@ -16,9 +16,6 @@ + + package io.livekit.android.webrtc + +-import android.javax.sdp.MediaDescription +-import android.javax.sdp.SdpFactory +-import android.javax.sdp.SessionDescription + import org.junit.Assert.assertEquals + import org.junit.Assert.assertNotNull + import org.junit.Assert.assertNull +@@ -26,17 +23,13 @@ import org.junit.Test + + class JainSdpUtilsTest { + +- private val sdpFactory = SdpFactory.getInstance() +- private fun createSessionDescription(): SessionDescription { +- return sdpFactory.createSessionDescription(DESCRIPTION) ++ private fun createMediaDescription(): SdpMediaSection { ++ return parseSdpSections(DESCRIPTION).mediaSections[1] + } + + @Test + fun getRtpAttributes() { +- val sdp = createSessionDescription() +- val mediaDescriptions = sdp.getMediaDescriptions(true) +- .filterIsInstance() +- val mediaDesc = mediaDescriptions[1] ++ val mediaDesc = createMediaDescription() + val rtps = mediaDesc.getRtps() + assertEquals(13, rtps.size) + +@@ -50,10 +43,7 @@ class JainSdpUtilsTest { + + @Test + fun getExtmapAttributes() { +- val sdp = createSessionDescription() +- val mediaDescriptions = sdp.getMediaDescriptions(true) +- .filterIsInstance() +- val mediaDesc = mediaDescriptions[1] ++ val mediaDesc = createMediaDescription() + val exts = mediaDesc.getExts() + + assertEquals(12, exts.size) +@@ -68,10 +58,7 @@ class JainSdpUtilsTest { + + @Test + fun getMsid() { +- val sdp = createSessionDescription() +- val mediaDescriptions = sdp.getMediaDescriptions(true) +- .filterIsInstance() +- val mediaDesc = mediaDescriptions[1] ++ val mediaDesc = createMediaDescription() + + val msid = mediaDesc.getMsid() + assertNotNull(msid) +@@ -80,10 +67,7 @@ class JainSdpUtilsTest { + + @Test + fun getFmtps() { +- val sdp = createSessionDescription() +- val mediaDescriptions = sdp.getMediaDescriptions(true) +- .filterIsInstance() +- val mediaDesc = mediaDescriptions[1] ++ val mediaDesc = createMediaDescription() + + val fmtps = mediaDesc.getFmtps() + .filter { (_, fmtp) -> fmtp.payload == 97L } diff --git a/protocol b/protocol index 8381f218..4c05a332 160000 --- a/protocol +++ b/protocol @@ -1 +1 @@ -Subproject commit 8381f2180c45ab926b3ebf19df0608f1dadcac1e +Subproject commit 4c05a3325ec35760bee1c0bfe57b7011604a124f From 287960ad64095d953470bfb23937673257c6c149 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Wed, 1 Jul 2026 07:55:36 +0800 Subject: [PATCH 02/14] fix: revert to SVC-only bitrate setting to avoid simulcast issues - Revert isVideoCodec back to isSVCCodec for bitrate registration - For simulcast, encodings are ordered smallest-to-largest, so encodings.first() returns the lowest layer's bitrate (e.g., 160kbps for H180), which would incorrectly cap all layers at that low value - SVC codecs (VP9, AV1) have a single encoding with the full bitrate, so this logic is safe for them - Remove unused isVideoCodec function - Remove accidentally committed munging.patch file Co-Authored-By: Claude Opus 4.5 --- .../android/room/PeerConnectionTransport.kt | 9 - .../room/participant/LocalParticipant.kt | 9 +- munging.patch | 591 ------------------ 3 files changed, 6 insertions(+), 603 deletions(-) delete mode 100644 munging.patch diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt index 04f405bd..d37ec7d6 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt @@ -510,15 +510,6 @@ internal fun isSVCCodec(codec: String?): Boolean { "vp9".equals(codec, ignoreCase = true)) } -internal fun isVideoCodec(codec: String?): Boolean { - return codec != null && - ("vp8".equals(codec, ignoreCase = true) || - "vp9".equals(codec, ignoreCase = true) || - "av1".equals(codec, ignoreCase = true) || - "h264".equals(codec, ignoreCase = true) || - "h265".equals(codec, ignoreCase = true)) -} - /** * @suppress */ diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt index 0a9829c4..5911928f 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt @@ -39,7 +39,6 @@ import io.livekit.android.room.SenderTransceiverHandle import io.livekit.android.room.TrackBitrateInfo import io.livekit.android.room.datastream.outgoing.OutgoingDataStreamManager import io.livekit.android.room.isSVCCodec -import io.livekit.android.room.isVideoCodec import io.livekit.android.room.rpc.RpcClientManager import io.livekit.android.room.rpc.RpcManager import io.livekit.android.room.rpc.RpcServerManager @@ -751,9 +750,13 @@ internal constructor( track.statsGetter = engine.createStatsGetter(transceiver.sender) val finalOptions = options - // Handle trackBitrates - apply start bitrate for all video codecs to prevent initial blurriness + // Handle trackBitrates - apply start bitrate for SVC codecs to prevent initial blurriness. + // Only SVC codecs (VP9, AV1) are supported here because they have a single encoding with + // the full bitrate. Simulcast codecs (VP8, H264) have multiple encodings ordered + // smallest-to-largest, so encodings.first() would incorrectly return the lowest layer's + // bitrate, which would cap all layers at that low value. if (encodings.isNotEmpty()) { - if (finalOptions is VideoTrackPublishOptions && isVideoCodec(finalOptions.videoCodec) && encodings.firstOrNull()?.maxBitrateBps != null) { + if (finalOptions is VideoTrackPublishOptions && isSVCCodec(finalOptions.videoCodec) && encodings.firstOrNull()?.maxBitrateBps != null) { engine.registerTrackBitrateInfo( cid = cid, TrackBitrateInfo( diff --git a/munging.patch b/munging.patch deleted file mode 100644 index 1fb38448..00000000 --- a/munging.patch +++ /dev/null @@ -1,591 +0,0 @@ -diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml -index 3501a8d..df624b7 100644 ---- a/gradle/libs.versions.toml -+++ b/gradle/libs.versions.toml -@@ -1,7 +1,6 @@ - [versions] - webrtc = "137.7151.05" - --androidJainSipRi = "1.3.0-91" - androidx-activity = "1.9.0" - androidx-camera = "1.4.2" - androidx-core = "1.13.1" -@@ -31,7 +30,6 @@ noise = "2.0.0" - lifecycleProcess = "2.8.7" - - [libraries] --android-jain-sip-ri = { module = "javax.sip:android-jain-sip-ri", version.ref = "androidJainSipRi" } - androidx-activity-ktx = { module = "androidx.activity:activity-ktx", version.ref = "androidx-activity" } - androidx-camera-core = { module = "androidx.camera:camera-core", version.ref = "androidx-camera" } - androidx-camera-camera2 = { module = "androidx.camera:camera-camera2", version.ref = "androidx-camera" } -@@ -111,4 +109,3 @@ androidx-material3 = { group = "androidx.compose.material3", name = "material3" - lifecycle-process = { group = "androidx.lifecycle", name = "lifecycle-process", version.ref = "lifecycleProcess" } - - [plugins] -- -diff --git a/livekit-android-sdk/build.gradle b/livekit-android-sdk/build.gradle -index 45a3f69..2c019e1 100644 ---- a/livekit-android-sdk/build.gradle -+++ b/livekit-android-sdk/build.gradle -@@ -154,8 +154,6 @@ dependencies { - implementation libs.androidx.core - implementation libs.protobuf.javalite - -- implementation libs.android.jain.sip.ri -- - implementation libs.dagger.lib - kapt libs.dagger.compiler - -diff --git a/livekit-android-sdk/consumer-rules.pro b/livekit-android-sdk/consumer-rules.pro -index a3d6542..6ae7318 100644 ---- a/livekit-android-sdk/consumer-rules.pro -+++ b/livekit-android-sdk/consumer-rules.pro -@@ -28,13 +28,6 @@ - @livekit.**.CalledByNativeUnchecked ; - } - --# NIST sdp parser --######################################### --# Preserve reflection used for Parser registrations ---keep class android.gov.nist.javax.sdp.parser.*Parser { *; } ---keep class android.gov.nist.javax.sdp.parser.ParserFactory { *; } ---keep class android.gov.nist.javax.sdp.parser.SDPParser { *; } -- - # Protobuf - ######################################### - -keep class * extends com.google.protobuf.GeneratedMessageLite { *; } -diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt b/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt -index 9f6da92..075f139 100644 ---- a/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt -+++ b/livekit-android-sdk/src/main/java/io/livekit/android/dagger/RTCModule.kt -@@ -17,7 +17,6 @@ - package io.livekit.android.dagger - - import android.content.Context --import android.javax.sdp.SdpFactory - import android.media.AudioAttributes - import android.media.MediaRecorder - import android.os.Build -@@ -408,8 +407,6 @@ internal object RTCModule { - @Named(InjectionNames.OPTIONS_VIDEO_HW_ACCEL) - fun videoHwAccel() = true - -- @Provides -- fun sdpFactory() = SdpFactory.getInstance() - } - - /** -diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt -index fb6f5a0..f1e533b 100644 ---- a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt -+++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt -@@ -16,8 +16,6 @@ - - package io.livekit.android.room - --import android.javax.sdp.MediaDescription --import android.javax.sdp.SdpFactory - import androidx.annotation.VisibleForTesting - import dagger.assisted.Assisted - import dagger.assisted.AssistedFactory -@@ -38,6 +36,8 @@ import io.livekit.android.webrtc.getFmtps - import io.livekit.android.webrtc.getMsid - import io.livekit.android.webrtc.getRtps - import io.livekit.android.webrtc.isConnected -+import io.livekit.android.webrtc.parseSdpSections -+import io.livekit.android.webrtc.SdpMediaSection - import io.livekit.android.webrtc.peerconnection.RTCThreadToken - import io.livekit.android.webrtc.peerconnection.executeBlockingOnRTCThread - import io.livekit.android.webrtc.peerconnection.launchBlockingOnRTCThread -@@ -76,7 +76,6 @@ constructor( - @Named(InjectionNames.DISPATCHER_IO) - private val ioDispatcher: CoroutineDispatcher, - connectionFactory: PeerConnectionFactory, -- private val sdpFactory: SdpFactory, - private val rtcThreadToken: RTCThreadToken, - ) { - private val coroutineScope = CoroutineScope(ioDispatcher + SupervisorJob()) -@@ -203,21 +202,16 @@ constructor( - return@launchRTCIfNotClosed - } - // munge sdp -- val sdpDescription = sdpFactory.createSessionDescription(sdpOffer.description) -- -- val mediaDescs = sdpDescription.getMediaDescriptions(true) -- for (mediaDesc in mediaDescs) { -- if (mediaDesc !is MediaDescription) { -- continue -- } -- if (mediaDesc.media.mediaType == "audio") { -+ val sdpSections = parseSdpSections(sdpOffer.description) -+ for (mediaDesc in sdpSections.mediaSections) { -+ if (mediaDesc.mediaType == "audio") { - // TODO -- } else if (mediaDesc.media.mediaType == "video") { -+ } else if (mediaDesc.mediaType == "video") { - ensureVideoDDExtensionForSVC(mediaDesc) - ensureCodecBitrates(mediaDesc, trackBitrates = trackBitrates) - } - } -- finalSdp = setMungedSdp(sdpOffer, sdpDescription.toString()) -+ finalSdp = setMungedSdp(sdpOffer, sdpSections.toSdpString()) - } - - finalSdp?.let { sdp -> -@@ -402,7 +396,7 @@ private const val DD_EXTENSION_URI = "https://aomediacodec.github.io/av1-rtp-spe - * @suppress - */ - @VisibleForTesting --fun ensureVideoDDExtensionForSVC(mediaDesc: MediaDescription) { -+fun ensureVideoDDExtensionForSVC(mediaDesc: SdpMediaSection) { - val codec = mediaDesc.getRtps() - .firstOrNull() - ?.second -@@ -426,13 +420,14 @@ fun ensureVideoDDExtensionForSVC(mediaDesc: MediaDescription) { - // Not found, add manually - if (!ddFound) { - mediaDesc.addAttribute( -+ "extmap", - SdpExt( - value = maxId + 1, - uri = DD_EXTENSION_URI, - config = null, - direction = null, - encryptUri = null, -- ).toAttributeField(), -+ ).toAttributeValue(), - ) - } - } -@@ -450,7 +445,7 @@ private const val startBitrateForSVC = 0.7 - */ - @VisibleForTesting - fun ensureCodecBitrates( -- media: MediaDescription, -+ media: SdpMediaSection, - trackBitrates: Map, - ) { - val msid = media.getMsid()?.value ?: return -@@ -482,7 +477,7 @@ fun ensureCodecBitrates( - newFmtpConfig = "$newFmtpConfig;x-google-max-bitrate=${trackBr.maxBitrate}" - } - if (fmtp.config != newFmtpConfig) { -- attribute.value = "${fmtp.payload} $newFmtpConfig" -+ attribute.mutableValue = "${fmtp.payload} $newFmtpConfig" - break - } - } -@@ -490,11 +485,12 @@ fun ensureCodecBitrates( - - if (!fmtpFound) { - media.addAttribute( -+ "fmtp", - SdpFmtp( - payload = codecPayload, - config = "x-google-start-bitrate=${trackBr.maxBitrate * startBitrateForSVC};" + - "x-google-max-bitrate=${trackBr.maxBitrate}", -- ).toAttributeField(), -+ ).toAttributeValue(), - ) - } - } -diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/webrtc/JainSdpUtils.kt b/livekit-android-sdk/src/main/java/io/livekit/android/webrtc/JainSdpUtils.kt -index c6e91c1..371669e 100644 ---- a/livekit-android-sdk/src/main/java/io/livekit/android/webrtc/JainSdpUtils.kt -+++ b/livekit-android-sdk/src/main/java/io/livekit/android/webrtc/JainSdpUtils.kt -@@ -16,8 +16,6 @@ - - package io.livekit.android.webrtc - --import android.gov.nist.javax.sdp.fields.AttributeField --import android.javax.sdp.MediaDescription - import io.livekit.android.util.LKLog - - /** -@@ -25,17 +23,142 @@ import io.livekit.android.util.LKLog - */ - data class SdpRtp(val payload: Long, val codec: String, val rate: Long?, val encoding: String?) - -+data class SdpAttributeRef( -+ val name: String, -+ val value: String, -+ private val onSetValue: (String) -> Unit, -+) { -+ var mutableValue: String = value -+ set(newValue) { -+ field = newValue -+ onSetValue(newValue) -+ } -+} -+ -+/** -+ * Represents a single `m=` section in SDP. -+ * -+ * @suppress -+ */ -+class SdpMediaSection internal constructor( -+ private val lines: MutableList, -+) { -+ internal fun toLines(): List = lines.toList() -+ -+ val mediaType: String? -+ get() { -+ val mediaLine = lines.firstOrNull()?.trim() ?: return null -+ if (!mediaLine.startsWith("m=")) return null -+ return mediaLine.removePrefix("m=") -+ .substringBefore(' ') -+ .ifEmpty { null } -+ } -+ -+ fun addAttribute(name: String, value: String? = null) { -+ if (value == null) { -+ lines.add("a=$name") -+ } else { -+ lines.add("a=$name:$value") -+ } -+ } -+ -+ fun getAttribute(name: String): String? { -+ return getAttributes() -+ .firstOrNull { it.name == name } -+ ?.value -+ } -+ -+ fun getAttributes(): List { -+ val output = mutableListOf() -+ lines.forEachIndexed { index, rawLine -> -+ if (!rawLine.startsWith("a=")) { -+ return@forEachIndexed -+ } -+ -+ val content = rawLine.removePrefix("a=") -+ val separator = content.indexOf(':') -+ val (attributeName, attributeValue) = if (separator < 0) { -+ content to "" -+ } else { -+ content.substring(0, separator) to content.substring(separator + 1) -+ } -+ -+ output.add( -+ SdpAttributeRef( -+ name = attributeName, -+ value = attributeValue, -+ ) { newValue -> -+ lines[index] = "a=$attributeName:$newValue" -+ }, -+ ) -+ } -+ return output -+ } -+} -+ - /** - * @suppress - */ --fun MediaDescription.getRtps(): List> { -- return getAttributes(true) -- .filterIsInstance() -- .filter { it.attribute.name == "rtpmap" } -+data class SdpSections( -+ val sessionLines: List, -+ val mediaSections: List, -+) { -+ fun toSdpString(): String { -+ return buildString { -+ (sessionLines + mediaSections.flatMap { it.toLines() }).forEachIndexed { index, line -> -+ if (index > 0) append("\r\n") -+ append(line) -+ } -+ append("\r\n") -+ } -+ } -+} -+ -+/** -+ * @suppress -+ */ -+fun parseSdpSections(description: String): SdpSections { -+ val normalizedLines = description -+ .replace("\r\n", "\n") -+ .replace('\r', '\n') -+ .split('\n') -+ .filter { it.isNotEmpty() } -+ -+ val sessionLines = mutableListOf() -+ val mediaSections = mutableListOf() -+ -+ var currentMediaSection: MutableList? = null -+ for (line in normalizedLines) { -+ if (line.startsWith("m=")) { -+ currentMediaSection?.let { mediaSections.add(SdpMediaSection(it)) } -+ currentMediaSection = mutableListOf(line) -+ continue -+ } -+ -+ if (currentMediaSection == null) { -+ sessionLines.add(line) -+ } else { -+ currentMediaSection.add(line) -+ } -+ } -+ currentMediaSection?.let { mediaSections.add(SdpMediaSection(it)) } -+ -+ return SdpSections( -+ sessionLines = sessionLines, -+ mediaSections = mediaSections, -+ ) -+} -+ -+/** -+ * @suppress -+ */ -+fun SdpMediaSection.getRtps(): List> { -+ return getAttributes() -+ .filter { it.name == "rtpmap" } - .mapNotNull { -- val rtp = tryParseRtp(it.value) -+ val rtp = tryParseRtp(it.mutableValue) - if (rtp == null) { -- LKLog.w { "could not parse rtpmap: ${it.encode()}" } -+ LKLog.w { "could not parse rtpmap: a=${it.name}:${it.mutableValue}" } - return@mapNotNull null - } - it to rtp -@@ -60,7 +183,7 @@ data class SdpMsid( - /** - * @suppress - */ --fun MediaDescription.getMsid(): SdpMsid? { -+fun SdpMediaSection.getMsid(): SdpMsid? { - val attribute = getAttribute("msid") ?: return null - return SdpMsid(attribute) - } -@@ -69,25 +192,21 @@ fun MediaDescription.getMsid(): SdpMsid? { - * @suppress - */ - data class SdpFmtp(val payload: Long, val config: String) { -- fun toAttributeField(): AttributeField { -- return AttributeField().apply { -- name = "fmtp" -- value = "$payload $config" -- } -+ fun toAttributeValue(): String { -+ return "$payload $config" - } - } - - /** - * @suppress - */ --fun MediaDescription.getFmtps(): List> { -- return getAttributes(true) -- .filterIsInstance() -- .filter { it.attribute.name == "fmtp" } -+fun SdpMediaSection.getFmtps(): List> { -+ return getAttributes() -+ .filter { it.name == "fmtp" } - .mapNotNull { -- val fmtp = tryParseFmtp(it.value) -+ val fmtp = tryParseFmtp(it.mutableValue) - if (fmtp == null) { -- LKLog.w { "could not parse fmtp: ${it.encode()}" } -+ LKLog.w { "could not parse fmtp: a=${it.name}:${it.mutableValue}" } - return@mapNotNull null - } - it to fmtp -@@ -105,21 +224,18 @@ internal fun tryParseFmtp(string: String): SdpFmtp? { - * @suppress - */ - data class SdpExt(val value: Long, val direction: String?, val encryptUri: String?, val uri: String, val config: String?) { -- fun toAttributeField(): AttributeField { -- return AttributeField().apply { -- name = "extmap" -- value = buildString { -- append(this@SdpExt.value) -- if (direction != null) { -- append(" $direction") -- } -- if (encryptUri != null) { -- append(" $encryptUri") -- } -- append(" $uri") -- if (config != null) { -- append(" $config") -- } -+ fun toAttributeValue(): String { -+ return buildString { -+ append(this@SdpExt.value) -+ if (direction != null) { -+ append(" $direction") -+ } -+ if (encryptUri != null) { -+ append(" $encryptUri") -+ } -+ append(" $uri") -+ if (config != null) { -+ append(" $config") - } - } - } -@@ -128,14 +244,13 @@ data class SdpExt(val value: Long, val direction: String?, val encryptUri: Strin - /** - * @suppress - */ --fun MediaDescription.getExts(): List> { -- return getAttributes(true) -- .filterIsInstance() -- .filter { it.attribute.name == "extmap" } -+fun SdpMediaSection.getExts(): List> { -+ return getAttributes() -+ .filter { it.name == "extmap" } - .mapNotNull { -- val ext = tryParseExt(it.value) -+ val ext = tryParseExt(it.mutableValue) - if (ext == null) { -- LKLog.w { "could not parse extmap: ${it.encode()}" } -+ LKLog.w { "could not parse extmap: a=${it.name}:${it.mutableValue}" } - return@mapNotNull null - } - it to ext -diff --git a/livekit-android-test/build.gradle b/livekit-android-test/build.gradle -index 30d84c2..a117d63 100644 ---- a/livekit-android-test/build.gradle -+++ b/livekit-android-test/build.gradle -@@ -82,7 +82,6 @@ dependencies { - api libs.audioswitch - implementation libs.androidx.annotation - api libs.protobuf.javalite -- implementation libs.android.jain.sip.ri - implementation libs.junit - implementation libs.robolectric - implementation libs.mockito.core -diff --git a/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt b/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt -index 0ed771c..5eb2761 100644 ---- a/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt -+++ b/livekit-android-test/src/main/java/io/livekit/android/test/mock/dagger/TestRTCModule.kt -@@ -17,7 +17,6 @@ - package io.livekit.android.test.mock.dagger - - import android.content.Context --import android.javax.sdp.SdpFactory - import dagger.Module - import dagger.Provides - import dagger.Reusable -@@ -129,9 +128,6 @@ object TestRTCModule { - @Named(InjectionNames.OPTIONS_VIDEO_HW_ACCEL) - fun videoHwAccel() = true - -- @Provides -- fun sdpFactory() = SdpFactory.getInstance() -- - @Provides - fun dataPacketCryptorManagerFactory(): DataPacketCryptorManager.Factory = object : DataPacketCryptorManager.Factory { - override fun create(keyProvider: KeyProvider): DataPacketCryptorManager { -diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt -index bfecd60..bd9db73 100644 ---- a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt -+++ b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt -@@ -16,11 +16,10 @@ - - package io.livekit.android.room - --import android.javax.sdp.MediaDescription --import android.javax.sdp.SdpFactory - import io.livekit.android.webrtc.JainSdpUtilsTest - import io.livekit.android.webrtc.getExts - import io.livekit.android.webrtc.getFmtps -+import io.livekit.android.webrtc.parseSdpSections - import org.junit.Assert.assertEquals - import org.junit.Assert.assertNotNull - import org.junit.Assert.assertNull -@@ -30,8 +29,7 @@ class SdpMungingTest { - - @Test - fun ensureVideoDDExtensionForSVCTest() { -- val sdp = SdpFactory.getInstance().createSessionDescription(NO_DD_DESCRIPTION) -- val mediaDescription = sdp.getMediaDescriptions(true).filterIsInstance()[1] -+ val mediaDescription = parseSdpSections(NO_DD_DESCRIPTION).mediaSections[1] - - ensureVideoDDExtensionForSVC(mediaDescription) - -@@ -53,8 +51,7 @@ class SdpMungingTest { - - @Test - fun ensureCodecBitratesTest() { -- val sdp = SdpFactory.getInstance().createSessionDescription(JainSdpUtilsTest.DESCRIPTION) -- val mediaDescription = sdp.getMediaDescriptions(true).filterIsInstance()[1] -+ val mediaDescription = parseSdpSections(JainSdpUtilsTest.DESCRIPTION).mediaSections[1] - - ensureCodecBitrates( - mediaDescription, -diff --git a/livekit-android-test/src/test/java/io/livekit/android/webrtc/JainSdpUtilsTest.kt b/livekit-android-test/src/test/java/io/livekit/android/webrtc/JainSdpUtilsTest.kt -index 18870af..f9f9a77 100644 ---- a/livekit-android-test/src/test/java/io/livekit/android/webrtc/JainSdpUtilsTest.kt -+++ b/livekit-android-test/src/test/java/io/livekit/android/webrtc/JainSdpUtilsTest.kt -@@ -16,9 +16,6 @@ - - package io.livekit.android.webrtc - --import android.javax.sdp.MediaDescription --import android.javax.sdp.SdpFactory --import android.javax.sdp.SessionDescription - import org.junit.Assert.assertEquals - import org.junit.Assert.assertNotNull - import org.junit.Assert.assertNull -@@ -26,17 +23,13 @@ import org.junit.Test - - class JainSdpUtilsTest { - -- private val sdpFactory = SdpFactory.getInstance() -- private fun createSessionDescription(): SessionDescription { -- return sdpFactory.createSessionDescription(DESCRIPTION) -+ private fun createMediaDescription(): SdpMediaSection { -+ return parseSdpSections(DESCRIPTION).mediaSections[1] - } - - @Test - fun getRtpAttributes() { -- val sdp = createSessionDescription() -- val mediaDescriptions = sdp.getMediaDescriptions(true) -- .filterIsInstance() -- val mediaDesc = mediaDescriptions[1] -+ val mediaDesc = createMediaDescription() - val rtps = mediaDesc.getRtps() - assertEquals(13, rtps.size) - -@@ -50,10 +43,7 @@ class JainSdpUtilsTest { - - @Test - fun getExtmapAttributes() { -- val sdp = createSessionDescription() -- val mediaDescriptions = sdp.getMediaDescriptions(true) -- .filterIsInstance() -- val mediaDesc = mediaDescriptions[1] -+ val mediaDesc = createMediaDescription() - val exts = mediaDesc.getExts() - - assertEquals(12, exts.size) -@@ -68,10 +58,7 @@ class JainSdpUtilsTest { - - @Test - fun getMsid() { -- val sdp = createSessionDescription() -- val mediaDescriptions = sdp.getMediaDescriptions(true) -- .filterIsInstance() -- val mediaDesc = mediaDescriptions[1] -+ val mediaDesc = createMediaDescription() - - val msid = mediaDesc.getMsid() - assertNotNull(msid) -@@ -80,10 +67,7 @@ class JainSdpUtilsTest { - - @Test - fun getFmtps() { -- val sdp = createSessionDescription() -- val mediaDescriptions = sdp.getMediaDescriptions(true) -- .filterIsInstance() -- val mediaDesc = mediaDescriptions[1] -+ val mediaDesc = createMediaDescription() - - val fmtps = mediaDesc.getFmtps() - .filter { (_, fmtp) -> fmtp.payload == 97L } From 7c77a42d85dfb88295a6f54f25b16283330073c3 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Thu, 16 Jul 2026 18:02:12 +0800 Subject: [PATCH 03/14] update the code to match the latest Rust / JS behavior --- .../android/room/PeerConnectionTransport.kt | 12 ++++++++-- .../room/participant/LocalParticipant.kt | 24 +++++++++---------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt index d37ec7d6..f74454a6 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt @@ -446,9 +446,13 @@ fun ensureVideoDDExtensionForSVC(mediaDesc: MediaDescription) { * Why 90%: Gives ~10% headroom for bandwidth estimation while starting close to target. * Why same for all codecs: Target bitrate already accounts for codec efficiency * (e.g., users set lower targets for VP9/AV1 knowing they're more efficient). + * Why cap at 1 Mbps: Prevents BWE from starting too aggressively on high bitrate tracks. */ private const val startBitrateMultiplier = 0.9 +/** Maximum x-google-start-bitrate in kbps. 1 Mbps prevents BWE from starting too aggressively. */ +private const val maxStartBitrateKbps = 1000L + /** * @suppress */ @@ -480,7 +484,9 @@ fun ensureCodecBitrates( fmtpFound = true var newFmtpConfig = fmtp.config if (!fmtp.config.contains("x-google-start-bitrate")) { - newFmtpConfig = "$newFmtpConfig;x-google-start-bitrate=${(trackBr.maxBitrate * startBitrateMultiplier).roundToLong()}" + // Use 90% of target bitrate, capped at 1 Mbps to prevent BWE from starting too aggressively + val startBitrate = minOf((trackBr.maxBitrate * startBitrateMultiplier).roundToLong(), maxStartBitrateKbps) + newFmtpConfig = "$newFmtpConfig;x-google-start-bitrate=$startBitrate" } if (!fmtp.config.contains("x-google-max-bitrate")) { newFmtpConfig = "$newFmtpConfig;x-google-max-bitrate=${trackBr.maxBitrate}" @@ -493,10 +499,12 @@ fun ensureCodecBitrates( } if (!fmtpFound) { + // Use 90% of target bitrate, capped at 1 Mbps to prevent BWE from starting too aggressively + val startBitrate = minOf((trackBr.maxBitrate * startBitrateMultiplier).roundToLong(), maxStartBitrateKbps) media.addAttribute( SdpFmtp( payload = codecPayload, - config = "x-google-start-bitrate=${trackBr.maxBitrate * startBitrateMultiplier};" + + config = "x-google-start-bitrate=$startBitrate;" + "x-google-max-bitrate=${trackBr.maxBitrate}", ).toAttributeField(), ) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt index 5911928f..a5d08a96 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt @@ -750,18 +750,16 @@ internal constructor( track.statsGetter = engine.createStatsGetter(transceiver.sender) val finalOptions = options - // Handle trackBitrates - apply start bitrate for SVC codecs to prevent initial blurriness. - // Only SVC codecs (VP9, AV1) are supported here because they have a single encoding with - // the full bitrate. Simulcast codecs (VP8, H264) have multiple encodings ordered - // smallest-to-largest, so encodings.first() would incorrectly return the lowest layer's - // bitrate, which would cap all layers at that low value. - if (encodings.isNotEmpty()) { - if (finalOptions is VideoTrackPublishOptions && isSVCCodec(finalOptions.videoCodec) && encodings.firstOrNull()?.maxBitrateBps != null) { + // Handle trackBitrates - apply start bitrate for all video codecs to prevent initial blurriness. + // Sum all encoding bitrates for simulcast (BWE needs to handle all layers combined). + if (encodings.isNotEmpty() && finalOptions is VideoTrackPublishOptions) { + val totalBitrateBps = encodings.sumOf { it.maxBitrateBps ?: 0 } + if (totalBitrateBps > 0) { engine.registerTrackBitrateInfo( cid = cid, TrackBitrateInfo( codec = finalOptions.videoCodec, - maxBitrate = (encodings.first().maxBitrateBps?.div(1000) ?: 0).toLong(), + maxBitrate = totalBitrateBps / 1000, ), ) } @@ -1595,8 +1593,9 @@ data class VideoTrackPublishDefaults( override val videoCodec: String = VideoCodec.VP8.codecName, override val scalabilityMode: String? = null, override val backupCodec: BackupVideoCodec? = null, - // Default to MAINTAIN_RESOLUTION to prevent initial video blurriness - override val degradationPreference: RtpParameters.DegradationPreference? = RtpParameters.DegradationPreference.MAINTAIN_RESOLUTION, + // Default is null - SDK applies source-based defaults at runtime: + // Camera: MAINTAIN_FRAMERATE, Screen share: MAINTAIN_RESOLUTION, Other: BALANCED + override val degradationPreference: RtpParameters.DegradationPreference? = null, override val simulcastLayers: List? = null, ) : BaseVideoTrackPublishOptions() @@ -1609,8 +1608,9 @@ data class VideoTrackPublishOptions( override val backupCodec: BackupVideoCodec? = null, override val source: Track.Source? = null, override val stream: String? = null, - // Default to MAINTAIN_RESOLUTION to prevent initial video blurriness - override val degradationPreference: RtpParameters.DegradationPreference? = RtpParameters.DegradationPreference.MAINTAIN_RESOLUTION, + // Default is null - SDK applies source-based defaults at runtime: + // Camera: MAINTAIN_FRAMERATE, Screen share: MAINTAIN_RESOLUTION, Other: BALANCED + override val degradationPreference: RtpParameters.DegradationPreference? = null, override val simulcastLayers: List? = null, ) : BaseVideoTrackPublishOptions(), TrackPublishOptions { constructor( From e65de2b48a8049f698ad762c65f9d6bb8f3717a7 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Sat, 18 Jul 2026 09:35:40 +0800 Subject: [PATCH 04/14] update the code to match latest Rust / JS SDKs behaviors --- .../android/room/PeerConnectionTransport.kt | 15 +++++++++++---- .../android/room/participant/LocalParticipant.kt | 14 ++++++++++---- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt index f74454a6..d678531d 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt @@ -478,14 +478,22 @@ fun ensureCodecBitrates( val codecPayload = rtp.payload val fmtps = media.getFmtps() + // Use 90% of target bitrate, capped at 1 Mbps for camera to prevent BWE from starting too aggressively + // Screen share is not capped since text/UI clarity requires high bitrate from the start + // TODO: dynamically adjust start bitrate based on network conditions (e.g., use previous BWE estimate) + val calculatedStartBitrate = (trackBr.maxBitrate * startBitrateMultiplier).roundToLong() + val startBitrate = if (trackBr.isScreenShare) { + calculatedStartBitrate + } else { + minOf(calculatedStartBitrate, maxStartBitrateKbps) + } + var fmtpFound = false for ((attribute, fmtp) in fmtps) { if (fmtp.payload == codecPayload) { fmtpFound = true var newFmtpConfig = fmtp.config if (!fmtp.config.contains("x-google-start-bitrate")) { - // Use 90% of target bitrate, capped at 1 Mbps to prevent BWE from starting too aggressively - val startBitrate = minOf((trackBr.maxBitrate * startBitrateMultiplier).roundToLong(), maxStartBitrateKbps) newFmtpConfig = "$newFmtpConfig;x-google-start-bitrate=$startBitrate" } if (!fmtp.config.contains("x-google-max-bitrate")) { @@ -499,8 +507,6 @@ fun ensureCodecBitrates( } if (!fmtpFound) { - // Use 90% of target bitrate, capped at 1 Mbps to prevent BWE from starting too aggressively - val startBitrate = minOf((trackBr.maxBitrate * startBitrateMultiplier).roundToLong(), maxStartBitrateKbps) media.addAttribute( SdpFmtp( payload = codecPayload, @@ -524,6 +530,7 @@ internal fun isSVCCodec(codec: String?): Boolean { data class TrackBitrateInfo( val codec: String, val maxBitrate: Long, + val isScreenShare: Boolean = false, ) /** diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt index a5d08a96..da64aecc 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt @@ -751,15 +751,21 @@ internal constructor( val finalOptions = options // Handle trackBitrates - apply start bitrate for all video codecs to prevent initial blurriness. - // Sum all encoding bitrates for simulcast (BWE needs to handle all layers combined). + // - SVC codecs: use first encoding's bitrate (single stream with built-in layers) + // - Simulcast: sum all encoding bitrates (independent streams, BWE needs total) if (encodings.isNotEmpty() && finalOptions is VideoTrackPublishOptions) { - val totalBitrateBps = encodings.sumOf { it.maxBitrateBps ?: 0 } - if (totalBitrateBps > 0) { + val targetBitrateBps = if (isSVCCodec(finalOptions.videoCodec)) { + encodings.firstOrNull()?.maxBitrateBps ?: 0 + } else { + encodings.sumOf { it.maxBitrateBps ?: 0 } + } + if (targetBitrateBps > 0) { engine.registerTrackBitrateInfo( cid = cid, TrackBitrateInfo( codec = finalOptions.videoCodec, - maxBitrate = totalBitrateBps / 1000, + maxBitrate = targetBitrateBps / 1000, + isScreenShare = trackSource == Track.Source.SCREEN_SHARE, ), ) } From b6a703157c3966bf39c088e8297fafdb9c235382 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Sat, 18 Jul 2026 14:33:36 +0800 Subject: [PATCH 05/14] fixed the build and cap the hint with 300kbps --- .../io/livekit/android/room/PeerConnectionTransport.kt | 8 ++++++++ .../livekit/android/room/participant/LocalParticipant.kt | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt index d678531d..b114d58d 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt @@ -453,6 +453,9 @@ private const val startBitrateMultiplier = 0.9 /** Maximum x-google-start-bitrate in kbps. 1 Mbps prevents BWE from starting too aggressively. */ private const val maxStartBitrateKbps = 1000L +/** Minimum target bitrate in kbps to apply start bitrate hint. Below this, the hint hurts more than it helps. */ +private const val minTargetBitrateKbps = 300L + /** * @suppress */ @@ -477,6 +480,11 @@ fun ensureCodecBitrates( ?: continue val codecPayload = rtp.payload + // Skip start bitrate hint for very low bitrate tracks - the hint hurts more than it helps + if (trackBr.maxBitrate < minTargetBitrateKbps) { + continue + } + val fmtps = media.getFmtps() // Use 90% of target bitrate, capped at 1 Mbps for camera to prevent BWE from starting too aggressively // Screen share is not capped since text/UI clarity requires high bitrate from the start diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt index da64aecc..32c544ba 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt @@ -754,10 +754,10 @@ internal constructor( // - SVC codecs: use first encoding's bitrate (single stream with built-in layers) // - Simulcast: sum all encoding bitrates (independent streams, BWE needs total) if (encodings.isNotEmpty() && finalOptions is VideoTrackPublishOptions) { - val targetBitrateBps = if (isSVCCodec(finalOptions.videoCodec)) { - encodings.firstOrNull()?.maxBitrateBps ?: 0 + val targetBitrateBps: Long = if (isSVCCodec(finalOptions.videoCodec)) { + (encodings.firstOrNull()?.maxBitrateBps ?: 0).toLong() } else { - encodings.sumOf { it.maxBitrateBps ?: 0 } + encodings.sumOf { (it.maxBitrateBps ?: 0).toLong() } } if (targetBitrateBps > 0) { engine.registerTrackBitrateInfo( From 2ab8a4128ed1f089e47e9902c61e10a64bd6a3c8 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Tue, 21 Jul 2026 14:15:13 +0800 Subject: [PATCH 06/14] fix the tests --- .../src/test/java/io/livekit/android/room/SdpMungingTest.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt index bfecd601..211c0230 100644 --- a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt +++ b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt @@ -56,13 +56,15 @@ class SdpMungingTest { val sdp = SdpFactory.getInstance().createSessionDescription(JainSdpUtilsTest.DESCRIPTION) val mediaDescription = sdp.getMediaDescriptions(true).filterIsInstance()[1] + // Use realistic bitrate: 1000 kbps (1 Mbps) + // With 0.9 multiplier: startBitrate = 900 kbps (below 1 Mbps cap) ensureCodecBitrates( mediaDescription, mapOf( TrackBitrateInfoKey.Cid("PA_Qwqk4y9fcD3G") to TrackBitrateInfo( "VP9", - 1000000L, + 1000L, ), ), ) @@ -71,7 +73,7 @@ class SdpMungingTest { .filter { (_, fmtp) -> fmtp.payload == 98L } .first() - assertEquals("profile-id=0;x-google-start-bitrate=700000;x-google-max-bitrate=1000000", vp9fmtp.config) + assertEquals("profile-id=0;x-google-start-bitrate=900;x-google-max-bitrate=1000", vp9fmtp.config) } companion object { From c01e21d41203eb43a0bc818a80228ec07662a3d4 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Fri, 24 Jul 2026 16:06:40 +0800 Subject: [PATCH 07/14] chore: restore protocol submodule pointer --- protocol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocol b/protocol index 4c05a332..8381f218 160000 --- a/protocol +++ b/protocol @@ -1 +1 @@ -Subproject commit 4c05a3325ec35760bee1c0bfe57b7011604a124f +Subproject commit 8381f2180c45ab926b3ebf19df0608f1dadcac1e From 4360fcfcc356212341c122f21e024cc759efe93a Mon Sep 17 00:00:00 2001 From: shijing xian Date: Mon, 3 Aug 2026 13:38:09 +0800 Subject: [PATCH 08/14] update the code not to set the maxBitrate, and set startBitrate only for the first track --- .../android/room/PeerConnectionTransport.kt | 191 +++++++++++++----- .../io/livekit/android/room/SdpMungingTest.kt | 69 ++++++- 2 files changed, 203 insertions(+), 57 deletions(-) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt index b114d58d..b57422e1 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt @@ -94,6 +94,10 @@ constructor( private var renegotiate = false private val trackBitrates = mutableMapOf() + // x-google-start-bitrate is a connection-level BWE hint in libwebrtc. Keep it + // available through data-channel/audio-only offers and consume it only after a + // local video m-section successfully gets the hint. + private var hasAppliedVideoStartBitrate = false private var isClosed = AtomicBoolean(false) private val latestOfferId = AtomicInteger(0) @@ -206,18 +210,37 @@ constructor( val sdpDescription = sdpFactory.createSessionDescription(sdpOffer.description) val mediaDescs = sdpDescription.getMediaDescriptions(true) + .filterIsInstance() + // The publisher PeerConnection may negotiate before any video is published + // (for example, data channel only or audio first). Those offers should not + // consume the video start hint. When the first video offer is created, use + // one connection-level value across all video m-sections so libwebrtc's + // last-writer-wins handling cannot depend on SDP m-section order. + val connectionStartBitrate = if (!hasAppliedVideoStartBitrate) { + computeConnectionStartBitrate(mediaDescs, trackBitrates) + } else { + null + } + var appliedVideoStartBitrate = false for (mediaDesc in mediaDescs) { - if (mediaDesc !is MediaDescription) { - continue - } if (mediaDesc.media.mediaType == "audio") { // TODO } else if (mediaDesc.media.mediaType == "video") { ensureVideoDDExtensionForSVC(mediaDesc) - ensureCodecBitrates(mediaDesc, trackBitrates = trackBitrates) + appliedVideoStartBitrate = ensureCodecBitrates( + mediaDesc, + trackBitrates = trackBitrates, + connectionStartBitrate = connectionStartBitrate, + ) || appliedVideoStartBitrate } } - finalSdp = setMungedSdp(sdpOffer, sdpDescription.toString()) + val mungedDescription = sdpDescription.toString() + finalSdp = setMungedSdp(sdpOffer, mungedDescription) + // setMungedSdp may fall back to the original SDP. Only mark the one-shot + // hint as used after the SDP with the hint is accepted locally. + if (appliedVideoStartBitrate && finalSdp?.description == mungedDescription) { + hasAppliedVideoStartBitrate = true + } } finalSdp?.let { sdp -> @@ -446,7 +469,18 @@ fun ensureVideoDDExtensionForSVC(mediaDesc: MediaDescription) { * Why 90%: Gives ~10% headroom for bandwidth estimation while starting close to target. * Why same for all codecs: Target bitrate already accounts for codec efficiency * (e.g., users set lower targets for VP9/AV1 knowing they're more efficient). - * Why cap at 1 Mbps: Prevents BWE from starting too aggressively on high bitrate tracks. + * Why cap camera at 1 Mbps: Prevents BWE from starting too aggressively on high bitrate tracks. + * + * libwebrtc applies these codec fmtp bitrate params to the shared Call, not just + * the m-section that carries them. To avoid last-writer-wins variance, each video + * m-section gets the same x-google-start-bitrate: the max hint among active video + * m-sections in the first offer that contains local video. Later renegotiations do + * not write it, because reapplying a start hint can reset an already-running + * bandwidth estimator. + * + * Do not write x-google-max-bitrate here. Per-track and per-layer caps are already + * applied with RtpParameters.Encoding.maxBitrateBps, while the SDP fmtp max becomes + * a Call-wide max_data_rate and can throttle unrelated concurrent tracks. */ private const val startBitrateMultiplier = 0.9 @@ -464,65 +498,110 @@ fun ensureCodecBitrates( media: MediaDescription, trackBitrates: Map, ) { - val msid = media.getMsid()?.value ?: return - for ((key, trackBr) in trackBitrates) { - if (key !is TrackBitrateInfoKey.Cid) { - continue - } + ensureCodecBitrates( + media = media, + trackBitrates = trackBitrates, + connectionStartBitrate = computeConnectionStartBitrate(trackBitrates.values), + ) +} - val (cid) = key - if (!msid.contains(cid)) { - continue +@VisibleForTesting +internal fun ensureCodecBitrates( + media: MediaDescription, + trackBitrates: Map, + connectionStartBitrate: Long?, +): Boolean { + // Returns true when this media section maps to a local video track and has or + // receives the connection-level start hint. + val startBitrate = connectionStartBitrate ?: return false + val (_, codecPayload) = findTrackCodecBitrateInfo(media, trackBitrates) ?: return false + + val fmtps = media.getFmtps() + var fmtpFound = false + for ((attribute, fmtp) in fmtps) { + if (fmtp.payload == codecPayload) { + fmtpFound = true + if (fmtp.config.contains("x-google-start-bitrate")) { + return true + } + attribute.value = "${fmtp.payload} ${fmtp.config};x-google-start-bitrate=$startBitrate" + break } + } - val (_, rtp) = media.getRtps() - .firstOrNull { (_, rtp) -> rtp.codec.equals(trackBr.codec, ignoreCase = true) } - ?: continue - val codecPayload = rtp.payload + if (!fmtpFound) { + media.addAttribute( + SdpFmtp( + payload = codecPayload, + config = "x-google-start-bitrate=$startBitrate", + ).toAttributeField(), + ) + } + return true +} + +private fun computeConnectionStartBitrate( + mediaDescriptions: Collection, + trackBitrates: Map, +): Long? { + // Use only video m-sections in the current SDP. trackBitrates can contain + // stale entries after unpublish, and those must not affect the connection hint. + return mediaDescriptions + .asSequence() + .filter { media -> media.media.mediaType == "video" } + .mapNotNull { media -> findTrackCodecBitrateInfo(media, trackBitrates)?.trackBitrateInfo } + .mapNotNull(::computeTrackStartBitrate) + .maxOrNull() +} - // Skip start bitrate hint for very low bitrate tracks - the hint hurts more than it helps - if (trackBr.maxBitrate < minTargetBitrateKbps) { +/** + * @suppress + */ +@VisibleForTesting +internal fun computeConnectionStartBitrate(trackBitrates: Collection): Long? { + return trackBitrates.mapNotNull(::computeTrackStartBitrate).maxOrNull() +} + +private data class TrackCodecBitrateInfo( + val trackBitrateInfo: TrackBitrateInfo, + val codecPayload: Long, +) + +private fun findTrackCodecBitrateInfo( + media: MediaDescription, + trackBitrates: Map, +): TrackCodecBitrateInfo? { + val msid = media.getMsid()?.value ?: return null + for ((key, trackBitrateInfo) in trackBitrates) { + if (key !is TrackBitrateInfoKey.Cid) { continue } - - val fmtps = media.getFmtps() - // Use 90% of target bitrate, capped at 1 Mbps for camera to prevent BWE from starting too aggressively - // Screen share is not capped since text/UI clarity requires high bitrate from the start - // TODO: dynamically adjust start bitrate based on network conditions (e.g., use previous BWE estimate) - val calculatedStartBitrate = (trackBr.maxBitrate * startBitrateMultiplier).roundToLong() - val startBitrate = if (trackBr.isScreenShare) { - calculatedStartBitrate - } else { - minOf(calculatedStartBitrate, maxStartBitrateKbps) + if (!msid.contains(key.value)) { + continue } + val (_, rtp) = media.getRtps() + .firstOrNull { (_, rtp) -> rtp.codec.equals(trackBitrateInfo.codec, ignoreCase = true) } + ?: continue + return TrackCodecBitrateInfo( + trackBitrateInfo = trackBitrateInfo, + codecPayload = rtp.payload, + ) + } + return null +} - var fmtpFound = false - for ((attribute, fmtp) in fmtps) { - if (fmtp.payload == codecPayload) { - fmtpFound = true - var newFmtpConfig = fmtp.config - if (!fmtp.config.contains("x-google-start-bitrate")) { - newFmtpConfig = "$newFmtpConfig;x-google-start-bitrate=$startBitrate" - } - if (!fmtp.config.contains("x-google-max-bitrate")) { - newFmtpConfig = "$newFmtpConfig;x-google-max-bitrate=${trackBr.maxBitrate}" - } - if (fmtp.config != newFmtpConfig) { - attribute.value = "${fmtp.payload} $newFmtpConfig" - break - } - } - } +private fun computeTrackStartBitrate(trackBr: TrackBitrateInfo): Long? { + if (trackBr.maxBitrate < minTargetBitrateKbps) { + return null + } - if (!fmtpFound) { - media.addAttribute( - SdpFmtp( - payload = codecPayload, - config = "x-google-start-bitrate=$startBitrate;" + - "x-google-max-bitrate=${trackBr.maxBitrate}", - ).toAttributeField(), - ) - } + // TODO: dynamically adjust start bitrate based on network conditions, such as + // using the previous BWE estimate. + val calculatedStartBitrate = (trackBr.maxBitrate * startBitrateMultiplier).roundToLong() + return if (trackBr.isScreenShare) { + calculatedStartBitrate + } else { + minOf(calculatedStartBitrate, maxStartBitrateKbps) } } diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt index 211c0230..a0fe4f0c 100644 --- a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt +++ b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt @@ -73,7 +73,74 @@ class SdpMungingTest { .filter { (_, fmtp) -> fmtp.payload == 98L } .first() - assertEquals("profile-id=0;x-google-start-bitrate=900;x-google-max-bitrate=1000", vp9fmtp.config) + assertEquals("profile-id=0;x-google-start-bitrate=900", vp9fmtp.config) + } + + @Test + fun ensureCodecBitratesUsesConnectionStartBitrateTest() { + val sdp = SdpFactory.getInstance().createSessionDescription(JainSdpUtilsTest.DESCRIPTION) + val mediaDescription = sdp.getMediaDescriptions(true).filterIsInstance()[1] + + ensureCodecBitrates( + mediaDescription, + mapOf( + TrackBitrateInfoKey.Cid("PA_Qwqk4y9fcD3G") to + TrackBitrateInfo( + "VP9", + 1000L, + ), + ), + connectionStartBitrate = 1000L, + ) + + val (_, vp9fmtp) = mediaDescription.getFmtps() + .filter { (_, fmtp) -> fmtp.payload == 98L } + .first() + + assertEquals("profile-id=0;x-google-start-bitrate=1000", vp9fmtp.config) + } + + @Test + fun ensureCodecBitratesSkipsStartBitrateTest() { + val sdp = SdpFactory.getInstance().createSessionDescription(JainSdpUtilsTest.DESCRIPTION) + val mediaDescription = sdp.getMediaDescriptions(true).filterIsInstance()[1] + + ensureCodecBitrates( + mediaDescription, + mapOf( + TrackBitrateInfoKey.Cid("PA_Qwqk4y9fcD3G") to + TrackBitrateInfo( + "VP9", + 1000L, + ), + ), + connectionStartBitrate = null, + ) + + val (_, vp9fmtp) = mediaDescription.getFmtps() + .filter { (_, fmtp) -> fmtp.payload == 98L } + .first() + + assertEquals("profile-id=0", vp9fmtp.config) + } + + @Test + fun computeConnectionStartBitrateTest() { + val startBitrate = computeConnectionStartBitrate( + listOf( + TrackBitrateInfo( + codec = "VP8", + maxBitrate = 2310L, + ), + TrackBitrateInfo( + codec = "VP8", + maxBitrate = 5000L, + isScreenShare = true, + ), + ), + ) + + assertEquals(4500L, startBitrate) } companion object { From 689fda838f0f7637380525f106c488f72c41772f Mon Sep 17 00:00:00 2001 From: shijing xian Date: Mon, 3 Aug 2026 14:57:09 +0800 Subject: [PATCH 09/14] Remove degradation preference changes from bitrate PR --- .../io/livekit/android/room/participant/LocalParticipant.kt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt index 32c544ba..781d806f 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt @@ -1599,8 +1599,6 @@ data class VideoTrackPublishDefaults( override val videoCodec: String = VideoCodec.VP8.codecName, override val scalabilityMode: String? = null, override val backupCodec: BackupVideoCodec? = null, - // Default is null - SDK applies source-based defaults at runtime: - // Camera: MAINTAIN_FRAMERATE, Screen share: MAINTAIN_RESOLUTION, Other: BALANCED override val degradationPreference: RtpParameters.DegradationPreference? = null, override val simulcastLayers: List? = null, ) : BaseVideoTrackPublishOptions() @@ -1614,8 +1612,6 @@ data class VideoTrackPublishOptions( override val backupCodec: BackupVideoCodec? = null, override val source: Track.Source? = null, override val stream: String? = null, - // Default is null - SDK applies source-based defaults at runtime: - // Camera: MAINTAIN_FRAMERATE, Screen share: MAINTAIN_RESOLUTION, Other: BALANCED override val degradationPreference: RtpParameters.DegradationPreference? = null, override val simulcastLayers: List? = null, ) : BaseVideoTrackPublishOptions(), TrackPublishOptions { From 19b77f98c620f57f179d1a16e2bd4f15ce2abe6f Mon Sep 17 00:00:00 2001 From: shijing xian Date: Mon, 3 Aug 2026 15:09:58 +0800 Subject: [PATCH 10/14] Clarify start bitrate target naming --- .../android/room/PeerConnectionTransport.kt | 6 +++--- .../android/room/participant/LocalParticipant.kt | 2 +- .../io/livekit/android/room/SdpMungingTest.kt | 16 ++++++++-------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt index b57422e1..6c6eaa6b 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt @@ -591,13 +591,13 @@ private fun findTrackCodecBitrateInfo( } private fun computeTrackStartBitrate(trackBr: TrackBitrateInfo): Long? { - if (trackBr.maxBitrate < minTargetBitrateKbps) { + if (trackBr.targetBitrateKbps < minTargetBitrateKbps) { return null } // TODO: dynamically adjust start bitrate based on network conditions, such as // using the previous BWE estimate. - val calculatedStartBitrate = (trackBr.maxBitrate * startBitrateMultiplier).roundToLong() + val calculatedStartBitrate = (trackBr.targetBitrateKbps * startBitrateMultiplier).roundToLong() return if (trackBr.isScreenShare) { calculatedStartBitrate } else { @@ -616,7 +616,7 @@ internal fun isSVCCodec(codec: String?): Boolean { */ data class TrackBitrateInfo( val codec: String, - val maxBitrate: Long, + val targetBitrateKbps: Long, val isScreenShare: Boolean = false, ) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt index 781d806f..145237bd 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt @@ -764,7 +764,7 @@ internal constructor( cid = cid, TrackBitrateInfo( codec = finalOptions.videoCodec, - maxBitrate = targetBitrateBps / 1000, + targetBitrateKbps = targetBitrateBps / 1000, isScreenShare = trackSource == Track.Source.SCREEN_SHARE, ), ) diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt index a0fe4f0c..41391e6d 100644 --- a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt +++ b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt @@ -63,8 +63,8 @@ class SdpMungingTest { mapOf( TrackBitrateInfoKey.Cid("PA_Qwqk4y9fcD3G") to TrackBitrateInfo( - "VP9", - 1000L, + codec = "VP9", + targetBitrateKbps = 1000L, ), ), ) @@ -86,8 +86,8 @@ class SdpMungingTest { mapOf( TrackBitrateInfoKey.Cid("PA_Qwqk4y9fcD3G") to TrackBitrateInfo( - "VP9", - 1000L, + codec = "VP9", + targetBitrateKbps = 1000L, ), ), connectionStartBitrate = 1000L, @@ -110,8 +110,8 @@ class SdpMungingTest { mapOf( TrackBitrateInfoKey.Cid("PA_Qwqk4y9fcD3G") to TrackBitrateInfo( - "VP9", - 1000L, + codec = "VP9", + targetBitrateKbps = 1000L, ), ), connectionStartBitrate = null, @@ -130,11 +130,11 @@ class SdpMungingTest { listOf( TrackBitrateInfo( codec = "VP8", - maxBitrate = 2310L, + targetBitrateKbps = 2310L, ), TrackBitrateInfo( codec = "VP8", - maxBitrate = 5000L, + targetBitrateKbps = 5000L, isScreenShare = true, ), ), From 674d5487f3ac9ea28cd265d9bc9af5b030e22a24 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Mon, 3 Aug 2026 15:24:29 +0800 Subject: [PATCH 11/14] Document max bitrate SDP behavior --- .../livekit/android/room/PeerConnectionTransport.kt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt index 6c6eaa6b..a25ddb5e 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt @@ -478,9 +478,14 @@ fun ensureVideoDDExtensionForSVC(mediaDesc: MediaDescription) { * not write it, because reapplying a start hint can reset an already-running * bandwidth estimator. * - * Do not write x-google-max-bitrate here. Per-track and per-layer caps are already - * applied with RtpParameters.Encoding.maxBitrateBps, while the SDP fmtp max becomes - * a Call-wide max_data_rate and can throttle unrelated concurrent tracks. + * Do not write x-google-max-bitrate here. libwebrtc promotes this SDP fmtp + * value into the shared Call max_data_rate, so one video m-section can cap the + * whole publisher connection and throttle unrelated concurrent tracks, such as + * camera plus screen share. The track-specific limit belongs in + * RtpParameters.Encoding.maxBitrateBps, where per-track and per-layer caps are + * already applied. Keep this behavior aligned across LiveKit SDKs by relying on + * encoding parameters for max bitrate and reserving SDP munging for the one + * connection-level start bitrate hint. */ private const val startBitrateMultiplier = 0.9 From f5a4fcd227de1c74c141fa3754430ed7a4e8991c Mon Sep 17 00:00:00 2001 From: shijing xian Date: Mon, 3 Aug 2026 15:33:43 +0800 Subject: [PATCH 12/14] Fix spotless formatting --- .../java/io/livekit/android/room/PeerConnectionTransport.kt | 1 + .../src/test/java/io/livekit/android/room/SdpMungingTest.kt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt index a25ddb5e..4426ed62 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt @@ -94,6 +94,7 @@ constructor( private var renegotiate = false private val trackBitrates = mutableMapOf() + // x-google-start-bitrate is a connection-level BWE hint in libwebrtc. Keep it // available through data-channel/audio-only offers and consume it only after a // local video m-section successfully gets the hint. diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt index 41391e6d..2c8cb83e 100644 --- a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt +++ b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt @@ -1,5 +1,5 @@ /* - * Copyright 2023-2024 LiveKit, Inc. + * Copyright 2023-2026 LiveKit, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From d10bc6187d53a73ee793db557cb29e3f2e01e658 Mon Sep 17 00:00:00 2001 From: shijing xian Date: Mon, 3 Aug 2026 15:48:45 +0800 Subject: [PATCH 13/14] Move bitrate fmtp rationale to munging helper --- .../android/room/PeerConnectionTransport.kt | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt index 4426ed62..d9ff63d6 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt @@ -471,22 +471,6 @@ fun ensureVideoDDExtensionForSVC(mediaDesc: MediaDescription) { * Why same for all codecs: Target bitrate already accounts for codec efficiency * (e.g., users set lower targets for VP9/AV1 knowing they're more efficient). * Why cap camera at 1 Mbps: Prevents BWE from starting too aggressively on high bitrate tracks. - * - * libwebrtc applies these codec fmtp bitrate params to the shared Call, not just - * the m-section that carries them. To avoid last-writer-wins variance, each video - * m-section gets the same x-google-start-bitrate: the max hint among active video - * m-sections in the first offer that contains local video. Later renegotiations do - * not write it, because reapplying a start hint can reset an already-running - * bandwidth estimator. - * - * Do not write x-google-max-bitrate here. libwebrtc promotes this SDP fmtp - * value into the shared Call max_data_rate, so one video m-section can cap the - * whole publisher connection and throttle unrelated concurrent tracks, such as - * camera plus screen share. The track-specific limit belongs in - * RtpParameters.Encoding.maxBitrateBps, where per-track and per-layer caps are - * already applied. Keep this behavior aligned across LiveKit SDKs by relying on - * encoding parameters for max bitrate and reserving SDP munging for the one - * connection-level start bitrate hint. */ private const val startBitrateMultiplier = 0.9 @@ -511,6 +495,23 @@ fun ensureCodecBitrates( ) } +/* + * libwebrtc applies these codec fmtp bitrate params to the shared Call, not just + * the m-section that carries them. To avoid last-writer-wins variance, each video + * m-section gets the same x-google-start-bitrate: the max hint among active video + * m-sections in the first offer that contains local video. Later renegotiations do + * not write it, because reapplying a start hint can reset an already-running + * bandwidth estimator. + * + * Do not write x-google-max-bitrate here. libwebrtc promotes this SDP fmtp + * value into the shared Call max_data_rate, so one video m-section can cap the + * whole publisher connection and throttle unrelated concurrent tracks, such as + * camera plus screen share. The track-specific limit belongs in + * RtpParameters.Encoding.maxBitrateBps, where per-track and per-layer caps are + * already applied. Keep this behavior aligned across LiveKit SDKs by relying on + * encoding parameters for max bitrate and reserving SDP munging for the one + * connection-level start bitrate hint. + */ @VisibleForTesting internal fun ensureCodecBitrates( media: MediaDescription, From b1fc06688356a5917ff581e038494eff60f4b0fd Mon Sep 17 00:00:00 2001 From: shijing xian Date: Mon, 14 Sep 2026 08:40:07 -0700 Subject: [PATCH 14/14] Make bitrate munging types internal and document the libwebrtc contract TrackBitrateInfo and TrackBitrateInfoKey were public only so the separate test module could reach them. friendPaths (#925) made that unnecessary, so mark them internal along with the two-arg ensureCodecBitrates that exposes them. Both were already @suppress'd and only reachable through a @VisibleForTesting helper, so this is not intended API. Compiled signatures are unchanged; internal is recorded in @Metadata only. Document isScreenShare and the kbps unit on TrackBitrateInfo, and record why the start bitrate is written once to every video m-section: libwebrtc pushes these codec fmtp params into the shared Call, retains start_bitrate_bps in RtpBitrateConfigurator, and re-applies it on network route changes, so rewriting it later either no-ops or restarts a converged estimator. Same for why x-google-max-bitrate is never written. Add the changeset, including the max-bitrate removal as a behavior change. Co-Authored-By: Claude Opus 5 (1M context) --- .changeset/video-start-bitrate-hint.md | 11 ++++ .../android/room/PeerConnectionTransport.kt | 59 ++++++++++++------- 2 files changed, 48 insertions(+), 22 deletions(-) create mode 100644 .changeset/video-start-bitrate-hint.md diff --git a/.changeset/video-start-bitrate-hint.md b/.changeset/video-start-bitrate-hint.md new file mode 100644 index 00000000..c5fadbac --- /dev/null +++ b/.changeset/video-start-bitrate-hint.md @@ -0,0 +1,11 @@ +--- +"client-sdk-android": patch +--- + +Seed the bandwidth estimator with `x-google-start-bitrate` for all video codecs, not just SVC, so published video reaches its target quality in the first second or two instead of ramping from ~300 kbps over 5-15 seconds. The hint is 90% of the track's target bitrate, capped at 1 Mbps for camera tracks (screen shares are exempt, since they are published at high bitrates for text legibility) and skipped below a 300 kbps target, where seeding high costs more than it gains. + +Because libwebrtc applies these codec fmtp parameters to the whole peer connection rather than the m-section carrying them, the SDK now writes a single connection-level value to every video m-section, once per publisher connection. Re-seeding a converged estimator is avoided: the value persists in libwebrtc's bitrate configurator and is automatically re-applied on network route changes, and a full reconnect builds a new peer connection and seeds it again. + +**Behavior change:** the SDK no longer writes `x-google-max-bitrate` into SDP. That value was promoted to a ceiling on total send bandwidth for the entire connection, so a camera publication could throttle a concurrent screen share. Per-track and per-layer limits continue to be enforced through `RtpParameters.Encoding.maxBitrateBps`, which is correctly scoped per encoding. Applications that relied on the SDP value as a connection-wide cap should set encoding bitrates instead. This matches client-sdk-js and the Rust SDK, neither of which writes it. + +`TrackBitrateInfo` and `TrackBitrateInfoKey` are now `internal`. They were never intended as public API (both were `@suppress`ed and only reachable through a `@VisibleForTesting` helper) and were public only to be visible from the test module, which is no longer necessary. diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt index d9ff63d6..f856b170 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt @@ -480,11 +480,8 @@ private const val maxStartBitrateKbps = 1000L /** Minimum target bitrate in kbps to apply start bitrate hint. Below this, the hint hurts more than it helps. */ private const val minTargetBitrateKbps = 300L -/** - * @suppress - */ @VisibleForTesting -fun ensureCodecBitrates( +internal fun ensureCodecBitrates( media: MediaDescription, trackBitrates: Map, ) { @@ -496,21 +493,27 @@ fun ensureCodecBitrates( } /* - * libwebrtc applies these codec fmtp bitrate params to the shared Call, not just - * the m-section that carries them. To avoid last-writer-wins variance, each video - * m-section gets the same x-google-start-bitrate: the max hint among active video - * m-sections in the first offer that contains local video. Later renegotiations do - * not write it, because reapplying a start hint can reset an already-running - * bandwidth estimator. + * These codec fmtp params are connection-scoped, not m-section-scoped. libwebrtc reads them per + * m-section (WebRtcVideoSendChannel::ApplyChangedParams -> GetBitrateConfigForCodec) but pushes the + * result into the shared Call via SetSdpBitrateParameters, where RtpBitrateConfigurator stores one + * config for the whole peer connection. Two m-sections carrying different values is last-writer-wins. + * + * Hence: one value, written to every video m-section, once. * - * Do not write x-google-max-bitrate here. libwebrtc promotes this SDP fmtp - * value into the shared Call max_data_rate, so one video m-section can cap the - * whole publisher connection and throttle unrelated concurrent tracks, such as - * camera plus screen share. The track-specific limit belongs in - * RtpParameters.Encoding.maxBitrateBps, where per-track and per-layer caps are - * already applied. Keep this behavior aligned across LiveKit SDKs by relying on - * encoding parameters for max bitrate and reserving SDP munging for the one - * connection-level start bitrate hint. + * Write it once because the value persists. RtpBitrateConfigurator keeps start_bitrate_bps in its + * stored config and re-applies it on every network route change (RtpTransportControllerSend:: + * OnNetworkRouteChanged reads GetConfig()), so a WiFi-to-cellular handover re-seeds the estimator + * from this hint with no renegotiation. Rewriting it later is at best a no-op (libwebrtc ignores an + * unchanged value, and only re-reads it when the send codec changes) and at worst restarts a + * converged bandwidth estimator, so a full reconnect -- a new peer connection, a new estimator -- is + * the only thing that should seed it again. + * + * Never write x-google-max-bitrate. The same Call-level promotion turns a per-track cap into a + * ceiling on total send bandwidth, so a 2.3 Mbps camera would starve a concurrent 3 Mbps screen + * share. libwebrtc carries a TODO conceding this is wrong ("codec max bitrate should probably not + * affect global call max bitrate"). Per-track and per-layer caps belong in + * RtpParameters.Encoding.maxBitrateBps, which is genuinely scoped per encoding. client-sdk-js and + * the Rust SDK never write it either. */ @VisibleForTesting internal fun ensureCodecBitrates( @@ -619,18 +622,30 @@ internal fun isSVCCodec(codec: String?): Boolean { } /** - * @suppress + * The bitrate a local video track was published at, used to derive the connection-level + * `x-google-start-bitrate` hint in [ensureCodecBitrates]. + * + * Carries no max bitrate: per-track and per-layer caps belong in + * [livekit.org.webrtc.RtpParameters.Encoding.maxBitrateBps], not in SDP. + * + * @param codec The codec the track is published with, matched against the m-section's rtpmap. + * @param targetBitrateKbps The track's target bitrate in **kbps** (not bps). For SVC this is the + * single encoding's bitrate; for simulcast it is the sum across layers, since the bandwidth + * estimator has to carry all of them. + * @param isScreenShare Whether the track is a screen share. Screen shares are exempt from the + * [maxStartBitrateKbps] cap: they are typically published at high bitrates for text legibility, + * and unlike camera content a conservative start is more costly than a brief overshoot. */ -data class TrackBitrateInfo( +internal data class TrackBitrateInfo( val codec: String, val targetBitrateKbps: Long, val isScreenShare: Boolean = false, ) /** - * @suppress + * Identifies the local track a [TrackBitrateInfo] belongs to. */ -sealed class TrackBitrateInfoKey { +internal sealed class TrackBitrateInfoKey { data class Cid(val value: String) : TrackBitrateInfoKey() data class Transceiver(val value: RtpTransceiver) : TrackBitrateInfoKey() }