diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt index efae279a..b114d58d 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/PeerConnectionTransport.kt @@ -437,13 +437,24 @@ fun ensureVideoDDExtensionForSVC(mediaDesc: MediaDescription) { } } -/* The svc codec (av1/vp9) would use a very low bitrate at the beginning and -increase slowly by the bandwidth estimator until it reach the target bitrate. The -process commonly cost more than 10 seconds cause subscriber will get blur video at -the first few seconds. So we use a 70% of target bitrate here as the start bitrate to -eliminate this issue. -*/ -private const val startBitrateForSVC = 0.7 +/* + * Video codecs use a very low bitrate at the beginning and increase slowly by + * the bandwidth estimator until they reach the target bitrate. The process commonly + * costs more than 10 seconds causing subscribers to get blurry video at the first + * few seconds. We use x-google-start-bitrate to hint the BWE to start higher. + * + * Why 90%: Gives ~10% headroom for bandwidth estimation while starting close to target. + * Why same for all codecs: Target bitrate already accounts for codec efficiency + * (e.g., users set lower targets for VP9/AV1 knowing they're more efficient). + * Why cap at 1 Mbps: Prevents BWE from starting too aggressively on high bitrate tracks. + */ +private const val startBitrateMultiplier = 0.9 + +/** Maximum x-google-start-bitrate in kbps. 1 Mbps prevents BWE from starting too aggressively. */ +private const val maxStartBitrateKbps = 1000L + +/** Minimum target bitrate in kbps to apply start bitrate hint. Below this, the hint hurts more than it helps. */ +private const val minTargetBitrateKbps = 300L /** * @suppress @@ -469,14 +480,29 @@ fun ensureCodecBitrates( ?: continue val codecPayload = rtp.payload + // Skip start bitrate hint for very low bitrate tracks - the hint hurts more than it helps + if (trackBr.maxBitrate < minTargetBitrateKbps) { + continue + } + val fmtps = media.getFmtps() + // Use 90% of target bitrate, capped at 1 Mbps for camera to prevent BWE from starting too aggressively + // Screen share is not capped since text/UI clarity requires high bitrate from the start + // TODO: dynamically adjust start bitrate based on network conditions (e.g., use previous BWE estimate) + val calculatedStartBitrate = (trackBr.maxBitrate * startBitrateMultiplier).roundToLong() + val startBitrate = if (trackBr.isScreenShare) { + calculatedStartBitrate + } else { + minOf(calculatedStartBitrate, maxStartBitrateKbps) + } + var fmtpFound = false for ((attribute, fmtp) in fmtps) { if (fmtp.payload == codecPayload) { fmtpFound = true var newFmtpConfig = fmtp.config if (!fmtp.config.contains("x-google-start-bitrate")) { - newFmtpConfig = "$newFmtpConfig;x-google-start-bitrate=${(trackBr.maxBitrate * startBitrateForSVC).roundToLong()}" + newFmtpConfig = "$newFmtpConfig;x-google-start-bitrate=$startBitrate" } if (!fmtp.config.contains("x-google-max-bitrate")) { newFmtpConfig = "$newFmtpConfig;x-google-max-bitrate=${trackBr.maxBitrate}" @@ -492,7 +518,7 @@ fun ensureCodecBitrates( media.addAttribute( SdpFmtp( payload = codecPayload, - config = "x-google-start-bitrate=${trackBr.maxBitrate * startBitrateForSVC};" + + config = "x-google-start-bitrate=$startBitrate;" + "x-google-max-bitrate=${trackBr.maxBitrate}", ).toAttributeField(), ) @@ -512,6 +538,7 @@ internal fun isSVCCodec(codec: String?): Boolean { data class TrackBitrateInfo( val codec: String, val maxBitrate: Long, + val isScreenShare: Boolean = false, ) /** diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt index 5daa840d..60b9781b 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/participant/LocalParticipant.kt @@ -697,14 +697,22 @@ internal constructor( track.statsGetter = engine.createStatsGetter(transceiver.sender) val finalOptions = options - // Handle trackBitrates - if (encodings.isNotEmpty()) { - if (finalOptions is VideoTrackPublishOptions && isSVCCodec(finalOptions.videoCodec) && encodings.firstOrNull()?.maxBitrateBps != null) { + // Handle trackBitrates - apply start bitrate for all video codecs to prevent initial blurriness. + // - SVC codecs: use first encoding's bitrate (single stream with built-in layers) + // - Simulcast: sum all encoding bitrates (independent streams, BWE needs total) + if (encodings.isNotEmpty() && finalOptions is VideoTrackPublishOptions) { + val targetBitrateBps: Long = if (isSVCCodec(finalOptions.videoCodec)) { + (encodings.firstOrNull()?.maxBitrateBps ?: 0).toLong() + } else { + encodings.sumOf { (it.maxBitrateBps ?: 0).toLong() } + } + if (targetBitrateBps > 0) { engine.registerTrackBitrateInfo( cid = cid, TrackBitrateInfo( codec = finalOptions.videoCodec, - maxBitrate = (encodings.first().maxBitrateBps?.div(1000) ?: 0).toLong(), + maxBitrate = targetBitrateBps / 1000, + isScreenShare = trackSource == Track.Source.SCREEN_SHARE, ), ) } @@ -716,7 +724,12 @@ internal constructor( (track as LocalVideoTrack).codec = finalOptions.videoCodec val rtpParameters = transceiver.sender.parameters + // Use provided degradation preference, or default based on track source: + // - Camera: MAINTAIN_FRAMERATE (smoother video for real-time communication) + // - Screen share: MAINTAIN_RESOLUTION (clarity is critical for text/UI) + // - Other/unknown: BALANCED rtpParameters.degradationPreference = finalOptions.degradationPreference + ?: getDefaultDegradationPreference(trackSource) transceiver.sender.parameters = rtpParameters } @@ -1456,10 +1469,17 @@ abstract class BaseVideoTrackPublishOptions { abstract val backupCodec: BackupVideoCodec? /** - * When bandwidth is constrained, this preference indicates which is preferred - * between degrading resolution vs. framerate. + * Controls how the encoder trades off between resolution and framerate + * when bandwidth is constrained. + * + * - MAINTAIN_FRAMERATE: Prioritizes framerate, reduces resolution if needed + * - MAINTAIN_RESOLUTION: Prioritizes resolution, drops frames if needed + * - BALANCED: Balances between both * - * null value indicates default value (maintain framerate). + * If not set (null), the SDK uses defaults based on track source: + * - Camera: MAINTAIN_FRAMERATE (smoother video for real-time communication) + * - Screen share: MAINTAIN_RESOLUTION (clarity is critical for text/UI) + * - Other/unknown: BALANCED */ abstract val degradationPreference: RtpParameters.DegradationPreference? @@ -1481,6 +1501,8 @@ data class VideoTrackPublishDefaults( override val videoCodec: String = VideoCodec.VP8.codecName, override val scalabilityMode: String? = null, override val backupCodec: BackupVideoCodec? = null, + // Default is null - SDK applies source-based defaults at runtime: + // Camera: MAINTAIN_FRAMERATE, Screen share: MAINTAIN_RESOLUTION, Other: BALANCED override val degradationPreference: RtpParameters.DegradationPreference? = null, override val simulcastLayers: List? = null, ) : BaseVideoTrackPublishOptions() @@ -1494,6 +1516,8 @@ data class VideoTrackPublishOptions( override val backupCodec: BackupVideoCodec? = null, override val source: Track.Source? = null, override val stream: String? = null, + // Default is null - SDK applies source-based defaults at runtime: + // Camera: MAINTAIN_FRAMERATE, Screen share: MAINTAIN_RESOLUTION, Other: BALANCED override val degradationPreference: RtpParameters.DegradationPreference? = null, override val simulcastLayers: List? = null, ) : BaseVideoTrackPublishOptions(), TrackPublishOptions { @@ -1661,6 +1685,21 @@ internal fun VideoTrackPublishOptions.hasBackupCodec(): Boolean { private val backupCodecs = listOf(VideoCodec.VP8.codecName, VideoCodec.H264.codecName) private fun isBackupCodec(codecName: String) = backupCodecs.contains(codecName) +/** + * Returns the appropriate degradation preference for a video track based on its source. + * + * - Camera: MAINTAIN_FRAMERATE (smoother video for real-time communication) + * - Screen share: MAINTAIN_RESOLUTION (clarity is critical for reading text/UI) + * - Other/unknown: BALANCED + */ +private fun getDefaultDegradationPreference(source: Track.Source): RtpParameters.DegradationPreference { + return when (source) { + Track.Source.CAMERA -> RtpParameters.DegradationPreference.MAINTAIN_FRAMERATE + Track.Source.SCREEN_SHARE -> RtpParameters.DegradationPreference.MAINTAIN_RESOLUTION + else -> RtpParameters.DegradationPreference.BALANCED + } +} + /** * A handler that processes an RPC request and returns a string * that will be sent back to the requester. The payload must diff --git a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt index bfecd601..211c0230 100644 --- a/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt +++ b/livekit-android-test/src/test/java/io/livekit/android/room/SdpMungingTest.kt @@ -56,13 +56,15 @@ class SdpMungingTest { val sdp = SdpFactory.getInstance().createSessionDescription(JainSdpUtilsTest.DESCRIPTION) val mediaDescription = sdp.getMediaDescriptions(true).filterIsInstance()[1] + // Use realistic bitrate: 1000 kbps (1 Mbps) + // With 0.9 multiplier: startBitrate = 900 kbps (below 1 Mbps cap) ensureCodecBitrates( mediaDescription, mapOf( TrackBitrateInfoKey.Cid("PA_Qwqk4y9fcD3G") to TrackBitrateInfo( "VP9", - 1000000L, + 1000L, ), ), ) @@ -71,7 +73,7 @@ class SdpMungingTest { .filter { (_, fmtp) -> fmtp.payload == 98L } .first() - assertEquals("profile-id=0;x-google-start-bitrate=700000;x-google-max-bitrate=1000000", vp9fmtp.config) + assertEquals("profile-id=0;x-google-start-bitrate=900;x-google-max-bitrate=1000", vp9fmtp.config) } companion object { diff --git a/protocol b/protocol index 8381f218..4c05a332 160000 --- a/protocol +++ b/protocol @@ -1 +1 @@ -Subproject commit 8381f2180c45ab926b3ebf19df0608f1dadcac1e +Subproject commit 4c05a3325ec35760bee1c0bfe57b7011604a124f