From 86bf0fd2c6d0a72834d6215b49017f92860285a0 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Tue, 12 Mar 2024 23:21:45 -0700 Subject: [PATCH 1/3] Fix FPS and latency bugs with VP9 screenshare. Also restored L3T3_KEY publishing for screen share. It was disabled previously. Chrome does not allow more than 5 fps with L1T3, and it has encoding bugs with L3T3 It has a different path for screenshare handling and it seems to be untested/buggy As a workaround, we are setting contentHint to force it to go through the same path as regular camera video. While this is not optimal, it delivers the performance that we need --- src/room/participant/LocalParticipant.ts | 11 ++++++++++- src/room/track/RemoteAudioTrack.ts | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index 3060a5b0c3..b47417e375 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -736,7 +736,16 @@ export default class LocalParticipant extends Participant { if (isSVCCodec(videoCodec)) { // vp9 svc with screenshare has problem to encode, always use L1T3 here if (track.source === Track.Source.ScreenShare && videoCodec === 'vp9') { - opts.scalabilityMode = 'L1T3'; + // Chrome does not allow more than 5 fps with L1T3, and it has encoding bugs with L3T3 + // It has a different path for screenshare handling and it seems to be untested/buggy + // As a workaround, we are setting contentHint to force it to go through the same + // path as regular camera video. While this is not optimal, it delivers the performance + // that we need + if ('contentHint' in track.mediaStreamTrack) { + track.mediaStreamTrack.contentHint = 'motion'; + } else { + opts.scalabilityMode = 'L1T3'; + } } // set scalabilityMode to 'L3T3_KEY' by default opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3_KEY'; diff --git a/src/room/track/RemoteAudioTrack.ts b/src/room/track/RemoteAudioTrack.ts index 37fd3d0f48..0abc4c5710 100644 --- a/src/room/track/RemoteAudioTrack.ts +++ b/src/room/track/RemoteAudioTrack.ts @@ -1,6 +1,6 @@ import { TrackEvent } from '../events'; -import { computeBitrate } from '../stats'; import type { AudioReceiverStats } from '../stats'; +import { computeBitrate } from '../stats'; import type { LoggerOptions } from '../types'; import { isReactNative, supportsSetSinkId } from '../utils'; import RemoteTrack from './RemoteTrack'; From efb231b55d873cc4f2dc02f4c06ce9703a21b729 Mon Sep 17 00:00:00 2001 From: David Zhao Date: Wed, 13 Mar 2024 00:22:19 -0700 Subject: [PATCH 2/3] changeset --- .changeset/smooth-glasses-jam.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/smooth-glasses-jam.md diff --git a/.changeset/smooth-glasses-jam.md b/.changeset/smooth-glasses-jam.md new file mode 100644 index 0000000000..96d7ad4db2 --- /dev/null +++ b/.changeset/smooth-glasses-jam.md @@ -0,0 +1,5 @@ +--- +"livekit-client": patch +--- + +Fix FPS and latency issues with VP9 screenshare From 7f7ba5adfd94321a20ed0e23b774126a1d9ddfdd Mon Sep 17 00:00:00 2001 From: David Zhao Date: Wed, 13 Mar 2024 09:48:43 -0700 Subject: [PATCH 3/3] added info log when contentHint is overridden --- src/room/participant/LocalParticipant.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index b47417e375..5eca4c7c1b 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -743,6 +743,10 @@ export default class LocalParticipant extends Participant { // that we need if ('contentHint' in track.mediaStreamTrack) { track.mediaStreamTrack.contentHint = 'motion'; + this.log.info('forcing contentHint to motion for screenshare with VP9', { + ...this.logContext, + ...getLogContextFromTrack(track), + }); } else { opts.scalabilityMode = 'L1T3'; }