From 8d39a95e92d74a0572a85e7845e76262d9f81f84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Mon, 15 May 2023 19:32:21 +0200 Subject: [PATCH] WebRTC congestion control (#24) * Move congestion control to server * Address MR comments * lint. * bring back webrtc stats sync with state. * update webrtc messages. * set bitrate & video auto for signal request. * clear up bitrate & video_auto. * add bitrate and audio for connect function. * reintroduce server side congestion control if video auto is false. * mov ecode around. * fix default video_auto. * revert bitrate addition. * remove video from signal provide. --------- Co-authored-by: Aleksandar Sukovic --- src/component/internal/connection.ts | 39 +++++++++++--------- src/component/internal/messages.ts | 9 ++--- src/component/internal/reconnector/webrtc.ts | 5 ++- src/component/main.vue | 24 +++++++++--- src/component/types/messages.ts | 3 +- src/component/types/state.ts | 2 +- src/page/components/events.vue | 32 ++++++++++------ 7 files changed, 70 insertions(+), 44 deletions(-) diff --git a/src/component/internal/connection.ts b/src/component/internal/connection.ts index 6871a0a2..dc719e06 100644 --- a/src/component/internal/connection.ts +++ b/src/component/internal/connection.ts @@ -38,8 +38,9 @@ export class NekoConnection extends EventEmitter { private _onDisconnectHandle: () => void private _onCloseHandle: (error?: Error) => void - private _webrtcCongestionControlHandle: (stats: WebRTCStats) => void + private _webrtcStatsHandle: (stats: WebRTCStats) => void private _webrtcStableHandle: (isStable: boolean) => void + private _webrtcCongestionControlHandle: (stats: WebRTCStats) => void // eslint-disable-next-line constructor( @@ -87,6 +88,12 @@ export class NekoConnection extends EventEmitter { r.on('close', this._onCloseHandle) }) + // synchronize webrtc stats with global state + this._webrtcStatsHandle = (stats: WebRTCStats) => { + Vue.set(this._state.webrtc, 'stats', stats) + } + this.webrtc.on('stats', this._webrtcStatsHandle) + // synchronize webrtc stable with global state this._webrtcStableHandle = (isStable: boolean) => { Vue.set(this._state.webrtc, 'stable', isStable) @@ -101,7 +108,8 @@ export class NekoConnection extends EventEmitter { let webrtcFallbackTimeout: number this._webrtcCongestionControlHandle = (stats: WebRTCStats) => { - Vue.set(this._state.webrtc, 'stats', stats) + // if automatic quality adjusting is turned off + if (this._state.webrtc.auto) return // when connection is paused, 0fps and muted track is expected if (stats.paused) return @@ -138,7 +146,7 @@ export class NekoConnection extends EventEmitter { // downgrade if lower video quality exists if (quality && this.webrtc.connected) { - this.setVideo(quality) + this.websocket.send(EVENT.SIGNAL_VIDEO, { video: quality }) } // try to perform ice restart, if available @@ -151,7 +159,6 @@ export class NekoConnection extends EventEmitter { this._reconnector.webrtc.reconnect() } } - this.webrtc.on('stats', this._webrtcCongestionControlHandle) } @@ -165,23 +172,11 @@ export class NekoConnection extends EventEmitter { this._reconnector.webrtc.config = this._state.webrtc.config } - public setVideo(video: string, bitrate: number = 0) { - if (video != '' && !this._state.webrtc.videos.includes(video)) { - throw new Error('video id not found') - } - - if (video == '' && bitrate == 0) { - throw new Error('video id and bitrate cannot be empty') - } - - this.websocket.send(EVENT.SIGNAL_VIDEO, { video, bitrate }) - } - public getLogger(scope?: string): Logger { return this.logger.new(scope) } - public open(video?: string) { + public open(video?: string, auto?: boolean) { if (this._open) { throw new Error('connection already open') } @@ -196,6 +191,13 @@ export class NekoConnection extends EventEmitter { Vue.set(this._state.webrtc, 'video', video) } + // if we didn't specify auto + if (typeof auto == 'undefined') { + // if we didn't specify video, set auto to true + auto = !video + } + Vue.set(this._state.webrtc, 'auto', auto) + Vue.set(this._state, 'status', 'connecting') // open all reconnectors with deferred connection @@ -230,9 +232,10 @@ export class NekoConnection extends EventEmitter { public destroy() { this.logger.destroy() + this.webrtc.off('stats', this._webrtcStatsHandle) + this.webrtc.off('stable', this._webrtcStableHandle) // TODO: Use server side congestion control. this.webrtc.off('stats', this._webrtcCongestionControlHandle) - this.webrtc.off('stable', this._webrtcStableHandle) // unbind events from all reconnectors Object.values(this._reconnector).forEach((r) => { diff --git a/src/component/internal/messages.ts b/src/component/internal/messages.ts index 956e1b91..9e1993b3 100644 --- a/src/component/internal/messages.ts +++ b/src/component/internal/messages.ts @@ -147,9 +147,8 @@ export class NekoMessages extends EventEmitter { // Signal Events ///////////////////////////// - protected async [EVENT.SIGNAL_PROVIDE]({ sdp, video, iceservers }: message.SignalProvide) { + protected async [EVENT.SIGNAL_PROVIDE]({ sdp, iceservers }: message.SignalProvide) { this._localLog.debug(`EVENT.SIGNAL_PROVIDE`) - Vue.set(this._state.connection.webrtc, 'video', video) // create WebRTC connection await this._connection.webrtc.connect(iceservers) @@ -194,10 +193,10 @@ export class NekoMessages extends EventEmitter { this.emit('connection.webrtc.sdp.candidate', 'remote', candidate) } - protected [EVENT.SIGNAL_VIDEO]({ video, bitrate }: message.SignalVideo) { - this._localLog.debug(`EVENT.SIGNAL_VIDEO`, { video, bitrate }) + protected [EVENT.SIGNAL_VIDEO]({ video, auto }: message.SignalVideo) { + this._localLog.debug(`EVENT.SIGNAL_VIDEO`, { video, auto }) Vue.set(this._state.connection.webrtc, 'video', video) - Vue.set(this._state.connection.webrtc, 'bitrate', bitrate) + Vue.set(this._state.connection.webrtc, 'auto', !!auto) } protected [EVENT.SIGNAL_CLOSE]() { diff --git a/src/component/internal/reconnector/webrtc.ts b/src/component/internal/reconnector/webrtc.ts index 587df0d4..bbddd24f 100644 --- a/src/component/internal/reconnector/webrtc.ts +++ b/src/component/internal/reconnector/webrtc.ts @@ -37,7 +37,10 @@ export class WebrtcReconnector extends ReconnectorAbstract { } if (this._websocket.connected) { - this._websocket.send(EVENT.SIGNAL_REQUEST, { video: this._state.webrtc.video }) + this._websocket.send(EVENT.SIGNAL_REQUEST, { + video: this._state.webrtc.video, + auto: this._state.webrtc.auto, + }) } } diff --git a/src/component/main.vue b/src/component/main.vue index 3a000f32..1e009f5f 100644 --- a/src/component/main.vue +++ b/src/component/main.vue @@ -170,7 +170,7 @@ }, stats: null, video: null, - bitrate: null, + auto: false, videos: [], }, screencast: true, // TODO: Should get by API call. @@ -383,7 +383,7 @@ } } - public connect(video?: string) { + public connect(video?: string, auto?: boolean) { if (!this.state.authenticated) { throw new Error('client not authenticated') } @@ -392,7 +392,7 @@ throw new Error('client is already connected') } - this.connection.open(video) + this.connection.open(video, auto) } public disconnect() { @@ -512,8 +512,19 @@ this.connection.websocket.send(EVENT.SCREEN_SET, { width, height, rate }) } - public setWebRTCVideo(video: string, bitrate: number = 0) { - this.connection.setVideo(video, bitrate) + public setWebRTCVideo(video?: string, auto?: boolean) { + // if video has been set, check if it exists + if (video && !this.state.connection.webrtc.videos.includes(video)) { + throw new Error('video id not found') + } + + // if we didn't specify auto + if (typeof auto == 'undefined') { + // if we didn't specify video, set auto to true + auto = !video + } + + this.connection.websocket.send(EVENT.SIGNAL_VIDEO, { video, auto }) } public addTrack(track: MediaStreamTrack, ...streams: MediaStream[]): RTCRtpSender { @@ -749,7 +760,6 @@ } // websocket - Vue.set(this.state.connection.webrtc, 'videos', []) Vue.set(this.state.control, 'clipboard', null) Vue.set(this.state.control, 'host_id', null) Vue.set(this.state.screen, 'size', { width: 1280, height: 720, rate: 30 }) @@ -768,6 +778,8 @@ // webrtc Vue.set(this.state.connection.webrtc, 'stats', null) Vue.set(this.state.connection.webrtc, 'video', null) + Vue.set(this.state.connection.webrtc, 'auto', false) + Vue.set(this.state.connection.webrtc, 'videos', []) Vue.set(this.state.connection, 'type', 'none') } } diff --git a/src/component/types/messages.ts b/src/component/types/messages.ts index e372191d..2bf56b10 100644 --- a/src/component/types/messages.ts +++ b/src/component/types/messages.ts @@ -45,7 +45,6 @@ export interface SystemDisconnect { export interface SignalProvide { sdp: string iceservers: ICEServer[] - video: string } export type SignalCandidate = RTCIceCandidateInit @@ -56,7 +55,7 @@ export interface SignalDescription { export interface SignalVideo { video: string - bitrate: number + auto: boolean } ///////////////////////////// diff --git a/src/component/types/state.ts b/src/component/types/state.ts index 2eaa8993..299f5f2b 100644 --- a/src/component/types/state.ts +++ b/src/component/types/state.ts @@ -39,7 +39,7 @@ export interface WebRTC { config: ReconnectorConfig stats: WebRTCStats | null video: string | null - bitrate: number | null + auto: boolean videos: string[] } diff --git a/src/page/components/events.vue b/src/page/components/events.vue index 84371ea5..c56e94dc 100644 --- a/src/page/components/events.vue +++ b/src/page/components/events.vue @@ -161,20 +161,28 @@ connection.webrtc.video {{ neko.state.connection.webrtc.video }} + + connection.webrtc.auto + +
+ {{ neko.state.connection.webrtc.auto }} + +
+ + connection.webrtc.videos Total {{ neko.state.connection.webrtc.videos.length }} videos. - - or - - @@ -376,13 +384,15 @@ - - screen.configurations - Session is not admin. - screen.sync - Session is not admin. - - + session_id {{ neko.state.session_id }}