WebRTC congestion control (#24)

* Move congestion control to server

* Address MR comments

* lint.

* bring back webrtc stats sync with state.

* update webrtc messages.

* set bitrate & video auto for signal request.

* clear up bitrate & video_auto.

* add bitrate and audio for connect function.

* reintroduce server side congestion control if video auto is false.

* mov ecode around.

* fix default video_auto.

* revert bitrate addition.

* remove video from signal provide.

---------

Co-authored-by: Aleksandar Sukovic <aleksandar.sukovic@gmail.com>

src/component/internal/connection.ts

@@ -38,8 +38,9 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
   private _onDisconnectHandle: () => void
   private _onCloseHandle: (error?: Error) => void
 
-  private _webrtcCongestionControlHandle: (stats: WebRTCStats) => void
+  private _webrtcStatsHandle: (stats: WebRTCStats) => void
   private _webrtcStableHandle: (isStable: boolean) => void
+  private _webrtcCongestionControlHandle: (stats: WebRTCStats) => void
 
   // eslint-disable-next-line
   constructor(
@@ -87,6 +88,12 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
       r.on('close', this._onCloseHandle)
     })
 
+    // synchronize webrtc stats with global state
+    this._webrtcStatsHandle = (stats: WebRTCStats) => {
+      Vue.set(this._state.webrtc, 'stats', stats)
+    }
+    this.webrtc.on('stats', this._webrtcStatsHandle)
+
     // synchronize webrtc stable with global state
     this._webrtcStableHandle = (isStable: boolean) => {
       Vue.set(this._state.webrtc, 'stable', isStable)
@@ -101,7 +108,8 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
     let webrtcFallbackTimeout: number
 
     this._webrtcCongestionControlHandle = (stats: WebRTCStats) => {
-      Vue.set(this._state.webrtc, 'stats', stats)
+      // if automatic quality adjusting is turned off
+      if (this._state.webrtc.auto) return
 
       // when connection is paused, 0fps and muted track is expected
       if (stats.paused) return
@@ -138,7 +146,7 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
 
         // downgrade if lower video quality exists
         if (quality && this.webrtc.connected) {
-          this.setVideo(quality)
+          this.websocket.send(EVENT.SIGNAL_VIDEO, { video: quality })
         }
 
         // try to perform ice restart, if available
@@ -151,7 +159,6 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
         this._reconnector.webrtc.reconnect()
       }
     }
-
     this.webrtc.on('stats', this._webrtcCongestionControlHandle)
   }
 
@@ -165,23 +172,11 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
     this._reconnector.webrtc.config = this._state.webrtc.config
   }
 
-  public setVideo(video: string, bitrate: number = 0) {
-    if (video != '' && !this._state.webrtc.videos.includes(video)) {
-      throw new Error('video id not found')
-    }
-
-    if (video == '' && bitrate == 0) {
-      throw new Error('video id and bitrate cannot be empty')
-    }
-
-    this.websocket.send(EVENT.SIGNAL_VIDEO, { video, bitrate })
-  }
-
   public getLogger(scope?: string): Logger {
     return this.logger.new(scope)
   }
 
-  public open(video?: string) {
+  public open(video?: string, auto?: boolean) {
     if (this._open) {
       throw new Error('connection already open')
     }
@@ -196,6 +191,13 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
       Vue.set(this._state.webrtc, 'video', video)
     }
 
+    // if we didn't specify auto
+    if (typeof auto == 'undefined') {
+      // if we didn't specify video, set auto to true
+      auto = !video
+    }
+    Vue.set(this._state.webrtc, 'auto', auto)
+
     Vue.set(this._state, 'status', 'connecting')
 
     // open all reconnectors with deferred connection
@@ -230,9 +232,10 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
   public destroy() {
     this.logger.destroy()
 
+    this.webrtc.off('stats', this._webrtcStatsHandle)
+    this.webrtc.off('stable', this._webrtcStableHandle)
     // TODO: Use server side congestion control.
     this.webrtc.off('stats', this._webrtcCongestionControlHandle)
-    this.webrtc.off('stable', this._webrtcStableHandle)
 
     // unbind events from all reconnectors
     Object.values(this._reconnector).forEach((r) => {

src/component/internal/messages.ts

@@ -147,9 +147,8 @@ export class NekoMessages extends EventEmitter<NekoEvents> {
   // Signal Events
   /////////////////////////////
 
-  protected async [EVENT.SIGNAL_PROVIDE]({ sdp, video, iceservers }: message.SignalProvide) {
+  protected async [EVENT.SIGNAL_PROVIDE]({ sdp, iceservers }: message.SignalProvide) {
     this._localLog.debug(`EVENT.SIGNAL_PROVIDE`)
-    Vue.set(this._state.connection.webrtc, 'video', video)
 
     // create WebRTC connection
     await this._connection.webrtc.connect(iceservers)
@@ -194,10 +193,10 @@ export class NekoMessages extends EventEmitter<NekoEvents> {
     this.emit('connection.webrtc.sdp.candidate', 'remote', candidate)
   }
 
-  protected [EVENT.SIGNAL_VIDEO]({ video, bitrate }: message.SignalVideo) {
-    this._localLog.debug(`EVENT.SIGNAL_VIDEO`, { video, bitrate })
+  protected [EVENT.SIGNAL_VIDEO]({ video, auto }: message.SignalVideo) {
+    this._localLog.debug(`EVENT.SIGNAL_VIDEO`, { video, auto })
     Vue.set(this._state.connection.webrtc, 'video', video)
-    Vue.set(this._state.connection.webrtc, 'bitrate', bitrate)
+    Vue.set(this._state.connection.webrtc, 'auto', !!auto)
   }
 
   protected [EVENT.SIGNAL_CLOSE]() {

src/component/internal/reconnector/webrtc.ts

@@ -37,7 +37,10 @@ export class WebrtcReconnector extends ReconnectorAbstract {
     }
 
     if (this._websocket.connected) {
-      this._websocket.send(EVENT.SIGNAL_REQUEST, { video: this._state.webrtc.video })
+      this._websocket.send(EVENT.SIGNAL_REQUEST, {
+        video: this._state.webrtc.video,
+        auto: this._state.webrtc.auto,
+      })
     }
   }
 

src/component/main.vue

@@ -170,7 +170,7 @@
           },
           stats: null,
           video: null,
-          bitrate: null,
+          auto: false,
           videos: [],
         },
         screencast: true, // TODO: Should get by API call.
@@ -383,7 +383,7 @@
       }
     }
 
-    public connect(video?: string) {
+    public connect(video?: string, auto?: boolean) {
       if (!this.state.authenticated) {
         throw new Error('client not authenticated')
       }
@@ -392,7 +392,7 @@
         throw new Error('client is already connected')
       }
 
-      this.connection.open(video)
+      this.connection.open(video, auto)
     }
 
     public disconnect() {
@@ -512,8 +512,19 @@
       this.connection.websocket.send(EVENT.SCREEN_SET, { width, height, rate })
     }
 
-    public setWebRTCVideo(video: string, bitrate: number = 0) {
-      this.connection.setVideo(video, bitrate)
+    public setWebRTCVideo(video?: string, auto?: boolean) {
+      // if video has been set, check if it exists
+      if (video && !this.state.connection.webrtc.videos.includes(video)) {
+        throw new Error('video id not found')
+      }
+
+      // if we didn't specify auto
+      if (typeof auto == 'undefined') {
+        // if we didn't specify video, set auto to true
+        auto = !video
+      }
+
+      this.connection.websocket.send(EVENT.SIGNAL_VIDEO, { video, auto })
     }
 
     public addTrack(track: MediaStreamTrack, ...streams: MediaStream[]): RTCRtpSender {
@@ -749,7 +760,6 @@
       }
 
       // websocket
-      Vue.set(this.state.connection.webrtc, 'videos', [])
       Vue.set(this.state.control, 'clipboard', null)
       Vue.set(this.state.control, 'host_id', null)
       Vue.set(this.state.screen, 'size', { width: 1280, height: 720, rate: 30 })
@@ -768,6 +778,8 @@
       // webrtc
       Vue.set(this.state.connection.webrtc, 'stats', null)
       Vue.set(this.state.connection.webrtc, 'video', null)
+      Vue.set(this.state.connection.webrtc, 'auto', false)
+      Vue.set(this.state.connection.webrtc, 'videos', [])
       Vue.set(this.state.connection, 'type', 'none')
     }
   }

src/component/types/messages.ts

@@ -45,7 +45,6 @@ export interface SystemDisconnect {
 export interface SignalProvide {
   sdp: string
   iceservers: ICEServer[]
-  video: string
 }
 
 export type SignalCandidate = RTCIceCandidateInit
@@ -56,7 +55,7 @@ export interface SignalDescription {
 
 export interface SignalVideo {
   video: string
-  bitrate: number
+  auto: boolean
 }
 
 /////////////////////////////

src/component/types/state.ts

@@ -39,7 +39,7 @@ export interface WebRTC {
   config: ReconnectorConfig
   stats: WebRTCStats | null
   video: string | null
-  bitrate: number | null
+  auto: boolean
   videos: string[]
 }
 

src/page/components/events.vue

@@ -161,20 +161,28 @@
         <th>connection.webrtc.video</th>
         <td>{{ neko.state.connection.webrtc.video }}</td>
       </tr>
+      <tr>
+        <th>connection.webrtc.auto</th>
+        <td>
+          <div class="space-between">
+            <span>{{ neko.state.connection.webrtc.auto }}</span>
+            <button @click="neko.setWebRTCVideo(undefined, !neko.state.connection.webrtc.auto)">
+              <i class="fas fa-toggle-on"></i>
+            </button>
+          </div>
+        </td>
+      </tr>
       <tr>
         <th rowspan="2">connection.webrtc.videos</th>
         <td>Total {{ neko.state.connection.webrtc.videos.length }} videos.</td>
       </tr>
       <tr>
         <td>
-          <select :value="neko.state.connection.webrtc.video" @input="neko.setWebRTCVideo($event.target.value)">
+          <select :value="neko.state.connection.webrtc.video" @input="neko.setWebRTCVideo($event.target.value, false)">
             <option v-for="video in neko.state.connection.webrtc.videos" :key="video" :value="video">
               {{ video }}
             </option>
           </select>
-          or
-          <input type="text" v-model="bitrate" style="width: 60px" placeholder="Bitrate" />
-          <button @click="neko.setWebRTCVideo('', Number(bitrate))">Set</button>
         </td>
       </tr>
       <tr>
@@ -376,13 +384,15 @@
           </td>
         </tr>
       </template>
-      <tr v-else>
+      <template v-else>
+        <tr>
           <th>screen.configurations</th>
-        <td>Session is not admin.</td>
-        <th>screen.sync</th>
-        <td>Session is not admin.</td>
+          <td rowspan="2" style="vertical-align: middle">Session is not admin.</td>
         </tr>
-
+        <tr>
+          <th>screen.sync</th>
+        </tr>
+      </template>
       <tr>
         <th>session_id</th>
         <td>{{ neko.state.session_id }}</td>