Refactor signaling for video and audio (#39)

* refactor webrtc video and audio.

* do not reconnect if video is disabled.

* export webrtc types.

src/component/internal/connection.ts

@@ -1,6 +1,7 @@
 import Vue from 'vue'
 import EventEmitter from 'eventemitter3'
 import * as EVENT from '../types/events'
+import * as webrtcTypes from '../types/webrtc'
 
 import { NekoWebSocket } from './websocket'
 import { NekoLoggerFactory } from './logger'
@@ -24,6 +25,7 @@ export interface NekoConnectionEvents {
 export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
   private _open = false
   private _closing = false
+  private _peerRequest?: webrtcTypes.PeerRequest
 
   public websocket = new NekoWebSocket()
   public logger = new NekoLoggerFactory(this.websocket)
@@ -62,7 +64,14 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
       }
 
       if (this.websocket.connected && !this.webrtc.connected) {
-        this._reconnector.webrtc.connect()
+        // if custom peer request is set, send custom peer request
+        if (this._peerRequest) {
+          this.websocket.send(EVENT.SIGNAL_REQUEST, this._peerRequest)
+          this._peerRequest = undefined
+        } else {
+          // otherwise use reconnectors connect method
+          this._reconnector.webrtc.connect()
+        }
       }
     }
 
@@ -109,10 +118,10 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
 
     this._webrtcCongestionControlHandle = (stats: WebRTCStats) => {
       // if automatic quality adjusting is turned off
-      if (this._state.webrtc.auto) return
+      if (this._state.webrtc.video.auto) return
 
-      // when connection is paused, 0fps and muted track is expected
-      if (stats.paused) return
+      // when connection is paused or video disabled, 0fps and muted track is expected
+      if (stats.paused || this._state.webrtc.video.disabled) return
 
       // if automatic quality adjusting is turned off
       if (!this._reconnector.webrtc.isOpen) return
@@ -121,7 +130,7 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
       if (this._state.webrtc.videos.length <= 1) return
 
       // current quality is not known
-      if (this._state.webrtc.video == null) return
+      if (this._state.webrtc.video.id == '') return
 
       // check if video is not playing smoothly
       if (stats.fps && stats.packetLoss < WEBRTC_RECONN_MAX_LOSS && !stats.muted) {
@@ -142,7 +151,7 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
 
         webrtcCongestion = 0
 
-        const quality = this._webrtcQualityDowngrade(this._state.webrtc.video)
+        const quality = this._webrtcQualityDowngrade(this._state.webrtc.video.id)
 
         // downgrade if lower video quality exists
         if (quality && this.webrtc.connected) {
@@ -176,27 +185,13 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
     return this.logger.new(scope)
   }
 
-  public open(video?: string, auto?: boolean) {
+  public open(peerRequest?: webrtcTypes.PeerRequest) {
     if (this._open) {
       throw new Error('connection already open')
     }
 
     this._open = true
-
-    if (video) {
-      if (!this._state.webrtc.videos.includes(video)) {
-        throw new Error('video id not found')
-      }
-
-      Vue.set(this._state.webrtc, 'video', video)
-    }
-
-    // if we didn't specify auto
-    if (typeof auto == 'undefined') {
-      // if we didn't specify video, set auto to true
-      auto = !video
-    }
-    Vue.set(this._state.webrtc, 'auto', auto)
+    this._peerRequest = peerRequest
 
     Vue.set(this._state, 'status', 'connecting')
 

src/component/internal/messages.ts

@@ -147,7 +147,7 @@ export class NekoMessages extends EventEmitter<NekoEvents> {
   // Signal Events
   /////////////////////////////
 
-  protected async [EVENT.SIGNAL_PROVIDE]({ sdp, iceservers }: message.SignalProvide) {
+  protected async [EVENT.SIGNAL_PROVIDE]({ sdp, iceservers, video, audio }: message.SignalProvide) {
     this._localLog.debug(`EVENT.SIGNAL_PROVIDE`)
 
     // create WebRTC connection
@@ -158,6 +158,9 @@ export class NekoMessages extends EventEmitter<NekoEvents> {
 
     // TODO: Return whole signal description (if answer / offer).
     this.emit('connection.webrtc.sdp', 'remote', sdp)
+
+    this[EVENT.SIGNAL_VIDEO](video)
+    this[EVENT.SIGNAL_AUDIO](audio)
   }
 
   protected async [EVENT.SIGNAL_OFFER]({ sdp }: message.SignalDescription) {
@@ -193,10 +196,16 @@ export class NekoMessages extends EventEmitter<NekoEvents> {
     this.emit('connection.webrtc.sdp.candidate', 'remote', candidate)
   }
 
-  protected [EVENT.SIGNAL_VIDEO]({ video, auto }: message.SignalVideo) {
-    this._localLog.debug(`EVENT.SIGNAL_VIDEO`, { video, auto })
-    Vue.set(this._state.connection.webrtc, 'video', video)
-    Vue.set(this._state.connection.webrtc, 'auto', !!auto)
+  protected [EVENT.SIGNAL_VIDEO]({ disabled, id, auto }: message.SignalVideo) {
+    this._localLog.debug(`EVENT.SIGNAL_VIDEO`, { disabled, id, auto })
+    Vue.set(this._state.connection.webrtc.video, 'disabled', disabled)
+    Vue.set(this._state.connection.webrtc.video, 'id', id)
+    Vue.set(this._state.connection.webrtc.video, 'auto', auto)
+  }
+
+  protected [EVENT.SIGNAL_AUDIO]({ disabled }: message.SignalAudio) {
+    this._localLog.debug(`EVENT.SIGNAL_AUDIO`, { disabled })
+    Vue.set(this._state.connection.webrtc.audio, 'disabled', disabled)
   }
 
   protected [EVENT.SIGNAL_CLOSE]() {

src/component/internal/reconnector/webrtc.ts

@@ -37,9 +37,25 @@ export class WebrtcReconnector extends ReconnectorAbstract {
     }
 
     if (this._websocket.connected) {
+      // use requests from state to connect with selected values
+
+      let selector = null
+      if (this._state.webrtc.video.id) {
+        selector = {
+          id: this._state.webrtc.video.id,
+          type: 'exact',
+        }
+      }
+
       this._websocket.send(EVENT.SIGNAL_REQUEST, {
-        video: this._state.webrtc.video,
-        auto: this._state.webrtc.auto,
+        video: {
+          disabled: this._state.webrtc.video.disabled,
+          selector,
+          auto: this._state.webrtc.video.auto,
+        },
+        audio: {
+          disabled: this._state.webrtc.audio.disabled,
+        },
       })
     }
   }

src/component/main.vue

@@ -74,7 +74,7 @@
 <script lang="ts">
   export * as ApiModels from './api/models'
   export * as StateModels from './types/state'
-  import * as EVENT from './types/events'
+  export * as webrtcTypes from './types/webrtc'
 
   import { Configuration } from './api/configuration'
   import { AxiosInstance } from 'axios'
@@ -89,6 +89,8 @@
   import { register as VideoRegister } from './internal/video'
 
   import { ReconnectorConfig } from './types/reconnector'
+  import * as EVENT from './types/events'
+  import * as webrtcTypes from './types/webrtc'
   import NekoState from './types/state'
   import { CursorDrawFunction, InactiveCursorDrawFunction, Dimension } from './types/cursors'
   import Overlay from './overlay.vue'
@@ -169,8 +171,14 @@
             backoff_ms: 1500,
           },
           stats: null,
-          video: null,
-          auto: false,
+          video: {
+            disabled: false,
+            id: '',
+            auto: false,
+          },
+          audio: {
+            disabled: false,
+          },
           videos: [],
         },
         screencast: true, // TODO: Should get by API call.
@@ -383,7 +391,7 @@
       }
     }
 
-    public connect(video?: string, auto?: boolean) {
+    public connect(peerRequest?: webrtcTypes.PeerRequest) {
       if (!this.state.authenticated) {
         throw new Error('client not authenticated')
       }
@@ -392,7 +400,7 @@
         throw new Error('client is already connected')
       }
 
-      this.connection.open(video, auto)
+      this.connection.open(peerRequest)
     }
 
     public disconnect() {
@@ -512,19 +520,12 @@
       this.connection.websocket.send(EVENT.SCREEN_SET, { width, height, rate })
     }
 
-    public setWebRTCVideo(video?: string, auto?: boolean) {
-      // if video has been set, check if it exists
-      if (video && !this.state.connection.webrtc.videos.includes(video)) {
-        throw new Error('video id not found')
-      }
+    public setWebRTCVideo(peerVideo: webrtcTypes.PeerVideoRequest) {
+      this.connection.websocket.send(EVENT.SIGNAL_VIDEO, peerVideo)
+    }
 
-      // if we didn't specify auto
-      if (typeof auto == 'undefined') {
-        // if we didn't specify video, set auto to true
-        auto = !video
-      }
-
-      this.connection.websocket.send(EVENT.SIGNAL_VIDEO, { video, auto })
+    public setWebRTCAudio(peerAudio: webrtcTypes.PeerAudioRequest) {
+      this.connection.websocket.send(EVENT.SIGNAL_AUDIO, peerAudio)
     }
 
     public addTrack(track: MediaStreamTrack, ...streams: MediaStream[]): RTCRtpSender {
@@ -778,8 +779,10 @@
 
       // webrtc
       Vue.set(this.state.connection.webrtc, 'stats', null)
-      Vue.set(this.state.connection.webrtc, 'video', null)
-      Vue.set(this.state.connection.webrtc, 'auto', false)
+      Vue.set(this.state.connection.webrtc.video, 'disabled', false)
+      Vue.set(this.state.connection.webrtc.video, 'id', '')
+      Vue.set(this.state.connection.webrtc.video, 'auto', false)
+      Vue.set(this.state.connection.webrtc.audio, 'disabled', false)
       Vue.set(this.state.connection.webrtc, 'videos', [])
       Vue.set(this.state.connection, 'type', 'none')
     }

src/component/types/events.ts

@@ -12,6 +12,7 @@ export const SIGNAL_ANSWER = 'signal/answer'
 export const SIGNAL_PROVIDE = 'signal/provide'
 export const SIGNAL_CANDIDATE = 'signal/candidate'
 export const SIGNAL_VIDEO = 'signal/video'
+export const SIGNAL_AUDIO = 'signal/audio'
 export const SIGNAL_CLOSE = 'signal/close'
 
 export const SESSION_CREATED = 'session/created'

src/component/types/messages.ts

@@ -1,5 +1,6 @@
 import { ICEServer } from '../internal/webrtc'
 import { Settings } from './state'
+import { PeerRequest, PeerVideo, PeerAudio } from './webrtc'
 
 /////////////////////////////
 // System
@@ -42,9 +43,13 @@ export interface SystemDisconnect {
 // Signal
 /////////////////////////////
 
+export type SignalRequest = PeerRequest
+
 export interface SignalProvide {
   sdp: string
   iceservers: ICEServer[]
+  video: PeerVideo
+  audio: PeerAudio
 }
 
 export type SignalCandidate = RTCIceCandidateInit
@@ -53,10 +58,9 @@ export interface SignalDescription {
   sdp: string
 }
 
-export interface SignalVideo {
-  video: string
-  auto: boolean
-}
+export type SignalVideo = PeerVideo
+
+export type SignalAudio = PeerAudio
 
 /////////////////////////////
 // Session

src/component/types/state.ts

@@ -38,8 +38,8 @@ export interface WebRTC {
   stable: boolean
   config: ReconnectorConfig
   stats: WebRTCStats | null
-  video: string | null
-  auto: boolean
+  video: PeerVideo
+  audio: PeerAudio
   videos: string[]
 }
 
@@ -47,6 +47,10 @@ export interface ReconnectorConfig extends reconnectorTypes.ReconnectorConfig {}
 
 export interface WebRTCStats extends webrtcTypes.WebRTCStats {}
 
+export interface PeerVideo extends webrtcTypes.PeerVideo {}
+
+export interface PeerAudio extends webrtcTypes.PeerAudio {}
+
 /////////////////////////////
 // Video
 /////////////////////////////

src/component/types/webrtc.ts

@@ -1,3 +1,36 @@
+export type StreamSelectorType = 'exact' | 'nearest' | 'lower' | 'higher'
+
+export interface StreamSelector {
+  type: StreamSelectorType
+  id?: string
+  bitrate?: number
+}
+
+export interface PeerRequest {
+  video?: PeerVideoRequest
+  audio?: PeerAudioRequest
+}
+
+export interface PeerVideo {
+  disabled: boolean
+  id: string
+  auto: boolean
+}
+
+export interface PeerVideoRequest {
+  disabled?: boolean
+  selector?: StreamSelector
+  auto?: boolean
+}
+
+export interface PeerAudio {
+  disabled: boolean
+}
+
+export interface PeerAudioRequest {
+  disabled?: boolean
+}
+
 export interface WebRTCStats {
   paused: boolean
   bitrate: number

src/page/components/events.vue

@@ -137,6 +137,13 @@
               >
                 webrtc is paused
               </td>
+              <td
+                colspan="2"
+                style="background: darkviolet; text-align: center"
+                v-else-if="neko.state.connection.webrtc.video.disabled"
+              >
+                video is disabled
+              </td>
               <td
                 colspan="2"
                 style="background: red; text-align: center"
@@ -158,15 +165,37 @@
         </td>
       </tr>
       <tr>
-        <th>connection.webrtc.video</th>
-        <td>{{ neko.state.connection.webrtc.video }}</td>
-      </tr>
-      <tr>
-        <th>connection.webrtc.auto</th>
+        <th title="connection.webrtc.video.disabled">connection.webrtc.video.disab..</th>
         <td>
           <div class="space-between">
-            <span>{{ neko.state.connection.webrtc.auto }}</span>
-            <button @click="neko.setWebRTCVideo(undefined, !neko.state.connection.webrtc.auto)">
+            <span>{{ neko.state.connection.webrtc.video.disabled }}</span>
+            <button @click="neko.setWebRTCVideo({ disabled: !neko.state.connection.webrtc.video.disabled })">
+              <i class="fas fa-toggle-on"></i>
+            </button>
+          </div>
+        </td>
+      </tr>
+      <tr>
+        <th>connection.webrtc.video.id</th>
+        <td>{{ neko.state.connection.webrtc.video.id }}</td>
+      </tr>
+      <tr>
+        <th>connection.webrtc.video.auto</th>
+        <td>
+          <div class="space-between">
+            <span>{{ neko.state.connection.webrtc.video.auto }}</span>
+            <button @click="neko.setWebRTCVideo({ auto: !neko.state.connection.webrtc.video.auto })">
+              <i class="fas fa-toggle-on"></i>
+            </button>
+          </div>
+        </td>
+      </tr>
+      <tr>
+        <th title="connection.webrtc.audio.disabled">connection.webrtc.audio.disab..</th>
+        <td>
+          <div class="space-between">
+            <span>{{ neko.state.connection.webrtc.audio.disabled }}</span>
+            <button @click="neko.setWebRTCAudio({ disabled: !neko.state.connection.webrtc.audio.disabled })">
               <i class="fas fa-toggle-on"></i>
             </button>
           </div>
@@ -178,7 +207,10 @@
       </tr>
       <tr>
         <td>
-          <select :value="neko.state.connection.webrtc.video" @input="neko.setWebRTCVideo($event.target.value, false)">
+          <select
+            :value="neko.state.connection.webrtc.video.id"
+            @input="neko.setWebRTCVideo({ selector: { id: $event.target.value } })"
+          >
             <option v-for="video in neko.state.connection.webrtc.videos" :key="video" :value="video">
               {{ video }}
             </option>