mirror of
https://github.com/m1k1o/neko.git
synced 2024-07-24 14:40:50 +12:00
WebRTC congestion control (#24)
* Move congestion control to server * Address MR comments * lint. * bring back webrtc stats sync with state. * update webrtc messages. * set bitrate & video auto for signal request. * clear up bitrate & video_auto. * add bitrate and audio for connect function. * reintroduce server side congestion control if video auto is false. * mov ecode around. * fix default video_auto. * revert bitrate addition. * remove video from signal provide. --------- Co-authored-by: Aleksandar Sukovic <aleksandar.sukovic@gmail.com>
This commit is contained in:
parent
da252dfd31
commit
8d39a95e92
@ -38,8 +38,9 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
|
|||||||
private _onDisconnectHandle: () => void
|
private _onDisconnectHandle: () => void
|
||||||
private _onCloseHandle: (error?: Error) => void
|
private _onCloseHandle: (error?: Error) => void
|
||||||
|
|
||||||
private _webrtcCongestionControlHandle: (stats: WebRTCStats) => void
|
private _webrtcStatsHandle: (stats: WebRTCStats) => void
|
||||||
private _webrtcStableHandle: (isStable: boolean) => void
|
private _webrtcStableHandle: (isStable: boolean) => void
|
||||||
|
private _webrtcCongestionControlHandle: (stats: WebRTCStats) => void
|
||||||
|
|
||||||
// eslint-disable-next-line
|
// eslint-disable-next-line
|
||||||
constructor(
|
constructor(
|
||||||
@ -87,6 +88,12 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
|
|||||||
r.on('close', this._onCloseHandle)
|
r.on('close', this._onCloseHandle)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// synchronize webrtc stats with global state
|
||||||
|
this._webrtcStatsHandle = (stats: WebRTCStats) => {
|
||||||
|
Vue.set(this._state.webrtc, 'stats', stats)
|
||||||
|
}
|
||||||
|
this.webrtc.on('stats', this._webrtcStatsHandle)
|
||||||
|
|
||||||
// synchronize webrtc stable with global state
|
// synchronize webrtc stable with global state
|
||||||
this._webrtcStableHandle = (isStable: boolean) => {
|
this._webrtcStableHandle = (isStable: boolean) => {
|
||||||
Vue.set(this._state.webrtc, 'stable', isStable)
|
Vue.set(this._state.webrtc, 'stable', isStable)
|
||||||
@ -101,7 +108,8 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
|
|||||||
let webrtcFallbackTimeout: number
|
let webrtcFallbackTimeout: number
|
||||||
|
|
||||||
this._webrtcCongestionControlHandle = (stats: WebRTCStats) => {
|
this._webrtcCongestionControlHandle = (stats: WebRTCStats) => {
|
||||||
Vue.set(this._state.webrtc, 'stats', stats)
|
// if automatic quality adjusting is turned off
|
||||||
|
if (this._state.webrtc.auto) return
|
||||||
|
|
||||||
// when connection is paused, 0fps and muted track is expected
|
// when connection is paused, 0fps and muted track is expected
|
||||||
if (stats.paused) return
|
if (stats.paused) return
|
||||||
@ -138,7 +146,7 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
|
|||||||
|
|
||||||
// downgrade if lower video quality exists
|
// downgrade if lower video quality exists
|
||||||
if (quality && this.webrtc.connected) {
|
if (quality && this.webrtc.connected) {
|
||||||
this.setVideo(quality)
|
this.websocket.send(EVENT.SIGNAL_VIDEO, { video: quality })
|
||||||
}
|
}
|
||||||
|
|
||||||
// try to perform ice restart, if available
|
// try to perform ice restart, if available
|
||||||
@ -151,7 +159,6 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
|
|||||||
this._reconnector.webrtc.reconnect()
|
this._reconnector.webrtc.reconnect()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
this.webrtc.on('stats', this._webrtcCongestionControlHandle)
|
this.webrtc.on('stats', this._webrtcCongestionControlHandle)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -165,23 +172,11 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
|
|||||||
this._reconnector.webrtc.config = this._state.webrtc.config
|
this._reconnector.webrtc.config = this._state.webrtc.config
|
||||||
}
|
}
|
||||||
|
|
||||||
public setVideo(video: string, bitrate: number = 0) {
|
|
||||||
if (video != '' && !this._state.webrtc.videos.includes(video)) {
|
|
||||||
throw new Error('video id not found')
|
|
||||||
}
|
|
||||||
|
|
||||||
if (video == '' && bitrate == 0) {
|
|
||||||
throw new Error('video id and bitrate cannot be empty')
|
|
||||||
}
|
|
||||||
|
|
||||||
this.websocket.send(EVENT.SIGNAL_VIDEO, { video, bitrate })
|
|
||||||
}
|
|
||||||
|
|
||||||
public getLogger(scope?: string): Logger {
|
public getLogger(scope?: string): Logger {
|
||||||
return this.logger.new(scope)
|
return this.logger.new(scope)
|
||||||
}
|
}
|
||||||
|
|
||||||
public open(video?: string) {
|
public open(video?: string, auto?: boolean) {
|
||||||
if (this._open) {
|
if (this._open) {
|
||||||
throw new Error('connection already open')
|
throw new Error('connection already open')
|
||||||
}
|
}
|
||||||
@ -196,6 +191,13 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
|
|||||||
Vue.set(this._state.webrtc, 'video', video)
|
Vue.set(this._state.webrtc, 'video', video)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// if we didn't specify auto
|
||||||
|
if (typeof auto == 'undefined') {
|
||||||
|
// if we didn't specify video, set auto to true
|
||||||
|
auto = !video
|
||||||
|
}
|
||||||
|
Vue.set(this._state.webrtc, 'auto', auto)
|
||||||
|
|
||||||
Vue.set(this._state, 'status', 'connecting')
|
Vue.set(this._state, 'status', 'connecting')
|
||||||
|
|
||||||
// open all reconnectors with deferred connection
|
// open all reconnectors with deferred connection
|
||||||
@ -230,9 +232,10 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
|
|||||||
public destroy() {
|
public destroy() {
|
||||||
this.logger.destroy()
|
this.logger.destroy()
|
||||||
|
|
||||||
|
this.webrtc.off('stats', this._webrtcStatsHandle)
|
||||||
|
this.webrtc.off('stable', this._webrtcStableHandle)
|
||||||
// TODO: Use server side congestion control.
|
// TODO: Use server side congestion control.
|
||||||
this.webrtc.off('stats', this._webrtcCongestionControlHandle)
|
this.webrtc.off('stats', this._webrtcCongestionControlHandle)
|
||||||
this.webrtc.off('stable', this._webrtcStableHandle)
|
|
||||||
|
|
||||||
// unbind events from all reconnectors
|
// unbind events from all reconnectors
|
||||||
Object.values(this._reconnector).forEach((r) => {
|
Object.values(this._reconnector).forEach((r) => {
|
||||||
|
@ -147,9 +147,8 @@ export class NekoMessages extends EventEmitter<NekoEvents> {
|
|||||||
// Signal Events
|
// Signal Events
|
||||||
/////////////////////////////
|
/////////////////////////////
|
||||||
|
|
||||||
protected async [EVENT.SIGNAL_PROVIDE]({ sdp, video, iceservers }: message.SignalProvide) {
|
protected async [EVENT.SIGNAL_PROVIDE]({ sdp, iceservers }: message.SignalProvide) {
|
||||||
this._localLog.debug(`EVENT.SIGNAL_PROVIDE`)
|
this._localLog.debug(`EVENT.SIGNAL_PROVIDE`)
|
||||||
Vue.set(this._state.connection.webrtc, 'video', video)
|
|
||||||
|
|
||||||
// create WebRTC connection
|
// create WebRTC connection
|
||||||
await this._connection.webrtc.connect(iceservers)
|
await this._connection.webrtc.connect(iceservers)
|
||||||
@ -194,10 +193,10 @@ export class NekoMessages extends EventEmitter<NekoEvents> {
|
|||||||
this.emit('connection.webrtc.sdp.candidate', 'remote', candidate)
|
this.emit('connection.webrtc.sdp.candidate', 'remote', candidate)
|
||||||
}
|
}
|
||||||
|
|
||||||
protected [EVENT.SIGNAL_VIDEO]({ video, bitrate }: message.SignalVideo) {
|
protected [EVENT.SIGNAL_VIDEO]({ video, auto }: message.SignalVideo) {
|
||||||
this._localLog.debug(`EVENT.SIGNAL_VIDEO`, { video, bitrate })
|
this._localLog.debug(`EVENT.SIGNAL_VIDEO`, { video, auto })
|
||||||
Vue.set(this._state.connection.webrtc, 'video', video)
|
Vue.set(this._state.connection.webrtc, 'video', video)
|
||||||
Vue.set(this._state.connection.webrtc, 'bitrate', bitrate)
|
Vue.set(this._state.connection.webrtc, 'auto', !!auto)
|
||||||
}
|
}
|
||||||
|
|
||||||
protected [EVENT.SIGNAL_CLOSE]() {
|
protected [EVENT.SIGNAL_CLOSE]() {
|
||||||
|
@ -37,7 +37,10 @@ export class WebrtcReconnector extends ReconnectorAbstract {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (this._websocket.connected) {
|
if (this._websocket.connected) {
|
||||||
this._websocket.send(EVENT.SIGNAL_REQUEST, { video: this._state.webrtc.video })
|
this._websocket.send(EVENT.SIGNAL_REQUEST, {
|
||||||
|
video: this._state.webrtc.video,
|
||||||
|
auto: this._state.webrtc.auto,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -170,7 +170,7 @@
|
|||||||
},
|
},
|
||||||
stats: null,
|
stats: null,
|
||||||
video: null,
|
video: null,
|
||||||
bitrate: null,
|
auto: false,
|
||||||
videos: [],
|
videos: [],
|
||||||
},
|
},
|
||||||
screencast: true, // TODO: Should get by API call.
|
screencast: true, // TODO: Should get by API call.
|
||||||
@ -383,7 +383,7 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public connect(video?: string) {
|
public connect(video?: string, auto?: boolean) {
|
||||||
if (!this.state.authenticated) {
|
if (!this.state.authenticated) {
|
||||||
throw new Error('client not authenticated')
|
throw new Error('client not authenticated')
|
||||||
}
|
}
|
||||||
@ -392,7 +392,7 @@
|
|||||||
throw new Error('client is already connected')
|
throw new Error('client is already connected')
|
||||||
}
|
}
|
||||||
|
|
||||||
this.connection.open(video)
|
this.connection.open(video, auto)
|
||||||
}
|
}
|
||||||
|
|
||||||
public disconnect() {
|
public disconnect() {
|
||||||
@ -512,8 +512,19 @@
|
|||||||
this.connection.websocket.send(EVENT.SCREEN_SET, { width, height, rate })
|
this.connection.websocket.send(EVENT.SCREEN_SET, { width, height, rate })
|
||||||
}
|
}
|
||||||
|
|
||||||
public setWebRTCVideo(video: string, bitrate: number = 0) {
|
public setWebRTCVideo(video?: string, auto?: boolean) {
|
||||||
this.connection.setVideo(video, bitrate)
|
// if video has been set, check if it exists
|
||||||
|
if (video && !this.state.connection.webrtc.videos.includes(video)) {
|
||||||
|
throw new Error('video id not found')
|
||||||
|
}
|
||||||
|
|
||||||
|
// if we didn't specify auto
|
||||||
|
if (typeof auto == 'undefined') {
|
||||||
|
// if we didn't specify video, set auto to true
|
||||||
|
auto = !video
|
||||||
|
}
|
||||||
|
|
||||||
|
this.connection.websocket.send(EVENT.SIGNAL_VIDEO, { video, auto })
|
||||||
}
|
}
|
||||||
|
|
||||||
public addTrack(track: MediaStreamTrack, ...streams: MediaStream[]): RTCRtpSender {
|
public addTrack(track: MediaStreamTrack, ...streams: MediaStream[]): RTCRtpSender {
|
||||||
@ -749,7 +760,6 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
// websocket
|
// websocket
|
||||||
Vue.set(this.state.connection.webrtc, 'videos', [])
|
|
||||||
Vue.set(this.state.control, 'clipboard', null)
|
Vue.set(this.state.control, 'clipboard', null)
|
||||||
Vue.set(this.state.control, 'host_id', null)
|
Vue.set(this.state.control, 'host_id', null)
|
||||||
Vue.set(this.state.screen, 'size', { width: 1280, height: 720, rate: 30 })
|
Vue.set(this.state.screen, 'size', { width: 1280, height: 720, rate: 30 })
|
||||||
@ -768,6 +778,8 @@
|
|||||||
// webrtc
|
// webrtc
|
||||||
Vue.set(this.state.connection.webrtc, 'stats', null)
|
Vue.set(this.state.connection.webrtc, 'stats', null)
|
||||||
Vue.set(this.state.connection.webrtc, 'video', null)
|
Vue.set(this.state.connection.webrtc, 'video', null)
|
||||||
|
Vue.set(this.state.connection.webrtc, 'auto', false)
|
||||||
|
Vue.set(this.state.connection.webrtc, 'videos', [])
|
||||||
Vue.set(this.state.connection, 'type', 'none')
|
Vue.set(this.state.connection, 'type', 'none')
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -45,7 +45,6 @@ export interface SystemDisconnect {
|
|||||||
export interface SignalProvide {
|
export interface SignalProvide {
|
||||||
sdp: string
|
sdp: string
|
||||||
iceservers: ICEServer[]
|
iceservers: ICEServer[]
|
||||||
video: string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export type SignalCandidate = RTCIceCandidateInit
|
export type SignalCandidate = RTCIceCandidateInit
|
||||||
@ -56,7 +55,7 @@ export interface SignalDescription {
|
|||||||
|
|
||||||
export interface SignalVideo {
|
export interface SignalVideo {
|
||||||
video: string
|
video: string
|
||||||
bitrate: number
|
auto: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
/////////////////////////////
|
/////////////////////////////
|
||||||
|
@ -39,7 +39,7 @@ export interface WebRTC {
|
|||||||
config: ReconnectorConfig
|
config: ReconnectorConfig
|
||||||
stats: WebRTCStats | null
|
stats: WebRTCStats | null
|
||||||
video: string | null
|
video: string | null
|
||||||
bitrate: number | null
|
auto: boolean
|
||||||
videos: string[]
|
videos: string[]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -161,20 +161,28 @@
|
|||||||
<th>connection.webrtc.video</th>
|
<th>connection.webrtc.video</th>
|
||||||
<td>{{ neko.state.connection.webrtc.video }}</td>
|
<td>{{ neko.state.connection.webrtc.video }}</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<th>connection.webrtc.auto</th>
|
||||||
|
<td>
|
||||||
|
<div class="space-between">
|
||||||
|
<span>{{ neko.state.connection.webrtc.auto }}</span>
|
||||||
|
<button @click="neko.setWebRTCVideo(undefined, !neko.state.connection.webrtc.auto)">
|
||||||
|
<i class="fas fa-toggle-on"></i>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<th rowspan="2">connection.webrtc.videos</th>
|
<th rowspan="2">connection.webrtc.videos</th>
|
||||||
<td>Total {{ neko.state.connection.webrtc.videos.length }} videos.</td>
|
<td>Total {{ neko.state.connection.webrtc.videos.length }} videos.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td>
|
<td>
|
||||||
<select :value="neko.state.connection.webrtc.video" @input="neko.setWebRTCVideo($event.target.value)">
|
<select :value="neko.state.connection.webrtc.video" @input="neko.setWebRTCVideo($event.target.value, false)">
|
||||||
<option v-for="video in neko.state.connection.webrtc.videos" :key="video" :value="video">
|
<option v-for="video in neko.state.connection.webrtc.videos" :key="video" :value="video">
|
||||||
{{ video }}
|
{{ video }}
|
||||||
</option>
|
</option>
|
||||||
</select>
|
</select>
|
||||||
or
|
|
||||||
<input type="text" v-model="bitrate" style="width: 60px" placeholder="Bitrate" />
|
|
||||||
<button @click="neko.setWebRTCVideo('', Number(bitrate))">Set</button>
|
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
@ -376,13 +384,15 @@
|
|||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
</template>
|
</template>
|
||||||
<tr v-else>
|
<template v-else>
|
||||||
<th>screen.configurations</th>
|
<tr>
|
||||||
<td>Session is not admin.</td>
|
<th>screen.configurations</th>
|
||||||
<th>screen.sync</th>
|
<td rowspan="2" style="vertical-align: middle">Session is not admin.</td>
|
||||||
<td>Session is not admin.</td>
|
</tr>
|
||||||
</tr>
|
<tr>
|
||||||
|
<th>screen.sync</th>
|
||||||
|
</tr>
|
||||||
|
</template>
|
||||||
<tr>
|
<tr>
|
||||||
<th>session_id</th>
|
<th>session_id</th>
|
||||||
<td>{{ neko.state.session_id }}</td>
|
<td>{{ neko.state.session_id }}</td>
|
||||||
|
Loading…
Reference in New Issue
Block a user