iridium/neko
2
2
Fork 0
mirror of https://github.com/m1k1o/neko.git synced 2024-07-24 14:40:50 +12:00
Code Issues Packages Projects Releases Wiki Activity

WebRTC congestion control (#24)

Browse Source 
* Move congestion control to server

* Address MR comments

* lint.

* bring back webrtc stats sync with state.

* update webrtc messages.

* set bitrate & video auto for signal request.

* clear up bitrate & video_auto.

* add bitrate and audio for connect function.

* reintroduce server side congestion control if video auto is false.

* mov ecode around.

* fix default video_auto.

* revert bitrate addition.

* remove video from signal provide.

---------

Co-authored-by: Aleksandar Sukovic <aleksandar.sukovic@gmail.com>
This commit is contained in:
Miroslav Šedivý 2023-05-15 19:32:21 +02:00 committed by GitHub
parent da252dfd31
commit 8d39a95e92
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 70 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/component/internal/connection.ts
View File

@ -38,8 +38,9 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
private _onDisconnectHandle: () => void
private _onCloseHandle: (error?: Error) => void
private _webrtcCongestionControlHandle: (stats: WebRTCStats) => void
private _webrtcStatsHandle: (stats: WebRTCStats) => void
private _webrtcStableHandle: (isStable: boolean) => void
private _webrtcCongestionControlHandle: (stats: WebRTCStats) => void
// eslint-disable-next-line
constructor(
@ -87,6 +88,12 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
r.on('close', this._onCloseHandle)
})
// synchronize webrtc stats with global state
this._webrtcStatsHandle = (stats: WebRTCStats) => {
Vue.set(this._state.webrtc, 'stats', stats)
}
this.webrtc.on('stats', this._webrtcStatsHandle)
// synchronize webrtc stable with global state
this._webrtcStableHandle = (isStable: boolean) => {
Vue.set(this._state.webrtc, 'stable', isStable)
@ -101,7 +108,8 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
let webrtcFallbackTimeout: number
this._webrtcCongestionControlHandle = (stats: WebRTCStats) => {
Vue.set(this._state.webrtc, 'stats', stats)
// if automatic quality adjusting is turned off
if (this._state.webrtc.auto) return
// when connection is paused, 0fps and muted track is expected
if (stats.paused) return
@ -138,7 +146,7 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
// downgrade if lower video quality exists
if (quality && this.webrtc.connected) {
this.setVideo(quality)
this.websocket.send(EVENT.SIGNAL_VIDEO, { video: quality })
}
// try to perform ice restart, if available
@ -151,7 +159,6 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
this._reconnector.webrtc.reconnect()
}
}
this.webrtc.on('stats', this._webrtcCongestionControlHandle)
}
@ -165,23 +172,11 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
this._reconnector.webrtc.config = this._state.webrtc.config
}
public setVideo(video: string, bitrate: number = 0) {
if (video != '' && !this._state.webrtc.videos.includes(video)) {
throw new Error('video id not found')
}
if (video == '' && bitrate == 0) {
throw new Error('video id and bitrate cannot be empty')
}
this.websocket.send(EVENT.SIGNAL_VIDEO, { video, bitrate })
}
public getLogger(scope?: string): Logger {
return this.logger.new(scope)
}
public open(video?: string) {
public open(video?: string, auto?: boolean) {
if (this._open) {
throw new Error('connection already open')
}
@ -196,6 +191,13 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
Vue.set(this._state.webrtc, 'video', video)
}
// if we didn't specify auto
if (typeof auto == 'undefined') {
// if we didn't specify video, set auto to true
auto = !video
}
Vue.set(this._state.webrtc, 'auto', auto)
Vue.set(this._state, 'status', 'connecting')
// open all reconnectors with deferred connection
@ -230,9 +232,10 @@ export class NekoConnection extends EventEmitter<NekoConnectionEvents> {
public destroy() {
this.logger.destroy()
this.webrtc.off('stats', this._webrtcStatsHandle)
this.webrtc.off('stable', this._webrtcStableHandle)
// TODO: Use server side congestion control.
this.webrtc.off('stats', this._webrtcCongestionControlHandle)
this.webrtc.off('stable', this._webrtcStableHandle)
// unbind events from all reconnectors
Object.values(this._reconnector).forEach((r) => {

9
src/component/internal/messages.ts
View File

@ -147,9 +147,8 @@ export class NekoMessages extends EventEmitter<NekoEvents> {
// Signal Events
/////////////////////////////
protected async [EVENT.SIGNAL_PROVIDE]({ sdp, video, iceservers }: message.SignalProvide) {
protected async [EVENT.SIGNAL_PROVIDE]({ sdp, iceservers }: message.SignalProvide) {
this._localLog.debug(`EVENT.SIGNAL_PROVIDE`)
Vue.set(this._state.connection.webrtc, 'video', video)
// create WebRTC connection
await this._connection.webrtc.connect(iceservers)
@ -194,10 +193,10 @@ export class NekoMessages extends EventEmitter<NekoEvents> {
this.emit('connection.webrtc.sdp.candidate', 'remote', candidate)
}
protected [EVENT.SIGNAL_VIDEO]({ video, bitrate }: message.SignalVideo) {
this._localLog.debug(`EVENT.SIGNAL_VIDEO`, { video, bitrate })
protected [EVENT.SIGNAL_VIDEO]({ video, auto }: message.SignalVideo) {
this._localLog.debug(`EVENT.SIGNAL_VIDEO`, { video, auto })
Vue.set(this._state.connection.webrtc, 'video', video)
Vue.set(this._state.connection.webrtc, 'bitrate', bitrate)
Vue.set(this._state.connection.webrtc, 'auto', !!auto)
}
protected [EVENT.SIGNAL_CLOSE]() {

5
src/component/internal/reconnector/webrtc.ts
View File

@ -37,7 +37,10 @@ export class WebrtcReconnector extends ReconnectorAbstract {
}
if (this._websocket.connected) {
this._websocket.send(EVENT.SIGNAL_REQUEST, { video: this._state.webrtc.video })
this._websocket.send(EVENT.SIGNAL_REQUEST, {
video: this._state.webrtc.video,
auto: this._state.webrtc.auto,
})
}
}

24
src/component/main.vue
View File

@ -170,7 +170,7 @@
},
stats: null,
video: null,
bitrate: null,
auto: false,
videos: [],
},
screencast: true, // TODO: Should get by API call.
@ -383,7 +383,7 @@
}
}
public connect(video?: string) {
public connect(video?: string, auto?: boolean) {
if (!this.state.authenticated) {
throw new Error('client not authenticated')
}
@ -392,7 +392,7 @@
throw new Error('client is already connected')
}
this.connection.open(video)
this.connection.open(video, auto)
}
public disconnect() {
@ -512,8 +512,19 @@
this.connection.websocket.send(EVENT.SCREEN_SET, { width, height, rate })
}
public setWebRTCVideo(video: string, bitrate: number = 0) {
this.connection.setVideo(video, bitrate)
public setWebRTCVideo(video?: string, auto?: boolean) {
// if video has been set, check if it exists
if (video && !this.state.connection.webrtc.videos.includes(video)) {
throw new Error('video id not found')
}
// if we didn't specify auto
if (typeof auto == 'undefined') {
// if we didn't specify video, set auto to true
auto = !video
}
this.connection.websocket.send(EVENT.SIGNAL_VIDEO, { video, auto })
}
public addTrack(track: MediaStreamTrack, ...streams: MediaStream[]): RTCRtpSender {
@ -749,7 +760,6 @@
}
// websocket
Vue.set(this.state.connection.webrtc, 'videos', [])
Vue.set(this.state.control, 'clipboard', null)
Vue.set(this.state.control, 'host_id', null)
Vue.set(this.state.screen, 'size', { width: 1280, height: 720, rate: 30 })
@ -768,6 +778,8 @@
// webrtc
Vue.set(this.state.connection.webrtc, 'stats', null)
Vue.set(this.state.connection.webrtc, 'video', null)
Vue.set(this.state.connection.webrtc, 'auto', false)
Vue.set(this.state.connection.webrtc, 'videos', [])
Vue.set(this.state.connection, 'type', 'none')
}
}

3
src/component/types/messages.ts
View File

@ -45,7 +45,6 @@ export interface SystemDisconnect {
export interface SignalProvide {
sdp: string
iceservers: ICEServer[]
video: string
}
export type SignalCandidate = RTCIceCandidateInit
@ -56,7 +55,7 @@ export interface SignalDescription {
export interface SignalVideo {
video: string
bitrate: number
auto: boolean
}
/////////////////////////////

2
src/component/types/state.ts
View File

@ -39,7 +39,7 @@ export interface WebRTC {
config: ReconnectorConfig
stats: WebRTCStats | null
video: string | null
bitrate: number | null
auto: boolean
videos: string[]
}

32
src/page/components/events.vue
View File

@ -161,20 +161,28 @@
<th>connection.webrtc.video</th>
<td>{{ neko.state.connection.webrtc.video }}</td>
</tr>
<tr>
<th>connection.webrtc.auto</th>
<td>
<div class="space-between">
<span>{{ neko.state.connection.webrtc.auto }}</span>
<button @click="neko.setWebRTCVideo(undefined, !neko.state.connection.webrtc.auto)">
<i class="fas fa-toggle-on"></i>
</button>
</div>
</td>
</tr>
<tr>
<th rowspan="2">connection.webrtc.videos</th>
<td>Total {{ neko.state.connection.webrtc.videos.length }} videos.</td>
</tr>
<tr>
<td>
<select :value="neko.state.connection.webrtc.video" @input="neko.setWebRTCVideo($event.target.value)">
<select :value="neko.state.connection.webrtc.video" @input="neko.setWebRTCVideo($event.target.value, false)">
<option v-for="video in neko.state.connection.webrtc.videos" :key="video" :value="video">
{{ video }}
</option>
</select>
or
<input type="text" v-model="bitrate" style="width: 60px" placeholder="Bitrate" />
<button @click="neko.setWebRTCVideo('', Number(bitrate))">Set</button>
</td>
</tr>
<tr>
@ -376,13 +384,15 @@
</td>
</tr>
</template>
<tr v-else>
<th>screen.configurations</th>
<td>Session is not admin.</td>
<th>screen.sync</th>
<td>Session is not admin.</td>
</tr>
<template v-else>
<tr>
<th>screen.configurations</th>
<td rowspan="2" style="vertical-align: middle">Session is not admin.</td>
</tr>
<tr>
<th>screen.sync</th>
</tr>
</template>
<tr>
<th>session_id</th>
<td>{{ neko.state.session_id }}</td>