From 33245fea03675d3d9ca5a53e14ba3e63d2314010 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= <sedivy.miro@gmail.com>
Date: Fri, 18 Mar 2022 23:56:27 +0100
Subject: [PATCH] webrtc fallback image video snap.

---
 src/component/internal/webrtc.ts |  9 +++++++++
 src/component/main.vue           | 20 ++++++++++++++++++++
 src/component/screencast.vue     |  8 ++++++++
 3 files changed, 37 insertions(+)

diff --git a/src/component/internal/webrtc.ts b/src/component/internal/webrtc.ts
index ec31345d..20bb9934 100644
--- a/src/component/internal/webrtc.ts
+++ b/src/component/internal/webrtc.ts
@@ -1,6 +1,7 @@
 import EventEmitter from 'eventemitter3'
 import { WebRTCStats, CursorPosition, CursorImage } from '../types/webrtc'
 import { Logger } from '../utils/logger'
+import { videoSnap } from '../utils/video-snap'
 
 export const OPCODE = {
   MOVE: 0x01,
@@ -25,11 +26,15 @@ export interface NekoWebRTCEvents {
   negotiation: (description: RTCSessionDescriptionInit) => void
   stable: (isStable: boolean) => void
   stats: (stats: WebRTCStats) => void
+  fallback: (image: string) => void // send last frame image URL as fallback
   ['cursor-position']: (data: CursorPosition) => void
   ['cursor-image']: (data: CursorImage) => void
 }
 
 export class NekoWebRTC extends EventEmitter<NekoWebRTCEvents> {
+  // used for creating snaps from video for fallback mode
+  public video!: HTMLVideoElement
+
   private _peer?: RTCPeerConnection
   private _channel?: RTCDataChannel
   private _track?: MediaStreamTrack
@@ -248,6 +253,10 @@ export class NekoWebRTC extends EventEmitter<NekoWebRTCEvents> {
       throw new Error('attempting to close nonexistent peer')
     }
 
+    // create and emit video snap before closing connection
+    const imageSrc = await videoSnap(this.video)
+    this.emit('fallback', imageSrc)
+
     this._peer.close()
   }
 
diff --git a/src/component/main.vue b/src/component/main.vue
index 235326d3..fd6a1d79 100644
--- a/src/component/main.vue
+++ b/src/component/main.vue
@@ -4,6 +4,7 @@
       <video ref="video" :autoplay="autoplay" :muted="autoplay" playsinline />
       <neko-screencast
         v-show="screencast && screencastReady"
+        :image="fallbackImage"
         :enabled="screencast || !state.connection.webrtc.stable"
         :api="api.room"
         @imageReady="screencastReady = $event"
@@ -100,6 +101,12 @@
     @Ref('container') readonly _container!: HTMLElement
     @Ref('video') readonly _video!: HTMLVideoElement
 
+    // fallback image for webrtc reconnections:
+    // chrome shows black screen when closing webrtc connection, that's why
+    // we need to grab video image before closing connection ans show that
+    // while reconnecting, to not see black screen
+    fallbackImage = ''
+
     api = new NekoApi()
     observer = new ResizeObserver(this.onResize.bind(this))
     canvasSize: Dimension = { width: 0, height: 0 }
@@ -433,6 +440,9 @@
       // component size change
       this.observer.observe(this._component)
 
+      // webrtc needs video tag to capture video snaps for fallback mode
+      this.connection.webrtc.video = this._video
+
       // video events
       VideoRegister(this._video, this.state.video)
 
@@ -441,6 +451,16 @@
         this.clear()
       })
 
+      // when webrtc emits fallback event, it means it is about to reconnect
+      // so we image that it provided (it is last frame of the video), we set
+      // it to the screencast module and pause video in order to show fallback
+      this.connection.webrtc.on('fallback', (image: string) => {
+        this.fallbackImage = image
+
+        // this ensures that fallback mode starts immediatly
+        this._video.pause()
+      })
+
       this.connection.webrtc.on('track', (event: RTCTrackEvent) => {
         const { track, streams } = event
         if (track.kind === 'audio') return
diff --git a/src/component/screencast.vue b/src/component/screencast.vue
index a5620982..58c299de 100644
--- a/src/component/screencast.vue
+++ b/src/component/screencast.vue
@@ -17,6 +17,14 @@
     private running = false
     private continue = false
 
+    @Prop()
+    private readonly image!: string
+
+    @Watch('image')
+    setImage(image: string) {
+      this.imageSrc = image
+    }
+
     @Prop()
     private readonly enabled!: boolean