cursor image/position from WebRTC.

src/component/internal/messages.ts

@@ -74,9 +74,6 @@ export class NekoMessages extends EventEmitter<NekoEvents> {
 
     this[EVENT.SCREEN_UPDATED](conf.screen_size)
     this[EVENT.CONTROL_HOST](conf.control_host)
-    if (conf.cursor_image) {
-      this[EVENT.CURSOR_IMAGE](conf.cursor_image)
-    }
   }
 
   protected [EVENT.SYSTEM_ADMIN]({ screen_sizes_list, broadcast_status }: message.SystemAdmin) {
@@ -186,20 +183,6 @@ export class NekoMessages extends EventEmitter<NekoEvents> {
     this.emit('room.clipboard.updated', text)
   }
 
-  /////////////////////////////
-  // Cursor Events
-  /////////////////////////////
-
-  protected [EVENT.CURSOR_IMAGE]({ uri, width, height, x, y }: message.CursorImage) {
-    this._log.debug('EVENT.CURSOR_IMAGE')
-    Vue.set(this.state.control.cursor, 'image', { uri, width, height, x, y })
-  }
-
-  protected [EVENT.CURSOR_POSITION]({ x, y }: message.CursorPosition) {
-    this._log.debug('EVENT.CURSOR_IMAGE')
-    Vue.set(this.state.control.cursor, 'position', { x, y })
-  }
-
   /////////////////////////////
   // Broadcast Events
   /////////////////////////////

src/component/internal/webrtc.ts

@@ -23,6 +23,8 @@ export interface NekoWebRTCEvents {
   track: (event: RTCTrackEvent) => void
   candidate: (candidate: RTCIceCandidateInit) => void
   stats: (stats: WebRTCStats) => void
+  ['cursor-position']: (data: { x: number; y: number }) => void
+  ['cursor-image']: (data: { width: number; height: number; x: number; y: number; uri: string }) => void
 }
 
 export class NekoWebRTC extends EventEmitter<NekoWebRTCEvents> {
@@ -204,9 +206,6 @@ export class NekoWebRTC extends EventEmitter<NekoWebRTCEvents> {
     this._channel!.send(buffer)
   }
 
-  // not-implemented
-  private onData(e: MessageEvent) {}
-
   private onTrack(event: RTCTrackEvent) {
     this._log.debug(`received ${event.track.kind} track from peer: ${event.track.id}`, event)
     const stream = event.streams[0]
@@ -229,6 +228,42 @@ export class NekoWebRTC extends EventEmitter<NekoWebRTCEvents> {
     this._channel.onclose = this.onDisconnected.bind(this, new Error('peer data channel closed'))
   }
 
+  // not-implemented
+  private onData(e: MessageEvent) {
+    const payload = new DataView(e.data)
+    const event = payload.getUint8(0)
+    const length = payload.getUint16(1)
+
+    switch (event) {
+      case 1:
+        this.emit('cursor-position', {
+          x: payload.getUint16(3),
+          y: payload.getUint16(5),
+        })
+        break
+      case 2:
+        const data = e.data.slice(11, length - 11)
+
+        // TODO: get string from server
+        const blob = new Blob([data], { type: 'image/png' })
+        const reader = new FileReader()
+        reader.onload = (e) => {
+          this.emit('cursor-image', {
+            width: payload.getUint16(3),
+            height: payload.getUint16(5),
+            x: payload.getUint16(7),
+            y: payload.getUint16(9),
+            uri: String(e.target!.result),
+          })
+        }
+        reader.readAsDataURL(blob)
+
+        break
+      default:
+        this._log.warn(`unhandled webrtc event '${event}'.`, payload)
+    }
+  }
+
   private onConnected() {
     if (!this.connected) {
       this._log.warn(`onConnected called while being disconnected`)

src/component/main.vue

@@ -4,7 +4,7 @@
       <video ref="video" :autoplay="autoplay" :muted="autoplay" />
       <neko-overlay
         :webrtc="webrtc"
-        :control="state.control"
+        :scroll="state.control.scroll"
         :screenSize="state.screen.size"
         :canvasSize="canvasSize"
         :isControling="controlling && watching"

src/component/overlay.vue

@@ -46,7 +46,7 @@
   import GuacamoleKeyboard from './utils/guacamole-keyboard'
   import { getFilesFromDataTansfer } from './utils/file-upload'
   import { NekoWebRTC } from './internal/webrtc'
-  import { Control } from './types/state'
+  import { Scroll } from './types/state'
 
   const inactiveCursorWin10 =
     'url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAACEUlEQVR4nOzWz6sSURQH8O+89zJ5C32LKbAgktCSaPpBSL' +
@@ -71,7 +71,7 @@
     private readonly webrtc!: NekoWebRTC
 
     @Prop()
-    private readonly control!: Control
+    private readonly scroll!: Scroll
 
     @Prop()
     private readonly screenSize!: { width: number; height: number }
@@ -90,11 +90,11 @@
         return inactiveCursorWin10
       }
 
-      if (!this.control.cursor.image) {
+      if (!this.cursorImage) {
         return 'auto'
       }
 
-      const { uri, x, y } = this.control.cursor.image
+      const { uri, x, y } = this.cursorImage
       return 'url(' + uri + ') ' + x + ' ' + y + ', auto'
     }
 
@@ -131,10 +131,16 @@
         this.webrtc.send('keyup', { key })
       }
       this.keyboard.listenTo(this._overlay)
+
+      this.webrtc.addListener('cursor-position', this.onCursorPosition)
+      this.webrtc.addListener('cursor-image', this.onCursorImage)
     }
 
     beforeDestroy() {
       // Guacamole Keyboard does not provide destroy functions
+
+      this.webrtc.removeListener('cursor-position', this.onCursorPosition)
+      this.webrtc.removeListener('cursor-image', this.onCursorImage)
     }
 
     getMousePos(clientX: number, clientY: number) {
@@ -146,11 +152,10 @@
       }
     }
 
-    private mousepos: { x: number; y: number } = { x: 0, y: 0 }
     setMousePos(e: MouseEvent) {
       const pos = this.getMousePos(e.clientX, e.clientY)
       this.webrtc.send('mousemove', pos)
-      Vue.set(this, 'mousepos', pos)
+      Vue.set(this, 'cursorPosition', pos)
     }
 
     onWheel(e: WheelEvent) {
@@ -161,13 +166,13 @@
       let x = e.deltaX
       let y = e.deltaY
 
-      if (this.control.scroll.inverse) {
+      if (this.scroll.inverse) {
         x *= -1
         y *= -1
       }
 
-      x = Math.min(Math.max(x, -this.control.scroll.sensitivity), this.control.scroll.sensitivity)
+      x = Math.min(Math.max(x, -this.scroll.sensitivity), this.scroll.sensitivity)
-      y = Math.min(Math.max(y, -this.control.scroll.sensitivity), this.control.scroll.sensitivity)
+      y = Math.min(Math.max(y, -this.scroll.sensitivity), this.scroll.sensitivity)
 
       this.setMousePos(e)
       this.webrtc.send('wheel', { x, y })
@@ -255,38 +260,67 @@
       }
     }
 
+    //
+    // canvas
+    //
+
+    private cursorImage: { width: number; height: number; x: number; y: number; uri: string } | null = null
+    private cursorElement: HTMLImageElement = new Image()
+    private cursorPosition: { x: number; y: number } | null = null
+
     @Watch('canvasSize')
     onCanvasSizeChange({ width, height }: { width: number; height: number }) {
       this._overlay.width = width
       this._overlay.height = height
+      this.canvasRedraw()
     }
 
-    private cursorElem: HTMLImageElement = new Image()
+    onCursorPosition(data: { x: number; y: number }) {
-    @Watch('control.cursor.image')
+      if (!this.isControling) {
-    onCursorImageChange({ uri }: { uri: string }) {
+        Vue.set(this, 'cursorPosition', data)
-      this.cursorElem.src = uri
+        this.canvasRedraw()
+      }
     }
 
-    @Watch('control.cursor.position')
+    onCursorImage(data: { width: number; height: number; x: number; y: number; uri: string }) {
-    onCursorPositionChange({ x, y }: { x: number; y: number }) {
+      Vue.set(this, 'cursorImage', data)
-      if (this.isControling || this.control.cursor.image == null) return
+
+      if (!this.isControling) {
+        this.cursorElement.src = data.uri
+        this.canvasRedraw()
+      }
+    }
+
+    canvasRedraw() {
+      if (this.cursorPosition == null || this.screenSize == null || this.cursorImage == null) return
 
       // get intrinsic dimensions
       const { width, height } = this._overlay
+      const { x, y } = this.cursorPosition
 
       // redraw cursor
       this._ctx.clearRect(0, 0, width, height)
       this._ctx.drawImage(
-        this.cursorElem,
+        this.cursorElement,
-        Math.round((x / this.screenSize.width) * width - this.control.cursor.image.x),
+        Math.round((x / this.screenSize.width) * width - this.cursorImage.x),
-        Math.round((y / this.screenSize.height) * height - this.control.cursor.image.y),
+        Math.round((y / this.screenSize.height) * height - this.cursorImage.y),
-        this.control.cursor.image.width,
+        this.cursorImage.width,
-        this.control.cursor.image.height,
+        this.cursorImage.height,
       )
     }
 
+    canvasClear() {
+      const { width, height } = this._overlay
+      this._ctx.clearRect(0, 0, width, height)
+    }
+
+    //
+    // implicit hosting
+    //
+
     private reqMouseDown: any | null = null
     private reqMouseUp: any | null = null
+
     @Watch('isControling')
     onControlChange(isControling: boolean) {
       if (isControling && this.reqMouseDown) {
@@ -299,10 +333,9 @@
       }
 
       if (isControling) {
-        const { width, height } = this._overlay
+        this.canvasClear()
-        this._ctx.clearRect(0, 0, width, height)
       } else {
-        this.onCursorPositionChange(this.mousepos)
+        this.canvasRedraw()
       }
 
       this.reqMouseDown = null

src/component/types/events.ts

@@ -30,9 +30,6 @@ export const CLIPBOARD_SET = 'clipboard/set'
 export const KEYBOARD_MODIFIERS = 'keyboard/modifiers'
 export const KEYBOARD_MAP = 'keyboard/map'
 
-export const CURSOR_IMAGE = 'cursor/image'
-export const CURSOR_POSITION = 'cursor/position'
-
 export const BORADCAST_STATUS = 'broadcast/status'
 
 export const SEND_UNICAST = 'send/unicast'

src/component/types/messages.ts

@@ -14,7 +14,6 @@ export interface SystemInit {
   screen_size: ScreenSize
   members: Record<string, MemberData>
   implicit_hosting: boolean
-  cursor_image: CursorImage | null
 }
 
 export interface SystemAdmin {
@@ -159,26 +158,6 @@ export interface KeyboardMap {
   variant: string
 }
 
-/////////////////////////////
-// Cursor
-/////////////////////////////
-
-export interface CursorImage {
-  event: string | undefined
-  uri: string
-  width: number
-  height: number
-  x: number
-  y: number
-}
-
-export interface CursorPosition {
-  event: string | undefined
-  member_id: string
-  x: number
-  y: number
-}
-
 /////////////////////////////
 // Broadcast
 /////////////////////////////

src/component/types/state.ts

@@ -49,7 +49,6 @@ export interface Video {
 /////////////////////////////
 export interface Control {
   scroll: Scroll
-  cursor: Cursor
   clipboard: Clipboard | null
   host_id: string | null
   implicit_hosting: boolean
@@ -60,24 +59,6 @@ export interface Scroll {
   sensitivity: number
 }
 
-export interface Cursor {
-  image: CursorImage | null
-  position: CursorPosition | null
-}
-
-export interface CursorImage {
-  uri: string
-  width: number
-  height: number
-  x: number
-  y: number
-}
-
-export interface CursorPosition {
-  x: number
-  y: number
-}
-
 export interface Clipboard {
   text: string
 }