Native touch events (#42)

* add webrtc touch events.

* bind touch events to overlay.

* we care only for changed touches.

* switch to int32.

* pressure uint16.

* add implicit control.

* add touch to controls.

* fix iteration of changedTouches.

* switch pressure to uint8.

* convert force to pressure.

* add hasTouchEvents.

* add touch_events to state.

* bind touch or gesture handler on demand.

* remove duplicate gesture detach.

src/component/internal/control.ts

@@ -33,6 +33,10 @@ export class NekoControl extends EventEmitter<NekoControlEvents> {
     return this._connection.webrtc.connected && this._state.is_host
   }
 
+  get hasTouchEvents() {
+    return this._state.touch_events
+  }
+
   public lock() {
     Vue.set(this._state, 'locked', true)
   }
@@ -112,6 +116,30 @@ export class NekoControl extends EventEmitter<NekoControlEvents> {
     }
   }
 
+  public touchBegin(touchId: number, pos: ControlPos, pressure: number) {
+    if (this.useWebrtc) {
+      this._connection.webrtc.send('touchbegin', { touchId, ...pos, pressure })
+    } else {
+      this._connection.websocket.send(EVENT.CONTROL_TOUCHBEGIN, { touchId, ...pos, pressure } as message.ControlTouch)
+    }
+  }
+
+  public touchUpdate(touchId: number, pos: ControlPos, pressure: number) {
+    if (this.useWebrtc) {
+      this._connection.webrtc.send('touchupdate', { touchId, ...pos, pressure })
+    } else {
+      this._connection.websocket.send(EVENT.CONTROL_TOUCHUPDATE, { touchId, ...pos, pressure } as message.ControlTouch)
+    }
+  }
+
+  public touchEnd(touchId: number, pos: ControlPos, pressure: number) {
+    if (this.useWebrtc) {
+      this._connection.webrtc.send('touchend', { touchId, ...pos, pressure })
+    } else {
+      this._connection.websocket.send(EVENT.CONTROL_TOUCHEND, { touchId, ...pos, pressure } as message.ControlTouch)
+    }
+  }
+
   public cut() {
     this._connection.websocket.send(EVENT.CONTROL_CUT)
   }

src/component/internal/messages.ts

@@ -104,6 +104,7 @@ export class NekoMessages extends EventEmitter<NekoEvents> {
   protected [EVENT.SYSTEM_INIT](conf: message.SystemInit) {
     this._localLog.debug(`EVENT.SYSTEM_INIT`)
     Vue.set(this._state, 'session_id', conf.session_id)
+    Vue.set(this._state.control, 'touch_events', conf.touch_events)
     Vue.set(this._state.connection, 'screencast', conf.screencast_enabled)
     Vue.set(this._state.connection.webrtc, 'videos', conf.webrtc.videos)
 

src/component/internal/webrtc.ts

@@ -13,6 +13,10 @@ export const OPCODE = {
   BTN_DOWN: 0x05,
   BTN_UP: 0x06,
   PING: 0x07,
+  // touch events
+  TOUCH_BEGIN: 0x08,
+  TOUCH_UPDATE: 0x09,
+  TOUCH_END: 0x0a,
 } as const
 
 export interface ICEServer {
@@ -361,6 +365,10 @@ export class NekoWebRTC extends EventEmitter<NekoWebRTCEvents> {
   public send(event: 'wheel' | 'mousemove', data: { x: number; y: number }): void
   public send(event: 'mousedown' | 'mouseup' | 'keydown' | 'keyup', data: { key: number }): void
   public send(event: 'ping', data: number): void
+  public send(
+    event: 'touchbegin' | 'touchupdate' | 'touchend',
+    data: { touchId: number; x: number; y: number; pressure: number },
+  ): void
   public send(event: string, data: any): void {
     if (typeof this._channel === 'undefined' || this._channel.readyState !== 'open') {
       this._log.warn(`attempting to send data, but data-channel is not open`, { event })
@@ -422,6 +430,24 @@ export class NekoWebRTC extends EventEmitter<NekoWebRTCEvents> {
         payload.setUint32(3, Math.trunc(data / maxUint32))
         payload.setUint32(7, data % maxUint32)
         break
+      case 'touchbegin':
+      case 'touchupdate':
+      case 'touchend':
+        buffer = new ArrayBuffer(16)
+        payload = new DataView(buffer)
+        if (event === 'touchbegin') {
+          payload.setUint8(0, OPCODE.TOUCH_BEGIN)
+        } else if (event === 'touchupdate') {
+          payload.setUint8(0, OPCODE.TOUCH_UPDATE)
+        } else if (event === 'touchend') {
+          payload.setUint8(0, OPCODE.TOUCH_END)
+        }
+        payload.setUint16(1, 13)
+        payload.setUint32(3, data.touchId)
+        payload.setInt32(7, data.x)
+        payload.setInt32(11, data.y)
+        payload.setUint8(15, data.pressure)
+        break
       default:
         this._log.warn(`unknown data event`, { event })
         return

src/component/main.vue

@@ -200,6 +200,7 @@
           layout: 'us',
           variant: '',
         },
+        touch_events: false,
         host_id: null,
         is_host: false,
         locked: false,

src/component/overlay.vue

@@ -194,11 +194,13 @@
 
       // Initialize GestureHandler
       this.gestureHandler = new GestureHandlerInit()
-      this.gestureHandler.attach(this._textarea)
 
-      this._textarea.addEventListener('gesturestart', this.onGestureHandler)
+      // bind touch handler using @Watch on hasTouchEvents
-      this._textarea.addEventListener('gesturemove', this.onGestureHandler)
+      // because we need to know if touch events are supported
-      this._textarea.addEventListener('gestureend', this.onGestureHandler)
+      // by the server before we can bind touch handler
+
+      // default value is false, so we can bind touch handler
+      this.bindGestureHandler()
 
       this.webrtc.addListener('cursor-position', this.onCursorPosition)
       this.webrtc.addListener('cursor-image', this.onCursorImage)
@@ -213,13 +215,11 @@
         this.keyboard.removeListener()
       }
 
-      if (this.gestureHandler) {
+      // unbind touch handler
-        this.gestureHandler.detach()
+      this.unbindTouchHandler()
-      }
 
-      this._textarea.removeEventListener('gesturestart', this.onGestureHandler)
+      // unbind gesture handler
-      this._textarea.removeEventListener('gesturemove', this.onGestureHandler)
+      this.unbindGestureHandler()
-      this._textarea.removeEventListener('gestureend', this.onGestureHandler)
 
       this.webrtc.removeListener('cursor-position', this.onCursorPosition)
       this.webrtc.removeListener('cursor-image', this.onCursorImage)
@@ -235,7 +235,78 @@
       }
     }
 
-    // Gesture state
+    //
+    // touch handler for native touch events
+    //
+
+    bindTouchHandler() {
+      this._textarea.addEventListener('touchstart', this.onTouchHandler, { passive: false })
+      this._textarea.addEventListener('touchmove', this.onTouchHandler, { passive: false })
+      this._textarea.addEventListener('touchend', this.onTouchHandler, { passive: false })
+      this._textarea.addEventListener('touchcancel', this.onTouchHandler, { passive: false })
+    }
+
+    unbindTouchHandler() {
+      this._textarea.removeEventListener('touchstart', this.onTouchHandler)
+      this._textarea.removeEventListener('touchmove', this.onTouchHandler)
+      this._textarea.removeEventListener('touchend', this.onTouchHandler)
+      this._textarea.removeEventListener('touchcancel', this.onTouchHandler)
+    }
+
+    onTouchHandler(ev: TouchEvent) {
+      // we cannot use implicitControlRequest because we don't have mouse event
+      if (!this.isControling) {
+        // if implicitControl is enabled, request control
+        if (this.implicitControl) {
+          this.control.request()
+        }
+        // otherwise, ignore event
+        return
+      }
+
+      ev.stopPropagation()
+      ev.preventDefault()
+
+      for (let i = 0; i < ev.changedTouches.length; i++) {
+        const touch = ev.changedTouches[i]
+        const pos = this.getMousePos(touch.clientX, touch.clientY)
+        // force is float value between 0 and 1
+        // pressure is integer value between 0 and 255
+        const pressure = Math.round(touch.force * 255)
+
+        switch (ev.type) {
+          case 'touchstart':
+            this.control.touchBegin(touch.identifier, pos, pressure)
+            break
+          case 'touchmove':
+            this.control.touchUpdate(touch.identifier, pos, pressure)
+            break
+          case 'touchend':
+          case 'touchcancel':
+            this.control.touchEnd(touch.identifier, pos, pressure)
+            break
+        }
+      }
+    }
+
+    //
+    // gesture handler for emulated mouse events
+    //
+
+    bindGestureHandler() {
+      this.gestureHandler.attach(this._textarea)
+      this._textarea.addEventListener('gesturestart', this.onGestureHandler)
+      this._textarea.addEventListener('gesturemove', this.onGestureHandler)
+      this._textarea.addEventListener('gestureend', this.onGestureHandler)
+    }
+
+    unbindGestureHandler() {
+      this.gestureHandler.detach()
+      this._textarea.removeEventListener('gesturestart', this.onGestureHandler)
+      this._textarea.removeEventListener('gesturemove', this.onGestureHandler)
+      this._textarea.removeEventListener('gestureend', this.onGestureHandler)
+    }
+
     private _gestureLastTapTime: any | null = null
     private _gestureFirstDoubleTapEv: any | null = null
     private _gestureLastMagnitudeX = 0
@@ -392,6 +463,21 @@
       }
     }
 
+    //
+    // touch and gesture handlers cannot be used together
+    //
+
+    @Watch('control.hasTouchEvents')
+    onTouchEventsChange() {
+      if (this.control.hasTouchEvents) {
+        this.unbindGestureHandler()
+        this.bindTouchHandler()
+      } else {
+        this.unbindTouchHandler()
+        this.bindGestureHandler()
+      }
+    }
+
     getMousePos(clientX: number, clientY: number) {
       const rect = this._overlay.getBoundingClientRect()
 

src/component/types/events.ts

@@ -34,6 +34,10 @@ export const CONTROL_BUTTONUP = 'control/buttonup'
 export const CONTROL_KEYPRESS = 'control/keypress'
 export const CONTROL_KEYDOWN = 'control/keydown'
 export const CONTROL_KEYUP = 'control/keyup'
+// touch
+export const CONTROL_TOUCHBEGIN = 'control/touchbegin'
+export const CONTROL_TOUCHUPDATE = 'control/touchupdate'
+export const CONTROL_TOUCHEND = 'control/touchend'
 // actions
 export const CONTROL_CUT = 'control/cut'
 export const CONTROL_COPY = 'control/copy'

src/component/types/messages.ts

@@ -18,6 +18,7 @@ export interface SystemInit {
   screen_size: ScreenSize
   sessions: Record<string, SessionData>
   settings: Settings
+  touch_events: boolean
   screencast_enabled: boolean
   webrtc: SystemWebRTC
 }
@@ -125,6 +126,11 @@ export interface ControlKey extends Partial<ControlPos> {
   keysym: number
 }
 
+export interface ControlTouch extends Partial<ControlPos> {
+  touchId: number
+  pressure: number
+}
+
 /////////////////////////////
 // Screen
 /////////////////////////////

src/component/types/state.ts

@@ -70,6 +70,7 @@ export interface Control {
   scroll: Scroll
   clipboard: Clipboard | null
   keyboard: Keyboard
+  touch_events: boolean
   host_id: string | null
   is_host: boolean
   locked: boolean

src/page/components/events.vue

@@ -334,6 +334,10 @@
           />
         </td>
       </tr>
+      <tr>
+        <th>control.touch_events</th>
+        <td>{{ neko.state.control.touch_events ? 'backend supports' : 'backend does not support' }}</td>
+      </tr>
       <tr>
         <th rowspan="2">control.host_id</th>
         <td>{{ neko.state.control.host_id }}</td>