Add support for touch gestures (#40)

* implement control using webrtc.

* overlay to use NekoControl.

* WIP.

* add is_host to control state.

* control use webrtc only if hosting.

* add proper links to 3rd party code.

* fix button events for gestureHandler.

* lint.

* fix implicit control gain for touch events.

src/component/internal/control.ts

@@ -26,6 +26,13 @@ export class NekoControl extends EventEmitter<NekoControlEvents> {
     super()
   }
 
+  get useWebrtc() {
+    // we want to use webrtc if we're connected and we're the host
+    // because webrtc is faster and it doesn't request control
+    // in contrast to the websocket
+    return this._connection.webrtc.connected && this._state.is_host
+  }
+
   public lock() {
     Vue.set(this._state, 'locked', true)
   }
@@ -43,36 +50,67 @@ export class NekoControl extends EventEmitter<NekoControlEvents> {
   }
 
   public move(pos: ControlPos) {
+    if (this.useWebrtc) {
+      this._connection.webrtc.send('mousemove', pos)
+    } else {
       this._connection.websocket.send(EVENT.CONTROL_MOVE, pos as message.ControlPos)
     }
-
-  public scroll(pos: ControlPos) {
-    this._connection.websocket.send(EVENT.CONTROL_SCROLL, pos as message.ControlPos)
   }
 
+  // TODO: rename pos to delta, and add a new pos parameter
+  public scroll(pos: ControlPos) {
+    if (this.useWebrtc) {
+      this._connection.webrtc.send('wheel', pos)
+    } else {
+      this._connection.websocket.send(EVENT.CONTROL_SCROLL, pos as message.ControlPos)
+    }
+  }
+
+  // buttonpress ensures that only one button is pressed at a time
   public buttonPress(code: number, pos?: ControlPos) {
     this._connection.websocket.send(EVENT.CONTROL_BUTTONPRESS, { code, ...pos } as message.ControlButton)
   }
 
   public buttonDown(code: number, pos?: ControlPos) {
+    if (this.useWebrtc) {
+      if (pos) this._connection.webrtc.send('mousemove', pos)
+      this._connection.webrtc.send('mousedown', { key: code })
+    } else {
       this._connection.websocket.send(EVENT.CONTROL_BUTTONDOWN, { code, ...pos } as message.ControlButton)
     }
+  }
 
   public buttonUp(code: number, pos?: ControlPos) {
+    if (this.useWebrtc) {
+      if (pos) this._connection.webrtc.send('mousemove', pos)
+      this._connection.webrtc.send('mouseup', { key: code })
+    } else {
       this._connection.websocket.send(EVENT.CONTROL_BUTTONUP, { code, ...pos } as message.ControlButton)
     }
+  }
 
+  // keypress ensures that only one key is pressed at a time
   public keyPress(keysym: number, pos?: ControlPos) {
     this._connection.websocket.send(EVENT.CONTROL_KEYPRESS, { keysym, ...pos } as message.ControlKey)
   }
 
   public keyDown(keysym: number, pos?: ControlPos) {
+    if (this.useWebrtc) {
+      if (pos) this._connection.webrtc.send('mousemove', pos)
+      this._connection.webrtc.send('keydown', { key: keysym })
+    } else {
       this._connection.websocket.send(EVENT.CONTROL_KEYDOWN, { keysym, ...pos } as message.ControlKey)
     }
+  }
 
   public keyUp(keysym: number, pos?: ControlPos) {
+    if (this.useWebrtc) {
+      if (pos) this._connection.webrtc.send('mousemove', pos)
+      this._connection.webrtc.send('keyup', { key: keysym })
+    } else {
       this._connection.websocket.send(EVENT.CONTROL_KEYUP, { keysym, ...pos } as message.ControlKey)
     }
+  }
 
   public cut() {
     this._connection.websocket.send(EVENT.CONTROL_CUT)

src/component/internal/messages.ts

@@ -262,6 +262,9 @@ export class NekoMessages extends EventEmitter<NekoEvents> {
       Vue.set(this._state.control, 'host_id', null)
     }
 
+    // save if user is host
+    Vue.set(this._state.control, 'is_host', has_host && this._state.control.host_id === this._state.session_id)
+
     this.emit('room.control.host', has_host, host_id)
   }
 

src/component/main.vue

@@ -24,7 +24,7 @@
         ref="overlay"
         v-show="!private_mode_enabled && state.connection.status != 'disconnected'"
         :style="{ pointerEvents: state.control.locked ? 'none' : 'auto' }"
-        :wsControl="control"
+        :control="control"
         :sessions="state.sessions"
         :hostId="state.control.host_id"
         :webrtc="connection.webrtc"
@@ -201,6 +201,7 @@
           variant: '',
         },
         host_id: null,
+        is_host: false,
         locked: false,
       },
       screen: {
@@ -763,6 +764,7 @@
       // websocket
       Vue.set(this.state.control, 'clipboard', null)
       Vue.set(this.state.control, 'host_id', null)
+      Vue.set(this.state.control, 'is_host', false)
       Vue.set(this.state.screen, 'size', { width: 1280, height: 720, rate: 30 })
       Vue.set(this.state.screen, 'configurations', [])
       Vue.set(this.state.screen, 'sync', false)

src/component/overlay.vue

@@ -6,8 +6,8 @@
       class="neko-overlay"
       :style="{ cursor }"
       v-model="textInput"
-      @click.stop.prevent="wsControl.emit('overlay.click', $event)"
+      @click.stop.prevent="control.emit('overlay.click', $event)"
-      @contextmenu.stop.prevent="wsControl.emit('overlay.contextmenu', $event)"
+      @contextmenu.stop.prevent="control.emit('overlay.contextmenu', $event)"
       @wheel.stop.prevent="onWheel"
       @mousemove.stop.prevent="onMouseMove"
       @mousedown.stop.prevent="onMouseDown"
@@ -47,6 +47,7 @@
   import { Vue, Component, Ref, Prop, Watch } from 'vue-property-decorator'
 
   import { KeyboardInterface, NewKeyboard } from './utils/keyboard'
+  import GestureHandlerInit, { GestureHandler } from './utils/gesturehandler'
   import { KeyTable, keySymsRemap } from './utils/keyboard-remapping'
   import { getFilesFromDataTansfer } from './utils/file-upload'
   import { NekoControl } from './internal/control'
@@ -55,8 +56,15 @@
   import { CursorPosition, CursorImage } from './types/webrtc'
   import { CursorDrawFunction, Dimension, KeyboardModifiers } from './types/cursors'
 
-  const WHEEL_STEP = 53 // Delta threshold for a mouse wheel step
+  // Wheel thresholds
-  const WHEEL_LINE_HEIGHT = 19
+  const WHEEL_STEP = 53 // Pixels needed for one step
+  const WHEEL_LINE_HEIGHT = 19 // Assumed pixels for one line step
+
+  // Gesture thresholds
+  const GESTURE_ZOOMSENS = 75
+  const GESTURE_SCRLSENS = 50
+  const DOUBLE_TAP_TIMEOUT = 1000
+  const DOUBLE_TAP_THRESHOLD = 50
 
   const MOUSE_MOVE_THROTTLE = 1000 / 60 // in ms, 60fps
   const INACTIVE_CURSOR_INTERVAL = 1000 / 4 // in ms, 4fps
@@ -72,12 +80,13 @@
     private canvasScale = window.devicePixelRatio
 
     private keyboard!: KeyboardInterface
+    private gestureHandler!: GestureHandler
     private textInput = ''
 
     private focused = false
 
     @Prop()
-    private readonly wsControl!: NekoControl
+    private readonly control!: NekoControl
 
     @Prop()
     private readonly sessions!: Record<string, Session>
@@ -165,12 +174,7 @@
         const isCtrlKey = key == KeyTable.XK_Control_L || key == KeyTable.XK_Control_R
         if (isCtrlKey) ctrlKey = key
 
-        if (this.webrtc.connected) {
+        this.control.keyDown(key)
-          this.webrtc.send('keydown', { key })
-        } else {
-          this.wsControl.keyDown(key)
-        }
-
         return isCtrlKey
       }
       this.keyboard.onkeyup = (key: number) => {
@@ -184,17 +188,17 @@
         const isCtrlKey = key == KeyTable.XK_Control_L || key == KeyTable.XK_Control_R
         if (isCtrlKey) ctrlKey = 0
 
-        if (this.webrtc.connected) {
+        this.control.keyUp(key)
-          this.webrtc.send('keyup', { key })
-        } else {
-          this.wsControl.keyUp(key)
-        }
       }
       this.keyboard.listenTo(this._textarea)
 
-      this._textarea.addEventListener('touchstart', this.onTouchHandler, { passive: false })
+      // Initialize GestureHandler
-      this._textarea.addEventListener('touchmove', this.onTouchHandler, { passive: false })
+      this.gestureHandler = new GestureHandlerInit()
-      this._textarea.addEventListener('touchend', this.onTouchHandler, { passive: false })
+      this.gestureHandler.attach(this._textarea)
+
+      this._textarea.addEventListener('gesturestart', this.onGestureHandler)
+      this._textarea.addEventListener('gesturemove', this.onGestureHandler)
+      this._textarea.addEventListener('gestureend', this.onGestureHandler)
 
       this.webrtc.addListener('cursor-position', this.onCursorPosition)
       this.webrtc.addListener('cursor-image', this.onCursorImage)
@@ -209,9 +213,13 @@
         this.keyboard.removeListener()
       }
 
-      this._textarea.removeEventListener('touchstart', this.onTouchHandler)
+      if (this.gestureHandler) {
-      this._textarea.removeEventListener('touchmove', this.onTouchHandler)
+        this.gestureHandler.detach()
-      this._textarea.removeEventListener('touchend', this.onTouchHandler)
+      }
+
+      this._textarea.removeEventListener('gesturestart', this.onGestureHandler)
+      this._textarea.removeEventListener('gesturemove', this.onGestureHandler)
+      this._textarea.removeEventListener('gestureend', this.onGestureHandler)
 
       this.webrtc.removeListener('cursor-position', this.onCursorPosition)
       this.webrtc.removeListener('cursor-image', this.onCursorImage)
@@ -227,34 +235,161 @@
       }
     }
 
-    onTouchHandler(e: TouchEvent) {
+    // Gesture state
-      let type = ''
+    private _gestureLastTapTime: any | null = null
-      switch (e.type) {
+    private _gestureFirstDoubleTapEv: any | null = null
-        case 'touchstart':
+    private _gestureLastMagnitudeX = 0
-          type = 'mousedown'
+    private _gestureLastMagnitudeY = 0
-          break
+
-        case 'touchmove':
+    _handleTapEvent(ev: any, code: number) {
-          type = 'mousemove'
+      let pos = this.getMousePos(ev.detail.clientX, ev.detail.clientY)
-          break
+
-        case 'touchend':
+      // If the user quickly taps multiple times we assume they meant to
-          type = 'mouseup'
+      // hit the same spot, so slightly adjust coordinates
-          break
+
-        default:
+      if (
-          // unknown event
+        this._gestureLastTapTime !== null &&
+        Date.now() - this._gestureLastTapTime < DOUBLE_TAP_TIMEOUT &&
+        this._gestureFirstDoubleTapEv.detail.type === ev.detail.type
+      ) {
+        let dx = this._gestureFirstDoubleTapEv.detail.clientX - ev.detail.clientX
+        let dy = this._gestureFirstDoubleTapEv.detail.clientY - ev.detail.clientY
+        let distance = Math.hypot(dx, dy)
+
+        if (distance < DOUBLE_TAP_THRESHOLD) {
+          pos = this.getMousePos(
+            this._gestureFirstDoubleTapEv.detail.clientX,
+            this._gestureFirstDoubleTapEv.detail.clientY,
+          )
+        } else {
+          this._gestureFirstDoubleTapEv = ev
+        }
+      } else {
+        this._gestureFirstDoubleTapEv = ev
+      }
+      this._gestureLastTapTime = Date.now()
+
+      this.control.buttonDown(code, pos)
+      this.control.buttonUp(code, pos)
+    }
+
+    // https://github.com/novnc/noVNC/blob/ca6527c1bf7131adccfdcc5028964a1e67f9018c/core/rfb.js#L1227-L1345
+    onGestureHandler(ev: any) {
+      // we cannot use implicitControlRequest because we don't have mouse event
+      if (!this.isControling) {
+        // if implicitControl is enabled, request control
+        if (this.implicitControl) {
+          this.control.request()
+        }
+        // otherwise, ignore event
         return
       }
 
-      const touch = e.changedTouches[0]
+      const pos = this.getMousePos(ev.detail.clientX, ev.detail.clientY)
-      touch.target.dispatchEvent(
-        new MouseEvent(type, {
-          button: 0, // currently only left button is supported
-          clientX: touch.clientX,
-          clientY: touch.clientY,
-        }),
-      )
 
-      e.preventDefault()
+      let magnitude
-      e.stopPropagation()
+      switch (ev.type) {
+        case 'gesturestart':
+          switch (ev.detail.type) {
+            case 'onetap':
+              this._handleTapEvent(ev, 1)
+              break
+            case 'twotap':
+              this._handleTapEvent(ev, 3)
+              break
+            case 'threetap':
+              this._handleTapEvent(ev, 2)
+              break
+            case 'drag':
+              this.control.buttonDown(1, pos)
+              break
+            case 'longpress':
+              this.control.buttonDown(3, pos)
+              break
+
+            case 'twodrag':
+              this._gestureLastMagnitudeX = ev.detail.magnitudeX
+              this._gestureLastMagnitudeY = ev.detail.magnitudeY
+              this.control.move(pos)
+              break
+            case 'pinch':
+              this._gestureLastMagnitudeX = Math.hypot(ev.detail.magnitudeX, ev.detail.magnitudeY)
+              this.control.move(pos)
+              break
+          }
+          break
+
+        case 'gesturemove':
+          switch (ev.detail.type) {
+            case 'onetap':
+            case 'twotap':
+            case 'threetap':
+              break
+            case 'drag':
+            case 'longpress':
+              this.control.move(pos)
+              break
+            case 'twodrag':
+              // Always scroll in the same position.
+              // We don't know if the mouse was moved so we need to move it
+              // every update.
+              this.control.move(pos)
+              while (ev.detail.magnitudeY - this._gestureLastMagnitudeY > GESTURE_SCRLSENS) {
+                this.control.scroll({ x: 0, y: 1 })
+                this._gestureLastMagnitudeY += GESTURE_SCRLSENS
+              }
+              while (ev.detail.magnitudeY - this._gestureLastMagnitudeY < -GESTURE_SCRLSENS) {
+                this.control.scroll({ x: 0, y: -1 })
+                this._gestureLastMagnitudeY -= GESTURE_SCRLSENS
+              }
+              while (ev.detail.magnitudeX - this._gestureLastMagnitudeX > GESTURE_SCRLSENS) {
+                this.control.scroll({ x: 1, y: 0 })
+                this._gestureLastMagnitudeX += GESTURE_SCRLSENS
+              }
+              while (ev.detail.magnitudeX - this._gestureLastMagnitudeX < -GESTURE_SCRLSENS) {
+                this.control.scroll({ x: -1, y: 0 })
+                this._gestureLastMagnitudeX -= GESTURE_SCRLSENS
+              }
+              break
+            case 'pinch':
+              // Always scroll in the same position.
+              // We don't know if the mouse was moved so we need to move it
+              // every update.
+              this.control.move(pos)
+              magnitude = Math.hypot(ev.detail.magnitudeX, ev.detail.magnitudeY)
+              if (Math.abs(magnitude - this._gestureLastMagnitudeX) > GESTURE_ZOOMSENS) {
+                this.control.keyDown(KeyTable.XK_Control_L)
+                while (magnitude - this._gestureLastMagnitudeX > GESTURE_ZOOMSENS) {
+                  this.control.scroll({ x: 0, y: 1 })
+                  this._gestureLastMagnitudeX += GESTURE_ZOOMSENS
+                }
+                while (magnitude - this._gestureLastMagnitudeX < -GESTURE_ZOOMSENS) {
+                  this.control.scroll({ x: 0, y: -1 })
+                  this._gestureLastMagnitudeX -= GESTURE_ZOOMSENS
+                }
+                this.control.keyUp(KeyTable.XK_Control_L)
+              }
+              break
+          }
+          break
+
+        case 'gestureend':
+          switch (ev.detail.type) {
+            case 'onetap':
+            case 'twotap':
+            case 'threetap':
+            case 'pinch':
+            case 'twodrag':
+              break
+            case 'drag':
+              this.control.buttonUp(1, pos)
+              break
+            case 'longpress':
+              this.control.buttonUp(3, pos)
+              break
+          }
+          break
+      }
     }
 
     getMousePos(clientX: number, clientY: number) {
@@ -268,7 +403,11 @@
 
     sendMousePos(e: MouseEvent) {
       const pos = this.getMousePos(e.clientX, e.clientY)
+      // not using NekoControl here because we want to avoid
+      // sending mousemove events over websocket
+      if (this.webrtc.connected) {
         this.webrtc.send('mousemove', pos)
+      } // otherwise, no events are sent
       this.cursorPosition = pos
     }
 
@@ -307,7 +446,7 @@
     @Watch('textInput')
     onTextInputChange() {
       if (this.textInput == '') return
-      this.wsControl.paste(this.textInput)
+      this.control.paste(this.textInput)
       this.textInput = ''
     }
 
@@ -365,12 +504,8 @@
       // skip if not scrolled
       if (x == 0 && y == 0) return
 
-      if (this.webrtc.connected) {
+      // TODO: add position for precision scrolling
-        this.sendMousePos(e)
+      this.control.scroll({ x, y })
-        this.webrtc.send('wheel', { x, y })
-      } else {
-        this.wsControl.scroll({ x, y })
-      }
     }
 
     lastMouseMove = 0
@@ -400,13 +535,8 @@
       }
 
       const key = e.button + 1
-      if (this.webrtc.connected) {
-        this.sendMousePos(e)
-        this.webrtc.send('mousedown', { key })
-      } else {
       const pos = this.getMousePos(e.clientX, e.clientY)
-        this.wsControl.buttonDown(key, pos)
+      this.control.buttonDown(key, pos)
-      }
     }
 
     onMouseUp(e: MouseEvent) {
@@ -420,13 +550,8 @@
       }
 
       const key = e.button + 1
-      if (this.webrtc.connected) {
-        this.sendMousePos(e)
-        this.webrtc.send('mouseup', { key })
-      } else {
       const pos = this.getMousePos(e.clientX, e.clientY)
-        this.wsControl.buttonUp(key, pos)
+      this.control.buttonUp(key, pos)
-      }
     }
 
     onMouseEnter(e: MouseEvent) {
@@ -474,7 +599,8 @@
         const files = await getFilesFromDataTansfer(dt)
         if (files.length === 0) return
 
-        this.$emit('uploadDrop', { ...this.getMousePos(e.clientX, e.clientY), files })
+        const pos = this.getMousePos(e.clientX, e.clientY)
+        this.$emit('uploadDrop', { ...pos, files })
       }
     }
 
@@ -510,9 +636,11 @@
     }
 
     sendInactiveMousePos() {
-      if (this.inactiveCursorPosition) {
+      if (this.inactiveCursorPosition && this.webrtc.connected) {
+        // not using NekoControl here, because inactive cursors are
+        // treated differently than moving the mouse while controling
         this.webrtc.send('mousemove', this.inactiveCursorPosition)
-      }
+      } // if webrtc is not connected, we don't need to send anything
     }
 
     //
@@ -715,7 +843,7 @@
       if (this.implicitControl && e.type === 'mousedown') {
         this.reqMouseDown = e
         this.reqMouseUp = null
-        this.wsControl.request()
+        this.control.request()
       }
 
       if (this.implicitControl && e.type === 'mouseup') {
@@ -726,7 +854,7 @@
     // unused
     implicitControlRelease() {
       if (this.implicitControl) {
-        this.wsControl.release()
+        this.control.release()
       }
     }
 

src/component/types/state.ts

@@ -71,6 +71,7 @@ export interface Control {
   clipboard: Clipboard | null
   keyboard: Keyboard
   host_id: string | null
+  is_host: boolean
   locked: boolean
 }
 

src/component/utils/gesturehandler.js

@@ -0,0 +1,567 @@
+/*
+ * noVNC: HTML5 VNC client
+ * Copyright (C) 2020 The noVNC Authors
+ * Licensed under MPL 2.0 (see LICENSE.txt)
+ *
+ * See README.md for usage and integration instructions.
+ *
+ */
+
+const GH_NOGESTURE = 0;
+const GH_ONETAP    = 1;
+const GH_TWOTAP    = 2;
+const GH_THREETAP  = 4;
+const GH_DRAG      = 8;
+const GH_LONGPRESS = 16;
+const GH_TWODRAG   = 32;
+const GH_PINCH     = 64;
+
+const GH_INITSTATE = 127;
+
+const GH_MOVE_THRESHOLD = 50;
+const GH_ANGLE_THRESHOLD = 90; // Degrees
+
+// Timeout when waiting for gestures (ms)
+const GH_MULTITOUCH_TIMEOUT = 250;
+
+// Maximum time between press and release for a tap (ms)
+const GH_TAP_TIMEOUT = 1000;
+
+// Timeout when waiting for longpress (ms)
+const GH_LONGPRESS_TIMEOUT = 1000;
+
+// Timeout when waiting to decide between PINCH and TWODRAG (ms)
+const GH_TWOTOUCH_TIMEOUT = 50;
+
+export default class GestureHandler {
+    constructor() {
+        this._target = null;
+
+        this._state = GH_INITSTATE;
+
+        this._tracked = [];
+        this._ignored = [];
+
+        this._waitingRelease = false;
+        this._releaseStart = 0.0;
+
+        this._longpressTimeoutId = null;
+        this._twoTouchTimeoutId = null;
+
+        this._boundEventHandler = this._eventHandler.bind(this);
+    }
+
+    attach(target) {
+        this.detach();
+
+        this._target = target;
+        this._target.addEventListener('touchstart',
+                                      this._boundEventHandler);
+        this._target.addEventListener('touchmove',
+                                      this._boundEventHandler);
+        this._target.addEventListener('touchend',
+                                      this._boundEventHandler);
+        this._target.addEventListener('touchcancel',
+                                      this._boundEventHandler);
+    }
+
+    detach() {
+        if (!this._target) {
+            return;
+        }
+
+        this._stopLongpressTimeout();
+        this._stopTwoTouchTimeout();
+
+        this._target.removeEventListener('touchstart',
+                                         this._boundEventHandler);
+        this._target.removeEventListener('touchmove',
+                                         this._boundEventHandler);
+        this._target.removeEventListener('touchend',
+                                         this._boundEventHandler);
+        this._target.removeEventListener('touchcancel',
+                                         this._boundEventHandler);
+        this._target = null;
+    }
+
+    _eventHandler(e) {
+        let fn;
+
+        e.stopPropagation();
+        e.preventDefault();
+
+        switch (e.type) {
+            case 'touchstart':
+                fn = this._touchStart;
+                break;
+            case 'touchmove':
+                fn = this._touchMove;
+                break;
+            case 'touchend':
+            case 'touchcancel':
+                fn = this._touchEnd;
+                break;
+        }
+
+        for (let i = 0; i < e.changedTouches.length; i++) {
+            let touch = e.changedTouches[i];
+            fn.call(this, touch.identifier, touch.clientX, touch.clientY);
+        }
+    }
+
+    _touchStart(id, x, y) {
+        // Ignore any new touches if there is already an active gesture,
+        // or we're in a cleanup state
+        if (this._hasDetectedGesture() || (this._state === GH_NOGESTURE)) {
+            this._ignored.push(id);
+            return;
+        }
+
+        // Did it take too long between touches that we should no longer
+        // consider this a single gesture?
+        if ((this._tracked.length > 0) &&
+            ((Date.now() - this._tracked[0].started) > GH_MULTITOUCH_TIMEOUT)) {
+            this._state = GH_NOGESTURE;
+            this._ignored.push(id);
+            return;
+        }
+
+        // If we're waiting for fingers to release then we should no longer
+        // recognize new touches
+        if (this._waitingRelease) {
+            this._state = GH_NOGESTURE;
+            this._ignored.push(id);
+            return;
+        }
+
+        this._tracked.push({
+            id: id,
+            started: Date.now(),
+            active: true,
+            firstX: x,
+            firstY: y,
+            lastX: x,
+            lastY: y,
+            angle: 0
+        });
+
+        switch (this._tracked.length) {
+            case 1:
+                this._startLongpressTimeout();
+                break;
+
+            case 2:
+                this._state &= ~(GH_ONETAP | GH_DRAG | GH_LONGPRESS);
+                this._stopLongpressTimeout();
+                break;
+
+            case 3:
+                this._state &= ~(GH_TWOTAP | GH_TWODRAG | GH_PINCH);
+                break;
+
+            default:
+                this._state = GH_NOGESTURE;
+        }
+    }
+
+    _touchMove(id, x, y) {
+        let touch = this._tracked.find(t => t.id === id);
+
+        // If this is an update for a touch we're not tracking, ignore it
+        if (touch === undefined) {
+            return;
+        }
+
+        // Update the touches last position with the event coordinates
+        touch.lastX = x;
+        touch.lastY = y;
+
+        let deltaX = x - touch.firstX;
+        let deltaY = y - touch.firstY;
+
+        // Update angle when the touch has moved
+        if ((touch.firstX !== touch.lastX) ||
+            (touch.firstY !== touch.lastY)) {
+            touch.angle = Math.atan2(deltaY, deltaX) * 180 / Math.PI;
+        }
+
+        if (!this._hasDetectedGesture()) {
+            // Ignore moves smaller than the minimum threshold
+            if (Math.hypot(deltaX, deltaY) < GH_MOVE_THRESHOLD) {
+                return;
+            }
+
+            // Can't be a tap or long press as we've seen movement
+            this._state &= ~(GH_ONETAP | GH_TWOTAP | GH_THREETAP | GH_LONGPRESS);
+            this._stopLongpressTimeout();
+
+            if (this._tracked.length !== 1) {
+                this._state &= ~(GH_DRAG);
+            }
+            if (this._tracked.length !== 2) {
+                this._state &= ~(GH_TWODRAG | GH_PINCH);
+            }
+
+            // We need to figure out which of our different two touch gestures
+            // this might be
+            if (this._tracked.length === 2) {
+
+                // The other touch is the one where the id doesn't match
+                let prevTouch = this._tracked.find(t => t.id !== id);
+
+                // How far the previous touch point has moved since start
+                let prevDeltaMove = Math.hypot(prevTouch.firstX - prevTouch.lastX,
+                                               prevTouch.firstY - prevTouch.lastY);
+
+                // We know that the current touch moved far enough,
+                // but unless both touches moved further than their
+                // threshold we don't want to disqualify any gestures
+                if (prevDeltaMove > GH_MOVE_THRESHOLD) {
+
+                    // The angle difference between the direction of the touch points
+                    let deltaAngle = Math.abs(touch.angle - prevTouch.angle);
+                    deltaAngle = Math.abs(((deltaAngle + 180) % 360) - 180);
+
+                    // PINCH or TWODRAG can be eliminated depending on the angle
+                    if (deltaAngle > GH_ANGLE_THRESHOLD) {
+                        this._state &= ~GH_TWODRAG;
+                    } else {
+                        this._state &= ~GH_PINCH;
+                    }
+
+                    if (this._isTwoTouchTimeoutRunning()) {
+                        this._stopTwoTouchTimeout();
+                    }
+                } else if (!this._isTwoTouchTimeoutRunning()) {
+                    // We can't determine the gesture right now, let's
+                    // wait and see if more events are on their way
+                    this._startTwoTouchTimeout();
+                }
+            }
+
+            if (!this._hasDetectedGesture()) {
+                return;
+            }
+
+            this._pushEvent('gesturestart');
+        }
+
+        this._pushEvent('gesturemove');
+    }
+
+    _touchEnd(id, x, y) {
+        // Check if this is an ignored touch
+        if (this._ignored.indexOf(id) !== -1) {
+            // Remove this touch from ignored
+            this._ignored.splice(this._ignored.indexOf(id), 1);
+
+            // And reset the state if there are no more touches
+            if ((this._ignored.length === 0) &&
+                (this._tracked.length === 0)) {
+                this._state = GH_INITSTATE;
+                this._waitingRelease = false;
+            }
+            return;
+        }
+
+        // We got a touchend before the timer triggered,
+        // this cannot result in a gesture anymore.
+        if (!this._hasDetectedGesture() &&
+            this._isTwoTouchTimeoutRunning()) {
+            this._stopTwoTouchTimeout();
+            this._state = GH_NOGESTURE;
+        }
+
+        // Some gestures don't trigger until a touch is released
+        if (!this._hasDetectedGesture()) {
+            // Can't be a gesture that relies on movement
+            this._state &= ~(GH_DRAG | GH_TWODRAG | GH_PINCH);
+            // Or something that relies on more time
+            this._state &= ~GH_LONGPRESS;
+            this._stopLongpressTimeout();
+
+            if (!this._waitingRelease) {
+                this._releaseStart = Date.now();
+                this._waitingRelease = true;
+
+                // Can't be a tap that requires more touches than we current have
+                switch (this._tracked.length) {
+                    case 1:
+                        this._state &= ~(GH_TWOTAP | GH_THREETAP);
+                        break;
+
+                    case 2:
+                        this._state &= ~(GH_ONETAP | GH_THREETAP);
+                        break;
+                }
+            }
+        }
+
+        // Waiting for all touches to release? (i.e. some tap)
+        if (this._waitingRelease) {
+            // Were all touches released at roughly the same time?
+            if ((Date.now() - this._releaseStart) > GH_MULTITOUCH_TIMEOUT) {
+                this._state = GH_NOGESTURE;
+            }
+
+            // Did too long time pass between press and release?
+            if (this._tracked.some(t => (Date.now() - t.started) > GH_TAP_TIMEOUT)) {
+                this._state = GH_NOGESTURE;
+            }
+
+            let touch = this._tracked.find(t => t.id === id);
+            touch.active = false;
+
+            // Are we still waiting for more releases?
+            if (this._hasDetectedGesture()) {
+                this._pushEvent('gesturestart');
+            } else {
+                // Have we reached a dead end?
+                if (this._state !== GH_NOGESTURE) {
+                    return;
+                }
+            }
+        }
+
+        if (this._hasDetectedGesture()) {
+            this._pushEvent('gestureend');
+        }
+
+        // Ignore any remaining touches until they are ended
+        for (let i = 0; i < this._tracked.length; i++) {
+            if (this._tracked[i].active) {
+                this._ignored.push(this._tracked[i].id);
+            }
+        }
+        this._tracked = [];
+
+        this._state = GH_NOGESTURE;
+
+        // Remove this touch from ignored if it's in there
+        if (this._ignored.indexOf(id) !== -1) {
+            this._ignored.splice(this._ignored.indexOf(id), 1);
+        }
+
+        // We reset the state if ignored is empty
+        if ((this._ignored.length === 0)) {
+            this._state = GH_INITSTATE;
+            this._waitingRelease = false;
+        }
+    }
+
+    _hasDetectedGesture() {
+        if (this._state === GH_NOGESTURE) {
+            return false;
+        }
+        // Check to see if the bitmask value is a power of 2
+        // (i.e. only one bit set). If it is, we have a state.
+        if (this._state & (this._state - 1)) {
+            return false;
+        }
+
+        // For taps we also need to have all touches released
+        // before we've fully detected the gesture
+        if (this._state & (GH_ONETAP | GH_TWOTAP | GH_THREETAP)) {
+            if (this._tracked.some(t => t.active)) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    _startLongpressTimeout() {
+        this._stopLongpressTimeout();
+        this._longpressTimeoutId = setTimeout(() => this._longpressTimeout(),
+                                              GH_LONGPRESS_TIMEOUT);
+    }
+
+    _stopLongpressTimeout() {
+        clearTimeout(this._longpressTimeoutId);
+        this._longpressTimeoutId = null;
+    }
+
+    _longpressTimeout() {
+        if (this._hasDetectedGesture()) {
+            throw new Error("A longpress gesture failed, conflict with a different gesture");
+        }
+
+        this._state = GH_LONGPRESS;
+        this._pushEvent('gesturestart');
+    }
+
+    _startTwoTouchTimeout() {
+        this._stopTwoTouchTimeout();
+        this._twoTouchTimeoutId = setTimeout(() => this._twoTouchTimeout(),
+                                             GH_TWOTOUCH_TIMEOUT);
+    }
+
+    _stopTwoTouchTimeout() {
+        clearTimeout(this._twoTouchTimeoutId);
+        this._twoTouchTimeoutId = null;
+    }
+
+    _isTwoTouchTimeoutRunning() {
+        return this._twoTouchTimeoutId !== null;
+    }
+
+    _twoTouchTimeout() {
+        if (this._tracked.length === 0) {
+            throw new Error("A pinch or two drag gesture failed, no tracked touches");
+        }
+
+        // How far each touch point has moved since start
+        let avgM = this._getAverageMovement();
+        let avgMoveH = Math.abs(avgM.x);
+        let avgMoveV = Math.abs(avgM.y);
+
+        // The difference in the distance between where
+        // the touch points started and where they are now
+        let avgD = this._getAverageDistance();
+        let deltaTouchDistance = Math.abs(Math.hypot(avgD.first.x, avgD.first.y) -
+                                          Math.hypot(avgD.last.x, avgD.last.y));
+
+        if ((avgMoveV < deltaTouchDistance) &&
+            (avgMoveH < deltaTouchDistance)) {
+            this._state = GH_PINCH;
+        } else {
+            this._state = GH_TWODRAG;
+        }
+
+        this._pushEvent('gesturestart');
+        this._pushEvent('gesturemove');
+    }
+
+    _pushEvent(type) {
+        let detail = { type: this._stateToGesture(this._state) };
+
+        // For most gesture events the current (average) position is the
+        // most useful
+        let avg = this._getPosition();
+        let pos = avg.last;
+
+        // However we have a slight distance to detect gestures, so for the
+        // first gesture event we want to use the first positions we saw
+        if (type === 'gesturestart') {
+            pos = avg.first;
+        }
+
+        // For these gestures, we always want the event coordinates
+        // to be where the gesture began, not the current touch location.
+        switch (this._state) {
+            case GH_TWODRAG:
+            case GH_PINCH:
+                pos = avg.first;
+                break;
+        }
+
+        detail['clientX'] = pos.x;
+        detail['clientY'] = pos.y;
+
+        // FIXME: other coordinates?
+
+        // Some gestures also have a magnitude
+        if (this._state === GH_PINCH) {
+            let distance = this._getAverageDistance();
+            if (type === 'gesturestart') {
+                detail['magnitudeX'] = distance.first.x;
+                detail['magnitudeY'] = distance.first.y;
+            } else {
+                detail['magnitudeX'] = distance.last.x;
+                detail['magnitudeY'] = distance.last.y;
+            }
+        } else if (this._state === GH_TWODRAG) {
+            if (type === 'gesturestart') {
+                detail['magnitudeX'] = 0.0;
+                detail['magnitudeY'] = 0.0;
+            } else {
+                let movement = this._getAverageMovement();
+                detail['magnitudeX'] = movement.x;
+                detail['magnitudeY'] = movement.y;
+            }
+        }
+
+        let gev = new CustomEvent(type, { detail: detail });
+        this._target.dispatchEvent(gev);
+    }
+
+    _stateToGesture(state) {
+        switch (state) {
+            case GH_ONETAP:
+                return 'onetap';
+            case GH_TWOTAP:
+                return 'twotap';
+            case GH_THREETAP:
+                return 'threetap';
+            case GH_DRAG:
+                return 'drag';
+            case GH_LONGPRESS:
+                return 'longpress';
+            case GH_TWODRAG:
+                return 'twodrag';
+            case GH_PINCH:
+                return 'pinch';
+        }
+
+        throw new Error("Unknown gesture state: " + state);
+    }
+
+    _getPosition() {
+        if (this._tracked.length === 0) {
+            throw new Error("Failed to get gesture position, no tracked touches");
+        }
+
+        let size = this._tracked.length;
+        let fx = 0, fy = 0, lx = 0, ly = 0;
+
+        for (let i = 0; i < this._tracked.length; i++) {
+            fx += this._tracked[i].firstX;
+            fy += this._tracked[i].firstY;
+            lx += this._tracked[i].lastX;
+            ly += this._tracked[i].lastY;
+        }
+
+        return { first: { x: fx / size,
+                          y: fy / size },
+                 last: { x: lx / size,
+                         y: ly / size } };
+    }
+
+    _getAverageMovement() {
+        if (this._tracked.length === 0) {
+            throw new Error("Failed to get gesture movement, no tracked touches");
+        }
+
+        let totalH, totalV;
+        totalH = totalV = 0;
+        let size = this._tracked.length;
+
+        for (let i = 0; i < this._tracked.length; i++) {
+            totalH += this._tracked[i].lastX - this._tracked[i].firstX;
+            totalV += this._tracked[i].lastY - this._tracked[i].firstY;
+        }
+
+        return { x: totalH / size,
+                 y: totalV / size };
+    }
+
+    _getAverageDistance() {
+        if (this._tracked.length === 0) {
+            throw new Error("Failed to get gesture distance, no tracked touches");
+        }
+
+        // Distance between the first and last tracked touches
+
+        let first = this._tracked[0];
+        let last = this._tracked[this._tracked.length - 1];
+
+        let fdx = Math.abs(last.firstX - first.firstX);
+        let fdy = Math.abs(last.firstY - first.firstY);
+
+        let ldx = Math.abs(last.lastX - first.lastX);
+        let ldy = Math.abs(last.lastY - first.lastY);
+
+        return { first: { x: fdx, y: fdy },
+                 last: { x: ldx, y: ldy } };
+    }
+}

src/component/utils/gesturehandler.ts

@@ -0,0 +1,14 @@
+// https://github.com/novnc/noVNC/blob/ca6527c1bf7131adccfdcc5028964a1e67f9018c/core/input/gesturehandler.js#L246
+import gh from './gesturehandler.js'
+
+const g = gh as GestureHandlerConstructor
+export default g
+
+interface GestureHandlerConstructor {
+  new (): GestureHandler
+}
+
+export interface GestureHandler {
+  attach(element: Element): void
+  detach(): void
+}

src/component/utils/keyboards/guacamole.ts

@@ -1,3 +1,4 @@
+// https://github.com/apache/guacamole-client/blob/1ca1161a68030565a37319ec6275556dfcd1a1af/guacamole-common-js/src/main/webapp/modules/Keyboard.js
 import GuacamoleKeyboard from './guacamole.js'
 
 export interface GuacamoleKeyboardInterface {

src/component/utils/keyboards/novnc.ts

@@ -1,3 +1,4 @@
+// https://github.com/novnc/noVNC/blob/ca6527c1bf7131adccfdcc5028964a1e67f9018c/core/input/keyboard.js
 import Keyboard from './novnc.js'
 
 export interface NoVncKeyboardInterface {