iridium/neko
2
2
Fork 0
mirror of https://github.com/m1k1o/neko.git synced 2024-07-24 14:40:50 +12:00
Code Issues Packages Projects Releases Wiki Activity

Add support for touch gestures (#40)

Browse Source 
* implement control using webrtc.

* overlay to use NekoControl.

* WIP.

* add is_host to control state.

* control use webrtc only if hosting.

* add proper links to 3rd party code.

* fix button events for gestureHandler.

* lint.

* fix implicit control gain for touch events.
This commit is contained in:
Miroslav Šedivý 2023-07-07 14:59:54 +02:00 committed by GitHub
parent 948cea7e00
commit 4918c62c9a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 836 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
src/component/internal/control.ts
View File

@ -26,6 +26,13 @@ export class NekoControl extends EventEmitter<NekoControlEvents> {
super()
}
get useWebrtc() {
// we want to use webrtc if we're connected and we're the host
// because webrtc is faster and it doesn't request control
// in contrast to the websocket
return this._connection.webrtc.connected && this._state.is_host
}
public lock() {
Vue.set(this._state, 'locked', true)
}
@ -43,35 +50,66 @@ export class NekoControl extends EventEmitter<NekoControlEvents> {
}
public move(pos: ControlPos) {
this._connection.websocket.send(EVENT.CONTROL_MOVE, pos as message.ControlPos)
if (this.useWebrtc) {
this._connection.webrtc.send('mousemove', pos)
} else {
this._connection.websocket.send(EVENT.CONTROL_MOVE, pos as message.ControlPos)
}
}
// TODO: rename pos to delta, and add a new pos parameter
public scroll(pos: ControlPos) {
this._connection.websocket.send(EVENT.CONTROL_SCROLL, pos as message.ControlPos)
if (this.useWebrtc) {
this._connection.webrtc.send('wheel', pos)
} else {
this._connection.websocket.send(EVENT.CONTROL_SCROLL, pos as message.ControlPos)
}
}
// buttonpress ensures that only one button is pressed at a time
public buttonPress(code: number, pos?: ControlPos) {
this._connection.websocket.send(EVENT.CONTROL_BUTTONPRESS, { code, ...pos } as message.ControlButton)
}
public buttonDown(code: number, pos?: ControlPos) {
this._connection.websocket.send(EVENT.CONTROL_BUTTONDOWN, { code, ...pos } as message.ControlButton)
if (this.useWebrtc) {
if (pos) this._connection.webrtc.send('mousemove', pos)
this._connection.webrtc.send('mousedown', { key: code })
} else {
this._connection.websocket.send(EVENT.CONTROL_BUTTONDOWN, { code, ...pos } as message.ControlButton)
}
}
public buttonUp(code: number, pos?: ControlPos) {
this._connection.websocket.send(EVENT.CONTROL_BUTTONUP, { code, ...pos } as message.ControlButton)
if (this.useWebrtc) {
if (pos) this._connection.webrtc.send('mousemove', pos)
this._connection.webrtc.send('mouseup', { key: code })
} else {
this._connection.websocket.send(EVENT.CONTROL_BUTTONUP, { code, ...pos } as message.ControlButton)
}
}
// keypress ensures that only one key is pressed at a time
public keyPress(keysym: number, pos?: ControlPos) {
this._connection.websocket.send(EVENT.CONTROL_KEYPRESS, { keysym, ...pos } as message.ControlKey)
}
public keyDown(keysym: number, pos?: ControlPos) {
this._connection.websocket.send(EVENT.CONTROL_KEYDOWN, { keysym, ...pos } as message.ControlKey)
if (this.useWebrtc) {
if (pos) this._connection.webrtc.send('mousemove', pos)
this._connection.webrtc.send('keydown', { key: keysym })
} else {
this._connection.websocket.send(EVENT.CONTROL_KEYDOWN, { keysym, ...pos } as message.ControlKey)
}
}
public keyUp(keysym: number, pos?: ControlPos) {
this._connection.websocket.send(EVENT.CONTROL_KEYUP, { keysym, ...pos } as message.ControlKey)
if (this.useWebrtc) {
if (pos) this._connection.webrtc.send('mousemove', pos)
this._connection.webrtc.send('keyup', { key: keysym })
} else {
this._connection.websocket.send(EVENT.CONTROL_KEYUP, { keysym, ...pos } as message.ControlKey)
}
}
public cut() {

3
src/component/internal/messages.ts
View File

@ -262,6 +262,9 @@ export class NekoMessages extends EventEmitter<NekoEvents> {
Vue.set(this._state.control, 'host_id', null)
}
// save if user is host
Vue.set(this._state.control, 'is_host', has_host && this._state.control.host_id === this._state.session_id)
this.emit('room.control.host', has_host, host_id)
}

4
src/component/main.vue
View File

@ -24,7 +24,7 @@
ref="overlay"
v-show="!private_mode_enabled && state.connection.status != 'disconnected'"
:style="{ pointerEvents: state.control.locked ? 'none' : 'auto' }"
:wsControl="control"
:control="control"
:sessions="state.sessions"
:hostId="state.control.host_id"
:webrtc="connection.webrtc"
@ -201,6 +201,7 @@
variant: '',
},
host_id: null,
is_host: false,
locked: false,
},
screen: {
@ -763,6 +764,7 @@
// websocket
Vue.set(this.state.control, 'clipboard', null)
Vue.set(this.state.control, 'host_id', null)
Vue.set(this.state.control, 'is_host', false)
Vue.set(this.state.screen, 'size', { width: 1280, height: 720, rate: 30 })
Vue.set(this.state.screen, 'configurations', [])
Vue.set(this.state.screen, 'sync', false)

276
src/component/overlay.vue
View File

@ -6,8 +6,8 @@
class="neko-overlay"
:style="{ cursor }"
v-model="textInput"
@click.stop.prevent="wsControl.emit('overlay.click', $event)"
@contextmenu.stop.prevent="wsControl.emit('overlay.contextmenu', $event)"
@click.stop.prevent="control.emit('overlay.click', $event)"
@contextmenu.stop.prevent="control.emit('overlay.contextmenu', $event)"
@wheel.stop.prevent="onWheel"
@mousemove.stop.prevent="onMouseMove"
@mousedown.stop.prevent="onMouseDown"
@ -47,6 +47,7 @@
import { Vue, Component, Ref, Prop, Watch } from 'vue-property-decorator'
import { KeyboardInterface, NewKeyboard } from './utils/keyboard'
import GestureHandlerInit, { GestureHandler } from './utils/gesturehandler'
import { KeyTable, keySymsRemap } from './utils/keyboard-remapping'
import { getFilesFromDataTansfer } from './utils/file-upload'
import { NekoControl } from './internal/control'
@ -55,8 +56,15 @@
import { CursorPosition, CursorImage } from './types/webrtc'
import { CursorDrawFunction, Dimension, KeyboardModifiers } from './types/cursors'
const WHEEL_STEP = 53 // Delta threshold for a mouse wheel step
const WHEEL_LINE_HEIGHT = 19
// Wheel thresholds
const WHEEL_STEP = 53 // Pixels needed for one step
const WHEEL_LINE_HEIGHT = 19 // Assumed pixels for one line step
// Gesture thresholds
const GESTURE_ZOOMSENS = 75
const GESTURE_SCRLSENS = 50
const DOUBLE_TAP_TIMEOUT = 1000
const DOUBLE_TAP_THRESHOLD = 50
const MOUSE_MOVE_THROTTLE = 1000 / 60 // in ms, 60fps
const INACTIVE_CURSOR_INTERVAL = 1000 / 4 // in ms, 4fps
@ -72,12 +80,13 @@
private canvasScale = window.devicePixelRatio
private keyboard!: KeyboardInterface
private gestureHandler!: GestureHandler
private textInput = ''
private focused = false
@Prop()
private readonly wsControl!: NekoControl
private readonly control!: NekoControl
@Prop()
private readonly sessions!: Record<string, Session>
@ -165,12 +174,7 @@
const isCtrlKey = key == KeyTable.XK_Control_L || key == KeyTable.XK_Control_R
if (isCtrlKey) ctrlKey = key
if (this.webrtc.connected) {
this.webrtc.send('keydown', { key })
} else {
this.wsControl.keyDown(key)
}
this.control.keyDown(key)
return isCtrlKey
}
this.keyboard.onkeyup = (key: number) => {
@ -184,17 +188,17 @@
const isCtrlKey = key == KeyTable.XK_Control_L || key == KeyTable.XK_Control_R
if (isCtrlKey) ctrlKey = 0
if (this.webrtc.connected) {
this.webrtc.send('keyup', { key })
} else {
this.wsControl.keyUp(key)
}
this.control.keyUp(key)
}
this.keyboard.listenTo(this._textarea)
this._textarea.addEventListener('touchstart', this.onTouchHandler, { passive: false })
this._textarea.addEventListener('touchmove', this.onTouchHandler, { passive: false })
this._textarea.addEventListener('touchend', this.onTouchHandler, { passive: false })
// Initialize GestureHandler
this.gestureHandler = new GestureHandlerInit()
this.gestureHandler.attach(this._textarea)
this._textarea.addEventListener('gesturestart', this.onGestureHandler)
this._textarea.addEventListener('gesturemove', this.onGestureHandler)
this._textarea.addEventListener('gestureend', this.onGestureHandler)
this.webrtc.addListener('cursor-position', this.onCursorPosition)
this.webrtc.addListener('cursor-image', this.onCursorImage)
@ -209,9 +213,13 @@
this.keyboard.removeListener()
}
this._textarea.removeEventListener('touchstart', this.onTouchHandler)
this._textarea.removeEventListener('touchmove', this.onTouchHandler)
this._textarea.removeEventListener('touchend', this.onTouchHandler)
if (this.gestureHandler) {
this.gestureHandler.detach()
}
this._textarea.removeEventListener('gesturestart', this.onGestureHandler)
this._textarea.removeEventListener('gesturemove', this.onGestureHandler)
this._textarea.removeEventListener('gestureend', this.onGestureHandler)
this.webrtc.removeListener('cursor-position', this.onCursorPosition)
this.webrtc.removeListener('cursor-image', this.onCursorImage)
@ -227,34 +235,161 @@
}
}
onTouchHandler(e: TouchEvent) {
let type = ''
switch (e.type) {
case 'touchstart':
type = 'mousedown'
break
case 'touchmove':
type = 'mousemove'
break
case 'touchend':
type = 'mouseup'
break
default:
// unknown event
return
// Gesture state
private _gestureLastTapTime: any | null = null
private _gestureFirstDoubleTapEv: any | null = null
private _gestureLastMagnitudeX = 0
private _gestureLastMagnitudeY = 0
_handleTapEvent(ev: any, code: number) {
let pos = this.getMousePos(ev.detail.clientX, ev.detail.clientY)
// If the user quickly taps multiple times we assume they meant to
// hit the same spot, so slightly adjust coordinates
if (
this._gestureLastTapTime !== null &&
Date.now() - this._gestureLastTapTime < DOUBLE_TAP_TIMEOUT &&
this._gestureFirstDoubleTapEv.detail.type === ev.detail.type
) {
let dx = this._gestureFirstDoubleTapEv.detail.clientX - ev.detail.clientX
let dy = this._gestureFirstDoubleTapEv.detail.clientY - ev.detail.clientY
let distance = Math.hypot(dx, dy)
if (distance < DOUBLE_TAP_THRESHOLD) {
pos = this.getMousePos(
this._gestureFirstDoubleTapEv.detail.clientX,
this._gestureFirstDoubleTapEv.detail.clientY,
)
} else {
this._gestureFirstDoubleTapEv = ev
}
} else {
this._gestureFirstDoubleTapEv = ev
}
this._gestureLastTapTime = Date.now()
this.control.buttonDown(code, pos)
this.control.buttonUp(code, pos)
}
// https://github.com/novnc/noVNC/blob/ca6527c1bf7131adccfdcc5028964a1e67f9018c/core/rfb.js#L1227-L1345
onGestureHandler(ev: any) {
// we cannot use implicitControlRequest because we don't have mouse event
if (!this.isControling) {
// if implicitControl is enabled, request control
if (this.implicitControl) {
this.control.request()
}
// otherwise, ignore event
return
}
const touch = e.changedTouches[0]
touch.target.dispatchEvent(
new MouseEvent(type, {
button: 0, // currently only left button is supported
clientX: touch.clientX,
clientY: touch.clientY,
}),
)
const pos = this.getMousePos(ev.detail.clientX, ev.detail.clientY)
e.preventDefault()
e.stopPropagation()
let magnitude
switch (ev.type) {
case 'gesturestart':
switch (ev.detail.type) {
case 'onetap':
this._handleTapEvent(ev, 1)
break
case 'twotap':
this._handleTapEvent(ev, 3)
break
case 'threetap':
this._handleTapEvent(ev, 2)
break
case 'drag':
this.control.buttonDown(1, pos)
break
case 'longpress':
this.control.buttonDown(3, pos)
break
case 'twodrag':
this._gestureLastMagnitudeX = ev.detail.magnitudeX
this._gestureLastMagnitudeY = ev.detail.magnitudeY
this.control.move(pos)
break
case 'pinch':
this._gestureLastMagnitudeX = Math.hypot(ev.detail.magnitudeX, ev.detail.magnitudeY)
this.control.move(pos)
break
}
break
case 'gesturemove':
switch (ev.detail.type) {
case 'onetap':
case 'twotap':
case 'threetap':
break
case 'drag':
case 'longpress':
this.control.move(pos)
break
case 'twodrag':
// Always scroll in the same position.
// We don't know if the mouse was moved so we need to move it
// every update.
this.control.move(pos)
while (ev.detail.magnitudeY - this._gestureLastMagnitudeY > GESTURE_SCRLSENS) {
this.control.scroll({ x: 0, y: 1 })
this._gestureLastMagnitudeY += GESTURE_SCRLSENS
}
while (ev.detail.magnitudeY - this._gestureLastMagnitudeY < -GESTURE_SCRLSENS) {
this.control.scroll({ x: 0, y: -1 })
this._gestureLastMagnitudeY -= GESTURE_SCRLSENS
}
while (ev.detail.magnitudeX - this._gestureLastMagnitudeX > GESTURE_SCRLSENS) {
this.control.scroll({ x: 1, y: 0 })
this._gestureLastMagnitudeX += GESTURE_SCRLSENS
}
while (ev.detail.magnitudeX - this._gestureLastMagnitudeX < -GESTURE_SCRLSENS) {
this.control.scroll({ x: -1, y: 0 })
this._gestureLastMagnitudeX -= GESTURE_SCRLSENS
}
break
case 'pinch':
// Always scroll in the same position.
// We don't know if the mouse was moved so we need to move it
// every update.
this.control.move(pos)
magnitude = Math.hypot(ev.detail.magnitudeX, ev.detail.magnitudeY)
if (Math.abs(magnitude - this._gestureLastMagnitudeX) > GESTURE_ZOOMSENS) {
this.control.keyDown(KeyTable.XK_Control_L)
while (magnitude - this._gestureLastMagnitudeX > GESTURE_ZOOMSENS) {
this.control.scroll({ x: 0, y: 1 })
this._gestureLastMagnitudeX += GESTURE_ZOOMSENS
}
while (magnitude - this._gestureLastMagnitudeX < -GESTURE_ZOOMSENS) {
this.control.scroll({ x: 0, y: -1 })
this._gestureLastMagnitudeX -= GESTURE_ZOOMSENS
}
this.control.keyUp(KeyTable.XK_Control_L)
}
break
}
break
case 'gestureend':
switch (ev.detail.type) {
case 'onetap':
case 'twotap':
case 'threetap':
case 'pinch':
case 'twodrag':
break
case 'drag':
this.control.buttonUp(1, pos)
break
case 'longpress':
this.control.buttonUp(3, pos)
break
}
break
}
}
getMousePos(clientX: number, clientY: number) {
@ -268,7 +403,11 @@
sendMousePos(e: MouseEvent) {
const pos = this.getMousePos(e.clientX, e.clientY)
this.webrtc.send('mousemove', pos)
// not using NekoControl here because we want to avoid
// sending mousemove events over websocket
if (this.webrtc.connected) {
this.webrtc.send('mousemove', pos)
} // otherwise, no events are sent
this.cursorPosition = pos
}
@ -307,7 +446,7 @@
@Watch('textInput')
onTextInputChange() {
if (this.textInput == '') return
this.wsControl.paste(this.textInput)
this.control.paste(this.textInput)
this.textInput = ''
}
@ -365,12 +504,8 @@
// skip if not scrolled
if (x == 0 && y == 0) return
if (this.webrtc.connected) {
this.sendMousePos(e)
this.webrtc.send('wheel', { x, y })
} else {
this.wsControl.scroll({ x, y })
}
// TODO: add position for precision scrolling
this.control.scroll({ x, y })
}
lastMouseMove = 0
@ -400,13 +535,8 @@
}
const key = e.button + 1
if (this.webrtc.connected) {
this.sendMousePos(e)
this.webrtc.send('mousedown', { key })
} else {
const pos = this.getMousePos(e.clientX, e.clientY)
this.wsControl.buttonDown(key, pos)
}
const pos = this.getMousePos(e.clientX, e.clientY)
this.control.buttonDown(key, pos)
}
onMouseUp(e: MouseEvent) {
@ -420,13 +550,8 @@
}
const key = e.button + 1
if (this.webrtc.connected) {
this.sendMousePos(e)
this.webrtc.send('mouseup', { key })
} else {
const pos = this.getMousePos(e.clientX, e.clientY)
this.wsControl.buttonUp(key, pos)
}
const pos = this.getMousePos(e.clientX, e.clientY)
this.control.buttonUp(key, pos)
}
onMouseEnter(e: MouseEvent) {
@ -474,7 +599,8 @@
const files = await getFilesFromDataTansfer(dt)
if (files.length === 0) return
this.$emit('uploadDrop', { ...this.getMousePos(e.clientX, e.clientY), files })
const pos = this.getMousePos(e.clientX, e.clientY)
this.$emit('uploadDrop', { ...pos, files })
}
}
@ -510,9 +636,11 @@
}
sendInactiveMousePos() {
if (this.inactiveCursorPosition) {
if (this.inactiveCursorPosition && this.webrtc.connected) {
// not using NekoControl here, because inactive cursors are
// treated differently than moving the mouse while controling
this.webrtc.send('mousemove', this.inactiveCursorPosition)
}
} // if webrtc is not connected, we don't need to send anything
}
//
@ -715,7 +843,7 @@
if (this.implicitControl && e.type === 'mousedown') {
this.reqMouseDown = e
this.reqMouseUp = null
this.wsControl.request()
this.control.request()
}
if (this.implicitControl && e.type === 'mouseup') {
@ -726,7 +854,7 @@
// unused
implicitControlRelease() {
if (this.implicitControl) {
this.wsControl.release()
this.control.release()
}
}

1
src/component/types/state.ts
View File

@ -71,6 +71,7 @@ export interface Control {
clipboard: Clipboard | null
keyboard: Keyboard
host_id: string | null
is_host: boolean
locked: boolean
}

567
src/component/utils/gesturehandler.js Normal file
View File

@ -0,0 +1,567 @@
/*
* noVNC: HTML5 VNC client
* Copyright (C) 2020 The noVNC Authors
* Licensed under MPL 2.0 (see LICENSE.txt)
*
* See README.md for usage and integration instructions.
*
*/
const GH_NOGESTURE = 0;
const GH_ONETAP = 1;
const GH_TWOTAP = 2;
const GH_THREETAP = 4;
const GH_DRAG = 8;
const GH_LONGPRESS = 16;
const GH_TWODRAG = 32;
const GH_PINCH = 64;
const GH_INITSTATE = 127;
const GH_MOVE_THRESHOLD = 50;
const GH_ANGLE_THRESHOLD = 90; // Degrees
// Timeout when waiting for gestures (ms)
const GH_MULTITOUCH_TIMEOUT = 250;
// Maximum time between press and release for a tap (ms)
const GH_TAP_TIMEOUT = 1000;
// Timeout when waiting for longpress (ms)
const GH_LONGPRESS_TIMEOUT = 1000;
// Timeout when waiting to decide between PINCH and TWODRAG (ms)
const GH_TWOTOUCH_TIMEOUT = 50;
export default class GestureHandler {
constructor() {
this._target = null;
this._state = GH_INITSTATE;
this._tracked = [];
this._ignored = [];
this._waitingRelease = false;
this._releaseStart = 0.0;
this._longpressTimeoutId = null;
this._twoTouchTimeoutId = null;
this._boundEventHandler = this._eventHandler.bind(this);
}
attach(target) {
this.detach();
this._target = target;
this._target.addEventListener('touchstart',
this._boundEventHandler);
this._target.addEventListener('touchmove',
this._boundEventHandler);
this._target.addEventListener('touchend',
this._boundEventHandler);
this._target.addEventListener('touchcancel',
this._boundEventHandler);
}
detach() {
if (!this._target) {
return;
}
this._stopLongpressTimeout();
this._stopTwoTouchTimeout();
this._target.removeEventListener('touchstart',
this._boundEventHandler);
this._target.removeEventListener('touchmove',
this._boundEventHandler);
this._target.removeEventListener('touchend',
this._boundEventHandler);
this._target.removeEventListener('touchcancel',
this._boundEventHandler);
this._target = null;
}
_eventHandler(e) {
let fn;
e.stopPropagation();
e.preventDefault();
switch (e.type) {
case 'touchstart':
fn = this._touchStart;
break;
case 'touchmove':
fn = this._touchMove;
break;
case 'touchend':
case 'touchcancel':
fn = this._touchEnd;
break;
}
for (let i = 0; i < e.changedTouches.length; i++) {
let touch = e.changedTouches[i];
fn.call(this, touch.identifier, touch.clientX, touch.clientY);
}
}
_touchStart(id, x, y) {
// Ignore any new touches if there is already an active gesture,
// or we're in a cleanup state
if (this._hasDetectedGesture() || (this._state === GH_NOGESTURE)) {
this._ignored.push(id);
return;
}
// Did it take too long between touches that we should no longer
// consider this a single gesture?
if ((this._tracked.length > 0) &&
((Date.now() - this._tracked[0].started) > GH_MULTITOUCH_TIMEOUT)) {
this._state = GH_NOGESTURE;
this._ignored.push(id);
return;
}
// If we're waiting for fingers to release then we should no longer
// recognize new touches
if (this._waitingRelease) {
this._state = GH_NOGESTURE;
this._ignored.push(id);
return;
}
this._tracked.push({
id: id,
started: Date.now(),
active: true,
firstX: x,
firstY: y,
lastX: x,
lastY: y,
angle: 0
});
switch (this._tracked.length) {
case 1:
this._startLongpressTimeout();
break;
case 2:
this._state &= ~(GH_ONETAP | GH_DRAG | GH_LONGPRESS);
this._stopLongpressTimeout();
break;
case 3:
this._state &= ~(GH_TWOTAP | GH_TWODRAG | GH_PINCH);
break;
default:
this._state = GH_NOGESTURE;
}
}
_touchMove(id, x, y) {
let touch = this._tracked.find(t => t.id === id);
// If this is an update for a touch we're not tracking, ignore it
if (touch === undefined) {
return;
}
// Update the touches last position with the event coordinates
touch.lastX = x;
touch.lastY = y;
let deltaX = x - touch.firstX;
let deltaY = y - touch.firstY;
// Update angle when the touch has moved
if ((touch.firstX !== touch.lastX) ||
(touch.firstY !== touch.lastY)) {
touch.angle = Math.atan2(deltaY, deltaX) * 180 / Math.PI;
}
if (!this._hasDetectedGesture()) {
// Ignore moves smaller than the minimum threshold
if (Math.hypot(deltaX, deltaY) < GH_MOVE_THRESHOLD) {
return;
}
// Can't be a tap or long press as we've seen movement
this._state &= ~(GH_ONETAP | GH_TWOTAP | GH_THREETAP | GH_LONGPRESS);
this._stopLongpressTimeout();
if (this._tracked.length !== 1) {
this._state &= ~(GH_DRAG);
}
if (this._tracked.length !== 2) {
this._state &= ~(GH_TWODRAG | GH_PINCH);
}
// We need to figure out which of our different two touch gestures
// this might be
if (this._tracked.length === 2) {
// The other touch is the one where the id doesn't match
let prevTouch = this._tracked.find(t => t.id !== id);
// How far the previous touch point has moved since start
let prevDeltaMove = Math.hypot(prevTouch.firstX - prevTouch.lastX,
prevTouch.firstY - prevTouch.lastY);
// We know that the current touch moved far enough,
// but unless both touches moved further than their
// threshold we don't want to disqualify any gestures
if (prevDeltaMove > GH_MOVE_THRESHOLD) {
// The angle difference between the direction of the touch points
let deltaAngle = Math.abs(touch.angle - prevTouch.angle);
deltaAngle = Math.abs(((deltaAngle + 180) % 360) - 180);
// PINCH or TWODRAG can be eliminated depending on the angle
if (deltaAngle > GH_ANGLE_THRESHOLD) {
this._state &= ~GH_TWODRAG;
} else {
this._state &= ~GH_PINCH;
}
if (this._isTwoTouchTimeoutRunning()) {
this._stopTwoTouchTimeout();
}
} else if (!this._isTwoTouchTimeoutRunning()) {
// We can't determine the gesture right now, let's
// wait and see if more events are on their way
this._startTwoTouchTimeout();
}
}
if (!this._hasDetectedGesture()) {
return;
}
this._pushEvent('gesturestart');
}
this._pushEvent('gesturemove');
}
_touchEnd(id, x, y) {
// Check if this is an ignored touch
if (this._ignored.indexOf(id) !== -1) {
// Remove this touch from ignored
this._ignored.splice(this._ignored.indexOf(id), 1);
// And reset the state if there are no more touches
if ((this._ignored.length === 0) &&
(this._tracked.length === 0)) {
this._state = GH_INITSTATE;
this._waitingRelease = false;
}
return;
}
// We got a touchend before the timer triggered,
// this cannot result in a gesture anymore.
if (!this._hasDetectedGesture() &&
this._isTwoTouchTimeoutRunning()) {
this._stopTwoTouchTimeout();
this._state = GH_NOGESTURE;
}
// Some gestures don't trigger until a touch is released
if (!this._hasDetectedGesture()) {
// Can't be a gesture that relies on movement
this._state &= ~(GH_DRAG | GH_TWODRAG | GH_PINCH);
// Or something that relies on more time
this._state &= ~GH_LONGPRESS;
this._stopLongpressTimeout();
if (!this._waitingRelease) {
this._releaseStart = Date.now();
this._waitingRelease = true;
// Can't be a tap that requires more touches than we current have
switch (this._tracked.length) {
case 1:
this._state &= ~(GH_TWOTAP | GH_THREETAP);
break;
case 2:
this._state &= ~(GH_ONETAP | GH_THREETAP);
break;
}
}
}
// Waiting for all touches to release? (i.e. some tap)
if (this._waitingRelease) {
// Were all touches released at roughly the same time?
if ((Date.now() - this._releaseStart) > GH_MULTITOUCH_TIMEOUT) {
this._state = GH_NOGESTURE;
}
// Did too long time pass between press and release?
if (this._tracked.some(t => (Date.now() - t.started) > GH_TAP_TIMEOUT)) {
this._state = GH_NOGESTURE;
}
let touch = this._tracked.find(t => t.id === id);
touch.active = false;
// Are we still waiting for more releases?
if (this._hasDetectedGesture()) {
this._pushEvent('gesturestart');
} else {
// Have we reached a dead end?
if (this._state !== GH_NOGESTURE) {
return;
}
}
}
if (this._hasDetectedGesture()) {
this._pushEvent('gestureend');
}
// Ignore any remaining touches until they are ended
for (let i = 0; i < this._tracked.length; i++) {
if (this._tracked[i].active) {
this._ignored.push(this._tracked[i].id);
}
}
this._tracked = [];
this._state = GH_NOGESTURE;
// Remove this touch from ignored if it's in there
if (this._ignored.indexOf(id) !== -1) {
this._ignored.splice(this._ignored.indexOf(id), 1);
}
// We reset the state if ignored is empty
if ((this._ignored.length === 0)) {
this._state = GH_INITSTATE;
this._waitingRelease = false;
}
}
_hasDetectedGesture() {
if (this._state === GH_NOGESTURE) {
return false;
}
// Check to see if the bitmask value is a power of 2
// (i.e. only one bit set). If it is, we have a state.
if (this._state & (this._state - 1)) {
return false;
}
// For taps we also need to have all touches released
// before we've fully detected the gesture
if (this._state & (GH_ONETAP | GH_TWOTAP | GH_THREETAP)) {
if (this._tracked.some(t => t.active)) {
return false;
}
}
return true;
}
_startLongpressTimeout() {
this._stopLongpressTimeout();
this._longpressTimeoutId = setTimeout(() => this._longpressTimeout(),
GH_LONGPRESS_TIMEOUT);
}
_stopLongpressTimeout() {
clearTimeout(this._longpressTimeoutId);
this._longpressTimeoutId = null;
}
_longpressTimeout() {
if (this._hasDetectedGesture()) {
throw new Error("A longpress gesture failed, conflict with a different gesture");
}
this._state = GH_LONGPRESS;
this._pushEvent('gesturestart');
}
_startTwoTouchTimeout() {
this._stopTwoTouchTimeout();
this._twoTouchTimeoutId = setTimeout(() => this._twoTouchTimeout(),
GH_TWOTOUCH_TIMEOUT);
}
_stopTwoTouchTimeout() {
clearTimeout(this._twoTouchTimeoutId);
this._twoTouchTimeoutId = null;
}
_isTwoTouchTimeoutRunning() {
return this._twoTouchTimeoutId !== null;
}
_twoTouchTimeout() {
if (this._tracked.length === 0) {
throw new Error("A pinch or two drag gesture failed, no tracked touches");
}
// How far each touch point has moved since start
let avgM = this._getAverageMovement();
let avgMoveH = Math.abs(avgM.x);
let avgMoveV = Math.abs(avgM.y);
// The difference in the distance between where
// the touch points started and where they are now
let avgD = this._getAverageDistance();
let deltaTouchDistance = Math.abs(Math.hypot(avgD.first.x, avgD.first.y) -
Math.hypot(avgD.last.x, avgD.last.y));
if ((avgMoveV < deltaTouchDistance) &&
(avgMoveH < deltaTouchDistance)) {
this._state = GH_PINCH;
} else {
this._state = GH_TWODRAG;
}
this._pushEvent('gesturestart');
this._pushEvent('gesturemove');
}
_pushEvent(type) {
let detail = { type: this._stateToGesture(this._state) };
// For most gesture events the current (average) position is the
// most useful
let avg = this._getPosition();
let pos = avg.last;
// However we have a slight distance to detect gestures, so for the
// first gesture event we want to use the first positions we saw
if (type === 'gesturestart') {
pos = avg.first;
}
// For these gestures, we always want the event coordinates
// to be where the gesture began, not the current touch location.
switch (this._state) {
case GH_TWODRAG:
case GH_PINCH:
pos = avg.first;
break;
}
detail['clientX'] = pos.x;
detail['clientY'] = pos.y;
// FIXME: other coordinates?
// Some gestures also have a magnitude
if (this._state === GH_PINCH) {
let distance = this._getAverageDistance();
if (type === 'gesturestart') {
detail['magnitudeX'] = distance.first.x;
detail['magnitudeY'] = distance.first.y;
} else {
detail['magnitudeX'] = distance.last.x;
detail['magnitudeY'] = distance.last.y;
}
} else if (this._state === GH_TWODRAG) {
if (type === 'gesturestart') {
detail['magnitudeX'] = 0.0;
detail['magnitudeY'] = 0.0;
} else {
let movement = this._getAverageMovement();
detail['magnitudeX'] = movement.x;
detail['magnitudeY'] = movement.y;
}
}
let gev = new CustomEvent(type, { detail: detail });
this._target.dispatchEvent(gev);
}
_stateToGesture(state) {
switch (state) {
case GH_ONETAP:
return 'onetap';
case GH_TWOTAP:
return 'twotap';
case GH_THREETAP:
return 'threetap';
case GH_DRAG:
return 'drag';
case GH_LONGPRESS:
return 'longpress';
case GH_TWODRAG:
return 'twodrag';
case GH_PINCH:
return 'pinch';
}
throw new Error("Unknown gesture state: " + state);
}
_getPosition() {
if (this._tracked.length === 0) {
throw new Error("Failed to get gesture position, no tracked touches");
}
let size = this._tracked.length;
let fx = 0, fy = 0, lx = 0, ly = 0;
for (let i = 0; i < this._tracked.length; i++) {
fx += this._tracked[i].firstX;
fy += this._tracked[i].firstY;
lx += this._tracked[i].lastX;
ly += this._tracked[i].lastY;
}
return { first: { x: fx / size,
y: fy / size },
last: { x: lx / size,
y: ly / size } };
}
_getAverageMovement() {
if (this._tracked.length === 0) {
throw new Error("Failed to get gesture movement, no tracked touches");
}
let totalH, totalV;
totalH = totalV = 0;
let size = this._tracked.length;
for (let i = 0; i < this._tracked.length; i++) {
totalH += this._tracked[i].lastX - this._tracked[i].firstX;
totalV += this._tracked[i].lastY - this._tracked[i].firstY;
}
return { x: totalH / size,
y: totalV / size };
}
_getAverageDistance() {
if (this._tracked.length === 0) {
throw new Error("Failed to get gesture distance, no tracked touches");
}
// Distance between the first and last tracked touches
let first = this._tracked[0];
let last = this._tracked[this._tracked.length - 1];
let fdx = Math.abs(last.firstX - first.firstX);
let fdy = Math.abs(last.firstY - first.firstY);
let ldx = Math.abs(last.lastX - first.lastX);
let ldy = Math.abs(last.lastY - first.lastY);
return { first: { x: fdx, y: fdy },
last: { x: ldx, y: ldy } };
}
}

14
src/component/utils/gesturehandler.ts Normal file
View File

@ -0,0 +1,14 @@
// https://github.com/novnc/noVNC/blob/ca6527c1bf7131adccfdcc5028964a1e67f9018c/core/input/gesturehandler.js#L246
import gh from './gesturehandler.js'
const g = gh as GestureHandlerConstructor
export default g
interface GestureHandlerConstructor {
new (): GestureHandler
}
export interface GestureHandler {
attach(element: Element): void
detach(): void
}

1
src/component/utils/keyboards/guacamole.ts
View File

@ -1,3 +1,4 @@
// https://github.com/apache/guacamole-client/blob/1ca1161a68030565a37319ec6275556dfcd1a1af/guacamole-common-js/src/main/webapp/modules/Keyboard.js
import GuacamoleKeyboard from './guacamole.js'
export interface GuacamoleKeyboardInterface {

1
src/component/utils/keyboards/novnc.ts
View File

@ -1,3 +1,4 @@
// https://github.com/novnc/noVNC/blob/ca6527c1bf7131adccfdcc5028964a1e67f9018c/core/input/keyboard.js
import Keyboard from './novnc.js'
export interface NoVncKeyboardInterface {