From 0d830998e5c76a816aaecd522bfc00ff2254a329 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Thu, 17 Aug 2023 16:16:08 +0200 Subject: [PATCH] Native touch events (#42) * add webrtc touch events. * bind touch events to overlay. * we care only for changed touches. * switch to int32. * pressure uint16. * add implicit control. * add touch to controls. * fix iteration of changedTouches. * switch pressure to uint8. * convert force to pressure. * add hasTouchEvents. * add touch_events to state. * bind touch or gesture handler on demand. * remove duplicate gesture detach. --- src/component/internal/control.ts | 28 ++++++++ src/component/internal/messages.ts | 1 + src/component/internal/webrtc.ts | 26 +++++++ src/component/main.vue | 1 + src/component/overlay.vue | 108 ++++++++++++++++++++++++++--- src/component/types/events.ts | 4 ++ src/component/types/messages.ts | 6 ++ src/component/types/state.ts | 1 + src/page/components/events.vue | 4 ++ 9 files changed, 168 insertions(+), 11 deletions(-) diff --git a/src/component/internal/control.ts b/src/component/internal/control.ts index 0f2e6c66..383e05c6 100644 --- a/src/component/internal/control.ts +++ b/src/component/internal/control.ts @@ -33,6 +33,10 @@ export class NekoControl extends EventEmitter { return this._connection.webrtc.connected && this._state.is_host } + get hasTouchEvents() { + return this._state.touch_events + } + public lock() { Vue.set(this._state, 'locked', true) } @@ -112,6 +116,30 @@ export class NekoControl extends EventEmitter { } } + public touchBegin(touchId: number, pos: ControlPos, pressure: number) { + if (this.useWebrtc) { + this._connection.webrtc.send('touchbegin', { touchId, ...pos, pressure }) + } else { + this._connection.websocket.send(EVENT.CONTROL_TOUCHBEGIN, { touchId, ...pos, pressure } as message.ControlTouch) + } + } + + public touchUpdate(touchId: number, pos: ControlPos, pressure: number) { + if (this.useWebrtc) { + this._connection.webrtc.send('touchupdate', { touchId, ...pos, pressure }) + } else { + this._connection.websocket.send(EVENT.CONTROL_TOUCHUPDATE, { touchId, ...pos, pressure } as message.ControlTouch) + } + } + + public touchEnd(touchId: number, pos: ControlPos, pressure: number) { + if (this.useWebrtc) { + this._connection.webrtc.send('touchend', { touchId, ...pos, pressure }) + } else { + this._connection.websocket.send(EVENT.CONTROL_TOUCHEND, { touchId, ...pos, pressure } as message.ControlTouch) + } + } + public cut() { this._connection.websocket.send(EVENT.CONTROL_CUT) } diff --git a/src/component/internal/messages.ts b/src/component/internal/messages.ts index 7080d1aa..8e6ead47 100644 --- a/src/component/internal/messages.ts +++ b/src/component/internal/messages.ts @@ -104,6 +104,7 @@ export class NekoMessages extends EventEmitter { protected [EVENT.SYSTEM_INIT](conf: message.SystemInit) { this._localLog.debug(`EVENT.SYSTEM_INIT`) Vue.set(this._state, 'session_id', conf.session_id) + Vue.set(this._state.control, 'touch_events', conf.touch_events) Vue.set(this._state.connection, 'screencast', conf.screencast_enabled) Vue.set(this._state.connection.webrtc, 'videos', conf.webrtc.videos) diff --git a/src/component/internal/webrtc.ts b/src/component/internal/webrtc.ts index 047a16a9..a8bf4710 100644 --- a/src/component/internal/webrtc.ts +++ b/src/component/internal/webrtc.ts @@ -13,6 +13,10 @@ export const OPCODE = { BTN_DOWN: 0x05, BTN_UP: 0x06, PING: 0x07, + // touch events + TOUCH_BEGIN: 0x08, + TOUCH_UPDATE: 0x09, + TOUCH_END: 0x0a, } as const export interface ICEServer { @@ -361,6 +365,10 @@ export class NekoWebRTC extends EventEmitter { public send(event: 'wheel' | 'mousemove', data: { x: number; y: number }): void public send(event: 'mousedown' | 'mouseup' | 'keydown' | 'keyup', data: { key: number }): void public send(event: 'ping', data: number): void + public send( + event: 'touchbegin' | 'touchupdate' | 'touchend', + data: { touchId: number; x: number; y: number; pressure: number }, + ): void public send(event: string, data: any): void { if (typeof this._channel === 'undefined' || this._channel.readyState !== 'open') { this._log.warn(`attempting to send data, but data-channel is not open`, { event }) @@ -422,6 +430,24 @@ export class NekoWebRTC extends EventEmitter { payload.setUint32(3, Math.trunc(data / maxUint32)) payload.setUint32(7, data % maxUint32) break + case 'touchbegin': + case 'touchupdate': + case 'touchend': + buffer = new ArrayBuffer(16) + payload = new DataView(buffer) + if (event === 'touchbegin') { + payload.setUint8(0, OPCODE.TOUCH_BEGIN) + } else if (event === 'touchupdate') { + payload.setUint8(0, OPCODE.TOUCH_UPDATE) + } else if (event === 'touchend') { + payload.setUint8(0, OPCODE.TOUCH_END) + } + payload.setUint16(1, 13) + payload.setUint32(3, data.touchId) + payload.setInt32(7, data.x) + payload.setInt32(11, data.y) + payload.setUint8(15, data.pressure) + break default: this._log.warn(`unknown data event`, { event }) return diff --git a/src/component/main.vue b/src/component/main.vue index d95b89dd..f2a6ec41 100644 --- a/src/component/main.vue +++ b/src/component/main.vue @@ -200,6 +200,7 @@ layout: 'us', variant: '', }, + touch_events: false, host_id: null, is_host: false, locked: false, diff --git a/src/component/overlay.vue b/src/component/overlay.vue index e5c56416..b1335ccb 100644 --- a/src/component/overlay.vue +++ b/src/component/overlay.vue @@ -194,11 +194,13 @@ // Initialize GestureHandler this.gestureHandler = new GestureHandlerInit() - this.gestureHandler.attach(this._textarea) - this._textarea.addEventListener('gesturestart', this.onGestureHandler) - this._textarea.addEventListener('gesturemove', this.onGestureHandler) - this._textarea.addEventListener('gestureend', this.onGestureHandler) + // bind touch handler using @Watch on hasTouchEvents + // because we need to know if touch events are supported + // by the server before we can bind touch handler + + // default value is false, so we can bind touch handler + this.bindGestureHandler() this.webrtc.addListener('cursor-position', this.onCursorPosition) this.webrtc.addListener('cursor-image', this.onCursorImage) @@ -213,13 +215,11 @@ this.keyboard.removeListener() } - if (this.gestureHandler) { - this.gestureHandler.detach() - } + // unbind touch handler + this.unbindTouchHandler() - this._textarea.removeEventListener('gesturestart', this.onGestureHandler) - this._textarea.removeEventListener('gesturemove', this.onGestureHandler) - this._textarea.removeEventListener('gestureend', this.onGestureHandler) + // unbind gesture handler + this.unbindGestureHandler() this.webrtc.removeListener('cursor-position', this.onCursorPosition) this.webrtc.removeListener('cursor-image', this.onCursorImage) @@ -235,7 +235,78 @@ } } - // Gesture state + // + // touch handler for native touch events + // + + bindTouchHandler() { + this._textarea.addEventListener('touchstart', this.onTouchHandler, { passive: false }) + this._textarea.addEventListener('touchmove', this.onTouchHandler, { passive: false }) + this._textarea.addEventListener('touchend', this.onTouchHandler, { passive: false }) + this._textarea.addEventListener('touchcancel', this.onTouchHandler, { passive: false }) + } + + unbindTouchHandler() { + this._textarea.removeEventListener('touchstart', this.onTouchHandler) + this._textarea.removeEventListener('touchmove', this.onTouchHandler) + this._textarea.removeEventListener('touchend', this.onTouchHandler) + this._textarea.removeEventListener('touchcancel', this.onTouchHandler) + } + + onTouchHandler(ev: TouchEvent) { + // we cannot use implicitControlRequest because we don't have mouse event + if (!this.isControling) { + // if implicitControl is enabled, request control + if (this.implicitControl) { + this.control.request() + } + // otherwise, ignore event + return + } + + ev.stopPropagation() + ev.preventDefault() + + for (let i = 0; i < ev.changedTouches.length; i++) { + const touch = ev.changedTouches[i] + const pos = this.getMousePos(touch.clientX, touch.clientY) + // force is float value between 0 and 1 + // pressure is integer value between 0 and 255 + const pressure = Math.round(touch.force * 255) + + switch (ev.type) { + case 'touchstart': + this.control.touchBegin(touch.identifier, pos, pressure) + break + case 'touchmove': + this.control.touchUpdate(touch.identifier, pos, pressure) + break + case 'touchend': + case 'touchcancel': + this.control.touchEnd(touch.identifier, pos, pressure) + break + } + } + } + + // + // gesture handler for emulated mouse events + // + + bindGestureHandler() { + this.gestureHandler.attach(this._textarea) + this._textarea.addEventListener('gesturestart', this.onGestureHandler) + this._textarea.addEventListener('gesturemove', this.onGestureHandler) + this._textarea.addEventListener('gestureend', this.onGestureHandler) + } + + unbindGestureHandler() { + this.gestureHandler.detach() + this._textarea.removeEventListener('gesturestart', this.onGestureHandler) + this._textarea.removeEventListener('gesturemove', this.onGestureHandler) + this._textarea.removeEventListener('gestureend', this.onGestureHandler) + } + private _gestureLastTapTime: any | null = null private _gestureFirstDoubleTapEv: any | null = null private _gestureLastMagnitudeX = 0 @@ -392,6 +463,21 @@ } } + // + // touch and gesture handlers cannot be used together + // + + @Watch('control.hasTouchEvents') + onTouchEventsChange() { + if (this.control.hasTouchEvents) { + this.unbindGestureHandler() + this.bindTouchHandler() + } else { + this.unbindTouchHandler() + this.bindGestureHandler() + } + } + getMousePos(clientX: number, clientY: number) { const rect = this._overlay.getBoundingClientRect() diff --git a/src/component/types/events.ts b/src/component/types/events.ts index 83eea355..f69ef52f 100644 --- a/src/component/types/events.ts +++ b/src/component/types/events.ts @@ -34,6 +34,10 @@ export const CONTROL_BUTTONUP = 'control/buttonup' export const CONTROL_KEYPRESS = 'control/keypress' export const CONTROL_KEYDOWN = 'control/keydown' export const CONTROL_KEYUP = 'control/keyup' +// touch +export const CONTROL_TOUCHBEGIN = 'control/touchbegin' +export const CONTROL_TOUCHUPDATE = 'control/touchupdate' +export const CONTROL_TOUCHEND = 'control/touchend' // actions export const CONTROL_CUT = 'control/cut' export const CONTROL_COPY = 'control/copy' diff --git a/src/component/types/messages.ts b/src/component/types/messages.ts index c6022e94..e5d04b4b 100644 --- a/src/component/types/messages.ts +++ b/src/component/types/messages.ts @@ -18,6 +18,7 @@ export interface SystemInit { screen_size: ScreenSize sessions: Record settings: Settings + touch_events: boolean screencast_enabled: boolean webrtc: SystemWebRTC } @@ -125,6 +126,11 @@ export interface ControlKey extends Partial { keysym: number } +export interface ControlTouch extends Partial { + touchId: number + pressure: number +} + ///////////////////////////// // Screen ///////////////////////////// diff --git a/src/component/types/state.ts b/src/component/types/state.ts index b9ed4b64..69f24ff5 100644 --- a/src/component/types/state.ts +++ b/src/component/types/state.ts @@ -70,6 +70,7 @@ export interface Control { scroll: Scroll clipboard: Clipboard | null keyboard: Keyboard + touch_events: boolean host_id: string | null is_host: boolean locked: boolean diff --git a/src/page/components/events.vue b/src/page/components/events.vue index 8d879455..d429c689 100644 --- a/src/page/components/events.vue +++ b/src/page/components/events.vue @@ -334,6 +334,10 @@ /> + + control.touch_events + {{ neko.state.control.touch_events ? 'backend supports' : 'backend does not support' }} + control.host_id {{ neko.state.control.host_id }}