From 91dee46db40c819bb86d6717120e7fc8005dd931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Mon, 11 Sep 2023 16:38:12 +0200 Subject: [PATCH] Add Xorg modifiers (#44) * add modifiers. * use modifiers. * scroll rename to delta and add ctrl key. --- .../api/models/keyboard-modifiers.ts | 38 ++++++++++++- src/component/internal/control.ts | 32 ++++++----- src/component/internal/webrtc.ts | 16 +++--- src/component/overlay.vue | 56 ++++++++++++------- src/component/types/cursors.ts | 10 +++- src/component/types/messages.ts | 8 ++- 6 files changed, 115 insertions(+), 45 deletions(-) diff --git a/src/component/api/models/keyboard-modifiers.ts b/src/component/api/models/keyboard-modifiers.ts index c12dc194..b8d283f3 100644 --- a/src/component/api/models/keyboard-modifiers.ts +++ b/src/component/api/models/keyboard-modifiers.ts @@ -25,12 +25,48 @@ export interface KeyboardModifiers { * @type {boolean} * @memberof KeyboardModifiers */ - 'numlock'?: boolean; + 'shift'?: boolean; /** * * @type {boolean} * @memberof KeyboardModifiers */ 'capslock'?: boolean; + /** + * + * @type {boolean} + * @memberof KeyboardModifiers + */ + 'control'?: boolean; + /** + * + * @type {boolean} + * @memberof KeyboardModifiers + */ + 'alt'?: boolean; + /** + * + * @type {boolean} + * @memberof KeyboardModifiers + */ + 'numlock'?: boolean; + /** + * + * @type {boolean} + * @memberof KeyboardModifiers + */ + 'meta'?: boolean; + /** + * + * @type {boolean} + * @memberof KeyboardModifiers + */ + 'super'?: boolean; + /** + * + * @type {boolean} + * @memberof KeyboardModifiers + */ + 'altgr'?: boolean; } diff --git a/src/component/internal/control.ts b/src/component/internal/control.ts index a3074265..290a38d0 100644 --- a/src/component/internal/control.ts +++ b/src/component/internal/control.ts @@ -17,6 +17,12 @@ export interface ControlPos { y: number } +export interface ControlScroll { + delta_x: number + delta_y: number + control_key?: boolean +} + export class NekoControl extends EventEmitter { // eslint-disable-next-line constructor( @@ -65,12 +71,12 @@ export class NekoControl extends EventEmitter { } } - // TODO: rename pos to delta, and add a new pos parameter - public scroll(pos: ControlPos) { + // TODO: add pos parameter + public scroll(scroll: ControlScroll) { if (this.useWebrtc) { - this._connection.webrtc.send('wheel', pos) + this._connection.webrtc.send('wheel', scroll) } else { - this._connection.websocket.send(EVENT.CONTROL_SCROLL, pos as message.ControlPos) + this._connection.websocket.send(EVENT.CONTROL_SCROLL, scroll as message.ControlScroll) } } @@ -120,27 +126,27 @@ export class NekoControl extends EventEmitter { } } - public touchBegin(touchId: number, pos: ControlPos, pressure: number) { + public touchBegin(touch_id: number, pos: ControlPos, pressure: number) { if (this.useWebrtc) { - this._connection.webrtc.send('touchbegin', { touchId, ...pos, pressure }) + this._connection.webrtc.send('touchbegin', { touch_id, ...pos, pressure }) } else { - this._connection.websocket.send(EVENT.CONTROL_TOUCHBEGIN, { touchId, ...pos, pressure } as message.ControlTouch) + this._connection.websocket.send(EVENT.CONTROL_TOUCHBEGIN, { touch_id, ...pos, pressure } as message.ControlTouch) } } - public touchUpdate(touchId: number, pos: ControlPos, pressure: number) { + public touchUpdate(touch_id: number, pos: ControlPos, pressure: number) { if (this.useWebrtc) { - this._connection.webrtc.send('touchupdate', { touchId, ...pos, pressure }) + this._connection.webrtc.send('touchupdate', { touch_id, ...pos, pressure }) } else { - this._connection.websocket.send(EVENT.CONTROL_TOUCHUPDATE, { touchId, ...pos, pressure } as message.ControlTouch) + this._connection.websocket.send(EVENT.CONTROL_TOUCHUPDATE, { touch_id, ...pos, pressure } as message.ControlTouch) } } - public touchEnd(touchId: number, pos: ControlPos, pressure: number) { + public touchEnd(touch_id: number, pos: ControlPos, pressure: number) { if (this.useWebrtc) { - this._connection.webrtc.send('touchend', { touchId, ...pos, pressure }) + this._connection.webrtc.send('touchend', { touch_id, ...pos, pressure }) } else { - this._connection.websocket.send(EVENT.CONTROL_TOUCHEND, { touchId, ...pos, pressure } as message.ControlTouch) + this._connection.websocket.send(EVENT.CONTROL_TOUCHEND, { touch_id, ...pos, pressure } as message.ControlTouch) } } diff --git a/src/component/internal/webrtc.ts b/src/component/internal/webrtc.ts index a8bf4710..eb999b47 100644 --- a/src/component/internal/webrtc.ts +++ b/src/component/internal/webrtc.ts @@ -362,12 +362,13 @@ export class NekoWebRTC extends EventEmitter { this._candidates = [] } - public send(event: 'wheel' | 'mousemove', data: { x: number; y: number }): void + public send(event: 'wheel', data: { delta_x: number; delta_y: number; control_key?: boolean }): void + public send(event: 'mousemove', data: { x: number; y: number }): void public send(event: 'mousedown' | 'mouseup' | 'keydown' | 'keyup', data: { key: number }): void public send(event: 'ping', data: number): void public send( event: 'touchbegin' | 'touchupdate' | 'touchend', - data: { touchId: number; x: number; y: number; pressure: number }, + data: { touch_id: number; x: number; y: number; pressure: number }, ): void public send(event: string, data: any): void { if (typeof this._channel === 'undefined' || this._channel.readyState !== 'open') { @@ -387,12 +388,13 @@ export class NekoWebRTC extends EventEmitter { payload.setUint16(5, data.y) break case 'wheel': - buffer = new ArrayBuffer(7) + buffer = new ArrayBuffer(8) payload = new DataView(buffer) payload.setUint8(0, OPCODE.SCROLL) - payload.setUint16(1, 4) - payload.setInt16(3, data.x) - payload.setInt16(5, data.y) + payload.setUint16(1, 5) + payload.setInt16(3, data.delta_x) + payload.setInt16(5, data.delta_y) + payload.setUint8(7, data.control_key ? 1 : 0) break case 'keydown': buffer = new ArrayBuffer(7) @@ -443,7 +445,7 @@ export class NekoWebRTC extends EventEmitter { payload.setUint8(0, OPCODE.TOUCH_END) } payload.setUint16(1, 13) - payload.setUint32(3, data.touchId) + payload.setUint32(3, data.touch_id) payload.setInt32(7, data.x) payload.setInt32(11, data.y) payload.setUint8(15, data.pressure) diff --git a/src/component/overlay.vue b/src/component/overlay.vue index 8dd49f93..8e526421 100644 --- a/src/component/overlay.vue +++ b/src/component/overlay.vue @@ -406,19 +406,19 @@ // every update. this.control.move(pos) while (ev.detail.magnitudeY - this._gestureLastMagnitudeY > GESTURE_SCRLSENS) { - this.control.scroll({ x: 0, y: 1 }) + this.control.scroll({ delta_x: 0, delta_y: 1 }) this._gestureLastMagnitudeY += GESTURE_SCRLSENS } while (ev.detail.magnitudeY - this._gestureLastMagnitudeY < -GESTURE_SCRLSENS) { - this.control.scroll({ x: 0, y: -1 }) + this.control.scroll({ delta_x: 0, delta_y: -1 }) this._gestureLastMagnitudeY -= GESTURE_SCRLSENS } while (ev.detail.magnitudeX - this._gestureLastMagnitudeX > GESTURE_SCRLSENS) { - this.control.scroll({ x: 1, y: 0 }) + this.control.scroll({ delta_x: 1, delta_y: 0 }) this._gestureLastMagnitudeX += GESTURE_SCRLSENS } while (ev.detail.magnitudeX - this._gestureLastMagnitudeX < -GESTURE_SCRLSENS) { - this.control.scroll({ x: -1, y: 0 }) + this.control.scroll({ delta_x: -1, delta_y: 0 }) this._gestureLastMagnitudeX -= GESTURE_SCRLSENS } break @@ -429,16 +429,14 @@ this.control.move(pos) magnitude = Math.hypot(ev.detail.magnitudeX, ev.detail.magnitudeY) if (Math.abs(magnitude - this._gestureLastMagnitudeX) > GESTURE_ZOOMSENS) { - this.control.keyDown(KeyTable.XK_Control_L) while (magnitude - this._gestureLastMagnitudeX > GESTURE_ZOOMSENS) { - this.control.scroll({ x: 0, y: 1 }) + this.control.scroll({ delta_x: 0, delta_y: 1, control_key: true }) this._gestureLastMagnitudeX += GESTURE_ZOOMSENS } while (magnitude - this._gestureLastMagnitudeX < -GESTURE_ZOOMSENS) { - this.control.scroll({ x: 0, y: -1 }) + this.control.scroll({ delta_x: 0, delta_y: -1, control_key: true }) this._gestureLastMagnitudeX -= GESTURE_ZOOMSENS } - this.control.keyUp(KeyTable.XK_Control_L) } break } @@ -484,6 +482,23 @@ } } + getModifierState(e: MouseEvent): KeyboardModifiers { + // we can only use locks, because when someone holds key outside + // of the renderer, and releases it inside, keyup event is not fired + // by guacamole keyboard and modifier state is not updated + + return { + //shift: e.getModifierState('Shift'), + capslock: e.getModifierState('CapsLock'), + //control: e.getModifierState('Control'), + //alt: e.getModifierState('Alt'), + numlock: e.getModifierState('NumLock'), + //meta: e.getModifierState('Meta'), + //super: e.getModifierState('Super'), + //altgr: e.getModifierState('AltGraph'), + } + } + getMousePos(clientX: number, clientY: number) { const rect = this._overlay.getBoundingClientRect() @@ -597,7 +612,11 @@ if (x == 0 && y == 0) return // TODO: add position for precision scrolling - this.control.scroll({ x, y }) + this.control.scroll({ + delta_x: x, + delta_y: y, + control_key: e.ctrlKey, + }) } lastMouseMove = 0 @@ -662,10 +681,7 @@ onMouseLeave(e: MouseEvent) { if (this.isControling) { // save current keyboard modifiers state - this.keyboardModifiers = { - capslock: e.getModifierState('CapsLock'), - numlock: e.getModifierState('NumLock'), - } + this.keyboardModifiers = this.getModifierState(e) } this.focused = false @@ -742,15 +758,13 @@ private keyboardModifiers: KeyboardModifiers | null = null updateKeyboardModifiers(e: MouseEvent) { - const capslock = e.getModifierState('CapsLock') - const numlock = e.getModifierState('NumLock') + const mods = this.getModifierState(e) + const newMods = Object.values(mods).join() + const oldMods = Object.values(this.keyboardModifiers || {}).join() - if ( - this.keyboardModifiers === null || - this.keyboardModifiers.capslock !== capslock || - this.keyboardModifiers.numlock !== numlock - ) { - this.$emit('updateKeyboardModifiers', { capslock, numlock }) + // update keyboard modifiers only if they changed + if (newMods !== oldMods) { + this.$emit('updateKeyboardModifiers', mods) } } diff --git a/src/component/types/cursors.ts b/src/component/types/cursors.ts index 95e246c8..9a725203 100644 --- a/src/component/types/cursors.ts +++ b/src/component/types/cursors.ts @@ -22,6 +22,12 @@ export interface Dimension { } export interface KeyboardModifiers { - capslock: boolean - numlock: boolean + shift?: boolean + capslock?: boolean + control?: boolean + alt?: boolean + numlock?: boolean + meta?: boolean + super?: boolean + altgr?: boolean } diff --git a/src/component/types/messages.ts b/src/component/types/messages.ts index e5d04b4b..172175fc 100644 --- a/src/component/types/messages.ts +++ b/src/component/types/messages.ts @@ -113,6 +113,12 @@ export interface ControlHost { host_id: string | undefined } +export interface ControlScroll { + delta_x: number + delta_y: number + control_key: boolean +} + export interface ControlPos { x: number y: number @@ -127,7 +133,7 @@ export interface ControlKey extends Partial { } export interface ControlTouch extends Partial { - touchId: number + touch_id: number pressure: number }