Merge pull request #92 from m1k1o/guacamole-kbd

Guacamole Keyboard support + choose layout

client/public/keyboard_layouts.json

@@ -0,0 +1 @@
+{"af":"Afghani","al":"Albanian","et":"Amharic","ma":"Arabic (Morocco)","sy":"Arabic (Syria)","am":"Armenian","az":"Azerbaijani","ml":"Bambara","bd":"Bangla","by":"Belarusian","be":"Belgian","dz":"Berber (Algeria, Latin characters)","ba":"Bosnian","bg":"Bulgarian","mm":"Burmese","hr":"Croatian","cz":"Czech","dk":"Danish","mv":"Dhivehi","nl":"Dutch","bt":"Dzongkha","au":"English (Australian)","cm":"English (Cameroon)","gh":"English (Ghana)","ng":"English (Nigeria)","za":"English (South Africa)","us":"English (US)","gb":"English (UK)","ee":"Estonian","fo":"Faroese","ph":"Filipino","fi":"Finnish","fr":"French","ca":"French (Canada)","cd":"French (Democratic Republic of the Congo)","gn":"French (Guinea)","tg":"French (Togo)","ge":"Georgian","de":"German","at":"German (Austria)","ch":"German (Switzerland)","gr":"Greek","il":"Hebrew","hu":"Hungarian","cn":"Chinese","is":"Icelandic","in":"Indian","id":"Indonesian (Jawi)","iq":"Iraqi","ie":"Irish","it":"Italian","jp":"Japanese","kz":"Kazakh","kh":"Khmer (Cambodia)","kr":"Korean","kg":"Kyrgyz","la":"Lao","lv":"Latvian","lt":"Lithuanian","mk":"Macedonian","my":"Malay (Jawi)","mt":"Maltese","md":"Moldavian","mn":"Mongolian","me":"Montenegrin","np":"Nepali","no":"Norwegian","ir":"Persian","pl":"Polish","pt":"Portuguese","br":"Portuguese (Brazil)","ro":"Romanian","ru":"Russian","rs":"Serbian","lk":"Sinhala (phonetic)","sk":"Slovak","si":"Slovenian","es":"Spanish","ke":"Swahili (Kenya)","tz":"Swahili (Tanzania)","se":"Swedish","tw":"Taiwanese","tj":"Tajik","th":"Thai","bw":"Tswana","tr":"Turkish","tm":"Turkmen","ua":"Ukrainian","pk":"Urdu (Pakistan)","uz":"Uzbek","vn":"Vietnamese","sn":"Wolof"}

client/src/components/settings.vue

@@ -35,6 +35,19 @@
           <span />
         </label>
       </li>
+      <li>
+        <span>{{ $t('setting.keyboard_layout') }}</span>
+        <label class="select">
+          <select v-model="keyboard_layout">
+            <option
+              v-for="(name, code) in keyboard_layouts_list"
+              :key="code"
+              :value="code"
+            >{{ name }}</option>
+          </select>
+          <span />
+        </label>
+      </li>
       <li v-if="connected">
         <button @click.stop.prevent="logout">{{ $t('logout') }}</button>
       </li>
@@ -182,6 +195,31 @@
             }
           }
         }
+
+        .select {
+          max-width: 120px;
+
+          select {
+            display: block;
+            width: 100%;
+            max-width: 100%;
+            padding: 4px;
+            margin: 0;
+            line-height: 30px;
+            font-weight: bold;
+            border: 0;
+            border-radius: 12px;
+
+            color: black;
+            background-color: $style-primary;
+
+            option {
+              font-weight: normal;
+              color: $text-normal;
+              background-color: $background-tertiary;
+            }
+          }
+        }
       }
     }
   }
@@ -236,6 +274,19 @@
       this.$accessor.settings.setSound(value)
     }
 
+    get keyboard_layouts_list() {
+      return this.$accessor.settings.keyboard_layouts_list
+    }
+
+    get keyboard_layout() {
+      return this.$accessor.settings.keyboard_layout
+    }
+
+    set keyboard_layout(value: string) {
+      this.$accessor.settings.setKeyboardLayout(value)
+      this.$accessor.remote.changeKeyboard()
+    }
+
     logout() {
       this.$accessor.logout()
     }

client/src/components/video.vue

@@ -20,8 +20,6 @@
           @mouseup.stop.prevent="onMouseUp"
           @mouseenter.stop.prevent="onMouseEnter"
           @mouseleave.stop.prevent="onMouseLeave"
-          @keydown.stop.prevent="onKeyDown"
-          @keyup.stop.prevent="onKeyUp"
         />
         <div v-if="!playing" class="player-overlay">
           <i @click.stop.prevent="toggle" v-if="playable" class="fas fa-play-circle" />
@@ -142,6 +140,8 @@
   import Emote from './emote.vue'
   import Resolution from './resolution.vue'
 
+  import GuacamoleKeyboard from '~/utils/guacamole-keyboard.ts'
+
   @Component({
     name: 'neko-video',
     components: {
@@ -158,10 +158,10 @@
     @Ref('video') readonly _video!: HTMLVideoElement
     @Ref('resolution') readonly _resolution!: any
 
+    private keyboard = GuacamoleKeyboard()
     private observer = new ResizeObserver(this.onResise.bind(this))
     private focused = false
     private fullscreen = false
-    private activeKeys: Set<number> = new Set()
 
     get admin() {
       return this.$accessor.user.admin
@@ -335,6 +335,24 @@
 
       document.addEventListener('focusin', this.onFocus.bind(this))
       document.addEventListener('focusout', this.onBlur.bind(this))
+
+      /* Initialize Guacamole Keyboard */
+      this.keyboard.onkeydown = (key: number) => {
+        if (!this.focused || !this.hosting || this.locked) {
+          return true
+        }
+
+        this.$client.sendData('keydown', { key })
+        return false
+      }
+      this.keyboard.onkeyup = (key: number) => {
+        if (!this.focused || !this.hosting || this.locked) {
+          return
+        }
+
+        this.$client.sendData('keyup', { key })
+      }
+      this.keyboard.listenTo(this._overlay)
     }
 
     beforeDestroy() {
@@ -342,6 +360,7 @@
       this.$accessor.video.setPlayable(false)
       document.removeEventListener('focusin', this.onFocus.bind(this))
       document.removeEventListener('focusout', this.onBlur.bind(this))
+      /* Guacamole Keyboard does not provide destroy functions */
     }
 
     play() {
@@ -409,10 +428,7 @@
         return
       }
 
-      for (let key of this.activeKeys) {
-        this.$client.sendData('keyup', { key })
-        this.activeKeys.delete(key)
-      }
+      this.keyboard.reset()
     }
 
     onMousePos(e: MouseEvent) {
@@ -449,7 +465,7 @@
         return
       }
       this.onMousePos(e)
-      this.$client.sendData('mousedown', { key: e.button })
+      this.$client.sendData('mousedown', { key: e.button + 1 })
     }
 
     onMouseUp(e: MouseEvent) {
@@ -457,7 +473,7 @@
         return
       }
       this.onMousePos(e)
-      this.$client.sendData('mouseup', { key: e.button })
+      this.$client.sendData('mouseup', { key: e.button + 1 })
     }
 
     onMouseMove(e: MouseEvent) {
@@ -477,44 +493,6 @@
       this.focused = false
     }
 
-    // frick you firefox
-    getCode(e: KeyboardEvent): number {
-      let key = e.keyCode
-      if (key === 59 && (e.key === ';' || e.key === ':')) {
-        key = 186
-      }
-
-      if (key === 61 && (e.key === '=' || e.key === '+')) {
-        key = 187
-      }
-
-      if (key === 173 && (e.key === '-' || e.key === '_')) {
-        key = 189
-      }
-
-      return key
-    }
-
-    onKeyDown(e: KeyboardEvent) {
-      if (!this.focused || !this.hosting || this.locked) {
-        return
-      }
-
-      let key = this.getCode(e)
-      this.$client.sendData('keydown', { key })
-      this.activeKeys.add(key)
-    }
-
-    onKeyUp(e: KeyboardEvent) {
-      if (!this.focused || !this.hosting || this.locked) {
-        return
-      }
-
-      let key = this.getCode(e)
-      this.$client.sendData('keyup', { key })
-      this.activeKeys.delete(key)
-    }
-
     onResise() {
       let height = 0
       if (!this.fullscreen) {

client/src/locale/en-us.ts

@@ -60,6 +60,7 @@ export const setting = {
   autoplay: 'Autoplay Video',
   ignore_emotes: 'Ignore Emotes',
   chat_sound: 'Play Chat Sound',
+  keyboard_layout: 'Change Keyboard Layout',
 }
 
 export const connection = {

client/src/neko/base.ts

@@ -123,19 +123,19 @@ export abstract class BaseClient extends EventEmitter<BaseEvents> {
         break
       case 'keydown':
       case 'mousedown':
-        buffer = new ArrayBuffer(5)
+        buffer = new ArrayBuffer(11)
         payload = new DataView(buffer)
         payload.setUint8(0, OPCODE.KEY_DOWN)
-        payload.setUint16(1, 1, true)
-        payload.setUint16(3, data.key, true)
+        payload.setUint16(1, 8, true)
+        payload.setBigUint64(3, BigInt(data.key), true)
         break
       case 'keyup':
       case 'mouseup':
-        buffer = new ArrayBuffer(5)
+        buffer = new ArrayBuffer(11)
         payload = new DataView(buffer)
         payload.setUint8(0, OPCODE.KEY_UP)
-        payload.setUint16(1, 1, true)
-        payload.setUint16(3, data.key, true)
+        payload.setUint16(1, 8, true)
+        payload.setBigUint64(3, BigInt(data.key), true)
         break
       default:
         this.emit('warn', `unknown data event: ${event}`)

client/src/neko/events.ts

@@ -27,6 +27,7 @@ export const EVENT = {
     REQUESTING: 'control/requesting',
     CLIPBOARD: 'control/clipboard',
     GIVE: 'control/give',
+    KEYBOARD: 'control/keyboard'
   },
   CHAT: {
     MESSAGE: 'chat/message',
@@ -67,6 +68,7 @@ export type ControlEvents =
   | typeof EVENT.CONTROL.REQUEST
   | typeof EVENT.CONTROL.GIVE
   | typeof EVENT.CONTROL.CLIPBOARD
+  | typeof EVENT.CONTROL.KEYBOARD
 
 export type SystemEvents = typeof EVENT.SYSTEM.DISCONNECT
 export type MemberEvents = typeof EVENT.MEMBER.LIST | typeof EVENT.MEMBER.CONNECTED | typeof EVENT.MEMBER.DISCONNECTED

client/src/neko/index.ts

@@ -165,6 +165,8 @@ export class NekoClient extends BaseClient implements EventEmitter<NekoEvents> {
   /////////////////////////////
   protected [EVENT.CONTROL.LOCKED]({ id }: ControlPayload) {
     this.$accessor.remote.setHost(id)
+    this.$accessor.remote.changeKeyboard()
+
     const member = this.member(id)
     if (!member) {
       return
@@ -251,6 +253,8 @@ export class NekoClient extends BaseClient implements EventEmitter<NekoEvents> {
     }
 
     this.$accessor.remote.setHost(member)
+    this.$accessor.remote.changeKeyboard()
+
     this.$accessor.chat.newMessage({
       id,
       content: this.$vue.$t('notifications.controls_given', {
@@ -431,6 +435,7 @@ export class NekoClient extends BaseClient implements EventEmitter<NekoEvents> {
 
   protected [EVENT.ADMIN.CONTROL]({ id, target }: AdminTargetPayload) {
     this.$accessor.remote.setHost(id)
+    this.$accessor.remote.changeKeyboard()
 
     if (!target) {
       this.$accessor.chat.newMessage({
@@ -495,6 +500,7 @@ export class NekoClient extends BaseClient implements EventEmitter<NekoEvents> {
     }
 
     this.$accessor.remote.setHost(member)
+    this.$accessor.remote.changeKeyboard()
 
     this.$accessor.chat.newMessage({
       id,

client/src/neko/messages.ts

@@ -30,6 +30,7 @@ export type WebSocketPayloads =
   | Member
   | ControlPayload
   | ControlClipboardPayload
+  | ControlKeyboardPayload
   | ChatPayload
   | ChatSendPayload
   | EmojiSendPayload
@@ -120,6 +121,10 @@ export interface ControlClipboardPayload {
   text: string
 }
 
+export interface ControlKeyboardPayload {
+  layout: string
+}
+
 /*
   CHAT PAYLOADS
 */

client/src/store/index.ts

@@ -55,6 +55,7 @@ export const actions = actionTree(
   {
     initialise(store) {
       accessor.emoji.initialise()
+      accessor.settings.initialise()
     },
 
     lock() {

client/src/store/remote.ts

@@ -133,5 +133,13 @@ export const actions = actionTree(
 
       $client.sendMessage(EVENT.ADMIN.GIVE, { id: member.id })
     },
+
+    changeKeyboard({ getters }) {
+      if (!accessor.connected || !getters.hosting) {
+        return
+      }
+
+      $client.sendMessage(EVENT.CONTROL.KEYBOARD, { layout: accessor.settings.keyboard_layout })
+    }
   },
 )

client/src/store/settings.ts

@@ -1,8 +1,13 @@
-import { getterTree, mutationTree } from 'typed-vuex'
+import { getterTree, mutationTree, actionTree } from 'typed-vuex'
 import { get, set } from '~/utils/localstorage'
+import { accessor } from '~/store'
 
 export const namespaced = true
 
+interface KeyboardLayouts {
+  [code: string]: string
+}
+
 export const state = () => {
   return {
     scroll: get<number>('scroll', 10),
@@ -10,6 +15,9 @@ export const state = () => {
     autoplay: get<boolean>('autoplay', true),
     ignore_emotes: get<boolean>('ignore_emotes', false),
     chat_sound: get<boolean>('chat_sound', true),
+    keyboard_layout: get<string>('keyboard_layout', 'us'),
+
+    keyboard_layouts_list: {} as KeyboardLayouts,
   }
 }
 
@@ -40,4 +48,28 @@ export const mutations = mutationTree(state, {
     state.chat_sound = value
     set('chat_sound', value)
   },
+
+  setKeyboardLayout(state, value: string) {
+    state.keyboard_layout = value
+    set('keyboard_layout', value)
+  },
+
+  setKeyboardLayoutsList(state, value: KeyboardLayouts) {
+    state.keyboard_layouts_list = value
+  },
 })
+
+export const actions = actionTree(
+  { state, getters, mutations },
+  {
+    initialise() {
+      $http
+        .get<KeyboardLayouts>('/keyboard_layouts.json')
+        .then((req) => {
+          accessor.settings.setKeyboardLayoutsList(req.data)
+          console.log(req.data)
+        })
+        .catch(console.error)
+    },
+  },
+)

client/src/utils/guacamole-keyboard.ts

@@ -0,0 +1,76 @@
+import GuacamoleKeyboard from './guacamole-keyboard.js'
+
+export interface GuacamoleKeyboardInterface {
+  /**
+   * Fired whenever the user presses a key with the element associated
+   * with this Guacamole.Keyboard in focus.
+   * 
+   * @event
+   * @param {Number} keysym The keysym of the key being pressed.
+   * @return {Boolean} true if the key event should be allowed through to the
+   *                   browser, false otherwise.
+   */
+  onkeydown?: (keysym: number) => boolean;
+
+  /**
+   * Fired whenever the user releases a key with the element associated
+   * with this Guacamole.Keyboard in focus.
+   * 
+   * @event
+   * @param {Number} keysym The keysym of the key being released.
+   */
+  onkeyup?: (keysym: number) => void;
+
+  /**
+   * Marks a key as pressed, firing the keydown event if registered. Key
+   * repeat for the pressed key will start after a delay if that key is
+   * not a modifier. The return value of this function depends on the
+   * return value of the keydown event handler, if any.
+   * 
+   * @param {Number} keysym The keysym of the key to press.
+   * @return {Boolean} true if event should NOT be canceled, false otherwise.
+   */
+  press: (keysym: number) => boolean;
+
+  /**
+   * Marks a key as released, firing the keyup event if registered.
+   * 
+   * @param {Number} keysym The keysym of the key to release.
+   */
+  release: (keysym: number) => void;
+  
+  /**
+   * Presses and releases the keys necessary to type the given string of
+   * text.
+   *
+   * @param {String} str
+   *     The string to type.
+   */
+  type: (str: string) => void;
+
+  /**
+   * Resets the state of this keyboard, releasing all keys, and firing keyup
+   * events for each released key.
+   */
+  reset: () => void;
+  
+  /**
+   * Attaches event listeners to the given Element, automatically translating
+   * received key, input, and composition events into simple keydown/keyup
+   * events signalled through this Guacamole.Keyboard's onkeydown and
+   * onkeyup handlers.
+   *
+   * @param {Element|Document} element
+   *     The Element to attach event listeners to for the sake of handling
+   *     key or input events.
+   */
+  listenTo: (element: Element | Document) => void;
+}
+
+export default function(element?: Element): GuacamoleKeyboardInterface {
+  var Keyboard = {};
+
+  GuacamoleKeyboard.bind(Keyboard, element)();
+
+  return Keyboard as GuacamoleKeyboardInterface;
+}