add interactivity.

src/Neko.vue

@@ -7,7 +7,15 @@
 
     <div ref="container" class="player-container">
       <video ref="video" />
-      <div ref="overlay" class="overlay" tabindex="0" @click.stop.prevent @contextmenu.stop.prevent />
+      <neko-overlay
+        v-if="websocket_state == 'connected' && webrtc_state == 'connected'"
+        :webrtc="webrtc"
+        :screenWidth="1280"
+        :screenHeight="720"
+        :scrollSensitivity="5"
+        :scrollInvert="true"
+        :isControling="true"
+      />
     </div>
   </div>
 </template>
@@ -31,14 +39,6 @@
         display: none !important;
       }
     }
-
-    .overlay {
-      position: absolute;
-      top: 0;
-      bottom: 0;
-      width: 100%;
-      height: 100%;
-    }
   }
 </style>
 
@@ -48,8 +48,13 @@
   import { NekoWebSocket } from './internal/websocket'
   import { NekoWebRTC } from './internal/webrtc'
 
+  import Overlay from '~/components/overlay.vue'
+
   @Component({
     name: 'neko',
+    components: {
+      'neko-overlay': Overlay,
+    },
   })
   export default class extends Vue {
     @Ref('component') readonly _component!: HTMLElement
@@ -58,20 +63,20 @@
 
     private observer = new ResizeObserver(this.onResize.bind(this))
 
-    protected _websocket?: NekoWebSocket
+    websocket: NekoWebSocket | null = null
-    protected _webrtc?: NekoWebRTC
+    webrtc: NekoWebRTC | null = null
 
     private websocket_state = 'disconnected'
     private webrtc_state = 'disconnected'
 
     public connect() {
       try {
-        this._websocket?.connect('ws://192.168.1.20:3000/', 'admin')
+        this.websocket?.connect('ws://192.168.1.20:3000/', 'admin')
       } catch (e) {}
     }
 
     public disconnect() {
-      this._websocket?.disconnect()
+      this.websocket?.disconnect()
     }
 
     mounted() {
@@ -80,13 +85,13 @@
       this.observer.observe(this._component)
 
       // WebSocket
-      this._websocket = new NekoWebSocket()
+      this.websocket = new NekoWebSocket()
-      this._websocket?.on('message', async (event: string, payload: any) => {
+      this.websocket?.on('message', async (event: string, payload: any) => {
         switch (event) {
           case 'signal/provide':
             try {
-              let sdp = await this._webrtc?.connect(payload.sdp, payload.lite, payload.ice)
+              let sdp = await this.webrtc?.connect(payload.sdp, payload.lite, payload.ice)
-              this._websocket?.send('signal/answer', { sdp, displayname: 'test' })
+              this.websocket?.send('signal/answer', { sdp, displayname: 'test' })
             } catch (e) {}
             break
           case 'screen/resolution':
@@ -98,20 +103,20 @@
             console.log(event, payload)
         }
       })
-      this._websocket?.on('connecting', () => {
+      this.websocket?.on('connecting', () => {
         this.websocket_state = 'connecting'
       })
-      this._websocket?.on('connected', () => {
+      this.websocket?.on('connected', () => {
         this.websocket_state = 'connected'
       })
-      this._websocket?.on('disconnected', () => {
+      this.websocket?.on('disconnected', () => {
         this.websocket_state = 'disconnected'
-        this._webrtc?.disconnect()
+        this.webrtc?.disconnect()
       })
 
       // WebRTC
-      this._webrtc = new NekoWebRTC()
+      this.webrtc = new NekoWebRTC()
-      this._webrtc?.on('track', (event: RTCTrackEvent) => {
+      this.webrtc?.on('track', (event: RTCTrackEvent) => {
         const { track, streams } = event
         if (track.kind === 'audio') {
           return
@@ -127,20 +132,20 @@
 
         this._video.play()
       })
-      this._webrtc?.on('connecting', () => {
+      this.webrtc?.on('connecting', () => {
         this.webrtc_state = 'connecting'
       })
-      this._webrtc?.on('connected', () => {
+      this.webrtc?.on('connected', () => {
         this.webrtc_state = 'connected'
       })
-      this._webrtc?.on('disconnected', () => {
+      this.webrtc?.on('disconnected', () => {
         this.webrtc_state = 'disconnected'
       })
     }
 
     destroyed() {
-      this._webrtc?.disconnect()
+      this.webrtc?.disconnect()
-      this._websocket?.disconnect()
+      this.websocket?.disconnect()
     }
 
     public onResize() {

src/components/overlay.vue

@@ -0,0 +1,171 @@
+<template>
+  <div
+    ref="overlay"
+    class="overlay"
+    tabindex="0"
+    @click.stop.prevent
+    @contextmenu.stop.prevent
+    @wheel.stop.prevent="onWheel"
+    @mousemove.stop.prevent="onMouseMove"
+    @mousedown.stop.prevent="onMouseDown"
+    @mouseup.stop.prevent="onMouseUp"
+    @mouseenter.stop.prevent="onMouseEnter"
+    @mouseleave.stop.prevent="onMouseLeave"
+  />
+</template>
+
+<style lang="scss" scoped>
+  .overlay {
+    position: absolute;
+    top: 0;
+    bottom: 0;
+    width: 100%;
+    height: 100%;
+  }
+</style>
+
+<script lang="ts">
+  import { Vue, Component, Ref, Prop } from 'vue-property-decorator'
+  import GuacamoleKeyboard from '~/utils/guacamole-keyboard.ts'
+  import { NekoWebRTC } from '~/internal/webrtc'
+
+  @Component({
+    name: 'neko-overlay',
+  })
+  export default class extends Vue {
+    @Ref('overlay') readonly _overlay!: HTMLElement
+
+    private keyboard = GuacamoleKeyboard()
+    private focused = false
+
+    @Prop()
+    private readonly webrtc!: NekoWebRTC
+
+    @Prop()
+    private readonly screenWidth!: number
+
+    @Prop()
+    private readonly screenHeight!: number
+
+    @Prop()
+    private readonly scrollSensitivity!: number
+
+    @Prop()
+    private readonly scrollInvert!: boolean
+
+    @Prop()
+    private readonly isControling!: boolean
+
+    mounted() {
+      // Initialize Guacamole Keyboard
+      this.keyboard.onkeydown = (key: number) => {
+        if (!this.focused || !this.isControling) {
+          return true
+        }
+
+        this.webrtc.send('keydown', { key })
+        return false
+      }
+      this.keyboard.onkeyup = (key: number) => {
+        if (!this.focused || !this.isControling) {
+          return
+        }
+
+        this.webrtc.send('keyup', { key })
+      }
+      this.keyboard.listenTo(this._overlay)
+    }
+
+    beforeDestroy() {
+      // Guacamole Keyboard does not provide destroy functions
+    }
+
+    setMousePos(e: MouseEvent) {
+      const rect = this._overlay.getBoundingClientRect()
+
+      this.webrtc.send('mousemove', {
+        x: Math.round((this.screenWidth / rect.width) * (e.clientX - rect.left)),
+        y: Math.round((this.screenHeight / rect.height) * (e.clientY - rect.top)),
+      })
+    }
+
+    onWheel(e: WheelEvent) {
+      if (!this.isControling) {
+        return
+      }
+
+      let x = e.deltaX
+      let y = e.deltaY
+
+      if (this.scrollInvert) {
+        x *= -1
+        y *= -1
+      }
+
+      x = Math.min(Math.max(x, -this.scrollSensitivity), this.scrollSensitivity)
+      y = Math.min(Math.max(y, -this.scrollSensitivity), this.scrollSensitivity)
+
+      this.setMousePos(e)
+      this.webrtc.send('wheel', { x, y })
+    }
+
+    onMouseMove(e: MouseEvent) {
+      if (!this.isControling) {
+        return
+      }
+
+      this.setMousePos(e)
+    }
+
+    onMouseDown(e: MouseEvent) {
+      if (!this.isControling) {
+        return
+      }
+
+      this.setMousePos(e)
+      this.webrtc.send('mousedown', { key: e.button + 1 })
+    }
+
+    onMouseUp(e: MouseEvent) {
+      if (!this.isControling) {
+        return
+      }
+
+      this.setMousePos(e)
+      this.webrtc.send('mouseup', { key: e.button + 1 })
+    }
+
+    onMouseEnter(e: MouseEvent) {
+      if (!this.isControling) {
+        return
+      }
+
+      // TODO: Refactor
+      //syncKeyboardModifierState({
+      //  capsLock: e.getModifierState('CapsLock'),
+      //  numLock: e.getModifierState('NumLock'),
+      //  scrollLock: e.getModifierState('ScrollLock'),
+      //})
+
+      this._overlay.focus()
+      this.focused = true
+    }
+
+    onMouseLeave(e: MouseEvent) {
+      if (!this.isControling) {
+        return
+      }
+
+      // TODO: Refactor
+      //setKeyboardModifierState({
+      //  capsLock: e.getModifierState('CapsLock'),
+      //  numLock: e.getModifierState('NumLock'),
+      //  scrollLock: e.getModifierState('ScrollLock'),
+      //})
+
+      this.keyboard.reset()
+      this._overlay.blur()
+      this.focused = false
+    }
+  }
+</script>

src/utils/guacamole-keyboard.ts

@@ -0,0 +1,76 @@
+import GuacamoleKeyboard from './guacamole-keyboard.js'
+
+export interface GuacamoleKeyboardInterface {
+  /**
+   * Fired whenever the user presses a key with the element associated
+   * with this Guacamole.Keyboard in focus.
+   *
+   * @event
+   * @param {Number} keysym The keysym of the key being pressed.
+   * @return {Boolean} true if the key event should be allowed through to the
+   *                   browser, false otherwise.
+   */
+  onkeydown?: (keysym: number) => boolean
+
+  /**
+   * Fired whenever the user releases a key with the element associated
+   * with this Guacamole.Keyboard in focus.
+   *
+   * @event
+   * @param {Number} keysym The keysym of the key being released.
+   */
+  onkeyup?: (keysym: number) => void
+
+  /**
+   * Marks a key as pressed, firing the keydown event if registered. Key
+   * repeat for the pressed key will start after a delay if that key is
+   * not a modifier. The return value of this function depends on the
+   * return value of the keydown event handler, if any.
+   *
+   * @param {Number} keysym The keysym of the key to press.
+   * @return {Boolean} true if event should NOT be canceled, false otherwise.
+   */
+  press: (keysym: number) => boolean
+
+  /**
+   * Marks a key as released, firing the keyup event if registered.
+   *
+   * @param {Number} keysym The keysym of the key to release.
+   */
+  release: (keysym: number) => void
+
+  /**
+   * Presses and releases the keys necessary to type the given string of
+   * text.
+   *
+   * @param {String} str
+   *     The string to type.
+   */
+  type: (str: string) => void
+
+  /**
+   * Resets the state of this keyboard, releasing all keys, and firing keyup
+   * events for each released key.
+   */
+  reset: () => void
+
+  /**
+   * Attaches event listeners to the given Element, automatically translating
+   * received key, input, and composition events into simple keydown/keyup
+   * events signalled through this Guacamole.Keyboard's onkeydown and
+   * onkeyup handlers.
+   *
+   * @param {Element|Document} element
+   *     The Element to attach event listeners to for the sake of handling
+   *     key or input events.
+   */
+  listenTo: (element: Element | Document) => void
+}
+
+export default function (element?: Element): GuacamoleKeyboardInterface {
+  var Keyboard = {}
+
+  GuacamoleKeyboard.bind(Keyboard, element)()
+
+  return Keyboard as GuacamoleKeyboardInterface
+}