Add Xorg modifiers (#57)

* implement additional modifiers to xorg.

* xorg modifiers to API.

* update modifiers api & add ws.

* scroll pos rename to delta and add ctrl key.

internal/api/room/keyboard.go

@@ -8,13 +8,11 @@ import (
 )
 
 type KeyboardMapData struct {
-	Layout  string `json:"layout"`
+	types.KeyboardMap
-	Variant string `json:"variant"`
 }
 
 type KeyboardModifiersData struct {
-	NumLock  *bool `json:"numlock"`
+	types.KeyboardModifiers
-	CapsLock *bool `json:"capslock"`
 }
 
 func (h *RoomHandler) keyboardMapSet(w http.ResponseWriter, r *http.Request) error {
@@ -23,11 +21,7 @@ func (h *RoomHandler) keyboardMapSet(w http.ResponseWriter, r *http.Request) err
 		return err
 	}
 
-	err := h.desktop.SetKeyboardMap(types.KeyboardMap{
+	err := h.desktop.SetKeyboardMap(data.KeyboardMap)
-		Layout:  data.Layout,
-		Variant: data.Variant,
-	})
-
 	if err != nil {
 		return utils.HttpInternalServerError().WithInternalErr(err)
 	}
@@ -37,14 +31,12 @@ func (h *RoomHandler) keyboardMapSet(w http.ResponseWriter, r *http.Request) err
 
 func (h *RoomHandler) keyboardMapGet(w http.ResponseWriter, r *http.Request) error {
 	data, err := h.desktop.GetKeyboardMap()
-
 	if err != nil {
 		return utils.HttpInternalServerError().WithInternalErr(err)
 	}
 
 	return utils.HttpSuccess(w, KeyboardMapData{
-		Layout:  data.Layout,
+		KeyboardMap: *data,
-		Variant: data.Variant,
 	})
 }
 
@@ -54,19 +46,12 @@ func (h *RoomHandler) keyboardModifiersSet(w http.ResponseWriter, r *http.Reques
 		return err
 	}
 
-	h.desktop.SetKeyboardModifiers(types.KeyboardModifiers{
+	h.desktop.SetKeyboardModifiers(data.KeyboardModifiers)
-		NumLock:  data.NumLock,
-		CapsLock: data.CapsLock,
-	})
-
 	return utils.HttpSuccess(w)
 }
 
 func (h *RoomHandler) keyboardModifiersGet(w http.ResponseWriter, r *http.Request) error {
-	data := h.desktop.GetKeyboardModifiers()
-
 	return utils.HttpSuccess(w, KeyboardModifiersData{
-		NumLock:  data.NumLock,
+		KeyboardModifiers: h.desktop.GetKeyboardModifiers(),
-		CapsLock: data.CapsLock,
 	})
 }

internal/desktop/xorg.go

@@ -18,8 +18,8 @@ func (manager *DesktopManagerCtx) GetCursorPosition() (int, int) {
 	return xorg.GetCursorPosition()
 }
 
-func (manager *DesktopManagerCtx) Scroll(x, y int) {
+func (manager *DesktopManagerCtx) Scroll(deltaX, deltaY int, controlKey bool) {
-	xorg.Scroll(x, y)
+	xorg.Scroll(deltaX, deltaY, controlKey)
 }
 
 func (manager *DesktopManagerCtx) ButtonDown(code uint32) error {
@@ -140,24 +140,56 @@ func (manager *DesktopManagerCtx) GetKeyboardMap() (*types.KeyboardMap, error) {
 }
 
 func (manager *DesktopManagerCtx) SetKeyboardModifiers(mod types.KeyboardModifiers) {
-	if mod.NumLock != nil {
+	if mod.Shift != nil {
-		xorg.SetKeyboardModifier(xorg.KbdModNumLock, *mod.NumLock)
+		xorg.SetKeyboardModifier(xorg.KbdModShift, *mod.Shift)
 	}
 
 	if mod.CapsLock != nil {
 		xorg.SetKeyboardModifier(xorg.KbdModCapsLock, *mod.CapsLock)
 	}
+
+	if mod.Control != nil {
+		xorg.SetKeyboardModifier(xorg.KbdModControl, *mod.Control)
+	}
+
+	if mod.Alt != nil {
+		xorg.SetKeyboardModifier(xorg.KbdModAlt, *mod.Alt)
+	}
+
+	if mod.NumLock != nil {
+		xorg.SetKeyboardModifier(xorg.KbdModNumLock, *mod.NumLock)
+	}
+
+	if mod.Meta != nil {
+		xorg.SetKeyboardModifier(xorg.KbdModMeta, *mod.Meta)
+	}
+
+	if mod.Super != nil {
+		xorg.SetKeyboardModifier(xorg.KbdModSuper, *mod.Super)
+	}
+
+	if mod.AltGr != nil {
+		xorg.SetKeyboardModifier(xorg.KbdModAltGr, *mod.AltGr)
+	}
 }
 
 func (manager *DesktopManagerCtx) GetKeyboardModifiers() types.KeyboardModifiers {
 	modifiers := xorg.GetKeyboardModifiers()
 
-	NumLock := (modifiers & xorg.KbdModNumLock) != 0
+	isset := func(mod xorg.KbdMod) *bool {
-	CapsLock := (modifiers & xorg.KbdModCapsLock) != 0
+		x := modifiers&mod != 0
+		return &x
+	}
 
 	return types.KeyboardModifiers{
-		NumLock:  &NumLock,
+		Shift:    isset(xorg.KbdModShift),
-		CapsLock: &CapsLock,
+		CapsLock: isset(xorg.KbdModCapsLock),
+		Control:  isset(xorg.KbdModControl),
+		Alt:      isset(xorg.KbdModAlt),
+		NumLock:  isset(xorg.KbdModNumLock),
+		Meta:     isset(xorg.KbdModMeta),
+		Super:    isset(xorg.KbdModSuper),
+		AltGr:    isset(xorg.KbdModAltGr),
 	}
 }
 

internal/webrtc/handler.go

@@ -97,16 +97,31 @@ func (manager *WebRTCManagerCtx) handle(
 
 	switch header.Event {
 	case payload.OP_SCROLL:
-		payload := &payload.Scroll{}
+		// TODO: remove this once the client is fixed
-		if err := binary.Read(buffer, binary.BigEndian, payload); err != nil {
+		if header.Length == 4 {
-			return err
+			payload := &payload.Scroll_Old{}
-		}
+			if err := binary.Read(buffer, binary.BigEndian, payload); err != nil {
+				return err
+			}
 
-		manager.desktop.Scroll(int(payload.X), int(payload.Y))
+			manager.desktop.Scroll(int(payload.X), int(payload.Y), false)
-		logger.Trace().
+			logger.Trace().
-			Int16("x", payload.X).
+				Int16("x", payload.X).
-			Int16("y", payload.Y).
+				Int16("y", payload.Y).
-			Msg("scroll")
+				Msg("scroll")
+		} else {
+			payload := &payload.Scroll{}
+			if err := binary.Read(buffer, binary.BigEndian, payload); err != nil {
+				return err
+			}
+
+			manager.desktop.Scroll(int(payload.DeltaX), int(payload.DeltaY), payload.ControlKey)
+			logger.Trace().
+				Int16("deltaX", payload.DeltaX).
+				Int16("deltaY", payload.DeltaY).
+				Bool("controlKey", payload.ControlKey).
+				Msg("scroll")
+		}
 	case payload.OP_KEY_DOWN:
 		payload := &payload.Key{}
 		if err := binary.Read(buffer, binary.BigEndian, payload); err != nil {

internal/webrtc/payload/receive.go

@@ -21,11 +21,18 @@ type Move struct {
 	Y uint16
 }
 
-type Scroll struct {
+// TODO: remove this once the client is fixed
+type Scroll_Old struct {
 	X int16
 	Y int16
 }
 
+type Scroll struct {
+	DeltaX     int16
+	DeltaY     int16
+	ControlKey bool
+}
+
 type Key struct {
 	Key uint32
 }

internal/websocket/handler/control.go

@@ -74,12 +74,18 @@ func (h *MessageHandlerCtx) controlMove(session types.Session, payload *message.
 	return nil
 }
 
-func (h *MessageHandlerCtx) controlScroll(session types.Session, payload *message.ControlPos) error {
+func (h *MessageHandlerCtx) controlScroll(session types.Session, payload *message.ControlScroll) error {
 	if err := h.controlRequest(session); err != nil && !errors.Is(err, ErrIsAlreadyTheHost) {
 		return err
 	}
 
-	h.desktop.Scroll(payload.X, payload.Y)
+	// TOOD: remove this once the client is fixed
+	if payload.DeltaX == 0 && payload.DeltaY == 0 {
+		payload.DeltaX = payload.X
+		payload.DeltaY = payload.Y
+	}
+
+	h.desktop.Scroll(payload.DeltaX, payload.DeltaY, payload.ControlKey)
 	return nil
 }
 

internal/websocket/handler/handler.go

@@ -88,7 +88,7 @@ func (h *MessageHandlerCtx) Message(session types.Session, data types.WebSocketM
 			return h.controlMove(session, payload)
 		})
 	case event.CONTROL_SCROLL:
-		payload := &message.ControlPos{}
+		payload := &message.ControlScroll{}
 		err = utils.Unmarshal(payload, data.Payload, func() error {
 			return h.controlScroll(session, payload)
 		})

internal/websocket/handler/keyboard.go

@@ -12,10 +12,7 @@ func (h *MessageHandlerCtx) keyboardMap(session types.Session, payload *message.
 		return errors.New("is not the host")
 	}
 
-	return h.desktop.SetKeyboardMap(types.KeyboardMap{
+	return h.desktop.SetKeyboardMap(payload.KeyboardMap)
-		Layout:  payload.Layout,
-		Variant: payload.Variant,
-	})
 }
 
 func (h *MessageHandlerCtx) keyboardModifiers(session types.Session, payload *message.KeyboardModifiers) error {
@@ -23,10 +20,6 @@ func (h *MessageHandlerCtx) keyboardModifiers(session types.Session, payload *me
 		return errors.New("is not the host")
 	}
 
-	h.desktop.SetKeyboardModifiers(types.KeyboardModifiers{
+	h.desktop.SetKeyboardModifiers(payload.KeyboardModifiers)
-		NumLock:  payload.NumLock,
-		CapsLock: payload.CapsLock,
-	})
-
 	return nil
 }

openapi.yaml

@@ -1087,10 +1087,22 @@ components:
     KeyboardModifiers:
       type: object
       properties:
-        numlock:
+        shift:
           type: boolean
         capslock:
           type: boolean
+        control:
+          type: boolean
+        alt:
+          type: boolean
+        numlock:
+          type: boolean
+        meta:
+          type: boolean
+        super:
+          type: boolean
+        altgr:
+          type: boolean
 
     ControlStatus:
       type: object

pkg/types/desktop.go

@@ -25,13 +25,19 @@ func (s ScreenSize) String() string {
 }
 
 type KeyboardModifiers struct {
-	NumLock  *bool
+	Shift    *bool `json:"shift"`
-	CapsLock *bool
+	CapsLock *bool `json:"capslock"`
+	Control  *bool `json:"control"`
+	Alt      *bool `json:"alt"`
+	NumLock  *bool `json:"numlock"`
+	Meta     *bool `json:"meta"`
+	Super    *bool `json:"super"`
+	AltGr    *bool `json:"altgr"`
 }
 
 type KeyboardMap struct {
-	Layout  string
+	Layout  string `json:"layout"`
-	Variant string
+	Variant string `json:"variant"`
 }
 
 type ClipboardText struct {
@@ -48,7 +54,7 @@ type DesktopManager interface {
 	// xorg
 	Move(x, y int)
 	GetCursorPosition() (int, int)
-	Scroll(x, y int)
+	Scroll(deltaX, deltaY int, controlKey bool)
 	ButtonDown(code uint32) error
 	KeyDown(code uint32) error
 	ButtonUp(code uint32) error

pkg/types/message/messages.go

@@ -115,6 +115,16 @@ type ControlHost struct {
 	HostID  string `json:"host_id,omitempty"`
 }
 
+type ControlScroll struct {
+	// TOOD: remove this once the client is fixed
+	X int `json:"x"`
+	Y int `json:"y"`
+
+	DeltaX     int  `json:"delta_x"`
+	DeltaY     int  `json:"delta_y"`
+	ControlKey bool `json:"control_key"`
+}
+
 type ControlPos struct {
 	X int `json:"x"`
 	Y int `json:"y"`
@@ -159,13 +169,11 @@ type ClipboardData struct {
 /////////////////////////////
 
 type KeyboardMap struct {
-	Layout  string `json:"layout"`
+	types.KeyboardMap
-	Variant string `json:"variant"`
 }
 
 type KeyboardModifiers struct {
-	CapsLock *bool `json:"capslock"`
+	types.KeyboardModifiers
-	NumLock  *bool `json:"numlock"`
 }
 
 /////////////////////////////

pkg/xorg/xorg.c

@@ -30,33 +30,33 @@ void XCursorPosition(int *x, int *y) {
   XQueryPointer(display, root, &root, &window, x, y, &i, &i, &mask);
 }
 
-void XScroll(int x, int y) {
+void XScroll(int deltaX, int deltaY) {
-  int ydir = 4; /* Button 4 is up, 5 is down. */
-  int xdir = 6;
-
   Display *display = getXDisplay();
 
-  if (y < 0) {
+  int ydir;
-    ydir = 5;
+  if (deltaY > 0) {
+    ydir = 4; // button 4 is up
+  } else {
+    ydir = 5; // button 5 is down
   }
 
-  if (x < 0) {
+  int xdir;
-    xdir = 7;
+  if (deltaX > 0) {
+    xdir = 6; // button 6 is right
+  } else {
+    xdir = 7; // button 7 is left
   }
 
-  int xi;
+  for (int i = 0; i < abs(deltaY); i++) {
-  int yi;
-
-  for (xi = 0; xi < abs(x); xi++) {
-    XTestFakeButtonEvent(display, xdir, 1, CurrentTime);
-    XTestFakeButtonEvent(display, xdir, 0, CurrentTime);
-  }
-
-  for (yi = 0; yi < abs(y); yi++) {
     XTestFakeButtonEvent(display, ydir, 1, CurrentTime);
     XTestFakeButtonEvent(display, ydir, 0, CurrentTime);
   }
 
+  for (int i = 0; i < abs(deltaX); i++) {
+    XTestFakeButtonEvent(display, xdir, 1, CurrentTime);
+    XTestFakeButtonEvent(display, xdir, 0, CurrentTime);
+  }
+
   XSync(display, 0);
 }
 
@@ -368,17 +368,19 @@ XRRModeInfo XCreateScreenModeInfo(int hdisplay, int vdisplay, short vrefresh) {
   return modeinfo;
 }
 
-void XSetKeyboardModifier(int mod, int on) {
+void XSetKeyboardModifier(unsigned char mod, int on) {
   Display *display = getXDisplay();
   XkbLockModifiers(display, XkbUseCoreKbd, mod, on ? mod : 0);
   XFlush(display);
 }
 
-char XGetKeyboardModifiers() {
+unsigned char XGetKeyboardModifiers() {
   XkbStateRec xkbState;
   Display *display = getXDisplay();
   XkbGetState(display, XkbUseCoreKbd, &xkbState);
-  return xkbState.locked_mods;
+  // XkbStateFieldFromRec() doesn't work properly because
+  // state.lookup_mods isn't properly updated, so we do this manually
+  return XkbBuildCoreState(XkbStateMods(&xkbState), xkbState.group);
 }
 
 XFixesCursorImage *XGetCursorImage(void) {

pkg/xorg/xorg.go

@@ -23,8 +23,14 @@ import (
 type KbdMod uint8
 
 const (
-	KbdModCapsLock KbdMod = 2
+	KbdModShift    KbdMod = C.ShiftMask
-	KbdModNumLock  KbdMod = 16
+	KbdModCapsLock KbdMod = C.LockMask
+	KbdModControl  KbdMod = C.ControlMask
+	KbdModAlt      KbdMod = C.Mod1Mask
+	KbdModNumLock  KbdMod = C.Mod2Mask
+	KbdModMeta     KbdMod = C.Mod3Mask
+	KbdModSuper    KbdMod = C.Mod4Mask
+	KbdModAltGr    KbdMod = C.Mod5Mask
 )
 
 type ScreenConfiguration struct {
@@ -82,11 +88,16 @@ func GetCursorPosition() (int, int) {
 	return int(x), int(y)
 }
 
-func Scroll(x, y int) {
+func Scroll(deltaX, deltaY int, controlKey bool) {
 	mu.Lock()
 	defer mu.Unlock()
 
-	C.XScroll(C.int(x), C.int(y))
+	if controlKey {
+		C.XSetKeyboardModifier(C.uchar(C.ControlMask), 1)
+		defer C.XSetKeyboardModifier(C.uchar(C.ControlMask), 0)
+	}
+
+	C.XScroll(C.int(deltaX), C.int(deltaY))
 }
 
 func ButtonDown(code uint32) error {
@@ -248,7 +259,7 @@ func SetKeyboardModifier(mod KbdMod, active bool) {
 		num = C.int(1)
 	}
 
-	C.XSetKeyboardModifier(C.int(mod), num)
+	C.XSetKeyboardModifier(C.uchar(mod), num)
 }
 
 func GetKeyboardModifiers() KbdMod {

pkg/xorg/xorg.h

@@ -22,7 +22,7 @@ void XDisplayClose(void);
 
 void XMove(int x, int y);
 void XCursorPosition(int *x, int *y);
-void XScroll(int x, int y);
+void XScroll(int deltaX, int deltaY);
 void XButton(unsigned int button, int down);
 
 typedef struct xkeyentry_t {
@@ -42,8 +42,8 @@ void XGetScreenConfigurations();
 void XCreateScreenMode(int width, int height, short rate);
 XRRModeInfo XCreateScreenModeInfo(int hdisplay, int vdisplay, short vrefresh);
 
-void XSetKeyboardModifier(int mod, int on);
+void XSetKeyboardModifier(unsigned char mod, int on);
-char XGetKeyboardModifiers();
+unsigned char XGetKeyboardModifiers();
 XFixesCursorImage *XGetCursorImage(void);
 
 char *XGetScreenshot(int *w, int *h);