xorg refactor.

server/internal/capture/manager.go

@@ -1,7 +1,6 @@
 package capture
 
 import (
-	"fmt"
 	"time"
 
 	"m1k1o/neko/internal/capture/gst"
@@ -127,7 +126,7 @@ func (manager *CaptureManagerCtx) Streaming() bool {
 
 func (manager *CaptureManagerCtx) createPipelines() {
 	// handle maximum fps
-	rate := int(manager.desktop.GetScreenSize().Rate)
+	rate := manager.desktop.GetScreenSize().Rate
 	if manager.config.MaxFPS != 0 && manager.config.MaxFPS < rate {
 		rate = manager.config.MaxFPS
 	}
@@ -158,11 +157,7 @@ func (manager *CaptureManagerCtx) createPipelines() {
 	}
 }
 
-func (manager *CaptureManagerCtx) ChangeResolution(width int, height int, rate int) error {
+func (manager *CaptureManagerCtx) ChangeResolution(width int, height int, rate int16) error {
-	if !xorg.ValidScreenSize(width, height, rate) {
-		return fmt.Errorf("unknown configuration")
-	}
-
 	manager.video.Stop()
 	manager.broadcast.Stop()
 

server/internal/capture/pipelines.go

@@ -52,7 +52,7 @@ func CreateRTMPPipeline(pipelineDevice string, pipelineDisplay string, pipelineS
 }
 
 // CreateAppPipeline creates a GStreamer Pipeline
-func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc string, fps int, bitrate uint, hwenc string) (*gst.Pipeline, error) {
+func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc string, fps int16, bitrate uint, hwenc string) (*gst.Pipeline, error) {
 	pipelineStr := " ! appsink name=appsink"
 
 	// if using custom pipeline

server/internal/config/capture.go

@@ -15,7 +15,7 @@ type Capture struct {
 	VideoCodec   string
 	VideoParams  string
 	VideoBitrate uint
-	MaxFPS       int
+	MaxFPS       int16
 }
 
 func (Capture) Init(cmd *cobra.Command) error {
@@ -135,5 +135,5 @@ func (s *Capture) Set() {
 	s.VideoParams = viper.GetString("video")
 	s.VideoBitrate = viper.GetUint("video_bitrate")
 
-	s.MaxFPS = viper.GetInt("max_fps")
+	s.MaxFPS = int16(viper.GetInt("max_fps"))
 }

server/internal/config/desktop.go

@@ -14,7 +14,7 @@ type Desktop struct {
 
 	ScreenWidth  int
 	ScreenHeight int
-	ScreenRate   int
+	ScreenRate   int16
 }
 
 func (Desktop) Init(cmd *cobra.Command) error {
@@ -45,7 +45,7 @@ func (s *Desktop) Set() {
 		if err1 == nil && err2 == nil && err3 == nil {
 			s.ScreenWidth = int(width)
 			s.ScreenHeight = int(height)
-			s.ScreenRate = int(rate)
+			s.ScreenRate = int16(rate)
 		}
 	}
 }

server/internal/desktop/manager.go

@@ -1,6 +1,7 @@
 package desktop
 
 import (
+	"fmt"
 	"sync"
 	"time"
 
@@ -28,14 +29,17 @@ func New(config *config.Desktop, broadcast types.BroadcastManager) *DesktopManag
 }
 
 func (manager *DesktopManagerCtx) Start() {
-	xorg.Display(manager.config.Display)
+	if xorg.DisplayOpen(manager.config.Display) {
-
+		manager.logger.Panic().Str("display", manager.config.Display).Msg("unable to open display")
-	if !xorg.ValidScreenSize(manager.config.ScreenWidth, manager.config.ScreenHeight, manager.config.ScreenRate) {
-		manager.logger.Warn().Msgf("invalid screen option %dx%d@%d", manager.config.ScreenWidth, manager.config.ScreenHeight, manager.config.ScreenRate)
-	} else if err := xorg.ChangeScreenSize(manager.config.ScreenWidth, manager.config.ScreenHeight, manager.config.ScreenRate); err != nil {
-		manager.logger.Warn().Err(err).Msg("unable to change screen size")
 	}
 
+	xorg.GetScreenConfigurations()
+
+	err := xorg.ChangeScreenSize(manager.config.ScreenWidth, manager.config.ScreenHeight, manager.config.ScreenRate)
+	manager.logger.Err(err).
+		Str("screen_size", fmt.Sprintf("%dx%d@%d", manager.config.ScreenWidth, manager.config.ScreenHeight, manager.config.ScreenRate)).
+		Msgf("setting initial screen size")
+
 	manager.wg.Add(1)
 
 	go func() {
@@ -61,5 +65,6 @@ func (manager *DesktopManagerCtx) Shutdown() error {
 	close(manager.shutdown)
 	manager.wg.Wait()
 
+	xorg.DisplayClose()
 	return nil
 }

server/internal/desktop/xorg.go

@@ -1,7 +1,10 @@
 package desktop
 
 import (
+	"image"
 	"os/exec"
+	"regexp"
+	"time"
 
 	"m1k1o/neko/internal/desktop/xorg"
 	"m1k1o/neko/internal/types"
@@ -11,26 +14,54 @@ func (manager *DesktopManagerCtx) Move(x, y int) {
 	xorg.Move(x, y)
 }
 
+func (manager *DesktopManagerCtx) GetCursorPosition() (int, int) {
+	return xorg.GetCursorPosition()
+}
+
 func (manager *DesktopManagerCtx) Scroll(x, y int) {
 	xorg.Scroll(x, y)
 }
 
-func (manager *DesktopManagerCtx) ButtonDown(code int) error {
+func (manager *DesktopManagerCtx) ButtonDown(code uint32) error {
 	return xorg.ButtonDown(code)
 }
 
-func (manager *DesktopManagerCtx) KeyDown(code uint64) error {
+func (manager *DesktopManagerCtx) KeyDown(code uint32) error {
 	return xorg.KeyDown(code)
 }
 
-func (manager *DesktopManagerCtx) ButtonUp(code int) error {
+func (manager *DesktopManagerCtx) ButtonUp(code uint32) error {
 	return xorg.ButtonUp(code)
 }
 
-func (manager *DesktopManagerCtx) KeyUp(code uint64) error {
+func (manager *DesktopManagerCtx) KeyUp(code uint32) error {
 	return xorg.KeyUp(code)
 }
 
+func (manager *DesktopManagerCtx) ButtonPress(code uint32) error {
+	xorg.ResetKeys()
+	defer xorg.ResetKeys()
+
+	return xorg.ButtonDown(code)
+}
+
+func (manager *DesktopManagerCtx) KeyPress(codes ...uint32) error {
+	xorg.ResetKeys()
+	defer xorg.ResetKeys()
+
+	for _, code := range codes {
+		if err := xorg.KeyDown(code); err != nil {
+			return err
+		}
+	}
+
+	if len(codes) > 1 {
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	return nil
+}
+
 func (manager *DesktopManagerCtx) ResetKeys() {
 	xorg.ResetKeys()
 }
@@ -39,14 +70,72 @@ func (manager *DesktopManagerCtx) ScreenConfigurations() map[int]types.ScreenCon
 	return xorg.ScreenConfigurations
 }
 
+func (manager *DesktopManagerCtx) SetScreenSize(size types.ScreenSize) error {
+	return xorg.ChangeScreenSize(size.Width, size.Height, size.Rate)
+}
+
 func (manager *DesktopManagerCtx) GetScreenSize() *types.ScreenSize {
 	return xorg.GetScreenSize()
 }
 
-func (manager *DesktopManagerCtx) SetKeyboardLayout(layout string) {
+func (manager *DesktopManagerCtx) SetKeyboardMap(kbd types.KeyboardMap) error {
-	_ = exec.Command("setxkbmap", layout).Run()
+	// TOOD: Use native API.
+	cmd := exec.Command("setxkbmap", "-layout", kbd.Layout, "-variant", kbd.Variant)
+	_, err := cmd.Output()
+	return err
 }
 
-func (manager *DesktopManagerCtx) SetKeyboardModifiers(NumLock int, CapsLock int, ScrollLock int) {
+func (manager *DesktopManagerCtx) GetKeyboardMap() (*types.KeyboardMap, error) {
-	xorg.SetKeyboardModifiers(NumLock, CapsLock, ScrollLock)
+	// TOOD: Use native API.
+	cmd := exec.Command("setxkbmap", "-query")
+	res, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+
+	kbd := types.KeyboardMap{}
+
+	re := regexp.MustCompile(`layout:\s+(.*)\n`)
+	arr := re.FindStringSubmatch(string(res))
+	if len(arr) > 1 {
+		kbd.Layout = arr[1]
+	}
+
+	re = regexp.MustCompile(`variant:\s+(.*)\n`)
+	arr = re.FindStringSubmatch(string(res))
+	if len(arr) > 1 {
+		kbd.Variant = arr[1]
+	}
+
+	return &kbd, nil
+}
+
+func (manager *DesktopManagerCtx) SetKeyboardModifiers(mod types.KeyboardModifiers) {
+	if mod.NumLock != nil {
+		xorg.SetKeyboardModifier(xorg.KbdModNumLock, *mod.NumLock)
+	}
+
+	if mod.CapsLock != nil {
+		xorg.SetKeyboardModifier(xorg.KbdModCapsLock, *mod.CapsLock)
+	}
+}
+
+func (manager *DesktopManagerCtx) GetKeyboardModifiers() types.KeyboardModifiers {
+	modifiers := xorg.GetKeyboardModifiers()
+
+	NumLock := (modifiers & xorg.KbdModNumLock) != 0
+	CapsLock := (modifiers & xorg.KbdModCapsLock) != 0
+
+	return types.KeyboardModifiers{
+		NumLock:  &NumLock,
+		CapsLock: &CapsLock,
+	}
+}
+
+func (manager *DesktopManagerCtx) GetCursorImage() *types.CursorImage {
+	return xorg.GetCursorImage()
+}
+
+func (manager *DesktopManagerCtx) GetScreenshotImage() *image.RGBA {
+	return xorg.GetScreenshotImage()
 }

server/internal/desktop/xorg/xorg.c

@@ -1,97 +1,35 @@
 #include "xorg.h"
 
 static Display *DISPLAY = NULL;
-static char *NAME = ":0.0";
-static int REGISTERED = 0;
-static int DIRTY = 0;
-
-xkeys_t *xKeysHead = NULL;
-
-void XKeysInsert(KeySym keysym, KeyCode keycode) {
-  xkeys_t *node = (xkeys_t *) malloc(sizeof(xkeys_t));
-
-  node->keysym = keysym;
-  node->keycode = keycode;
-  node->next = xKeysHead;
-  xKeysHead = node;
-}
-
-KeyCode XKeysPop(KeySym keysym) {
-  KeyCode keycode = 0;
-  xkeys_t *node = xKeysHead,
-          *previous = NULL;
-  int i = 0;
-
-  while (node) {
-    if (node->keysym == keysym) {
-      keycode = node->keycode;
-
-      if (!previous)
-        xKeysHead = node->next;
-      else
-        previous->next = node->next;
-
-      free(node);
-      return keycode;
-    }
-
-    previous = node;
-    node = node->next;
-    if (i++ > 120) {
-      // this should NEVER HAPPEN
-      fprintf(stderr, "[FATAL-ERROR] XKeysPop() in xorg.c: reached maximum loop limit! Something is wrong\n");
-      break;
-    }
-  }
-
-  return 0;
-}
 
 Display *getXDisplay(void) {
-  /* Close the display if displayName has changed */
-  if (DIRTY) {
-    XDisplayClose();
-    DIRTY = 0;
-  }
-
-  if (DISPLAY == NULL) {
-    /* First try the user set displayName */
-    DISPLAY = XOpenDisplay(NAME);
-
-    /* Then try using environment variable DISPLAY */
-    if (DISPLAY == NULL) {
-      DISPLAY = XOpenDisplay(NULL);
-    }
-
-    if (DISPLAY == NULL) {
-      fprintf(stderr, "[FATAL-ERROR] XKeysPop() in xorg.c: Could not open main display!");
-    } else if (!REGISTERED) {
-      atexit(&XDisplayClose);
-      REGISTERED = 1;
-    }
-  }
-
   return DISPLAY;
 }
 
-void XDisplayClose(void) {
+int XDisplayOpen(char *name) {
-  if (DISPLAY != NULL) {
+  DISPLAY = XOpenDisplay(name);
-    XCloseDisplay(DISPLAY);
+  return DISPLAY == NULL;
-    DISPLAY = NULL;
-  }
 }
 
-void XDisplaySet(char *input) {
+void XDisplayClose(void) {
-  NAME = strdup(input);
+  XCloseDisplay(DISPLAY);
-  DIRTY = 1;
 }
 
 void XMove(int x, int y) {
   Display *display = getXDisplay();
-  XWarpPointer(display, None, DefaultRootWindow(display),  0, 0, 0, 0, x, y);
+  XWarpPointer(display, None, DefaultRootWindow(display), 0, 0, 0, 0, x, y);
   XSync(display, 0);
 }
 
+void XCursorPosition(int *x, int *y) {
+  Display *display = getXDisplay();
+  Window root = DefaultRootWindow(display);
+  Window window;
+  int i;
+  unsigned mask;
+  XQueryPointer(display, root, &root, &window, x, y, &i, &i, &mask);
+}
+
 void XScroll(int x, int y) {
   int ydir = 4; /* Button 4 is up, 5 is down. */
   int xdir = 6;
@@ -123,15 +61,55 @@ void XScroll(int x, int y) {
 }
 
 void XButton(unsigned int button, int down) {
-  if (button != 0) {
+  if (button == 0)
-    Display *display = getXDisplay();
+    return;
-    XTestFakeButtonEvent(display, button, down, CurrentTime);
+
-    XSync(display, 0);
+  Display *display = getXDisplay();
-  }
+  XTestFakeButtonEvent(display, button, down, CurrentTime);
+  XSync(display, 0);
 }
 
-// From: https://github.com/TigerVNC/tigervnc/blob/0946e298075f8f7b6d63e552297a787c5f84d27c/unix/x0vncserver/XDesktop.cxx#L343-L379
+static xkeyentry_t *xKeysHead = NULL;
-KeyCode XkbKeysymToKeycode(Display *dpy, KeySym keysym) {
+
+void XKeyEntryAdd(KeySym keysym, KeyCode keycode) {
+  xkeyentry_t *entry = (xkeyentry_t *) malloc(sizeof(xkeyentry_t));
+  if (entry == NULL)
+    return;
+
+  entry->keysym = keysym;
+  entry->keycode = keycode;
+  entry->next = xKeysHead;
+  xKeysHead = entry;
+}
+
+KeyCode XKeyEntryGet(KeySym keysym) {
+  xkeyentry_t *prev = NULL;
+  xkeyentry_t *curr = xKeysHead;
+
+  KeyCode keycode = 0;
+  while (curr != NULL) {
+    if (curr->keysym == keysym) {
+      keycode = curr->keycode;
+
+      if (prev == NULL) {
+        xKeysHead = curr->next;
+      } else {
+        prev->next = curr->next;
+      }
+
+      free(curr);
+      return keycode;
+    }
+
+    prev = curr;
+    curr = curr->next;
+  }
+
+  return 0;
+}
+
+// From https://github.com/TigerVNC/tigervnc/blob/0946e298075f8f7b6d63e552297a787c5f84d27c/unix/x0vncserver/XDesktop.cxx#L343-L379
+KeyCode XkbKeysymToKeycode(Display* dpy, KeySym keysym) {
   XkbDescPtr xkb;
   XkbStateRec state;
   unsigned int mods;
@@ -169,51 +147,52 @@ KeyCode XkbKeysymToKeycode(Display *dpy, KeySym keysym) {
   return keycode;
 }
 
-void XKey(KeySym key, int down) {
+void XKey(KeySym keysym, int down) {
+  if (keysym == 0)
+    return;
+
   Display *display = getXDisplay();
-  KeyCode code = 0;
+  KeyCode keycode = 0;
 
   if (!down)
-    code = XKeysPop(key);
+    keycode = XKeyEntryGet(keysym);
 
-  if (!code)
+  if (keycode == 0)
-    code = XkbKeysymToKeycode(display, key);
+    keycode = XkbKeysymToKeycode(display, keysym);
 
-  if (!code) {
+  // Map non-existing keysyms to new keycodes
+  if (keycode == 0) {
     int min, max, numcodes;
     XDisplayKeycodes(display, &min, &max);
     XGetKeyboardMapping(display, min, max-min, &numcodes);
 
-    code = (max-min+1)*numcodes;
+    keycode = (max-min+1)*numcodes;
     KeySym keysym_list[numcodes];
-    for (int i=0;i<numcodes;i++) keysym_list[i] = key;
+    for(int i=0;i<numcodes;i++) keysym_list[i] = keysym;
-    XChangeKeyboardMapping(display, code, numcodes, keysym_list, 1);
+    XChangeKeyboardMapping(display, keycode, numcodes, keysym_list, 1);
   }
 
-  if (!code)
-    return;
-
   if (down)
-    XKeysInsert(key, code);
+    XKeyEntryAdd(keysym, keycode);
 
-  XTestFakeKeyEvent(display, code, down, CurrentTime);
+  XTestFakeKeyEvent(display, keycode, down, CurrentTime);
   XSync(display, 0);
 }
 
 void XGetScreenConfigurations() {
-  Display       *display = getXDisplay();
+  Display *display = getXDisplay();
-  Window        root = RootWindow(display, 0);
+  Window root = RootWindow(display, 0);
   XRRScreenSize *xrrs;
-  int           num_sizes;
+  int num_sizes;
 
   xrrs = XRRSizes(display, 0, &num_sizes);
-  for(int i = 0; i < num_sizes; i ++) {
+  for (int i = 0; i < num_sizes; i++) {
     short *rates;
-    int   num_rates;
+    int num_rates;
 
     goCreateScreenSize(i, xrrs[i].width, xrrs[i].height, xrrs[i].mwidth, xrrs[i].mheight);
     rates = XRRRates(display, 0, i, &num_rates);
-    for (int j = 0; j < num_rates; j ++) {
+    for (int j = 0; j < num_rates; j++) {
       goSetScreenRates(i, j, rates[j]);
     }
   }
@@ -227,34 +206,61 @@ void XSetScreenConfiguration(int index, short rate) {
 
 int XGetScreenSize() {
   Display *display = getXDisplay();
-  XRRScreenConfiguration *conf  = XRRGetScreenInfo(display, RootWindow(display, 0));
+  XRRScreenConfiguration *conf = XRRGetScreenInfo(display, RootWindow(display, 0));
   Rotation original_rotation;
   return XRRConfigCurrentConfiguration(conf, &original_rotation);
 }
 
 short XGetScreenRate() {
   Display *display = getXDisplay();
-  XRRScreenConfiguration *conf  = XRRGetScreenInfo(display, RootWindow(display, 0));
+  XRRScreenConfiguration *conf = XRRGetScreenInfo(display, RootWindow(display, 0));
   return XRRConfigCurrentRate(conf);
 }
 
-void SetKeyboardModifiers(int num_lock, int caps_lock, int scroll_lock) {
+void XSetKeyboardModifier(int mod, int on) {
   Display *display = getXDisplay();
-
+  XkbLockModifiers(display, XkbUseCoreKbd, mod, on ? mod : 0);
-  if (num_lock != -1) {
-    XkbLockModifiers(display, XkbUseCoreKbd, 16, num_lock * 16);
-  }
-
-  if (caps_lock != -1) {
-    XkbLockModifiers(display, XkbUseCoreKbd, 2, caps_lock * 2);
-  }
-
-  if (scroll_lock != -1) {
-    XKeyboardControl values;
-    values.led_mode = scroll_lock ? LedModeOn : LedModeOff;
-    values.led = 3;
-    XChangeKeyboardControl(display, KBLedMode, &values);
-  }
-
   XFlush(display);
 }
+
+char XGetKeyboardModifiers() {
+  XkbStateRec xkbState;
+  Display *display = getXDisplay();
+  XkbGetState(display, XkbUseCoreKbd, &xkbState);
+  return xkbState.locked_mods;
+}
+
+XFixesCursorImage *XGetCursorImage(void) {
+  Display *display = getXDisplay();
+  return XFixesGetCursorImage(display);
+}
+
+char *XGetScreenshot(int *w, int *h) {
+  Display *display = getXDisplay();
+  Window root = DefaultRootWindow(display);
+
+  XWindowAttributes attr;
+  XGetWindowAttributes(display, root, &attr);
+  int width = attr.width;
+  int height = attr.height;
+
+  XImage *ximage = XGetImage(display, root, 0, 0, width, height, AllPlanes, ZPixmap);
+
+  *w = width;
+  *h = height;
+  char *pixels = (char *)malloc(width * height * 3);
+
+	for (int row = 0; row < height; row++) {
+		for (int col = 0; col < width; col++) {
+			int pos = ((row * width) + col) * 3;
+      unsigned long pixel = XGetPixel(ximage, col, row);
+
+			pixels[pos]   = (pixel & ximage->red_mask)   >> 16;
+			pixels[pos+1] = (pixel & ximage->green_mask) >> 8;
+			pixels[pos+2] =  pixel & ximage->blue_mask;
+    }
+  }
+
+  XDestroyImage(ximage);
+  return pixels;
+}

server/internal/desktop/xorg/xorg.go

@@ -1,8 +1,7 @@
 package xorg
 
 /*
-#cgo CFLAGS: -I/usr/local/include/
+#cgo LDFLAGS: -lX11 -lXrandr -lXtst -lXfixes
-#cgo LDFLAGS: -lX11 -lXtst -lXrandr -lxcb
 
 #include "xorg.h"
 */
@@ -10,6 +9,8 @@ import "C"
 
 import (
 	"fmt"
+	"image"
+	"image/color"
 	"sync"
 	"time"
 	"unsafe"
@@ -17,24 +18,44 @@ import (
 	"m1k1o/neko/internal/types"
 )
 
+//go:generate ./keysymdef.sh
+
+type KbdMod uint8
+
+const (
+	KbdModCapsLock KbdMod = 2
+	KbdModNumLock  KbdMod = 16
+)
+
 var ScreenConfigurations = make(map[int]types.ScreenConfiguration)
 
-var debounce_button = make(map[int]time.Time)
+var debounce_button = make(map[uint32]time.Time)
-var debounce_key = make(map[uint64]time.Time)
+var debounce_key = make(map[uint32]time.Time)
 var mu = sync.Mutex{}
 
-func init() {
+func GetScreenConfigurations() {
+	mu.Lock()
+	defer mu.Unlock()
+
 	C.XGetScreenConfigurations()
 }
 
-func Display(display string) {
+func DisplayOpen(display string) bool {
 	mu.Lock()
 	defer mu.Unlock()
 
 	displayUnsafe := C.CString(display)
 	defer C.free(unsafe.Pointer(displayUnsafe))
 
-	C.XDisplaySet(displayUnsafe)
+	ok := C.XDisplayOpen(displayUnsafe)
+	return int(ok) == 1
+}
+
+func DisplayClose() {
+	mu.Lock()
+	defer mu.Unlock()
+
+	C.XDisplayClose()
 }
 
 func Move(x, y int) {
@@ -44,6 +65,17 @@ func Move(x, y int) {
 	C.XMove(C.int(x), C.int(y))
 }
 
+func GetCursorPosition() (int, int) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	var x C.int
+	var y C.int
+	C.XCursorPosition(&x, &y)
+
+	return int(x), int(y)
+}
+
 func Scroll(x, y int) {
 	mu.Lock()
 	defer mu.Unlock()
@@ -51,7 +83,7 @@ func Scroll(x, y int) {
 	C.XScroll(C.int(x), C.int(y))
 }
 
-func ButtonDown(code int) error {
+func ButtonDown(code uint32) error {
 	mu.Lock()
 	defer mu.Unlock()
 
@@ -65,7 +97,7 @@ func ButtonDown(code int) error {
 	return nil
 }
 
-func KeyDown(code uint64) error {
+func KeyDown(code uint32) error {
 	mu.Lock()
 	defer mu.Unlock()
 
@@ -79,7 +111,7 @@ func KeyDown(code uint64) error {
 	return nil
 }
 
-func ButtonUp(code int) error {
+func ButtonUp(code uint32) error {
 	mu.Lock()
 	defer mu.Unlock()
 
@@ -93,7 +125,7 @@ func ButtonUp(code int) error {
 	return nil
 }
 
-func KeyUp(code uint64) error {
+func KeyUp(code uint32) error {
 	mu.Lock()
 	defer mu.Unlock()
 
@@ -108,60 +140,52 @@ func KeyUp(code uint64) error {
 }
 
 func ResetKeys() {
-	for code := range debounce_button {
+	mu.Lock()
-		_ = ButtonUp(code)
+	defer mu.Unlock()
 
+	for code := range debounce_button {
+		C.XButton(C.uint(code), C.int(0))
 		delete(debounce_button, code)
 	}
-	for code := range debounce_key {
-		_ = KeyUp(code)
 
+	for code := range debounce_key {
+		C.XKey(C.KeySym(code), C.int(0))
 		delete(debounce_key, code)
 	}
 }
 
 func CheckKeys(duration time.Duration) {
+	mu.Lock()
+	defer mu.Unlock()
+
 	t := time.Now()
 	for code, start := range debounce_button {
 		if t.Sub(start) < duration {
 			continue
 		}
-		_ = ButtonUp(code)
 
+		C.XButton(C.uint(code), C.int(0))
 		delete(debounce_button, code)
 	}
+
 	for code, start := range debounce_key {
 		if t.Sub(start) < duration {
 			continue
 		}
-		_ = KeyUp(code)
 
+		C.XKey(C.KeySym(code), C.int(0))
 		delete(debounce_key, code)
 	}
 }
 
-func ValidScreenSize(width int, height int, rate int) bool {
+func ChangeScreenSize(width int, height int, rate int16) error {
-	for _, size := range ScreenConfigurations {
-		if size.Width == width && size.Height == height {
-			for _, fps := range size.Rates {
-				if int16(rate) == fps {
-					return true
-				}
-			}
-		}
-	}
-
-	return false
-}
-
-func ChangeScreenSize(width int, height int, rate int) error {
 	mu.Lock()
 	defer mu.Unlock()
 
 	for index, size := range ScreenConfigurations {
 		if size.Width == width && size.Height == height {
 			for _, fps := range size.Rates {
-				if int16(rate) == fps {
+				if rate == fps {
 					C.XSetScreenConfiguration(C.int(index), C.short(fps))
 					return nil
 				}
@@ -169,7 +193,7 @@ func ChangeScreenSize(width int, height int, rate int) error {
 		}
 	}
 
-	return fmt.Errorf("unknown configuration")
+	return fmt.Errorf("unknown screen configuration %dx%d@%d", width, height, rate)
 }
 
 func GetScreenSize() *types.ScreenSize {
@@ -190,11 +214,88 @@ func GetScreenSize() *types.ScreenSize {
 	return nil
 }
 
-func SetKeyboardModifiers(num_lock int, caps_lock int, scroll_lock int) {
+func SetKeyboardModifier(mod KbdMod, active bool) {
 	mu.Lock()
 	defer mu.Unlock()
 
-	C.SetKeyboardModifiers(C.int(num_lock), C.int(caps_lock), C.int(scroll_lock))
+	num := C.int(0)
+	if active {
+		num = C.int(1)
+	}
+
+	C.XSetKeyboardModifier(C.int(mod), num)
+}
+
+func GetKeyboardModifiers() KbdMod {
+	mu.Lock()
+	defer mu.Unlock()
+
+	return KbdMod(C.XGetKeyboardModifiers())
+}
+
+func GetCursorImage() *types.CursorImage {
+	mu.Lock()
+	defer mu.Unlock()
+
+	cur := C.XGetCursorImage()
+	defer C.XFree(unsafe.Pointer(cur))
+
+	width := int(cur.width)
+	height := int(cur.height)
+
+	// Xlib stores 32-bit data in longs, even if longs are 64-bits long.
+	pixels := C.GoBytes(unsafe.Pointer(cur.pixels), C.int(width*height*8))
+
+	img := image.NewRGBA(image.Rect(0, 0, width, height))
+	for y := 0; y < height; y++ {
+		for x := 0; x < width; x++ {
+			pos := ((y * width) + x) * 8
+
+			img.SetRGBA(x, y, color.RGBA{
+				A: pixels[pos+3],
+				R: pixels[pos+2],
+				G: pixels[pos+1],
+				B: pixels[pos+0],
+			})
+		}
+	}
+
+	return &types.CursorImage{
+		Width:  uint16(width),
+		Height: uint16(height),
+		Xhot:   uint16(cur.xhot),
+		Yhot:   uint16(cur.yhot),
+		Serial: uint64(cur.cursor_serial),
+		Image:  img,
+	}
+}
+
+func GetScreenshotImage() *image.RGBA {
+	mu.Lock()
+	defer mu.Unlock()
+
+	var w, h C.int
+	pixelsUnsafe := C.XGetScreenshot(&w, &h)
+	pixels := C.GoBytes(unsafe.Pointer(pixelsUnsafe), w*h*3)
+	defer C.free(unsafe.Pointer(pixelsUnsafe))
+
+	width := int(w)
+	height := int(h)
+	img := image.NewRGBA(image.Rect(0, 0, width, height))
+	for row := 0; row < height; row++ {
+		for col := 0; col < width; col++ {
+			pos := ((row * width) + col) * 3
+
+			img.SetRGBA(col, row, color.RGBA{
+				R: uint8(pixels[pos]),
+				G: uint8(pixels[pos+1]),
+				B: uint8(pixels[pos+2]),
+				A: 0xFF,
+			})
+		}
+	}
+
+	return img
 }
 
 //export goCreateScreenSize
@@ -207,6 +308,13 @@ func goCreateScreenSize(index C.int, width C.int, height C.int, mwidth C.int, mh
 }
 
 //export goSetScreenRates
-func goSetScreenRates(index C.int, rate_index C.int, rate C.short) {
+func goSetScreenRates(index C.int, rate_index C.int, rateC C.short) {
-	ScreenConfigurations[int(index)].Rates[int(rate_index)] = int16(rate)
+	rate := int16(rateC)
+
+	// filter out all irrelevant rates
+	if rate > 60 || (rate > 30 && rate%10 != 0) {
+		return
+	}
+
+	ScreenConfigurations[int(index)].Rates[int(rate_index)] = rate
 }

server/internal/desktop/xorg/xorg.h

@@ -1,39 +1,43 @@
 #pragma once
 
+#include <X11/Xlib.h>
 #include <X11/XKBlib.h>
+#include <X11/Xutil.h>
 #include <X11/extensions/Xrandr.h>
 #include <X11/extensions/XTest.h>
-#include <stdlib.h> /* For free() */
+#include <X11/extensions/Xfixes.h>
-#include <stdio.h> /* For fputs() */
+#include <stdlib.h>
-#include <string.h> /* For strdup() */
 
 extern void goCreateScreenSize(int index, int width, int height, int mwidth, int mheight);
 extern void goSetScreenRates(int index, int rate_index, short rate);
 
-typedef struct xkeys_t {
-  KeySym keysym;
-  KeyCode keycode;
-  struct xkeys_t *next;
-} xkeys_t;
-
-/* Returns the main display, closed either on exit or when closeMainDisplay()
-* is invoked. This removes a bit of the overhead of calling XOpenDisplay() &
-* XCloseDisplay() everytime the main display needs to be used.
-*
-* Note that this is almost certainly not thread safe. */
 Display *getXDisplay(void);
+int XDisplayOpen(char *input);
+void XDisplayClose(void);
 
 void XMove(int x, int y);
+void XCursorPosition(int *x, int *y);
 void XScroll(int x, int y);
 void XButton(unsigned int button, int down);
-void XKey(unsigned long key, int down);
+
+typedef struct xkeyentry_t {
+  KeySym keysym;
+  KeyCode keycode;
+  struct xkeyentry_t *next;
+} xkeyentry_t;
+
+static void XKeyEntryAdd(KeySym keysym, KeyCode keycode);
+static KeyCode XKeyEntryGet(KeySym keysym);
+static KeyCode XkbKeysymToKeycode(Display *dpy, KeySym keysym);
+void XKey(KeySym keysym, int down);
 
 void XGetScreenConfigurations();
 void XSetScreenConfiguration(int index, short rate);
 int XGetScreenSize();
 short XGetScreenRate();
 
-void XDisplayClose(void);
+void XSetKeyboardModifier(int mod, int on);
-void XDisplaySet(char *input);
+char XGetKeyboardModifiers();
+XFixesCursorImage *XGetCursorImage(void);
 
-void SetKeyboardModifiers(int num_lock, int caps_lock, int scroll_lock);
+char *XGetScreenshot(int *w, int *h);

server/internal/types/capture.go

@@ -10,5 +10,5 @@ type CaptureManager interface {
 	StartStream()
 	StopStream()
 	Streaming() bool
-	ChangeResolution(width int, height int, rate int) error
+	ChangeResolution(width int, height int, rate int16) error
 }

server/internal/types/desktop.go

@@ -1,5 +1,16 @@
 package types
 
+import "image"
+
+type CursorImage struct {
+	Width  uint16
+	Height uint16
+	Xhot   uint16
+	Yhot   uint16
+	Serial uint64
+	Image  *image.RGBA
+}
+
 type ScreenSize struct {
 	Width  int   `json:"width"`
 	Height int   `json:"height"`
@@ -12,22 +23,42 @@ type ScreenConfiguration struct {
 	Rates  map[int]int16 `json:"rates"`
 }
 
+type KeyboardModifiers struct {
+	NumLock  *bool
+	CapsLock *bool
+}
+
+type KeyboardMap struct {
+	Layout  string
+	Variant string
+}
+
 type DesktopManager interface {
 	Start()
 	Shutdown() error
+
 	// clipboard
 	ReadClipboard() string
 	WriteClipboard(data string)
+
 	// xorg
 	Move(x, y int)
+	GetCursorPosition() (int, int)
 	Scroll(x, y int)
-	ButtonDown(code int) error
+	ButtonDown(code uint32) error
-	KeyDown(code uint64) error
+	KeyDown(code uint32) error
-	ButtonUp(code int) error
+	ButtonUp(code uint32) error
-	KeyUp(code uint64) error
+	KeyUp(code uint32) error
+	ButtonPress(code uint32) error
+	KeyPress(codes ...uint32) error
 	ResetKeys()
 	ScreenConfigurations() map[int]ScreenConfiguration
+	SetScreenSize(ScreenSize) error
 	GetScreenSize() *ScreenSize
-	SetKeyboardLayout(layout string)
+	SetKeyboardMap(KeyboardMap) error
-	SetKeyboardModifiers(NumLock int, CapsLock int, ScrollLock int)
+	GetKeyboardMap() (*KeyboardMap, error)
+	SetKeyboardModifiers(mod KeyboardModifiers)
+	GetKeyboardModifiers() KeyboardModifiers
+	GetCursorImage() *CursorImage
+	GetScreenshotImage() *image.RGBA
 }

server/internal/types/message/messages.go

@@ -70,7 +70,7 @@ type Keyboard struct {
 	Layout     *string `json:"layout,omitempty"`
 	CapsLock   *bool   `json:"capsLock,omitempty"`
 	NumLock    *bool   `json:"numLock,omitempty"`
-	ScrollLock *bool   `json:"scrollLock,omitempty"`
+	ScrollLock *bool   `json:"scrollLock,omitempty"` // TODO: ScrollLock is deprecated.
 }
 
 type Control struct {
@@ -128,7 +128,7 @@ type ScreenResolution struct {
 	ID     string `json:"id,omitempty"`
 	Width  int    `json:"width"`
 	Height int    `json:"height"`
-	Rate   int    `json:"rate"`
+	Rate   int16  `json:"rate"`
 }
 
 type ScreenConfigurations struct {

server/internal/webrtc/handle.go

@@ -35,7 +35,7 @@ type PayloadScroll struct {
 
 type PayloadKey struct {
 	PayloadHeader
-	Key uint64
+	Key uint64 // TODO: uint32
 }
 
 func (manager *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) error {
@@ -85,7 +85,7 @@ func (manager *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) e
 		}
 
 		if payload.Key < 8 {
-			err := manager.desktop.ButtonDown(int(payload.Key))
+			err := manager.desktop.ButtonDown(uint32(payload.Key))
 			if err != nil {
 				manager.logger.Warn().Err(err).Msg("button down failed")
 				return nil
@@ -93,7 +93,7 @@ func (manager *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) e
 
 			manager.logger.Debug().Msgf("button down %d", payload.Key)
 		} else {
-			err := manager.desktop.KeyDown(uint64(payload.Key))
+			err := manager.desktop.KeyDown(uint32(payload.Key))
 			if err != nil {
 				manager.logger.Warn().Err(err).Msg("key down failed")
 				return nil
@@ -109,7 +109,7 @@ func (manager *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) e
 		}
 
 		if payload.Key < 8 {
-			err := manager.desktop.ButtonUp(int(payload.Key))
+			err := manager.desktop.ButtonUp(uint32(payload.Key))
 			if err != nil {
 				manager.logger.Warn().Err(err).Msg("button up failed")
 				return nil
@@ -117,7 +117,7 @@ func (manager *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) e
 
 			manager.logger.Debug().Msgf("button up %d", payload.Key)
 		} else {
-			err := manager.desktop.KeyUp(uint64(payload.Key))
+			err := manager.desktop.KeyUp(uint32(payload.Key))
 			if err != nil {
 				manager.logger.Warn().Err(err).Msg("key up failed")
 				return nil

server/internal/websocket/handler/control.go

@@ -140,41 +140,18 @@ func (h *MessageHandler) controlKeyboard(id string, session types.Session, paylo
 		return nil
 	}
 
+	h.desktop.SetKeyboardModifiers(types.KeyboardModifiers{
+		NumLock:  payload.NumLock,
+		CapsLock: payload.CapsLock,
+		// TODO: ScrollLock is deprecated.
+	})
+
 	// change layout
 	if payload.Layout != nil {
-		h.desktop.SetKeyboardLayout(*payload.Layout)
+		return h.desktop.SetKeyboardMap(types.KeyboardMap{
+			Layout: *payload.Layout,
+		})
 	}
 
-	// set num lock
-	var NumLock = 0
-	if payload.NumLock == nil {
-		NumLock = -1
-	} else if *payload.NumLock {
-		NumLock = 1
-	}
-
-	// set caps lock
-	var CapsLock = 0
-	if payload.CapsLock == nil {
-		CapsLock = -1
-	} else if *payload.CapsLock {
-		CapsLock = 1
-	}
-
-	// set scroll lock
-	var ScrollLock = 0
-	if payload.ScrollLock == nil {
-		ScrollLock = -1
-	} else if *payload.ScrollLock {
-		ScrollLock = 1
-	}
-
-	h.logger.Debug().
-		Int("NumLock", NumLock).
-		Int("CapsLock", CapsLock).
-		Int("ScrollLock", ScrollLock).
-		Msg("setting keyboard modifiers")
-
-	h.desktop.SetKeyboardModifiers(NumLock, CapsLock, ScrollLock)
 	return nil
 }

server/internal/websocket/handler/screen.go

@@ -38,7 +38,7 @@ func (h *MessageHandler) screenResolution(id string, session types.Session) erro
 			Event:  event.SCREEN_RESOLUTION,
 			Width:  size.Width,
 			Height: size.Height,
-			Rate:   int(size.Rate),
+			Rate:   size.Rate,
 		}); err != nil {
 			h.logger.Warn().Err(err).Msgf("sending event %s has failed", event.SCREEN_RESOLUTION)
 			return err