diff --git a/internal/desktop/xorg/xorg.c b/internal/desktop/xorg/xorg.c
index 8d446440..dfc59667 100644
--- a/internal/desktop/xorg/xorg.c
+++ b/internal/desktop/xorg/xorg.c
@@ -68,25 +68,114 @@ void XButton(unsigned int button, int down) {
   XSync(display, 0);
 }
 
-void XKey(unsigned long key, int down) {
-  if (key == 0) return;
+static xkeyentry_t *pKeysHead = NULL;
+
+void XKeyEntryAdd(KeySym keysym, KeyCode keycode) {
+  xkeyentry_t *entry = (xkeyentry_t *) malloc(sizeof(xkeyentry_t));
+  if (entry == NULL) {
+    return;
+  }
+
+  entry->keysym = keysym;
+  entry->keycode = keycode;
+  entry->next = pKeysHead;
+  pKeysHead = entry;
+}
+
+KeyCode XKeyEntryGet(KeySym keysym) {
+  xkeyentry_t *prev = NULL;
+  xkeyentry_t *curr = pKeysHead;
+
+  KeyCode keycode = 0;
+  while (curr != NULL) {
+    if (curr->keysym == keysym) {
+      keycode = curr->keycode;
+
+      if (prev == NULL) {
+        pKeysHead = curr->next;
+      } else {
+        prev->next = curr->next;
+      }
+
+      free(curr);
+      return keycode;
+    }
+
+    prev = curr;
+    curr = curr->next;
+  }
+
+  return 0;
+}
+
+// From https://github.com/TigerVNC/tigervnc/blob/0946e298075f8f7b6d63e552297a787c5f84d27c/unix/x0vncserver/XDesktop.cxx#L343-L379
+KeyCode KbdXKeysymToKeycode(Display *dpy, KeySym keysym) {
+  XkbDescPtr xkb;
+  XkbStateRec state;
+  unsigned int mods;
+  unsigned keycode;
+
+  xkb = XkbGetMap(dpy, XkbAllComponentsMask, XkbUseCoreKbd);
+  if (!xkb)
+    return 0;
+
+  XkbGetState(dpy, XkbUseCoreKbd, &state);
+  // XkbStateFieldFromRec() doesn't work properly because
+  // state.lookup_mods isn't properly updated, so we do this manually
+  mods = XkbBuildCoreState(XkbStateMods(&state), state.group);
+
+  for (keycode = xkb->min_key_code;
+       keycode <= xkb->max_key_code;
+       keycode++) {
+    KeySym cursym;
+    unsigned int out_mods;
+    XkbTranslateKeyCode(xkb, keycode, mods, &out_mods, &cursym);
+    if (cursym == keysym)
+      break;
+  }
+
+  if (keycode > xkb->max_key_code)
+    keycode = 0;
+
+  XkbFreeKeyboard(xkb, XkbAllComponentsMask, True);
+
+  // Shift+Tab is usually ISO_Left_Tab, but RFB hides this fact. Do
+  // another attempt if we failed the initial lookup
+  if ((keycode == 0) && (keysym == XK_Tab) && (mods & ShiftMask))
+    return KbdXKeysymToKeycode(dpy, XK_ISO_Left_Tab);
+
+  return keycode;
+}
+
+void XKey(KeySym keysym, int down) {
+  if (keysym == 0)
+    return;
 
   Display *display = getXDisplay();
-  KeyCode code = XKeysymToKeycode(display, key);
+  KeyCode keycode = 0;
+
+  if (!down)
+    keycode = XKeyEntryGet(keysym);
+
+  if (keycode == 0)
+    keycode = KbdXKeysymToKeycode(display, keysym);
 
   // Map non-existing keysyms to new keycodes
-  if (code == 0) {
+  if (keycode == 0) {
     int min, max, numcodes;
     XDisplayKeycodes(display, &min, &max);
     XGetKeyboardMapping(display, min, max-min, &numcodes);
 
-    code = (max-min+1)*numcodes;
+    keycode = (max-min+1)*numcodes;
     KeySym keysym_list[numcodes];
-    for(int i=0;i<numcodes;i++) keysym_list[i] = key;
-    XChangeKeyboardMapping(display, code, numcodes, keysym_list, 1);
+    for(int i=0;i<numcodes;i++) keysym_list[i] = keysym;
+    XChangeKeyboardMapping(display, keycode, numcodes, keysym_list, 1);
   }
 
-  XTestFakeKeyEvent(display, code, down, CurrentTime);
+  if (down)
+    XKeyEntryAdd(keysym, keycode);
+
+  XTestFakeKeyEvent(display, keycode, down, CurrentTime);
   XSync(display, 0);
 }
 
diff --git a/internal/desktop/xorg/xorg.go b/internal/desktop/xorg/xorg.go
index 34146f3b..357b7474 100644
--- a/internal/desktop/xorg/xorg.go
+++ b/internal/desktop/xorg/xorg.go
@@ -106,7 +106,7 @@ func KeyDown(code uint32) error {
 
 	debounce_key[code] = time.Now()
 
-	C.XKey(C.ulong(code), C.int(1))
+	C.XKey(C.KeySym(code), C.int(1))
 	return nil
 }
 
@@ -134,7 +134,7 @@ func KeyUp(code uint32) error {
 
 	delete(debounce_key, code)
 
-	C.XKey(C.ulong(code), C.int(0))
+	C.XKey(C.KeySym(code), C.int(0))
 	return nil
 }
 
@@ -148,7 +148,7 @@ func ResetKeys() {
 	}
 
 	for code := range debounce_key {
-		C.XKey(C.ulong(code), C.int(0))
+		C.XKey(C.KeySym(code), C.int(0))
 		delete(debounce_key, code)
 	}
 }
@@ -172,7 +172,7 @@ func CheckKeys(duration time.Duration) {
 			continue
 		}
 
-		C.XKey(C.ulong(code), C.int(0))
+		C.XKey(C.KeySym(code), C.int(0))
 		delete(debounce_key, code)
 	}
 }
diff --git a/internal/desktop/xorg/xorg.h b/internal/desktop/xorg/xorg.h
index 8fad29c9..2b96b377 100644
--- a/internal/desktop/xorg/xorg.h
+++ b/internal/desktop/xorg/xorg.h
@@ -19,7 +19,17 @@ void XMove(int x, int y);
 void XCursorPosition(int *x, int *y);
 void XScroll(int x, int y);
 void XButton(unsigned int button, int down);
-void XKey(unsigned long key, int down);
+
+typedef struct xkeyentry_t {
+  KeySym keysym;
+  KeyCode keycode;
+  struct xkeyentry_t *next;
+} xkeyentry_t;
+
+static void XKeyEntryAdd(KeySym keysym, KeyCode keycode);
+static KeyCode XKeyEntryGet(KeySym keysym);
+static KeyCode XKeysymToKeycodeRespectingModifiers(Display *display, KeySym keysym);
+void XKey(KeySym keysym, int down);
 
 void XGetScreenConfigurations();
 void XSetScreenConfiguration(int index, short rate);