seperate remote desktop from webrtc

server/cmd/serve.go

@@ -19,6 +19,7 @@ func init() {
 	configs := []config.Config{
 		neko.Service.Server,
 		neko.Service.WebRTC,
+		neko.Service.Remote,
 		neko.Service.WebSocket,
 	}
 

server/internal/remote/manager.go

@@ -0,0 +1,208 @@
+package remote
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/kataras/go-events"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"n.eko.moe/neko/internal/gst"
+	"n.eko.moe/neko/internal/types"
+	"n.eko.moe/neko/internal/types/config"
+	"n.eko.moe/neko/internal/xorg"
+)
+
+type RemoteManager struct {
+	logger   zerolog.Logger
+	video    *gst.Pipeline
+	audio    *gst.Pipeline
+	config   *config.Remote
+	cleanup  *time.Ticker
+	shutdown chan bool
+	emmiter  events.EventEmmiter
+}
+
+func New(config *config.Remote) *RemoteManager {
+	return &RemoteManager{
+		logger:   log.With().Str("module", "remote").Logger(),
+		cleanup:  time.NewTicker(1 * time.Second),
+		shutdown: make(chan bool),
+		emmiter:  events.New(),
+		config:   config,
+	}
+}
+
+func (manager *RemoteManager) VideoCodec() string {
+	return manager.config.VideoCodec
+}
+
+func (manager *RemoteManager) AudioCodec() string {
+	return manager.config.AudioCodec
+}
+
+func (manager *RemoteManager) Start() {
+	var err error
+	manager.video, err = gst.CreatePipeline(
+		manager.config.VideoCodec,
+		manager.config.Display,
+		manager.config.VideoParams,
+	)
+	if err != nil {
+		manager.logger.Panic().Err(err).Msg("unable to create video pipeline")
+	}
+
+	manager.audio, err = gst.CreatePipeline(
+		manager.config.AudioCodec,
+		manager.config.Device,
+		manager.config.AudioParams,
+	)
+	if err != nil {
+		manager.logger.Panic().Err(err).Msg("unable to screate audio pipeline")
+	}
+
+	manager.StartStream()
+
+	go func() {
+		defer func() {
+			manager.logger.Info().Msg("shutdown")
+		}()
+
+		for {
+			select {
+			case <-manager.shutdown:
+				return
+			case sample := <-manager.video.Sample:
+				manager.emmiter.Emit("video", sample)
+			case sample := <-manager.audio.Sample:
+				manager.emmiter.Emit("audio", sample)
+			case <-manager.cleanup.C:
+				xorg.CheckKeys(time.Second * 10)
+			}
+		}
+	}()
+}
+
+func (manager *RemoteManager) Shutdown() error {
+	manager.logger.Info().Msgf("remote shutting down")
+	manager.video.Stop()
+	manager.audio.Stop()
+	manager.cleanup.Stop()
+	manager.shutdown <- true
+	return nil
+}
+
+func (manager *RemoteManager) OnVideoFrame(listener func(sample types.Sample)) {
+	manager.emmiter.On("video", func(payload ...interface{}) {
+		listener(payload[0].(types.Sample))
+	})
+}
+
+func (manager *RemoteManager) OnAudioFrame(listener func(sample types.Sample)) {
+	manager.emmiter.On("audio", func(payload ...interface{}) {
+		listener(payload[0].(types.Sample))
+	})
+}
+
+func (manager *RemoteManager) StartStream() {
+	manager.logger.Info().
+		Str("video_display", manager.config.Display).
+		Str("video_codec", manager.config.VideoCodec).
+		Str("audio_device", manager.config.Device).
+		Str("audio_codec", manager.config.AudioCodec).
+		Str("audio_pipeline_src", manager.audio.Src).
+		Str("video_pipeline_src", manager.video.Src).
+		Str("screen_resolution", fmt.Sprintf("%dx%d@%d", manager.config.ScreenWidth, manager.config.ScreenHeight, manager.config.ScreenRate)).
+		Msgf("pipelines starting...")
+
+	xorg.Display(manager.config.Display)
+
+	if !xorg.ValidScreenSize(manager.config.ScreenWidth, manager.config.ScreenHeight, manager.config.ScreenRate) {
+		manager.logger.Warn().Msgf("invalid screen option %dx%d@%d", manager.config.ScreenWidth, manager.config.ScreenHeight, manager.config.ScreenRate)
+	} else {
+		if err := xorg.ChangeScreenSize(manager.config.ScreenWidth, manager.config.ScreenHeight, manager.config.ScreenRate); err != nil {
+			manager.logger.Warn().Err(err).Msg("unable to change screen size")
+		}
+	}
+
+	manager.video.Start()
+	manager.audio.Start()
+}
+
+func (manager *RemoteManager) StopStream() {
+	manager.video.Stop()
+	manager.audio.Stop()
+}
+
+func (manager *RemoteManager) ChangeResolution(width int, height int, rate int) error {
+	if !xorg.ValidScreenSize(width, height, rate) {
+		return fmt.Errorf("unknown configuration")
+	}
+
+	manager.video.Stop()
+	defer func() {
+		manager.video.Start()
+		manager.logger.Info().Msg("starting video pipeline...")
+	}()
+
+	if err := xorg.ChangeScreenSize(width, height, rate); err != nil {
+		return err
+	}
+
+	video, err := gst.CreatePipeline(
+		manager.config.VideoCodec,
+		manager.config.Display,
+		manager.config.VideoParams,
+	)
+
+	if err != nil {
+		manager.logger.Panic().Err(err).Msg("unable to create new video pipeline")
+	}
+
+	manager.video = video
+	return nil
+}
+
+func (manager *RemoteManager) Move(x, y int) {
+	xorg.Move(x, y)
+}
+
+func (manager *RemoteManager) Scroll(x, y int) {
+	xorg.Scroll(x, y)
+}
+
+func (manager *RemoteManager) ButtonDown(code int) (*types.Button, error) {
+	return xorg.ButtonDown(code)
+}
+
+func (manager *RemoteManager) KeyDown(code int) (*types.Key, error) {
+	return xorg.KeyDown(code)
+}
+
+func (manager *RemoteManager) ButtonUp(code int) (*types.Button, error) {
+	return xorg.ButtonUp(code)
+}
+
+func (manager *RemoteManager) KeyUp(code int) (*types.Key, error) {
+	return xorg.KeyUp(code)
+}
+
+func (manager *RemoteManager) ReadClipboard() string {
+	return xorg.ReadClipboard()
+}
+
+func (manager *RemoteManager) WriteClipboard(data string) {
+	xorg.WriteClipboard(data)
+}
+
+func (manager *RemoteManager) ResetKeys() {
+	xorg.ResetKeys()
+}
+
+func (manager *RemoteManager) ScreenConfigurations() map[int]types.ScreenConfiguration {
+	return xorg.ScreenConfigurations
+}
+
+func (manager *RemoteManager) GetScreenSize() *types.ScreenSize {
+	return xorg.GetScreenSize()
+}

server/internal/types/config/remote.go

@@ -0,0 +1,136 @@
+package config
+
+import (
+	"regexp"
+	"strconv"
+
+	"github.com/pion/webrtc/v2"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+type Remote struct {
+	Display      string
+	Device       string
+	AudioCodec   string
+	AudioParams  string
+	VideoCodec   string
+	VideoParams  string
+	ScreenWidth  int
+	ScreenHeight int
+	ScreenRate   int
+}
+
+func (Remote) Init(cmd *cobra.Command) error {
+	cmd.PersistentFlags().String("display", ":99.0", "XDisplay to capture")
+	if err := viper.BindPFlag("display", cmd.PersistentFlags().Lookup("display")); err != nil {
+		return err
+	}
+
+	cmd.PersistentFlags().String("device", "auto_null.monitor", "audio device to capture")
+	if err := viper.BindPFlag("device", cmd.PersistentFlags().Lookup("device")); err != nil {
+		return err
+	}
+
+	cmd.PersistentFlags().String("audio", "", "audio codec parameters to use for streaming (unused)")
+	if err := viper.BindPFlag("aparams", cmd.PersistentFlags().Lookup("audio")); err != nil {
+		return err
+	}
+
+	cmd.PersistentFlags().String("video", "", "video codec parameters to use for streaming (unused)")
+	if err := viper.BindPFlag("vparams", cmd.PersistentFlags().Lookup("video")); err != nil {
+		return err
+	}
+
+	cmd.PersistentFlags().String("screen", "1280x720@30", "default screen resolution and framerate")
+	if err := viper.BindPFlag("vparams", cmd.PersistentFlags().Lookup("video")); err != nil {
+		return err
+	}
+
+	// video codecs
+	cmd.PersistentFlags().Bool("vp8", false, "use VP8 video codec")
+	if err := viper.BindPFlag("vp8", cmd.PersistentFlags().Lookup("vp8")); err != nil {
+		return err
+	}
+
+	cmd.PersistentFlags().Bool("vp9", false, "use VP9 video codec")
+	if err := viper.BindPFlag("vp9", cmd.PersistentFlags().Lookup("vp9")); err != nil {
+		return err
+	}
+
+	cmd.PersistentFlags().Bool("h264", false, "use H264 video codec")
+	if err := viper.BindPFlag("h264", cmd.PersistentFlags().Lookup("h264")); err != nil {
+		return err
+	}
+
+	// audio codecs
+	cmd.PersistentFlags().Bool("opus", false, "use Opus audio codec")
+	if err := viper.BindPFlag("opus", cmd.PersistentFlags().Lookup("opus")); err != nil {
+		return err
+	}
+
+	cmd.PersistentFlags().Bool("g722", false, "use G722 audio codec")
+	if err := viper.BindPFlag("g722", cmd.PersistentFlags().Lookup("g722")); err != nil {
+		return err
+	}
+
+	cmd.PersistentFlags().Bool("pcmu", false, "use PCMU audio codec")
+	if err := viper.BindPFlag("pcmu", cmd.PersistentFlags().Lookup("pcmu")); err != nil {
+		return err
+	}
+
+	cmd.PersistentFlags().Bool("pcma", false, "use PCMA audio codec")
+	if err := viper.BindPFlag("pcma", cmd.PersistentFlags().Lookup("pcma")); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Remote) Set() {
+	videoCodec := webrtc.VP8
+	if viper.GetBool("vp8") {
+		videoCodec = webrtc.VP8
+	} else if viper.GetBool("vp9") {
+		videoCodec = webrtc.VP9
+	} else if viper.GetBool("h264") {
+		videoCodec = webrtc.H264
+	}
+
+	audioCodec := webrtc.Opus
+	if viper.GetBool("opus") {
+		audioCodec = webrtc.Opus
+	} else if viper.GetBool("g722") {
+		audioCodec = webrtc.G722
+	} else if viper.GetBool("pcmu") {
+		audioCodec = webrtc.PCMU
+	} else if viper.GetBool("pcma") {
+		audioCodec = webrtc.PCMA
+	}
+
+	s.Device = viper.GetString("device")
+	s.AudioCodec = audioCodec
+	s.AudioParams = viper.GetString("aparams")
+	s.Display = viper.GetString("display")
+	s.VideoCodec = videoCodec
+	s.VideoParams = viper.GetString("vparams")
+
+	s.ScreenWidth = 1280
+	s.ScreenHeight = 720
+	s.ScreenRate = 30
+
+	r := regexp.MustCompile(`([0-9]{1,4})x([0-9]{1,4})@([0-9]{1,3})`)
+	res := r.FindStringSubmatch(viper.GetString("screen"))
+
+	if len(res) > 0 {
+		width, err1 := strconv.ParseInt(res[1], 10, 64)
+		height, err2 := strconv.ParseInt(res[1], 10, 64)
+		rate, err3 := strconv.ParseInt(res[1], 10, 64)
+
+		if err1 == nil && err2 == nil && err3 == nil {
+			s.ScreenWidth = int(width)
+			s.ScreenHeight = int(height)
+			s.ScreenRate = int(rate)
+		}
+	}
+}

server/internal/types/config/webrtc.go

@@ -1,101 +1,28 @@
 package config
 
 import (
-	"regexp"
 	"strconv"
 	"strings"
 
-	"github.com/pion/webrtc/v2"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"n.eko.moe/neko/internal/utils"
 )
 
 type WebRTC struct {
-	Device       string
-	AudioCodec   string
-	AudioParams  string
-	Display      string
 	ICELite      bool
 	ICEServers   []string
-	VideoCodec   string
-	VideoParams  string
 	EphemeralMin uint16
 	EphemeralMax uint16
 	NAT1To1IPs   []string
-	ScreenWidth  int
-	ScreenHeight int
-	ScreenRate   int
 }
 
 func (WebRTC) Init(cmd *cobra.Command) error {
-	cmd.PersistentFlags().String("device", "auto_null.monitor", "audio device to capture")
-	if err := viper.BindPFlag("device", cmd.PersistentFlags().Lookup("device")); err != nil {
-		return err
-	}
-
-	cmd.PersistentFlags().String("audio", "", "audio codec parameters to use for streaming (unused)")
-	if err := viper.BindPFlag("aparams", cmd.PersistentFlags().Lookup("audio")); err != nil {
-		return err
-	}
-
-	cmd.PersistentFlags().String("display", ":99.0", "XDisplay to capture")
-	if err := viper.BindPFlag("display", cmd.PersistentFlags().Lookup("display")); err != nil {
-		return err
-	}
-
-	cmd.PersistentFlags().String("video", "", "video codec parameters to use for streaming (unused)")
-	if err := viper.BindPFlag("vparams", cmd.PersistentFlags().Lookup("video")); err != nil {
-		return err
-	}
-
-	cmd.PersistentFlags().String("screen", "1280x720@30", "default screen resolution and framerate")
-	if err := viper.BindPFlag("vparams", cmd.PersistentFlags().Lookup("video")); err != nil {
-		return err
-	}
-
 	cmd.PersistentFlags().String("epr", "59000-59100", "limits the pool of ephemeral ports that ICE UDP connections can allocate from")
 	if err := viper.BindPFlag("epr", cmd.PersistentFlags().Lookup("epr")); err != nil {
 		return err
 	}
 
-	// video codecs
-	cmd.PersistentFlags().Bool("vp8", false, "use VP8 video codec")
-	if err := viper.BindPFlag("vp8", cmd.PersistentFlags().Lookup("vp8")); err != nil {
-		return err
-	}
-
-	cmd.PersistentFlags().Bool("vp9", false, "use VP9 video codec")
-	if err := viper.BindPFlag("vp9", cmd.PersistentFlags().Lookup("vp9")); err != nil {
-		return err
-	}
-
-	cmd.PersistentFlags().Bool("h264", false, "use H264 video codec")
-	if err := viper.BindPFlag("h264", cmd.PersistentFlags().Lookup("h264")); err != nil {
-		return err
-	}
-
-	// audio codecs
-	cmd.PersistentFlags().Bool("opus", false, "use Opus audio codec")
-	if err := viper.BindPFlag("opus", cmd.PersistentFlags().Lookup("opus")); err != nil {
-		return err
-	}
-
-	cmd.PersistentFlags().Bool("g722", false, "use G722 audio codec")
-	if err := viper.BindPFlag("g722", cmd.PersistentFlags().Lookup("g722")); err != nil {
-		return err
-	}
-
-	cmd.PersistentFlags().Bool("pcmu", false, "use PCMU audio codec")
-	if err := viper.BindPFlag("pcmu", cmd.PersistentFlags().Lookup("pcmu")); err != nil {
-		return err
-	}
-
-	cmd.PersistentFlags().Bool("pcma", false, "use PCMA audio codec")
-	if err := viper.BindPFlag("pcma", cmd.PersistentFlags().Lookup("pcma")); err != nil {
-		return err
-	}
-
 	cmd.PersistentFlags().StringSlice("nat1to1", []string{}, "sets a list of external IP addresses of 1:1 (D)NAT and a candidate type for which the external IP address is used")
 	if err := viper.BindPFlag("nat1to1", cmd.PersistentFlags().Lookup("nat1to1")); err != nil {
 		return err
@@ -115,35 +42,8 @@ func (WebRTC) Init(cmd *cobra.Command) error {
 }
 
 func (s *WebRTC) Set() {
-	videoCodec := webrtc.VP8
-	if viper.GetBool("vp8") {
-		videoCodec = webrtc.VP8
-	} else if viper.GetBool("vp9") {
-		videoCodec = webrtc.VP9
-	} else if viper.GetBool("h264") {
-		videoCodec = webrtc.H264
-	}
-
-	audioCodec := webrtc.Opus
-	if viper.GetBool("opus") {
-		audioCodec = webrtc.Opus
-	} else if viper.GetBool("g722") {
-		audioCodec = webrtc.G722
-	} else if viper.GetBool("pcmu") {
-		audioCodec = webrtc.PCMU
-	} else if viper.GetBool("pcma") {
-		audioCodec = webrtc.PCMA
-	}
-
 	s.ICELite = viper.GetBool("icelite")
 	s.ICEServers = viper.GetStringSlice("iceserver")
-
-	s.Device = viper.GetString("device")
-	s.AudioCodec = audioCodec
-	s.AudioParams = viper.GetString("aparams")
-	s.Display = viper.GetString("display")
-	s.VideoCodec = videoCodec
-	s.VideoParams = viper.GetString("vparams")
 	s.NAT1To1IPs = viper.GetStringSlice("nat1to1")
 
 	if len(s.NAT1To1IPs) == 0 {
@@ -176,23 +76,4 @@ func (s *WebRTC) Set() {
 		s.EphemeralMin = min
 		s.EphemeralMax = max
 	}
-
-	s.ScreenWidth = 1280
-	s.ScreenHeight = 720
-	s.ScreenRate = 30
-
-	r := regexp.MustCompile(`([0-9]{1,4})x([0-9]{1,4})@([0-9]{1,3})`)
-	res := r.FindStringSubmatch(viper.GetString("screen"))
-
-	if len(res) > 0 {
-		width, err1 := strconv.ParseInt(res[1], 10, 64)
-		height, err2 := strconv.ParseInt(res[1], 10, 64)
-		rate, err3 := strconv.ParseInt(res[1], 10, 64)
-
-		if err1 == nil && err2 == nil && err3 == nil {
-			s.ScreenWidth = int(width)
-			s.ScreenHeight = int(height)
-			s.ScreenRate = int(rate)
-		}
-	}
 }

server/internal/types/keys.go

@@ -0,0 +1,14 @@
+package types
+
+type Button struct {
+	Name   string
+	Code   int
+	Keysym int
+}
+
+type Key struct {
+	Name   string
+	Value  string
+	Code   int
+	Keysym int
+}

server/internal/types/remote.go

@@ -0,0 +1,24 @@
+package types
+
+type RemoteManager interface {
+	VideoCodec() string
+	AudioCodec() string
+	Start()
+	Shutdown() error
+	OnVideoFrame(listener func(sample Sample))
+	OnAudioFrame(listener func(sample Sample))
+	StartStream()
+	StopStream()
+	ChangeResolution(width int, height int, rate int) error
+	GetScreenSize() *ScreenSize
+	ScreenConfigurations() map[int]ScreenConfiguration
+	Move(x, y int)
+	Scroll(x, y int)
+	ButtonDown(code int) (*Button, error)
+	KeyDown(code int) (*Key, error)
+	ButtonUp(code int) (*Button, error)
+	KeyUp(code int) (*Key, error)
+	ReadClipboard() string
+	WriteClipboard(data string)
+	ResetKeys()
+}

server/internal/types/webrtc.go

@@ -9,7 +9,6 @@ type WebRTCManager interface {
 	Start()
 	Shutdown() error
 	CreatePeer(id string, session Session) (string, bool, []string, error)
-	ChangeScreenSize(width int, height int, rate int) error
 }
 
 type Peer interface {

server/internal/webrtc/handle.go

@@ -6,7 +6,6 @@ import (
 	"strconv"
 
 	"github.com/pion/webrtc/v2"
-	"n.eko.moe/neko/internal/xorg"
 )
 
 const OP_MOVE = 0x01
@@ -37,8 +36,8 @@ type PayloadKey struct {
 	Key uint16
 }
 
-func (m *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) error {
+func (manager *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) error {
-	if !m.sessions.IsHost(id) {
+	if !manager.sessions.IsHost(id) {
 		return nil
 	}
 
@@ -63,7 +62,7 @@ func (m *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) error {
 			return err
 		}
 
-		xorg.Move(int(payload.X), int(payload.Y))
+		manager.remote.Move(int(payload.X), int(payload.Y))
 		break
 	case OP_SCROLL:
 		payload := &PayloadScroll{}
@@ -71,13 +70,13 @@ func (m *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) error {
 			return err
 		}
 
-		m.logger.
+		manager.logger.
 			Debug().
 			Str("x", strconv.Itoa(int(payload.X))).
 			Str("y", strconv.Itoa(int(payload.Y))).
 			Msg("scroll")
 
-		xorg.Scroll(int(payload.X), int(payload.Y))
+		manager.remote.Scroll(int(payload.X), int(payload.Y))
 		break
 	case OP_KEY_DOWN:
 		payload := &PayloadKey{}
@@ -86,21 +85,21 @@ func (m *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) error {
 		}
 
 		if payload.Key < 8 {
-			button, err := xorg.ButtonDown(int(payload.Key))
+			button, err := manager.remote.ButtonDown(int(payload.Key))
 			if err != nil {
-				m.logger.Warn().Err(err).Msg("key down failed")
+				manager.logger.Warn().Err(err).Msg("key down failed")
 				return nil
 			}
 
-			m.logger.Debug().Msgf("button down %s(%d)", button.Name, payload.Key)
+			manager.logger.Debug().Msgf("button down %s(%d)", button.Name, payload.Key)
 		} else {
-			key, err := xorg.KeyDown(int(payload.Key))
+			key, err := manager.remote.KeyDown(int(payload.Key))
 			if err != nil {
-				m.logger.Warn().Err(err).Msg("key down failed")
+				manager.logger.Warn().Err(err).Msg("key down failed")
 				return nil
 			}
 
-			m.logger.Debug().Msgf("key down %s(%d)", key.Name, payload.Key)
+			manager.logger.Debug().Msgf("key down %s(%d)", key.Name, payload.Key)
 		}
 
 		break
@@ -112,21 +111,21 @@ func (m *WebRTCManager) handle(id string, msg webrtc.DataChannelMessage) error {
 		}
 
 		if payload.Key < 8 {
-			button, err := xorg.ButtonUp(int(payload.Key))
+			button, err := manager.remote.ButtonUp(int(payload.Key))
 			if err != nil {
-				m.logger.Warn().Err(err).Msg("button up failed")
+				manager.logger.Warn().Err(err).Msg("button up failed")
 				return nil
 			}
 
-			m.logger.Debug().Msgf("button up %s(%d)", button.Name, payload.Key)
+			manager.logger.Debug().Msgf("button up %s(%d)", button.Name, payload.Key)
 		} else {
-			key, err := xorg.KeyUp(int(payload.Key))
+			key, err := manager.remote.KeyUp(int(payload.Key))
 			if err != nil {
-				m.logger.Warn().Err(err).Msg("keyup failed")
+				manager.logger.Warn().Err(err).Msg("keyup failed")
 				return nil
 			}
 
-			m.logger.Debug().Msgf("key up %s(%d)", key.Name, payload.Key)
+			manager.logger.Debug().Msgf("key up %s(%d)", key.Name, payload.Key)
 		}
 		break
 	case OP_KEY_CLK:

server/internal/webrtc/tracks.go

@@ -1,48 +0,0 @@
-package webrtc
-
-import (
-	"fmt"
-	"math/rand"
-
-	"github.com/pion/webrtc/v2"
-	"n.eko.moe/neko/internal/gst"
-)
-
-func (m *WebRTCManager) createTrack(codecName string, pipelineDevice string, pipelineSrc string) (*gst.Pipeline, *webrtc.Track, *webrtc.RTPCodec, error) {
-	pipeline, err := gst.CreatePipeline(
-		codecName,
-		pipelineDevice,
-		pipelineSrc,
-	)
-
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	var codec *webrtc.RTPCodec
-	switch codecName {
-	case webrtc.VP8:
-		codec = webrtc.NewRTPVP8Codec(webrtc.DefaultPayloadTypeVP8, 90000)
-	case webrtc.VP9:
-		codec = webrtc.NewRTPVP9Codec(webrtc.DefaultPayloadTypeVP9, 90000)
-	case webrtc.H264:
-		codec = webrtc.NewRTPH264Codec(webrtc.DefaultPayloadTypeH264, 90000)
-	case webrtc.Opus:
-		codec = webrtc.NewRTPOpusCodec(webrtc.DefaultPayloadTypeOpus, 48000)
-	case webrtc.G722:
-		codec = webrtc.NewRTPG722Codec(webrtc.DefaultPayloadTypeG722, 8000)
-	case webrtc.PCMU:
-		codec = webrtc.NewRTPPCMUCodec(webrtc.DefaultPayloadTypePCMU, 8000)
-	case webrtc.PCMA:
-		codec = webrtc.NewRTPPCMACodec(webrtc.DefaultPayloadTypePCMA, 8000)
-	default:
-		return nil, nil, nil, fmt.Errorf("unknown codec %s", codecName)
-	}
-
-	track, err := webrtc.NewTrack(codec.PayloadType, rand.Uint32(), "stream", "stream", codec)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	return pipeline, track, codec, nil
-}

server/internal/webrtc/webrtc.go

@@ -3,25 +3,22 @@ package webrtc
 import (
 	"fmt"
 	"io"
+	"math/rand"
 	"strings"
-	"time"
 
 	"github.com/pion/webrtc/v2"
 	"github.com/pion/webrtc/v2/pkg/media"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 
-	"n.eko.moe/neko/internal/gst"
 	"n.eko.moe/neko/internal/types"
 	"n.eko.moe/neko/internal/types/config"
-	"n.eko.moe/neko/internal/xorg"
 )
 
-func New(sessions types.SessionManager, config *config.WebRTC) *WebRTCManager {
+func New(sessions types.SessionManager, remote types.RemoteManager, config *config.WebRTC) *WebRTCManager {
 	return &WebRTCManager{
 		logger:   log.With().Str("module", "webrtc").Logger(),
-		cleanup:  time.NewTicker(1 * time.Second),
+		remote:   remote,
-		shutdown: make(chan bool),
 		sessions: sessions,
 		config:   config,
 	}
@@ -31,105 +28,55 @@ type WebRTCManager struct {
 	logger     zerolog.Logger
 	videoTrack *webrtc.Track
 	audioTrack *webrtc.Track
-	videoPipeline *gst.Pipeline
-	audioPipeline *gst.Pipeline
 	videoCodec *webrtc.RTPCodec
 	audioCodec *webrtc.RTPCodec
 	sessions   types.SessionManager
-	cleanup       *time.Ticker
+	remote     types.RemoteManager
 	config     *config.WebRTC
-	shutdown      chan bool
 }
 
-func (m *WebRTCManager) Start() {
+func (manager *WebRTCManager) Start() {
-	// Set display and change to default resolution
-	xorg.Display(m.config.Display)
-	if !xorg.ValidScreenSize(m.config.ScreenWidth, m.config.ScreenHeight, m.config.ScreenRate) {
-		m.logger.Warn().Msgf("invalid screen option %dx%d@%d", m.config.ScreenWidth, m.config.ScreenHeight, m.config.ScreenRate)
-	} else {
-		if err := xorg.ChangeScreenSize(m.config.ScreenWidth, m.config.ScreenHeight, m.config.ScreenRate); err != nil {
-			m.logger.Warn().Err(err).Msg("unable to change screen size")
-		}
-	}
-
 	var err error
-	m.videoPipeline, m.videoTrack, m.videoCodec, err = m.createTrack(m.config.VideoCodec, m.config.Display, m.config.VideoParams)
+	manager.audioTrack, manager.audioCodec, err = manager.createTrack(manager.remote.AudioCodec())
 	if err != nil {
-		m.logger.Panic().Err(err).Msg("unable to start webrtc manager")
+		manager.logger.Panic().Err(err).Msg("unable to create audio track")
 	}
 
-	m.audioPipeline, m.audioTrack, m.audioCodec, err = m.createTrack(m.config.AudioCodec, m.config.Device, m.config.AudioParams)
+	manager.remote.OnAudioFrame(func(sample types.Sample) {
+		if err := manager.audioTrack.WriteSample(media.Sample(sample)); err != nil && err != io.ErrClosedPipe {
+			manager.logger.Warn().Err(err).Msg("audio pipeline failed to write")
+		}
+	})
+
+	manager.videoTrack, manager.videoCodec, err = manager.createTrack(manager.remote.VideoCodec())
 	if err != nil {
-		m.logger.Panic().Err(err).Msg("unable to start webrtc manager")
+		manager.logger.Panic().Err(err).Msg("unable to create video track")
 	}
 
-	go func() {
+	manager.remote.OnVideoFrame(func(sample types.Sample) {
-		defer func() {
+		if err := manager.videoTrack.WriteSample(media.Sample(sample)); err != nil && err != io.ErrClosedPipe {
-			m.logger.Info().Msg("shutdown")
+			manager.logger.Warn().Err(err).Msg("video pipeline failed to write")
-		}()
-
-		for {
-			select {
-			case <-m.shutdown:
-				return
-			case sample := <-m.videoPipeline.Sample:
-				if err := m.videoTrack.WriteSample(media.Sample(sample)); err != nil && err != io.ErrClosedPipe {
-					m.logger.Warn().Err(err).Msg("video pipeline failed to write")
 		}
-			case sample := <-m.audioPipeline.Sample:
-				if err := m.audioTrack.WriteSample(media.Sample(sample)); err != nil && err != io.ErrClosedPipe {
-					m.logger.Warn().Err(err).Msg("audio pipeline failed to write")
-				}
-			case <-m.cleanup.C:
-				xorg.CheckKeys(time.Second * 10)
-			}
-		}
-	}()
-
-	m.videoPipeline.Start()
-	m.audioPipeline.Start()
-
-	m.sessions.OnHostCleared(func(id string) {
-		xorg.ResetKeys()
 	})
 
-	m.sessions.OnCreated(func(id string, session types.Session) {
+	manager.logger.Info().
-		m.logger.Debug().Str("id", id).Msg("session created")
+		Str("ice_lite", fmt.Sprintf("%t", manager.config.ICELite)).
-	})
+		Str("ice_servers", strings.Join(manager.config.ICEServers, ",")).
-
+		Str("ephemeral_port_range", fmt.Sprintf("%d-%d", manager.config.EphemeralMin, manager.config.EphemeralMax)).
-	m.sessions.OnDestroy(func(id string) {
+		Str("nat_ips", strings.Join(manager.config.NAT1To1IPs, ",")).
-		m.logger.Debug().Str("id", id).Msg("session destroyed")
+		Msgf("webrtc starting")
-	})
-
-	// TODO: log resolution, bit rate and codec parameters
-	m.logger.Info().
-		Str("video_display", m.config.Display).
-		Str("video_codec", m.config.VideoCodec).
-		Str("audio_device", m.config.Device).
-		Str("audio_codec", m.config.AudioCodec).
-		Str("audio_pipeline_src", m.audioPipeline.Src).
-		Str("video_pipeline_src", m.videoPipeline.Src).
-		Str("ice_lite", fmt.Sprintf("%t", m.config.ICELite)).
-		Str("ice_servers", strings.Join(m.config.ICEServers, ",")).
-		Str("ephemeral_port_range", fmt.Sprintf("%d-%d", m.config.EphemeralMin, m.config.EphemeralMax)).
-		Str("nat_ips", strings.Join(m.config.NAT1To1IPs, ",")).
-		Msgf("webrtc streaming")
 }
 
-func (m *WebRTCManager) Shutdown() error {
+func (manager *WebRTCManager) Shutdown() error {
-	m.logger.Info().Msgf("webrtc shutting down")
+	manager.logger.Info().Msgf("webrtc shutting down")
-	m.videoPipeline.Stop()
-	m.audioPipeline.Stop()
-	m.cleanup.Stop()
-	m.shutdown <- true
 	return nil
 }
 
-func (m *WebRTCManager) CreatePeer(id string, session types.Session) (string, bool, []string, error) {
+func (manager *WebRTCManager) CreatePeer(id string, session types.Session) (string, bool, []string, error) {
 	configuration := &webrtc.Configuration{
 		ICEServers: []webrtc.ICEServer{
 			{
-				URLs: m.config.ICEServers,
+				URLs: manager.config.ICEServers,
 			},
 		},
 		SDPSemantics: webrtc.SDPSemanticsUnifiedPlanWithFallback,
@@ -137,25 +84,25 @@ func (m *WebRTCManager) CreatePeer(id string, session types.Session) (string, bo
 
 	settings := webrtc.SettingEngine{
 		LoggerFactory: loggerFactory{
-			logger: m.logger,
+			logger: manager.logger,
 		},
 	}
 
-	if m.config.ICELite {
+	if manager.config.ICELite {
 		configuration = &webrtc.Configuration{
 			SDPSemantics: webrtc.SDPSemanticsUnifiedPlanWithFallback,
 		}
 		settings.SetLite(true)
 	}
 
-	settings.SetEphemeralUDPPortRange(m.config.EphemeralMin, m.config.EphemeralMax)
+	settings.SetEphemeralUDPPortRange(manager.config.EphemeralMin, manager.config.EphemeralMax)
-	settings.SetNAT1To1IPs(m.config.NAT1To1IPs, webrtc.ICECandidateTypeHost)
+	settings.SetNAT1To1IPs(manager.config.NAT1To1IPs, webrtc.ICECandidateTypeHost)
 
 	// Create MediaEngine based off sdp
 	engine := webrtc.MediaEngine{}
-	// engine.RegisterDefaultCodecs()
+
-	engine.RegisterCodec(m.audioCodec)
+	engine.RegisterCodec(manager.audioCodec)
-	engine.RegisterCodec(m.videoCodec)
+	engine.RegisterCodec(manager.videoCodec)
 
 	// Create API with MediaEngine and SettingEngine
 	api := webrtc.NewAPI(webrtc.WithMediaEngine(engine), webrtc.WithSettingEngine(settings))
@@ -163,30 +110,30 @@ func (m *WebRTCManager) CreatePeer(id string, session types.Session) (string, bo
 	// Create new peer connection
 	connection, err := api.NewPeerConnection(*configuration)
 	if err != nil {
-		return "", m.config.ICELite, m.config.ICEServers, err
+		return "", manager.config.ICELite, manager.config.ICEServers, err
 	}
 
-	if _, err = connection.AddTransceiverFromTrack(m.videoTrack, webrtc.RtpTransceiverInit{
+	if _, err = connection.AddTransceiverFromTrack(manager.videoTrack, webrtc.RtpTransceiverInit{
 		Direction: webrtc.RTPTransceiverDirectionSendonly,
 	}); err != nil {
-		return "", m.config.ICELite, m.config.ICEServers, err
+		return "", manager.config.ICELite, manager.config.ICEServers, err
 	}
 
-	if _, err = connection.AddTransceiverFromTrack(m.audioTrack, webrtc.RtpTransceiverInit{
+	if _, err = connection.AddTransceiverFromTrack(manager.audioTrack, webrtc.RtpTransceiverInit{
 		Direction: webrtc.RTPTransceiverDirectionSendonly,
 	}); err != nil {
-		return "", m.config.ICELite, m.config.ICEServers, err
+		return "", manager.config.ICELite, manager.config.ICEServers, err
 	}
 
 	description, err := connection.CreateOffer(nil)
 	if err != nil {
-		return "", m.config.ICELite, m.config.ICEServers, err
+		return "", manager.config.ICELite, manager.config.ICEServers, err
 	}
 
 	connection.OnDataChannel(func(d *webrtc.DataChannel) {
 		d.OnMessage(func(msg webrtc.DataChannelMessage) {
-			if err = m.handle(id, msg); err != nil {
+			if err = manager.handle(id, msg); err != nil {
-				m.logger.Warn().Err(err).Msg("data handle failed")
+				manager.logger.Warn().Err(err).Msg("data handle failed")
 			}
 		})
 	})
@@ -196,14 +143,14 @@ func (m *WebRTCManager) CreatePeer(id string, session types.Session) (string, bo
 		switch state {
 		case webrtc.PeerConnectionStateDisconnected:
 		case webrtc.PeerConnectionStateFailed:
-			m.logger.Info().Str("id", id).Msg("peer disconnected")
+			manager.logger.Info().Str("id", id).Msg("peer disconnected")
-			m.sessions.Destroy(id)
+			manager.sessions.Destroy(id)
 			break
 		case webrtc.PeerConnectionStateConnected:
-			m.logger.Info().Str("id", id).Msg("peer connected")
+			manager.logger.Info().Str("id", id).Msg("peer connected")
 			if err = session.SetConnected(true); err != nil {
-				m.logger.Warn().Err(err).Msg("unable to set connected on peer")
+				manager.logger.Warn().Err(err).Msg("unable to set connected on peer")
-				m.sessions.Destroy(id)
+				manager.sessions.Destroy(id)
 			}
 			break
 		}
@@ -213,43 +160,42 @@ func (m *WebRTCManager) CreatePeer(id string, session types.Session) (string, bo
 		id:            id,
 		api:           api,
 		engine:        &engine,
-		manager:       m,
+		manager:       manager,
 		settings:      &settings,
 		connection:    connection,
 		configuration: configuration,
 	}); err != nil {
-		return "", m.config.ICELite, m.config.ICEServers, err
+		return "", manager.config.ICELite, manager.config.ICEServers, err
 	}
 
-	return description.SDP, m.config.ICELite, m.config.ICEServers, nil
+	return description.SDP, manager.config.ICELite, manager.config.ICEServers, nil
 }
 
-func (m *WebRTCManager) ChangeScreenSize(width int, height int, rate int) error {
+func (m *WebRTCManager) createTrack(codecName string) (*webrtc.Track, *webrtc.RTPCodec, error) {
-	if !xorg.ValidScreenSize(width, height, rate) {
+	var codec *webrtc.RTPCodec
-		return fmt.Errorf("unknown configuration")
+	switch codecName {
+	case webrtc.VP8:
+		codec = webrtc.NewRTPVP8Codec(webrtc.DefaultPayloadTypeVP8, 90000)
+	case webrtc.VP9:
+		codec = webrtc.NewRTPVP9Codec(webrtc.DefaultPayloadTypeVP9, 90000)
+	case webrtc.H264:
+		codec = webrtc.NewRTPH264Codec(webrtc.DefaultPayloadTypeH264, 90000)
+	case webrtc.Opus:
+		codec = webrtc.NewRTPOpusCodec(webrtc.DefaultPayloadTypeOpus, 48000)
+	case webrtc.G722:
+		codec = webrtc.NewRTPG722Codec(webrtc.DefaultPayloadTypeG722, 8000)
+	case webrtc.PCMU:
+		codec = webrtc.NewRTPPCMUCodec(webrtc.DefaultPayloadTypePCMU, 8000)
+	case webrtc.PCMA:
+		codec = webrtc.NewRTPPCMACodec(webrtc.DefaultPayloadTypePCMA, 8000)
+	default:
+		return nil, nil, fmt.Errorf("unknown codec %s", codecName)
 	}
 
-	m.videoPipeline.Stop()
+	track, err := webrtc.NewTrack(codec.PayloadType, rand.Uint32(), "stream", "stream", codec)
-	defer func() {
-		m.videoPipeline.Start()
-		m.logger.Info().Msg("starting pipeline")
-	}()
-
-	if err := xorg.ChangeScreenSize(width, height, rate); err != nil {
-		return err
-	}
-
-	videoPipeline, err := gst.CreatePipeline(
-		m.config.VideoCodec,
-		m.config.Display,
-		m.config.VideoParams,
-	)
-
 	if err != nil {
-		m.logger.Panic().Err(err).Msg("unable to create new video pipeline")
+		return nil, nil, err
 	}
 
-	m.videoPipeline = videoPipeline
+	return track, codec, nil
-
-	return nil
 }

server/internal/websocket/control.go

@@ -4,7 +4,6 @@ import (
 	"n.eko.moe/neko/internal/types"
 	"n.eko.moe/neko/internal/types/event"
 	"n.eko.moe/neko/internal/types/message"
-	"n.eko.moe/neko/internal/xorg"
 )
 
 func (h *MessageHandler) controlRelease(id string, session types.Session) error {
@@ -113,6 +112,6 @@ func (h *MessageHandler) controlClipboard(id string, session types.Session, payl
 		return nil
 	}
 
-	xorg.WriteClipboard(payload.Text)
+	h.remote.WriteClipboard(payload.Text)
 	return nil
 }

server/internal/websocket/handler.go

@@ -16,6 +16,7 @@ type MessageHandler struct {
 	logger   zerolog.Logger
 	sessions types.SessionManager
 	webrtc   types.WebRTCManager
+	remote   types.RemoteManager
 	banned   map[string]bool
 	locked   bool
 }

server/internal/websocket/screen.go

@@ -4,7 +4,6 @@ import (
 	"n.eko.moe/neko/internal/types"
 	"n.eko.moe/neko/internal/types/event"
 	"n.eko.moe/neko/internal/types/message"
-	"n.eko.moe/neko/internal/xorg"
 )
 
 func (h *MessageHandler) screenSet(id string, session types.Session, payload *message.ScreenResolution) error {
@@ -13,7 +12,7 @@ func (h *MessageHandler) screenSet(id string, session types.Session, payload *me
 		return nil
 	}
 
-	if err := h.webrtc.ChangeScreenSize(payload.Width, payload.Height, payload.Rate); err != nil {
+	if err := h.remote.ChangeResolution(payload.Width, payload.Height, payload.Rate); err != nil {
 		h.logger.Warn().Err(err).Msgf("unable to change screen size")
 		return err
 	}
@@ -34,7 +33,7 @@ func (h *MessageHandler) screenSet(id string, session types.Session, payload *me
 }
 
 func (h *MessageHandler) screenResolution(id string, session types.Session) error {
-	if size := xorg.GetScreenSize(); size != nil {
+	if size := h.remote.GetScreenSize(); size != nil {
 		if err := session.Send(message.ScreenResolution{
 			Event:  event.SCREEN_RESOLUTION,
 			Width:  size.Width,
@@ -57,7 +56,7 @@ func (h *MessageHandler) screenConfigurations(id string, session types.Session)
 
 	if err := session.Send(message.ScreenConfigurations{
 		Event:          event.SCREEN_CONFIGURATIONS,
-		Configurations: xorg.ScreenConfigurations,
+		Configurations: h.remote.ScreenConfigurations(),
 	}); err != nil {
 		h.logger.Warn().Err(err).Msgf("sending event %s has failed", event.SCREEN_CONFIGURATIONS)
 		return err

server/internal/websocket/websocket.go

@@ -14,16 +14,16 @@ import (
 	"n.eko.moe/neko/internal/types/event"
 	"n.eko.moe/neko/internal/types/message"
 	"n.eko.moe/neko/internal/utils"
-	"n.eko.moe/neko/internal/xorg"
 )
 
-func New(sessions types.SessionManager, webrtc types.WebRTCManager, conf *config.WebSocket) *WebSocketHandler {
+func New(sessions types.SessionManager, remote types.RemoteManager, webrtc types.WebRTCManager, conf *config.WebSocket) *WebSocketHandler {
 	logger := log.With().Str("module", "websocket").Logger()
 
 	return &WebSocketHandler{
 		logger:   logger,
 		conf:     conf,
 		sessions: sessions,
+		remote:   remote,
 		upgrader: websocket.Upgrader{
 			CheckOrigin: func(r *http.Request) bool {
 				return true
@@ -31,6 +31,7 @@ func New(sessions types.SessionManager, webrtc types.WebRTCManager, conf *config
 		},
 		handler: &MessageHandler{
 			logger:   logger.With().Str("subsystem", "handler").Logger(),
+			remote:   remote,
 			sessions: sessions,
 			webrtc:   webrtc,
 			banned:   make(map[string]bool),
@@ -46,6 +47,7 @@ type WebSocketHandler struct {
 	logger   zerolog.Logger
 	upgrader websocket.Upgrader
 	sessions types.SessionManager
+	remote   types.RemoteManager
 	conf     *config.WebSocket
 	handler  *MessageHandler
 	shutdown chan bool
@@ -81,7 +83,7 @@ func (ws *WebSocketHandler) Start() error {
 			ws.logger.Info().Msg("shutdown")
 		}()
 
-		current := xorg.ReadClipboard()
+		current := ws.remote.ReadClipboard()
 
 		for {
 			select {
@@ -89,7 +91,7 @@ func (ws *WebSocketHandler) Start() error {
 				return
 			default:
 				if ws.sessions.HasHost() {
-					text := xorg.ReadClipboard()
+					text := ws.remote.ReadClipboard()
 					if text != current {
 						session, ok := ws.sessions.GetHost()
 						if ok {

server/internal/xorg/keycode/button.go

@@ -1,48 +1,44 @@
 package keycode
 
-type Button struct {
+import "n.eko.moe/neko/internal/types"
-	Name   string
-	Code   int
-	Keysym int
-}
 
-var LEFT_BUTTON = Button{
+var LEFT_BUTTON = types.Button{
 	Name:   "LEFT",
 	Code:   0,
 	Keysym: 1,
 }
 
-var CENTER_BUTTON = Button{
+var CENTER_BUTTON = types.Button{
 	Name:   "CENTER",
 	Code:   1,
 	Keysym: 2,
 }
 
-var RIGHT_BUTTON = Button{
+var RIGHT_BUTTON = types.Button{
 	Name:   "RIGHT",
 	Code:   2,
 	Keysym: 3,
 }
 
-var SCROLL_UP_BUTTON = Button{
+var SCROLL_UP_BUTTON = types.Button{
 	Name:   "SCROLL_UP",
 	Code:   3,
 	Keysym: 4,
 }
 
-var SCROLL_DOWN_BUTTON = Button{
+var SCROLL_DOWN_BUTTON = types.Button{
 	Name:   "SCROLL_DOWN",
 	Code:   4,
 	Keysym: 5,
 }
 
-var SCROLL_LEFT_BUTTON = Button{
+var SCROLL_LEFT_BUTTON = types.Button{
 	Name:   "SCROLL_LEFT",
 	Code:   5,
 	Keysym: 6,
 }
 
-var SCROLL_RIGHT_BUTTON = Button{
+var SCROLL_RIGHT_BUTTON = types.Button{
 	Name:   "SCROLL_RIGHT",
 	Code:   6,
 	Keysym: 7,

server/internal/xorg/keycode/keys.go

@@ -1,699 +1,694 @@
 package keycode
 
-type Key struct {
+import "n.eko.moe/neko/internal/types"
-	Name   string
-	Value  string
-	Code   int
-	Keysym int
-}
 
-var BACKSPACE = Key{
+var BACKSPACE = types.Key{
 	Name:   "BACKSPACE",
 	Value:  "BackSpace",
 	Code:   8,
 	Keysym: int(0xff08),
 }
 
-var TAB = Key{
+var TAB = types.Key{
 	Name:   "TAB",
 	Value:  "Tab",
 	Code:   9,
 	Keysym: int(0xFF09),
 }
 
-var CLEAR = Key{
+var CLEAR = types.Key{
 	Name:   "CLEAR",
 	Value:  "Clear",
 	Code:   12,
 	Keysym: int(0xFF0B),
 }
 
-var ENTER = Key{
+var ENTER = types.Key{
 	Name:   "ENTER",
 	Value:  "Enter",
 	Code:   13,
 	Keysym: int(0xFF0D),
 }
 
-var SHIFT = Key{
+var SHIFT = types.Key{
 	Name:   "SHIFT",
 	Value:  "Shift",
 	Code:   16,
 	Keysym: int(0xFFE1),
 }
 
-var CTRL = Key{
+var CTRL = types.Key{
 	Name:   "CTRL",
 	Value:  "Ctrl",
 	Code:   17,
 	Keysym: int(0xFFE3),
 }
 
-var ALT = Key{
+var ALT = types.Key{
 	Name:   "ALT",
 	Value:  "Alt",
 	Code:   18,
 	Keysym: int(0xFFE9),
 }
 
-var PAUSE = Key{
+var PAUSE = types.Key{
 	Name:   "PAUSE",
 	Value:  "Pause",
 	Code:   19,
 	Keysym: int(0xFF13),
 }
 
-var CAPS_LOCK = Key{
+var CAPS_LOCK = types.Key{
 	Name:   "CAPS_LOCK",
 	Value:  "Caps Lock",
 	Code:   20,
 	Keysym: int(0xFFE5),
 }
 
-var ESCAPE = Key{
+var ESCAPE = types.Key{
 	Name:   "ESCAPE",
 	Value:  "Escape",
 	Code:   27,
 	Keysym: int(0xFF1B),
 }
 
-var SPACE = Key{
+var SPACE = types.Key{
 	Name:   "SPACE",
 	Value:  " ",
 	Code:   32,
 	Keysym: int(0x0020),
 }
 
-var PAGE_UP = Key{
+var PAGE_UP = types.Key{
 	Name:   "PAGE_UP",
 	Value:  "Page Up",
 	Code:   33,
 	Keysym: int(0xFF55),
 }
 
-var PAGE_DOWN = Key{
+var PAGE_DOWN = types.Key{
 	Name:   "PAGE_DOWN",
 	Value:  "Page Down",
 	Code:   34,
 	Keysym: int(0xFF56),
 }
 
-var END = Key{
+var END = types.Key{
 	Name:   "END",
 	Value:  "End",
 	Code:   35,
 	Keysym: int(0xFF57),
 }
 
-var HOME = Key{
+var HOME = types.Key{
 	Name:   "HOME",
 	Value:  "Home",
 	Code:   36,
 	Keysym: int(0xFF50),
 }
 
-var LEFT_ARROW = Key{
+var LEFT_ARROW = types.Key{
 	Name:   "LEFT_ARROW",
 	Value:  "Left Arrow",
 	Code:   37,
 	Keysym: int(0xFF51),
 }
 
-var UP_ARROW = Key{
+var UP_ARROW = types.Key{
 	Name:   "UP_ARROW",
 	Value:  "Up Arrow",
 	Code:   38,
 	Keysym: int(0xFF52),
 }
 
-var RIGHT_ARROW = Key{
+var RIGHT_ARROW = types.Key{
 	Name:   "RIGHT_ARROW",
 	Value:  "Right Arrow",
 	Code:   39,
 	Keysym: int(0xFF53),
 }
 
-var DOWN_ARROW = Key{
+var DOWN_ARROW = types.Key{
 	Name:   "DOWN_ARROW",
 	Value:  "Down Arrow",
 	Code:   40,
 	Keysym: int(0xFF54),
 }
 
-var INSERT = Key{
+var INSERT = types.Key{
 	Name:   "INSERT",
 	Value:  "Insert",
 	Code:   45,
 	Keysym: int(0xFF63),
 }
 
-var DELETE = Key{
+var DELETE = types.Key{
 	Name:   "DELETE",
 	Value:  "Delete",
 	Code:   46,
 	Keysym: int(0xFFFF),
 }
 
-var KEY_0 = Key{
+var KEY_0 = types.Key{
 	Name:   "KEY_0",
 	Value:  "0",
 	Code:   48,
 	Keysym: int(0x0030),
 }
 
-var KEY_1 = Key{
+var KEY_1 = types.Key{
 	Name:   "KEY_1",
 	Value:  "1",
 	Code:   49,
 	Keysym: int(0x0031),
 }
 
-var KEY_2 = Key{
+var KEY_2 = types.Key{
 	Name:   "KEY_2",
 	Value:  "2",
 	Code:   50,
 	Keysym: int(0x0032),
 }
 
-var KEY_3 = Key{
+var KEY_3 = types.Key{
 	Name:   "KEY_3",
 	Value:  "3",
 	Code:   51,
 	Keysym: int(0x0033),
 }
 
-var KEY_4 = Key{
+var KEY_4 = types.Key{
 	Name:   "KEY_4",
 	Value:  "4",
 	Code:   52,
 	Keysym: int(0x0034),
 }
 
-var KEY_5 = Key{
+var KEY_5 = types.Key{
 	Name:   "KEY_5",
 	Value:  "5",
 	Code:   53,
 	Keysym: int(0x0035),
 }
 
-var KEY_6 = Key{
+var KEY_6 = types.Key{
 	Name:   "KEY_6",
 	Value:  "6",
 	Code:   54,
 	Keysym: int(0x0036),
 }
 
-var KEY_7 = Key{
+var KEY_7 = types.Key{
 	Name:   "KEY_7",
 	Value:  "7",
 	Code:   55,
 	Keysym: int(0x0037),
 }
 
-var KEY_8 = Key{
+var KEY_8 = types.Key{
 	Name:   "KEY_8",
 	Value:  "8",
 	Code:   56,
 	Keysym: int(0x0038),
 }
 
-var KEY_9 = Key{
+var KEY_9 = types.Key{
 	Name:   "KEY_9",
 	Value:  "9",
 	Code:   57,
 	Keysym: int(0x0039),
 }
 
-var KEY_A = Key{
+var KEY_A = types.Key{
 	Name:   "KEY_A",
 	Value:  "a",
 	Code:   65,
 	Keysym: int(0x0061),
 }
 
-var KEY_B = Key{
+var KEY_B = types.Key{
 	Name:   "KEY_B",
 	Value:  "b",
 	Code:   66,
 	Keysym: int(0x0062),
 }
 
-var KEY_C = Key{
+var KEY_C = types.Key{
 	Name:   "KEY_C",
 	Value:  "c",
 	Code:   67,
 	Keysym: int(0x0063),
 }
 
-var KEY_D = Key{
+var KEY_D = types.Key{
 	Name:   "KEY_D",
 	Value:  "d",
 	Code:   68,
 	Keysym: int(0x0064),
 }
 
-var KEY_E = Key{
+var KEY_E = types.Key{
 	Name:   "KEY_E",
 	Value:  "e",
 	Code:   69,
 	Keysym: int(0x0065),
 }
 
-var KEY_F = Key{
+var KEY_F = types.Key{
 	Name:   "KEY_F",
 	Value:  "f",
 	Code:   70,
 	Keysym: int(0x0066),
 }
 
-var KEY_G = Key{
+var KEY_G = types.Key{
 	Name:   "KEY_G",
 	Value:  "g",
 	Code:   71,
 	Keysym: int(0x0067),
 }
 
-var KEY_H = Key{
+var KEY_H = types.Key{
 	Name:   "KEY_H",
 	Value:  "h",
 	Code:   72,
 	Keysym: int(0x0068),
 }
 
-var KEY_I = Key{
+var KEY_I = types.Key{
 	Name:   "KEY_I",
 	Value:  "i",
 	Code:   73,
 	Keysym: int(0x0069),
 }
 
-var KEY_J = Key{
+var KEY_J = types.Key{
 	Name:   "KEY_J",
 	Value:  "j",
 	Code:   74,
 	Keysym: int(0x006a),
 }
 
-var KEY_K = Key{
+var KEY_K = types.Key{
 	Name:   "KEY_K",
 	Value:  "k",
 	Code:   75,
 	Keysym: int(0x006b),
 }
 
-var KEY_L = Key{
+var KEY_L = types.Key{
 	Name:   "KEY_L",
 	Value:  "l",
 	Code:   76,
 	Keysym: int(0x006c),
 }
 
-var KEY_M = Key{
+var KEY_M = types.Key{
 	Name:   "KEY_M",
 	Value:  "m",
 	Code:   77,
 	Keysym: int(0x006d),
 }
 
-var KEY_N = Key{
+var KEY_N = types.Key{
 	Name:   "KEY_N",
 	Value:  "n",
 	Code:   78,
 	Keysym: int(0x006e),
 }
 
-var KEY_O = Key{
+var KEY_O = types.Key{
 	Name:   "KEY_O",
 	Value:  "o",
 	Code:   79,
 	Keysym: int(0x006f),
 }
 
-var KEY_P = Key{
+var KEY_P = types.Key{
 	Name:   "KEY_P",
 	Value:  "p",
 	Code:   80,
 	Keysym: int(0x0070),
 }
 
-var KEY_Q = Key{
+var KEY_Q = types.Key{
 	Name:   "KEY_Q",
 	Value:  "q",
 	Code:   81,
 	Keysym: int(0x0071),
 }
 
-var KEY_R = Key{
+var KEY_R = types.Key{
 	Name:   "KEY_R",
 	Value:  "r",
 	Code:   82,
 	Keysym: int(0x0072),
 }
 
-var KEY_S = Key{
+var KEY_S = types.Key{
 	Name:   "KEY_S",
 	Value:  "s",
 	Code:   83,
 	Keysym: int(0x0073),
 }
 
-var KEY_T = Key{
+var KEY_T = types.Key{
 	Name:   "KEY_T",
 	Value:  "t",
 	Code:   84,
 	Keysym: int(0x0074),
 }
 
-var KEY_U = Key{
+var KEY_U = types.Key{
 	Name:   "KEY_U",
 	Value:  "u",
 	Code:   85,
 	Keysym: int(0x0075),
 }
 
-var KEY_V = Key{
+var KEY_V = types.Key{
 	Name:   "KEY_V",
 	Value:  "v",
 	Code:   86,
 	Keysym: int(0x0076),
 }
 
-var KEY_W = Key{
+var KEY_W = types.Key{
 	Name:   "KEY_W",
 	Value:  "w",
 	Code:   87,
 	Keysym: int(0x0077),
 }
 
-var KEY_X = Key{
+var KEY_X = types.Key{
 	Name:   "KEY_X",
 	Value:  "x",
 	Code:   88,
 	Keysym: int(0x0078),
 }
 
-var KEY_Y = Key{
+var KEY_Y = types.Key{
 	Name:   "KEY_Y",
 	Value:  "y",
 	Code:   89,
 	Keysym: int(0x0079),
 }
 
-var KEY_Z = Key{
+var KEY_Z = types.Key{
 	Name:   "KEY_Z",
 	Value:  "z",
 	Code:   90,
 	Keysym: int(0x007a),
 }
 
-var WIN_LEFT = Key{
+var WIN_LEFT = types.Key{
 	Name:   "WIN_LEFT",
 	Value:  "Win Left",
 	Code:   91,
 	Keysym: int(0xFFEB),
 }
 
-var WIN_RIGHT = Key{
+var WIN_RIGHT = types.Key{
 	Name:   "WIN_RIGHT",
 	Value:  "Win Right",
 	Code:   92,
 	Keysym: int(0xFF67),
 }
 
-var PAD_0 = Key{
+var PAD_0 = types.Key{
 	Name:   "PAD_0",
 	Value:  "Num Pad 0",
 	Code:   96,
 	Keysym: int(0xFFB0),
 }
 
-var PAD_1 = Key{
+var PAD_1 = types.Key{
 	Name:   "PAD_1",
 	Value:  "Num Pad 1",
 	Code:   97,
 	Keysym: int(0xFFB1),
 }
 
-var PAD_2 = Key{
+var PAD_2 = types.Key{
 	Name:   "PAD_2",
 	Value:  "Num Pad 2",
 	Code:   98,
 	Keysym: int(0xFFB2),
 }
 
-var PAD_3 = Key{
+var PAD_3 = types.Key{
 	Name:   "PAD_3",
 	Value:  "Num Pad 3",
 	Code:   99,
 	Keysym: int(0xFFB3),
 }
 
-var PAD_4 = Key{
+var PAD_4 = types.Key{
 	Name:   "PAD_4",
 	Value:  "Num Pad 4",
 	Code:   100,
 	Keysym: int(0xFFB4),
 }
 
-var PAD_5 = Key{
+var PAD_5 = types.Key{
 	Name:   "PAD_5",
 	Value:  "Num Pad 5",
 	Code:   101,
 	Keysym: int(0xFFB5),
 }
 
-var PAD_6 = Key{
+var PAD_6 = types.Key{
 	Name:   "PAD_6",
 	Value:  "Num Pad 6",
 	Code:   102,
 	Keysym: int(0xFFB6),
 }
 
-var PAD_7 = Key{
+var PAD_7 = types.Key{
 	Name:   "PAD_7",
 	Value:  "Num Pad 7",
 	Code:   103,
 	Keysym: int(0xFFB7),
 }
 
-var PAD_8 = Key{
+var PAD_8 = types.Key{
 	Name:   "PAD_8",
 	Value:  "Num Pad 8",
 	Code:   104,
 	Keysym: int(0xFFB8),
 }
 
-var PAD_9 = Key{
+var PAD_9 = types.Key{
 	Name:   "PAD_9",
 	Value:  "Num Pad 9",
 	Code:   105,
 	Keysym: int(0xFFB9),
 }
 
-var MULTIPLY = Key{
+var MULTIPLY = types.Key{
 	Name:   "MULTIPLY",
 	Value:  "*",
 	Code:   106,
 	Keysym: int(0xFFAA),
 }
 
-var ADD = Key{
+var ADD = types.Key{
 	Name:   "ADD",
 	Value:  "+",
 	Code:   107,
 	Keysym: int(0xFFAB),
 }
 
-var SUBTRACT = Key{
+var SUBTRACT = types.Key{
 	Name:   "SUBTRACT",
 	Value:  "-",
 	Code:   109,
 	Keysym: int(0xFFAD),
 }
 
-var DECIMAL = Key{
+var DECIMAL = types.Key{
 	Name:   "DECIMAL",
 	Value:  ".",
 	Code:   110,
 	Keysym: int(0xFFAE),
 }
 
-var DIVIDE = Key{
+var DIVIDE = types.Key{
 	Name:   "DIVIDE",
 	Value:  "/",
 	Code:   111,
 	Keysym: int(0xFFAF),
 }
 
-var KEY_F1 = Key{
+var KEY_F1 = types.Key{
 	Name:   "KEY_F1",
 	Value:  "f1",
 	Code:   112,
 	Keysym: int(0xFFBE),
 }
 
-var KEY_F2 = Key{
+var KEY_F2 = types.Key{
 	Name:   "KEY_F2",
 	Value:  "f2",
 	Code:   113,
 	Keysym: int(0xFFBF),
 }
 
-var KEY_F3 = Key{
+var KEY_F3 = types.Key{
 	Name:   "KEY_F3",
 	Value:  "f3",
 	Code:   114,
 	Keysym: int(0xFFC0),
 }
 
-var KEY_F4 = Key{
+var KEY_F4 = types.Key{
 	Name:   "KEY_F4",
 	Value:  "f4",
 	Code:   115,
 	Keysym: int(0xFFC1),
 }
 
-var KEY_F5 = Key{
+var KEY_F5 = types.Key{
 	Name:   "KEY_F5",
 	Value:  "f5",
 	Code:   116,
 	Keysym: int(0xFFC2),
 }
 
-var KEY_F6 = Key{
+var KEY_F6 = types.Key{
 	Name:   "KEY_F6",
 	Value:  "f6",
 	Code:   117,
 	Keysym: int(0xFFC3),
 }
 
-var KEY_F7 = Key{
+var KEY_F7 = types.Key{
 	Name:   "KEY_F7",
 	Value:  "f7",
 	Code:   118,
 	Keysym: int(0xFFC4),
 }
 
-var KEY_F8 = Key{
+var KEY_F8 = types.Key{
 	Name:   "KEY_F8",
 	Value:  "f8",
 	Code:   119,
 	Keysym: int(0xFFC5),
 }
 
-var KEY_F9 = Key{
+var KEY_F9 = types.Key{
 	Name:   "KEY_F9",
 	Value:  "f9",
 	Code:   120,
 	Keysym: int(0xFFC6),
 }
 
-var KEY_F10 = Key{
+var KEY_F10 = types.Key{
 	Name:   "KEY_F10",
 	Value:  "f10",
 	Code:   121,
 	Keysym: int(0xFFC7),
 }
 
-var KEY_F11 = Key{
+var KEY_F11 = types.Key{
 	Name:   "KEY_F11",
 	Value:  "f11",
 	Code:   122,
 	Keysym: int(0xFFC8),
 }
 
-var KEY_F12 = Key{
+var KEY_F12 = types.Key{
 	Name:   "KEY_F12",
 	Value:  "f12",
 	Code:   123,
 	Keysym: int(0xFFC9),
 }
 
-var NUM_LOCK = Key{
+var NUM_LOCK = types.Key{
 	Name:   "NUM_LOCK",
 	Value:  "Num Lock",
 	Code:   144,
 	Keysym: int(0xFF7F),
 }
 
-var SCROLL_LOCK = Key{
+var SCROLL_LOCK = types.Key{
 	Name:   "SCROLL_LOCK",
 	Value:  "Scroll Lock",
 	Code:   145,
 	Keysym: int(0xFF14),
 }
 
-var SEMI_COLON = Key{
+var SEMI_COLON = types.Key{
 	Name:   "SEMI_COLON",
 	Value:  ";",
 	Code:   186,
 	Keysym: int(0x003b),
 }
 
-var EQUAL = Key{
+var EQUAL = types.Key{
 	Name:   "EQUAL",
 	Value:  "=",
 	Code:   187,
 	Keysym: int(0x003d),
 }
 
-var COMMA = Key{
+var COMMA = types.Key{
 	Name:   "COMMA",
 	Value:  ",",
 	Code:   188,
 	Keysym: int(0x002c),
 }
 
-var DASH = Key{
+var DASH = types.Key{
 	Name:   "DASH",
 	Value:  "-",
 	Code:   189,
 	Keysym: int(0x002d),
 }
 
-var PERIOD = Key{
+var PERIOD = types.Key{
 	Name:   "PERIOD",
 	Value:  ".",
 	Code:   190,
 	Keysym: int(0x002e),
 }
 
-var FORWARD_SLASH = Key{
+var FORWARD_SLASH = types.Key{
 	Name:   "FORWARD_SLASH",
 	Value:  "/",
 	Code:   191,
 	Keysym: int(0x002f),
 }
 
-var GRAVE = Key{
+var GRAVE = types.Key{
 	Name:   "GRAVE",
 	Value:  "`",
 	Code:   192,
 	Keysym: int(0x0060),
 }
 
-var OPEN_BRACKET = Key{
+var OPEN_BRACKET = types.Key{
 	Name:   "OPEN_BRACKET",
 	Value:  "[",
 	Code:   219,
 	Keysym: int(0x005b),
 }
 
-var BACK_SLASH = Key{
+var BACK_SLASH = types.Key{
 	Name:   "BACK_SLASH",
 	Value:  "\\",
 	Code:   220,
 	Keysym: int(0x005c),
 }
 
-var CLOSE_BRAKET = Key{
+var CLOSE_BRAKET = types.Key{
 	Name:   "CLOSE_BRAKET",
 	Value:  "]",
 	Code:   221,
 	Keysym: int(0x005d),
 }
 
-var SINGLE_QUOTE = Key{
+var SINGLE_QUOTE = types.Key{
 	Name:   "SINGLE_QUOTE",
 	Value:  "'",
 	Code:   222,

server/internal/xorg/xorg.go

@@ -27,8 +27,8 @@ import (
 var ScreenConfigurations = make(map[int]types.ScreenConfiguration)
 
 var debounce = make(map[int]time.Time)
-var buttons = make(map[int]keycode.Button)
+var buttons = make(map[int]types.Button)
-var keys = make(map[int]keycode.Key)
+var keys = make(map[int]types.Key)
 var mu = sync.Mutex{}
 
 func init() {
@@ -167,7 +167,7 @@ func Scroll(x, y int) {
 	C.XScroll(C.int(x), C.int(y))
 }
 
-func ButtonDown(code int) (*keycode.Button, error) {
+func ButtonDown(code int) (*types.Button, error) {
 	mu.Lock()
 	defer mu.Unlock()
 
@@ -186,7 +186,7 @@ func ButtonDown(code int) (*keycode.Button, error) {
 	return &button, nil
 }
 
-func KeyDown(code int) (*keycode.Key, error) {
+func KeyDown(code int) (*types.Key, error) {
 	mu.Lock()
 	defer mu.Unlock()
 
@@ -205,7 +205,7 @@ func KeyDown(code int) (*keycode.Key, error) {
 	return &key, nil
 }
 
-func ButtonUp(code int) (*keycode.Button, error) {
+func ButtonUp(code int) (*types.Button, error) {
 	mu.Lock()
 	defer mu.Unlock()
 
@@ -224,7 +224,7 @@ func ButtonUp(code int) (*keycode.Button, error) {
 	return &button, nil
 }
 
-func KeyUp(code int) (*keycode.Key, error) {
+func KeyUp(code int) (*types.Key, error) {
 	mu.Lock()
 	defer mu.Unlock()
 

server/neko.go

@@ -7,6 +7,7 @@ import (
 	"runtime"
 
 	"n.eko.moe/neko/internal/http"
+	"n.eko.moe/neko/internal/remote"
 	"n.eko.moe/neko/internal/session"
 	"n.eko.moe/neko/internal/types/config"
 	"n.eko.moe/neko/internal/webrtc"
@@ -59,6 +60,7 @@ func init() {
 		},
 		Root:      &config.Root{},
 		Server:    &config.Server{},
+		Remote:    &config.Remote{},
 		WebRTC:    &config.WebRTC{},
 		WebSocket: &config.WebSocket{},
 	}
@@ -96,13 +98,15 @@ func (i *Version) Details() string {
 type Neko struct {
 	Version   *Version
 	Root      *config.Root
+	Remote    *config.Remote
 	Server    *config.Server
 	WebRTC    *config.WebRTC
 	WebSocket *config.WebSocket
 
 	logger           zerolog.Logger
 	server           *http.Server
-	sessions         *session.SessionManager
+	sessionManager   *session.SessionManager
+	remoteManager    *remote.RemoteManager
 	webRTCManager    *webrtc.WebRTCManager
 	webSocketHandler *websocket.WebSocketHandler
 }
@@ -112,24 +116,34 @@ func (neko *Neko) Preflight() {
 }
 
 func (neko *Neko) Start() {
-	sessions := session.New()
+	sessionManager := session.New()
 
-	webRTCManager := webrtc.New(sessions, neko.WebRTC)
+	remoteManager := remote.New(neko.Remote)
+	remoteManager.Start()
+
+	webRTCManager := webrtc.New(sessionManager, remoteManager, neko.WebRTC)
 	webRTCManager.Start()
 
-	webSocketHandler := websocket.New(sessions, webRTCManager, neko.WebSocket)
+	webSocketHandler := websocket.New(sessionManager, remoteManager, webRTCManager, neko.WebSocket)
 	webSocketHandler.Start()
 
 	server := http.New(neko.Server, webSocketHandler)
 	server.Start()
 
-	neko.sessions = sessions
+	neko.sessionManager = sessionManager
+	neko.remoteManager = remoteManager
 	neko.webRTCManager = webRTCManager
 	neko.webSocketHandler = webSocketHandler
 	neko.server = server
 }
 
 func (neko *Neko) Shutdown() {
+	if err := neko.remoteManager.Shutdown(); err != nil {
+		neko.logger.Err(err).Msg("remote manager shutdown with an error")
+	} else {
+		neko.logger.Debug().Msg("remote manager shutdown")
+	}
+
 	if err := neko.webRTCManager.Shutdown(); err != nil {
 		neko.logger.Err(err).Msg("webrtc manager shutdown with an error")
 	} else {