From 56a5dcf77f6a10445716485dc717acb901372561 Mon Sep 17 00:00:00 2001 From: Craig Date: Sat, 25 Jan 2020 14:29:52 +0000 Subject: [PATCH] clipboard sync and some minor fixes --- .docker/build | 5 +- client/src/components/video.vue | 34 +++++- client/src/neko/base.ts | 5 + client/src/neko/events.ts | 2 + client/src/neko/index.ts | 5 + client/src/neko/messages.ts | 5 + client/src/store/remote.ts | 15 +++ server/internal/config/webrtc.go | 2 +- server/internal/gst/gst.go | 6 +- server/internal/hid/clipboard/clipboard.go | 124 +++++++++++++++++++++ server/internal/hid/hid.c | 48 ++++---- server/internal/hid/hid.go | 19 ++-- server/internal/hid/hid.h | 11 +- server/internal/session/session.go | 16 ++- server/internal/types/event/events.go | 1 + server/internal/types/message/messages.go | 5 + server/internal/webrtc/logger.go | 31 +++++- server/internal/webrtc/tracks.go | 4 +- server/internal/webrtc/webrtc.go | 42 ++++--- server/internal/websocket/control.go | 12 ++ server/internal/websocket/handler.go | 6 + server/internal/websocket/websocket.go | 49 +++++--- 22 files changed, 359 insertions(+), 88 deletions(-) create mode 100644 server/internal/hid/clipboard/clipboard.go diff --git a/.docker/build b/.docker/build index 0d80fa5..4e3c437 100755 --- a/.docker/build +++ b/.docker/build @@ -10,6 +10,9 @@ cd ../ sudo docker build -f Dockerfile -t nurdism/neko . # sudo docker push nurdism/neko:latest +# sudo docker run -e NEKO_BIND='0.0.0.0:9000' --shm-size=1gb nurdism/neko:latest +# sudo docker run -p 8080:8080 -p 59000-65000:59000-65000/udp --network host --shm-size=1gb nurdism/neko:latest # sudo docker run -p 8080:8080 --shm-size=1gb nurdism/neko:latest # sudo docker run --network host --shm-size=1gb nurdism/neko:latest -# sudo docker run --network host --shm-size=1gb -it nurdism/neko:latest /bin/bash \ No newline at end of file +# sudo docker run --network host --shm-size=1gb -it nurdism/neko:latest /bin/bash +# \ No newline at end of file diff --git a/client/src/components/video.vue b/client/src/components/video.vue index 0b5877d..eac674b 100644 --- a/client/src/components/video.vue +++ b/client/src/components/video.vue @@ -196,6 +196,10 @@ return this.$accessor.settings.scroll_invert } + get clipboard() { + return this.$accessor.remote.clipboard + } + @Watch('volume') onVolumeChanged(volume: number) { if (this._video) { @@ -233,6 +237,11 @@ } } + @Watch('clipboard') + onClipboardChanged(clipboard: string) { + navigator.clipboard.writeText(clipboard).catch(console.error) + } + mounted() { this._container.addEventListener('resize', this.onResise) this.onVolumeChanged(this.volume) @@ -249,7 +258,9 @@ this._video.addEventListener('canplaythrough', () => { this.$accessor.video.setPlayable(true) if (this.autoplay) { - this.$accessor.video.play() + this.$nextTick(() => { + this.$accessor.video.play() + }) } }) @@ -261,11 +272,14 @@ console.error(event.error) this.$accessor.video.setPlayable(false) }) + + document.addEventListener('focusin', this.onFocus.bind(this)) } beforeDestroy() { this.observer.disconnect() this.$accessor.video.setPlayable(false) + document.removeEventListener('focusin', this.onFocus.bind(this)) } play() { @@ -308,6 +322,23 @@ this.onResise() } + onFocus() { + if (!document.hasFocus()) { + return + } + + if (this.hosting) { + navigator.clipboard + .readText() + .then(text => { + if (this.clipboard !== text) { + this.$accessor.remote.sendClipboard(text) + } + }) + .catch(console.error) + } + } + onMousePos(e: MouseEvent) { const { w, h } = this.$accessor.video.resolution const rect = this._overlay.getBoundingClientRect() @@ -362,6 +393,7 @@ onMouseEnter(e: MouseEvent) { this._overlay.focus() + this.onFocus() this.focused = true } diff --git a/client/src/neko/base.ts b/client/src/neko/base.ts index fec5d1e..33f2010 100644 --- a/client/src/neko/base.ts +++ b/client/src/neko/base.ts @@ -165,6 +165,11 @@ export abstract class BaseClient extends EventEmitter { this.emit('debug', `peer connection state chagned: ${this._state}`) switch (this._state) { + case 'checking': + if (this._timeout) { + clearTimeout(this._timeout) + } + break case 'connected': this.onConnected() break diff --git a/client/src/neko/events.ts b/client/src/neko/events.ts index 524d6e2..6791c1e 100644 --- a/client/src/neko/events.ts +++ b/client/src/neko/events.ts @@ -29,6 +29,7 @@ export const EVENT = { RELEASE: 'control/release', REQUEST: 'control/request', REQUESTING: 'control/requesting', + CLIPBOARD: 'control/clipboard', GIVE: 'control/give', }, CHAT: { @@ -64,6 +65,7 @@ export type ControlEvents = | typeof EVENT.CONTROL.RELEASE | typeof EVENT.CONTROL.REQUEST | typeof EVENT.CONTROL.GIVE + | typeof EVENT.CONTROL.CLIPBOARD export type SystemEvents = typeof EVENT.SYSTEM.DISCONNECT export type IdentityEvents = typeof EVENT.IDENTITY.PROVIDE | typeof EVENT.IDENTITY.DETAILS diff --git a/client/src/neko/index.ts b/client/src/neko/index.ts index 0eb05b8..80020c4 100644 --- a/client/src/neko/index.ts +++ b/client/src/neko/index.ts @@ -17,6 +17,7 @@ import { EmotePayload, AdminPayload, AdminTargetPayload, + ControlClipboardPayload, } from './messages' interface NekoEvents extends BaseEvents {} @@ -254,6 +255,10 @@ export class NekoClient extends BaseClient implements EventEmitter { }) } + protected [EVENT.CONTROL.CLIPBOARD]({ text }: ControlClipboardPayload) { + this.$accessor.remote.setClipboard(text) + } + ///////////////////////////// // Chat Events ///////////////////////////// diff --git a/client/src/neko/messages.ts b/client/src/neko/messages.ts index 76b5367..42c68c8 100644 --- a/client/src/neko/messages.ts +++ b/client/src/neko/messages.ts @@ -27,6 +27,7 @@ export type WebSocketPayloads = | MemberListPayload | Member | ControlPayload + | ControlClipboardPayload | ChatPayload | ChatSendPayload | EmojiSendPayload @@ -110,6 +111,10 @@ export interface ControlTargetPayload { target: string } +export interface ControlClipboardPayload { + text: string +} + /* CHAT PAYLOADS */ diff --git a/client/src/store/remote.ts b/client/src/store/remote.ts index c6c20e8..1d50a1a 100644 --- a/client/src/store/remote.ts +++ b/client/src/store/remote.ts @@ -7,6 +7,7 @@ export const namespaced = true export const state = () => ({ id: '', + clipboard: '', }) export const getters = getterTree(state, { @@ -30,8 +31,13 @@ export const mutations = mutationTree(state, { } }, + setClipboard(state, clipboard: string) { + state.clipboard = clipboard + }, + clear(state) { state.id = '' + state.clipboard = '' }, }) @@ -41,6 +47,15 @@ export const actions = actionTree( initialise({ commit }) { // }, + + sendClipboard({ getters }, clipboard: string) { + if (!accessor.connected || !getters.hosting) { + return + } + + $client.sendMessage(EVENT.CONTROL.CLIPBOARD, { text: clipboard }) + }, + toggle({ getters }) { if (!accessor.connected) { return diff --git a/server/internal/config/webrtc.go b/server/internal/config/webrtc.go index 1118759..642662d 100644 --- a/server/internal/config/webrtc.go +++ b/server/internal/config/webrtc.go @@ -86,7 +86,7 @@ func (s *WebRTC) Set() { videoCodec = webrtc.H264 } - audioCodec := webrtc.VP8 + audioCodec := webrtc.Opus if viper.GetBool("opus") { audioCodec = webrtc.Opus } else if viper.GetBool("g722") { diff --git a/server/internal/gst/gst.go b/server/internal/gst/gst.go index 6ef7dcb..d86da45 100644 --- a/server/internal/gst/gst.go +++ b/server/internal/gst/gst.go @@ -88,9 +88,6 @@ func CreatePipeline(codecName string, pipelineSrc string) (*Pipeline, error) { } case webrtc.H264: - // https://gstreamer.freedesktop.org/documentation/x264/index.html?gi-language=c - // gstreamer1.0-plugins-ugly - // video/x-raw,format=I420 ! x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream // https://gstreamer.freedesktop.org/documentation/openh264/openh264enc.html?gi-language=c#openh264enc // gstreamer1.0-plugins-bad @@ -102,6 +99,9 @@ func CreatePipeline(codecName string, pipelineSrc string) (*Pipeline, error) { return nil, err } + // https://gstreamer.freedesktop.org/documentation/x264/index.html?gi-language=c + // gstreamer1.0-plugins-ugly + // video/x-raw,format=I420 ! x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream if err := CheckPlugins([]string{"openh264"}); err != nil { pipelineStr = pipelineSrc + " ! video/x-raw,format=I420 ! x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream ! " + pipelineStr diff --git a/server/internal/hid/clipboard/clipboard.go b/server/internal/hid/clipboard/clipboard.go new file mode 100644 index 0000000..628aeef --- /dev/null +++ b/server/internal/hid/clipboard/clipboard.go @@ -0,0 +1,124 @@ +package clipboard + +import ( + "errors" + "os" + "os/exec" +) + +const ( + xsel = "xsel" + xclip = "xclip" + wlcopy = "wl-copy" + wlpaste = "wl-paste" + termuxClipboardGet = "termux-clipboard-get" + termuxClipboardSet = "termux-clipboard-set" +) + +var ( + Unsupported bool + Primary bool + + pasteCmdArgs []string + copyCmdArgs []string + + xselPasteArgs = []string{xsel, "--output", "--clipboard"} + xselCopyArgs = []string{xsel, "--input", "--clipboard"} + + xclipPasteArgs = []string{xclip, "-out", "-selection", "clipboard"} + xclipCopyArgs = []string{xclip, "-in", "-selection", "clipboard"} + + wlpasteArgs = []string{wlpaste, "--no-newline"} + wlcopyArgs = []string{wlcopy} + + termuxPasteArgs = []string{termuxClipboardGet} + termuxCopyArgs = []string{termuxClipboardSet} + + missingCommands = errors.New("No clipboard utilities available. Please install xsel, xclip, wl-clipboard or Termux:API add-on for termux-clipboard-get/set.") +) + +func init() { + if os.Getenv("WAYLAND_DISPLAY") != "" { + pasteCmdArgs = wlpasteArgs + copyCmdArgs = wlcopyArgs + + if _, err := exec.LookPath(wlcopy); err == nil { + if _, err := exec.LookPath(wlpaste); err == nil { + return + } + } + } + + pasteCmdArgs = xclipPasteArgs + copyCmdArgs = xclipCopyArgs + + if _, err := exec.LookPath(xclip); err == nil { + return + } + + pasteCmdArgs = xselPasteArgs + copyCmdArgs = xselCopyArgs + + if _, err := exec.LookPath(xsel); err == nil { + return + } + + pasteCmdArgs = termuxPasteArgs + copyCmdArgs = termuxCopyArgs + + if _, err := exec.LookPath(termuxClipboardSet); err == nil { + if _, err := exec.LookPath(termuxClipboardGet); err == nil { + return + } + } + + Unsupported = true +} + +func getPasteCommand() *exec.Cmd { + if Primary { + pasteCmdArgs = pasteCmdArgs[:1] + } + return exec.Command(pasteCmdArgs[0], pasteCmdArgs[1:]...) +} + +func getCopyCommand() *exec.Cmd { + if Primary { + copyCmdArgs = copyCmdArgs[:1] + } + return exec.Command(copyCmdArgs[0], copyCmdArgs[1:]...) +} + +func ReadAll() (string, error) { + if Unsupported { + return "", missingCommands + } + pasteCmd := getPasteCommand() + out, err := pasteCmd.Output() + if err != nil { + return "", err + } + return string(out), nil +} + +func WriteAll(text string) error { + if Unsupported { + return missingCommands + } + copyCmd := getCopyCommand() + in, err := copyCmd.StdinPipe() + if err != nil { + return err + } + + if err := copyCmd.Start(); err != nil { + return err + } + if _, err := in.Write([]byte(text)); err != nil { + return err + } + if err := in.Close(); err != nil { + return err + } + return copyCmd.Wait() +} diff --git a/server/internal/hid/hid.c b/server/internal/hid/hid.c index 56926b1..0c2aa9a 100644 --- a/server/internal/hid/hid.c +++ b/server/internal/hid/hid.c @@ -1,56 +1,56 @@ #include "hid.h" -static Display *display = NULL; -static char *name = ":0.0"; -static int registered = 0; -static int dirty = 0; +static Display *DISPLAY = NULL; +static char *NAME = ":0.0"; +static int REGISTERED = 0; +static int DIRTY = 0; Display *getXDisplay(void) { /* Close the display if displayName has changed */ - if (dirty) { + if (DIRTY) { closeXDisplay(); - dirty = 0; + DIRTY = 0; } - if (display == NULL) { + if (DISPLAY == NULL) { /* First try the user set displayName */ - display = XOpenDisplay(name); + DISPLAY = XOpenDisplay(NAME); /* Then try using environment variable DISPLAY */ - if (display == NULL) { - display = XOpenDisplay(NULL); + if (DISPLAY == NULL) { + DISPLAY = XOpenDisplay(NULL); } - if (display == NULL) { + if (DISPLAY == NULL) { fputs("Could not open main display\n", stderr); - } else if (!registered) { + } else if (!REGISTERED) { atexit(&closeXDisplay); - registered = 1; + REGISTERED = 1; } } - return display; + return DISPLAY; } void closeXDisplay(void) { - if (display != NULL) { - XCloseDisplay(display); - display = NULL; + if (DISPLAY != NULL) { + XCloseDisplay(DISPLAY); + DISPLAY = NULL; } } void setXDisplay(char *input) { - name = strdup(input); - dirty = 1; + NAME = strdup(input); + DIRTY = 1; } -void mouseMove(int x, int y) { +void XMove(int x, int y) { Display *display = getXDisplay(); XWarpPointer(display, None, DefaultRootWindow(display), 0, 0, 0, 0, x, y); XSync(display, 0); } -void mouseScroll(int x, int y) { +void XScroll(int x, int y) { int ydir = 4; /* Button 4 is up, 5 is down. */ int xdir = 6; @@ -59,7 +59,7 @@ void mouseScroll(int x, int y) { if (y < 0) { ydir = 5; } - + if (x < 0) { xdir = 7; } @@ -80,13 +80,13 @@ void mouseScroll(int x, int y) { XSync(display, 0); } -void mouseEvent(unsigned int button, int down) { +void XButton(unsigned int button, int down) { Display *display = getXDisplay(); XTestFakeButtonEvent(display, button, down, CurrentTime); XSync(display, 0); } -void keyEvent(unsigned long key, int down) { +void XKey(unsigned long key, int down) { Display *display = getXDisplay(); KeyCode code = XKeysymToKeycode(display, key); XTestFakeKeyEvent(display, code, down, CurrentTime); diff --git a/server/internal/hid/hid.go b/server/internal/hid/hid.go index 6f18f45..0c9950c 100644 --- a/server/internal/hid/hid.go +++ b/server/internal/hid/hid.go @@ -134,11 +134,11 @@ func Display(display string) { } func Move(x, y int) { - C.mouseMove(C.int(x), C.int(y)) + C.XMove(C.int(x), C.int(y)) } func Scroll(x, y int) { - C.mouseScroll(C.int(x), C.int(y)) + C.XScroll(C.int(x), C.int(y)) } func ButtonDown(code int) (*keycode.Button, error) { @@ -153,7 +153,7 @@ func ButtonDown(code int) (*keycode.Button, error) { debounce[code] = time.Now() - C.mouseEvent(C.uint(button.Keysym), C.int(1)) + C.XButton(C.uint(button.Keysym), C.int(1)) return &button, nil } @@ -168,12 +168,11 @@ func KeyDown(code int) (*keycode.Key, error) { } debounce[code] = time.Now() - - C.keyEvent(C.ulong(key.Keysym), C.int(1)) + + C.XKey(C.ulong(key.Keysym), C.int(1)) return &key, nil } - func ButtonUp(code int) (*keycode.Button, error) { button, ok := buttons[code] if !ok { @@ -186,7 +185,7 @@ func ButtonUp(code int) (*keycode.Button, error) { delete(debounce, code) - C.mouseEvent(C.uint(button.Keysym), C.int(0)) + C.XButton(C.uint(button.Keysym), C.int(0)) return &button, nil } @@ -202,13 +201,13 @@ func KeyUp(code int) (*keycode.Key, error) { delete(debounce, code) - C.keyEvent(C.ulong(key.Keysym), C.int(0)) + C.XKey(C.ulong(key.Keysym), C.int(0)) return &key, nil } func Reset() { for key := range debounce { - if (key < 8) { + if key < 8 { ButtonUp(key) } else { KeyUp(key) @@ -225,7 +224,7 @@ func Check(duration time.Duration) { continue } - if (key < 8) { + if key < 8 { ButtonUp(key) } else { KeyUp(key) diff --git a/server/internal/hid/hid.h b/server/internal/hid/hid.h index 58a607b..55b78df 100644 --- a/server/internal/hid/hid.h +++ b/server/internal/hid/hid.h @@ -16,12 +16,13 @@ * * Note that this is almost certainly not thread safe. */ Display *getXDisplay(void); - void closeXDisplay(void); - void mouseMove(int x, int y); - void mouseScroll(int x, int y); - void mouseEvent(unsigned int button, int down); - void keyEvent(unsigned long key, int down); + void XMove(int x, int y); + void XScroll(int x, int y); + void XButton(unsigned int button, int down); + void XKey(unsigned long key, int down); + + void closeXDisplay(void); #ifdef __cplusplus extern "C" { diff --git a/server/internal/session/session.go b/server/internal/session/session.go index 78280f2..f411d6d 100644 --- a/server/internal/session/session.go +++ b/server/internal/session/session.go @@ -64,6 +64,8 @@ func (session *Session) SetMuted(muted bool) { func (session *Session) SetName(name string) error { session.name = name + session.connected = true + session.manager.emmiter.Emit("connected", session.id, session) return nil } @@ -74,8 +76,6 @@ func (session *Session) SetSocket(socket types.WebScoket) error { func (session *Session) SetPeer(peer types.Peer) error { session.peer = peer - session.connected = true - session.manager.emmiter.Emit("connected", session.id, session) return nil } @@ -122,12 +122,16 @@ func (session *Session) WriteAudioSample(sample types.Sample) error { } func (session *Session) destroy() error { - if err := session.socket.Destroy(); err != nil { - return err + if session.socket != nil { + if err := session.socket.Destroy(); err != nil { + return err + } } - if err := session.peer.Destroy(); err != nil { - return err + if session.peer != nil { + if err := session.peer.Destroy(); err != nil { + return err + } } return nil diff --git a/server/internal/types/event/events.go b/server/internal/types/event/events.go index 522aabf..723501b 100644 --- a/server/internal/types/event/events.go +++ b/server/internal/types/event/events.go @@ -17,6 +17,7 @@ const CONTROL_RELEASE = "control/release" const CONTROL_REQUEST = "control/request" const CONTROL_REQUESTING = "control/requesting" const CONTROL_GIVE = "control/give" +const CONTROL_CLIPBOARD = "control/clipboard" const CHAT_MESSAGE = "chat/message" const CHAT_EMOTE = "chat/emote" diff --git a/server/internal/types/message/messages.go b/server/internal/types/message/messages.go index 11fdcd9..4542b5c 100644 --- a/server/internal/types/message/messages.go +++ b/server/internal/types/message/messages.go @@ -42,6 +42,11 @@ type MemberDisconnected struct { ID string `json:"id"` } +type Clipboard struct { + Event string `json:"event"` + Text string `json:"text"` +} + type Control struct { Event string `json:"event"` ID string `json:"id"` diff --git a/server/internal/webrtc/logger.go b/server/internal/webrtc/logger.go index 6468e34..5276486 100644 --- a/server/internal/webrtc/logger.go +++ b/server/internal/webrtc/logger.go @@ -8,8 +8,22 @@ import ( "github.com/rs/zerolog" ) +type nulllog struct{} + +func (l nulllog) Trace(msg string) {} +func (l nulllog) Tracef(format string, args ...interface{}) {} +func (l nulllog) Debug(msg string) {} +func (l nulllog) Debugf(format string, args ...interface{}) {} +func (l nulllog) Info(msg string) {} +func (l nulllog) Infof(format string, args ...interface{}) {} +func (l nulllog) Warn(msg string) {} +func (l nulllog) Warnf(format string, args ...interface{}) {} +func (l nulllog) Error(msg string) {} +func (l nulllog) Errorf(format string, args ...interface{}) {} + type logger struct { - logger zerolog.Logger + logger zerolog.Logger + subsystem string } func (l logger) Trace(msg string) { l.logger.Trace().Msg(msg) } @@ -18,16 +32,18 @@ func (l logger) Debug(msg string) { l.logger.Debug().Ms func (l logger) Debugf(format string, args ...interface{}) { l.logger.Debug().Msgf(format, args...) } func (l logger) Info(msg string) { if strings.Contains(msg, "packetio.Buffer is full") { - l.logger.Panic().Msg(msg) + //l.logger.Panic().Msg(msg) + return } l.logger.Info().Msg(msg) } func (l logger) Infof(format string, args ...interface{}) { msg := fmt.Sprintf(format, args...) if strings.Contains(msg, "packetio.Buffer is full") { - l.logger.Panic().Msg(msg) + // l.logger.Panic().Msg(msg) + return } - l.logger.Info().Msgf(format, args...) + l.logger.Info().Msg(msg) } func (l logger) Warn(msg string) { l.logger.Warn().Msg(msg) } func (l logger) Warnf(format string, args ...interface{}) { l.logger.Warn().Msgf(format, args...) } @@ -39,7 +55,12 @@ type loggerFactory struct { } func (l loggerFactory) NewLogger(subsystem string) logging.LeveledLogger { + if subsystem == "sctp" { + return nulllog{} + } + return logger{ - logger: l.logger.With().Str("subsystem", subsystem).Logger(), + subsystem: subsystem, + logger: l.logger.With().Str("subsystem", subsystem).Logger(), } } diff --git a/server/internal/webrtc/tracks.go b/server/internal/webrtc/tracks.go index e00b59b..236421c 100644 --- a/server/internal/webrtc/tracks.go +++ b/server/internal/webrtc/tracks.go @@ -17,7 +17,7 @@ func (m *WebRTCManager) createVideoTrack(engine webrtc.MediaEngine) (*webrtc.Tra } if codec == nil || codec.PayloadType == 0 { - return nil, fmt.Errorf("remote peer does not support %s", m.videoPipeline.CodecName) + return nil, fmt.Errorf("remote peer does not support video codec %s", m.videoPipeline.CodecName) } return webrtc.NewTrack(codec.PayloadType, rand.Uint32(), "stream", "stream", codec) @@ -33,7 +33,7 @@ func (m *WebRTCManager) createAudioTrack(engine webrtc.MediaEngine) (*webrtc.Tra } if codec == nil || codec.PayloadType == 0 { - return nil, fmt.Errorf("remote peer does not support %s", m.audioPipeline.CodecName) + return nil, fmt.Errorf("remote peer does not support audio codec %s", m.audioPipeline.CodecName) } return webrtc.NewTrack(codec.PayloadType, rand.Uint32(), "stream", "stream", codec) diff --git a/server/internal/webrtc/webrtc.go b/server/internal/webrtc/webrtc.go index b3e9d3d..0f8ce00 100644 --- a/server/internal/webrtc/webrtc.go +++ b/server/internal/webrtc/webrtc.go @@ -133,32 +133,32 @@ func (m *WebRTCManager) Shutdown() error { } func (m *WebRTCManager) CreatePeer(id string, sdp string) (string, types.Peer, error) { - // create SessionDescription + // Create SessionDescription description := webrtc.SessionDescription{ SDP: sdp, Type: webrtc.SDPTypeOffer, } - // create MediaEngine based off sdp + // Create MediaEngine based off sdp engine := webrtc.MediaEngine{} engine.PopulateFromSDP(description) - // create API with MediaEngine and SettingEngine + // Create API with MediaEngine and SettingEngine api := webrtc.NewAPI(webrtc.WithMediaEngine(engine), webrtc.WithSettingEngine(m.setings)) - // create new peer connection + // Create new peer connection connection, err := api.NewPeerConnection(*m.configuration) if err != nil { return "", nil, err } - // create video track + // Create video track video, err := m.createVideoTrack(engine) if err != nil { return "", nil, err } - videoTransceiver, err := connection.AddTransceiverFromTrack(video, webrtc.RtpTransceiverInit{ + _, err = connection.AddTransceiverFromTrack(video, webrtc.RtpTransceiverInit{ Direction: webrtc.RTPTransceiverDirectionSendonly, }) @@ -166,13 +166,13 @@ func (m *WebRTCManager) CreatePeer(id string, sdp string) (string, types.Peer, e return "", nil, err } - // create audio track + // Create audio track audio, err := m.createAudioTrack(engine) if err != nil { return "", nil, err } - audioTransceiver, err := connection.AddTransceiverFromTrack(audio, webrtc.RtpTransceiverInit{ + _, err = connection.AddTransceiverFromTrack(audio, webrtc.RtpTransceiverInit{ Direction: webrtc.RTPTransceiverDirectionSendonly, }) @@ -182,15 +182,25 @@ func (m *WebRTCManager) CreatePeer(id string, sdp string) (string, types.Peer, e // clear the Transceiver bufers go func() { - for { - if _, err := audioTransceiver.Sender.ReadRTCP(); err != nil { - return - } + defer func() { + m.logger.Warn().Msgf("ReadRTCP shutting down") + }() - if _, err = videoTransceiver.Sender.ReadRTCP(); err != nil { - return - } - } + /* + for { + packet, err := videoTransceiver.Sender.ReadRTCP() + if err != nil { + return + } + m.logger.Debug().Msgf("vReadRTCP %v", packet) + + packet, err = audioTransceiver.Sender.ReadRTCP() + if err != nil { + return + } + m.logger.Debug().Msgf("aReadRTCP %v", packet) + } + */ }() // set remote description diff --git a/server/internal/websocket/control.go b/server/internal/websocket/control.go index ca7c6ce..c34af5c 100644 --- a/server/internal/websocket/control.go +++ b/server/internal/websocket/control.go @@ -1,6 +1,7 @@ package websocket import ( + "n.eko.moe/neko/internal/hid/clipboard" "n.eko.moe/neko/internal/types" "n.eko.moe/neko/internal/types/event" "n.eko.moe/neko/internal/types/message" @@ -104,3 +105,14 @@ func (h *MessageHandler) controlGive(id string, session types.Session, payload * return nil } + +func (h *MessageHandler) controlClipboard(id string, session types.Session, payload *message.Clipboard) error { + // check if session is host + if !h.sessions.IsHost(id) { + h.logger.Debug().Str("id", id).Msg("is not the host") + return nil + } + + clipboard.WriteAll(payload.Text) + return nil +} diff --git a/server/internal/websocket/handler.go b/server/internal/websocket/handler.go index ba10c48..d850271 100644 --- a/server/internal/websocket/handler.go +++ b/server/internal/websocket/handler.go @@ -82,6 +82,12 @@ func (h *MessageHandler) Message(id string, raw []byte) error { utils.Unmarshal(payload, raw, func() error { return h.controlGive(id, session, payload) }), "%s failed", header.Event) + case event.CONTROL_CLIPBOARD: + payload := &message.Clipboard{} + return errors.Wrapf( + utils.Unmarshal(payload, raw, func() error { + return h.controlClipboard(id, session, payload) + }), "%s failed", header.Event) // Chat Events case event.CHAT_MESSAGE: diff --git a/server/internal/websocket/websocket.go b/server/internal/websocket/websocket.go index 339d6a9..e24953e 100644 --- a/server/internal/websocket/websocket.go +++ b/server/internal/websocket/websocket.go @@ -10,6 +10,7 @@ import ( "github.com/rs/zerolog/log" "n.eko.moe/neko/internal/config" + "n.eko.moe/neko/internal/hid/clipboard" "n.eko.moe/neko/internal/types" "n.eko.moe/neko/internal/types/event" "n.eko.moe/neko/internal/types/message" @@ -51,20 +52,6 @@ type WebSocketHandler struct { } func (ws *WebSocketHandler) Start() error { - - go func() { - defer func() { - ws.logger.Info().Msg("shutdown") - }() - - for { - select { - case <-ws.shutdown: - return - } - } - }() - ws.sessions.OnCreated(func(id string, session types.Session) { if err := ws.handler.SessionCreated(id, session); err != nil { ws.logger.Warn().Str("id", id).Err(err).Msg("session created with and error") @@ -89,6 +76,40 @@ func (ws *WebSocketHandler) Start() error { } }) + go func() { + defer func() { + ws.logger.Info().Msg("shutdown") + }() + + current := "" + clip, err := clipboard.ReadAll() + if err == nil && clip != current { + current = clip + } + + for { + select { + case <-ws.shutdown: + return + default: + if ws.sessions.HasHost() { + clip, err := clipboard.ReadAll() + if err == nil && clip != current { + session, ok := ws.sessions.GetHost() + if ok { + session.Send(message.Clipboard{ + Event: event.CONTROL_CLIPBOARD, + Text: clip, + }) + } + current = clip + } + } + time.Sleep(100 * time.Millisecond) + } + } + }() + return nil }