From 9daf83cc5213fbc5758bf66c5cb0b41b96329028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Sat, 18 Mar 2023 00:49:25 +0100 Subject: [PATCH] fix webrtc client gathering, #259. --- client/src/neko/base.ts | 28 ++++++++++++++++---- docs/changelog.md | 1 + server/internal/session/session.go | 19 +++++++++---- server/internal/types/session.go | 3 ++- server/internal/types/webrtc.go | 1 + server/internal/webrtc/peer.go | 11 ++++++++ server/internal/webrtc/webrtc.go | 14 +++++----- server/internal/websocket/handler/handler.go | 6 +++++ server/internal/websocket/handler/signal.go | 4 +++ 9 files changed, 70 insertions(+), 17 deletions(-) diff --git a/client/src/neko/base.ts b/client/src/neko/base.ts index 57baef86..43267b1d 100644 --- a/client/src/neko/base.ts +++ b/client/src/neko/base.ts @@ -203,11 +203,12 @@ export abstract class BaseClient extends EventEmitter { return } - this._peer = new RTCPeerConnection() if (lite !== true) { this._peer = new RTCPeerConnection({ iceServers: servers, }) + } else { + this._peer = new RTCPeerConnection() } this._peer.onconnectionstatechange = () => { @@ -251,11 +252,28 @@ export abstract class BaseClient extends EventEmitter { this._peer.ontrack = this.onTrack.bind(this) + this._peer.onicecandidate = (event: RTCPeerConnectionIceEvent) => { + if (!event.candidate) { + this.emit('debug', `sent all local ICE candidates`) + return + } + + const init = event.candidate.toJSON() + this.emit('debug', `sending local ICE candidate`, init) + + this._ws!.send( + JSON.stringify({ + event: EVENT.SIGNAL.CANDIDATE, + data: JSON.stringify(init), + }), + ) + } + this._peer.onnegotiationneeded = async () => { this.emit('warn', `negotiation is needed`) const d = await this._peer!.createOffer() - this._peer!.setLocalDescription(d) + await this._peer!.setLocalDescription(d) this._ws!.send( JSON.stringify({ @@ -277,10 +295,10 @@ export abstract class BaseClient extends EventEmitter { return } - this._peer.setRemoteDescription({ type: 'offer', sdp }) + await this._peer.setRemoteDescription({ type: 'offer', sdp }) for (const candidate of this._candidates) { - this._peer.addIceCandidate(candidate) + await this._peer.addIceCandidate(candidate) } this._candidates = [] @@ -310,7 +328,7 @@ export abstract class BaseClient extends EventEmitter { return } - this._peer.setRemoteDescription({ type: 'answer', sdp }) + await this._peer.setRemoteDescription({ type: 'answer', sdp }) } private async onMessage(e: MessageEvent) { diff --git a/docs/changelog.md b/docs/changelog.md index bfc6cf19..bcf05dde 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -12,6 +12,7 @@ - Fixed stereo problem in chromium-based browsers, where it was only as mono by adding `stereo=1` to opus SDP to clients answer. - Fixed keysym mapping for unknown keycodes, which was causing some key combinations to not work on some keyboards. - Fixed a bug where `max_fps=0` would lead to an invalid pipeline. +- Fixed client side webrtc ICE gathering, so that neko can be used without exposed ports, only with STUN and TURN servers. ### Misc - Updated to go 1.19 and Node 18, removed go-events as dependency (by @mbattista). diff --git a/server/internal/session/session.go b/server/internal/session/session.go index 1b1e0f75..96aaf1a3 100644 --- a/server/internal/session/session.go +++ b/server/internal/session/session.go @@ -131,6 +131,17 @@ func (session *Session) SignalLocalAnswer(sdp string) error { }) } +func (session *Session) SignalLocalCandidate(data string) error { + if session.socket == nil { + return nil + } + session.peer.SetCandidate(data) + return session.socket.Send(&message.SignalCandidate{ + Event: event.SIGNAL_CANDIDATE, + Data: data, + }) +} + func (session *Session) SignalRemoteOffer(sdp string) error { if session.peer == nil { return nil @@ -154,14 +165,12 @@ func (session *Session) SignalRemoteAnswer(sdp string) error { return session.peer.SetAnswer(sdp) } -func (session *Session) SignalCandidate(data string) error { +func (session *Session) SignalRemoteCandidate(data string) error { if session.socket == nil { return nil } - return session.socket.Send(&message.SignalCandidate{ - Event: event.SIGNAL_CANDIDATE, - Data: data, - }) + session.logger.Info().Msg("signal update - RemoteCandidate") + return session.peer.SetCandidate(data) } func (session *Session) destroy() error { diff --git a/server/internal/types/session.go b/server/internal/types/session.go index 9b0f339d..4217c996 100644 --- a/server/internal/types/session.go +++ b/server/internal/types/session.go @@ -40,9 +40,10 @@ type Session interface { Send(v interface{}) error SignalLocalOffer(sdp string) error SignalLocalAnswer(sdp string) error + SignalLocalCandidate(data string) error SignalRemoteOffer(sdp string) error SignalRemoteAnswer(sdp string) error - SignalCandidate(data string) error + SignalRemoteCandidate(data string) error } type SessionManager interface { diff --git a/server/internal/types/webrtc.go b/server/internal/types/webrtc.go index 26595607..7db122c9 100644 --- a/server/internal/types/webrtc.go +++ b/server/internal/types/webrtc.go @@ -21,6 +21,7 @@ type Peer interface { CreateAnswer() (string, error) SetOffer(sdp string) error SetAnswer(sdp string) error + SetCandidate(candidateString string) error WriteData(v interface{}) error Destroy() error } diff --git a/server/internal/webrtc/peer.go b/server/internal/webrtc/peer.go index 9ff6d774..e109eba1 100644 --- a/server/internal/webrtc/peer.go +++ b/server/internal/webrtc/peer.go @@ -1,6 +1,7 @@ package webrtc import ( + "encoding/json" "sync" "github.com/pion/webrtc/v3" @@ -49,6 +50,16 @@ func (peer *Peer) SetAnswer(sdp string) error { return peer.connection.SetRemoteDescription(webrtc.SessionDescription{SDP: sdp, Type: webrtc.SDPTypeAnswer}) } +func (peer *Peer) SetCandidate(candidateString string) error { + var candidate webrtc.ICECandidateInit + err := json.Unmarshal([]byte(candidateString), &candidate) + if err != nil { + return err + } + + return peer.connection.AddICECandidate(candidate) +} + func (peer *Peer) WriteData(v interface{}) error { peer.mu.Lock() defer peer.mu.Unlock() diff --git a/server/internal/webrtc/webrtc.go b/server/internal/webrtc/webrtc.go index 693f5729..fe278cae 100644 --- a/server/internal/webrtc/webrtc.go +++ b/server/internal/webrtc/webrtc.go @@ -123,7 +123,6 @@ func (manager *WebRTCManager) initAPI() error { LoggerFactory: logger, } - _ = settings.SetEphemeralUDPPortRange(manager.config.EphemeralMin, manager.config.EphemeralMax) settings.SetNAT1To1IPs(manager.config.NAT1To1IPs, webrtc.ICECandidateTypeHost) settings.SetICETimeouts(6*time.Second, 6*time.Second, 3*time.Second) settings.SetSRTPReplayProtectionWindow(512) @@ -168,12 +167,15 @@ func (manager *WebRTCManager) initAPI() error { networkType = append(networkType, webrtc.NetworkTypeUDP4) manager.logger.Info().Int("port", manager.config.UDPMUX).Msg("using UDP MUX") + } else if manager.config.EphemeralMax != 0 { + _ = settings.SetEphemeralUDPPortRange(manager.config.EphemeralMin, manager.config.EphemeralMax) + networkType = append(networkType, + webrtc.NetworkTypeUDP4, + webrtc.NetworkTypeUDP6, + ) } - // Enable support for TCP and UDP ICE candidates - if len(networkType) > 0 { - settings.SetNetworkTypes(networkType) - } + settings.SetNetworkTypes(networkType) // Create MediaEngine with selected codecs engine := webrtc.MediaEngine{} @@ -299,7 +301,7 @@ func (manager *WebRTCManager) CreatePeer(id string, session types.Session) (type return } - if err := session.SignalCandidate(string(candidateString)); err != nil { + if err := session.SignalLocalCandidate(string(candidateString)); err != nil { manager.logger.Warn().Err(err).Msg("sending SignalCandidate failed") return } diff --git a/server/internal/websocket/handler/handler.go b/server/internal/websocket/handler/handler.go index 039ba492..6996f8ed 100644 --- a/server/internal/websocket/handler/handler.go +++ b/server/internal/websocket/handler/handler.go @@ -87,6 +87,12 @@ func (h *MessageHandler) Message(id string, raw []byte) error { utils.Unmarshal(payload, raw, func() error { return h.signalRemoteAnswer(id, session, payload) }), "%s failed", header.Event) + case event.SIGNAL_CANDIDATE: + payload := &message.SignalCandidate{} + return errors.Wrapf( + utils.Unmarshal(payload, raw, func() error { + return h.signalRemoteCandidate(id, session, payload) + }), "%s failed", header.Event) // Control Events case event.CONTROL_RELEASE: diff --git a/server/internal/websocket/handler/signal.go b/server/internal/websocket/handler/signal.go index 409d3a7e..da1f2e0b 100644 --- a/server/internal/websocket/handler/signal.go +++ b/server/internal/websocket/handler/signal.go @@ -45,3 +45,7 @@ func (h *MessageHandler) signalRemoteAnswer(id string, session types.Session, pa return nil } + +func (h *MessageHandler) signalRemoteCandidate(id string, session types.Session, payload *message.SignalCandidate) error { + return session.SignalRemoteCandidate(payload.Data) +}