neko/server/internal/webrtc/manager.go

577 lines
15 KiB
Go
Raw Normal View History

package webrtc
import (
"fmt"
2022-02-10 09:45:03 +13:00
"net"
2021-02-15 02:40:17 +13:00
"strings"
"sync"
2022-02-13 06:55:56 +13:00
"sync/atomic"
2021-02-26 05:16:37 +13:00
"time"
2022-02-10 09:45:03 +13:00
"github.com/pion/ice/v2"
"github.com/pion/interceptor"
"github.com/pion/interceptor/pkg/cc"
"github.com/pion/interceptor/pkg/gcc"
2021-11-29 11:16:26 +13:00
"github.com/pion/rtcp"
2021-02-02 11:50:18 +13:00
"github.com/pion/webrtc/v3"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/demodesk/neko/internal/config"
"github.com/demodesk/neko/internal/webrtc/cursor"
"github.com/demodesk/neko/internal/webrtc/pionlog"
"github.com/demodesk/neko/pkg/types"
"github.com/demodesk/neko/pkg/types/codec"
"github.com/demodesk/neko/pkg/types/event"
"github.com/demodesk/neko/pkg/types/message"
"github.com/demodesk/neko/pkg/utils"
)
2023-04-17 09:34:02 +12:00
const (
// size of receiving channel used to buffer incoming TCP packets
tcpReadChanBufferSize = 50
2021-02-26 05:16:37 +13:00
2023-04-17 09:34:02 +12:00
// size of buffer used to buffer outgoing TCP packets. Default is 4MB
tcpWriteBufferSizeInBytes = 4 * 1024 * 1024
2021-02-26 05:16:37 +13:00
2023-04-17 09:34:02 +12:00
// the duration without network activity before a Agent is considered disconnected. Default is 5 Seconds
disconnectedTimeout = 4 * time.Second
2021-02-26 05:16:37 +13:00
2023-04-17 09:34:02 +12:00
// the duration without network activity before a Agent is considered failed after disconnected. Default is 25 Seconds
failedTimeout = 6 * time.Second
2022-01-07 08:41:44 +13:00
2023-04-17 09:34:02 +12:00
// how often the ICE Agent sends extra traffic if there is no activity, if media is flowing no traffic will be sent. Default is 2 seconds
keepAliveInterval = 2 * time.Second
// send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval
rtcpPLIInterval = 3 * time.Second
)
2020-11-02 04:09:48 +13:00
func New(desktop types.DesktopManager, capture types.CaptureManager, config *config.WebRTC) *WebRTCManagerCtx {
2023-04-17 10:42:29 +12:00
logger := log.With().Str("module", "webrtc").Logger()
configuration := webrtc.Configuration{
2023-04-17 09:34:02 +12:00
SDPSemantics: webrtc.SDPSemanticsUnifiedPlan,
}
if !config.ICELite {
ICEServers := []webrtc.ICEServer{}
for _, server := range config.ICEServersBackend {
var credential any
if server.Credential != "" {
credential = server.Credential
} else {
credential = false
}
ICEServers = append(ICEServers, webrtc.ICEServer{
URLs: server.URLs,
Username: server.Username,
Credential: credential,
})
}
configuration.ICEServers = ICEServers
}
2020-11-02 04:09:48 +13:00
return &WebRTCManagerCtx{
2023-04-17 10:42:29 +12:00
logger: logger,
2022-06-26 06:12:42 +12:00
config: config,
2023-04-11 07:33:53 +12:00
metrics: newMetricsManager(),
2021-09-27 09:26:22 +13:00
webrtcConfiguration: configuration,
2021-09-27 09:26:22 +13:00
desktop: desktop,
capture: capture,
2023-04-17 10:42:29 +12:00
curImage: cursor.NewImage(logger, desktop),
curPosition: cursor.NewPosition(logger),
}
}
2020-11-02 04:09:48 +13:00
type WebRTCManagerCtx struct {
2022-06-26 06:12:42 +12:00
logger zerolog.Logger
config *config.WebRTC
2023-04-11 07:33:53 +12:00
metrics *metricsManager
2022-06-26 06:12:42 +12:00
peerId int32
2021-09-27 09:26:22 +13:00
desktop types.DesktopManager
capture types.CaptureManager
2023-04-17 10:42:29 +12:00
curImage cursor.Image
curPosition cursor.Position
2022-01-07 10:28:34 +13:00
webrtcConfiguration webrtc.Configuration
2022-02-10 09:45:03 +13:00
tcpMux ice.TCPMux
udpMux ice.UDPMux
2022-01-07 10:28:34 +13:00
camStop, micStop *func()
}
2020-11-02 04:09:48 +13:00
func (manager *WebRTCManagerCtx) Start() {
2021-08-30 04:59:46 +12:00
manager.curImage.Start()
2022-02-10 09:45:03 +13:00
logger := pionlog.New(manager.logger)
// add TCP Mux listener
if manager.config.TCPMux > 0 {
tcpListener, err := net.ListenTCP("tcp", &net.TCPAddr{
IP: net.IP{0, 0, 0, 0},
Port: manager.config.TCPMux,
})
if err != nil {
2023-04-17 09:34:02 +12:00
manager.logger.Fatal().Err(err).Msg("unable to setup ice TCP mux")
2022-02-10 09:45:03 +13:00
}
manager.tcpMux = ice.NewTCPMuxDefault(ice.TCPMuxParams{
Listener: tcpListener,
Logger: logger.NewLogger("ice-tcp"),
2023-04-17 09:34:02 +12:00
ReadBufferSize: tcpReadChanBufferSize,
WriteBufferSize: tcpWriteBufferSizeInBytes,
})
2022-02-10 09:45:03 +13:00
}
// add UDP Mux listener
if manager.config.UDPMux > 0 {
var err error
manager.udpMux, err = ice.NewMultiUDPMuxFromPort(manager.config.UDPMux,
ice.UDPMuxFromPortWithLogger(logger.NewLogger("ice-udp")),
)
2022-02-10 09:45:03 +13:00
if err != nil {
2023-04-17 09:34:02 +12:00
manager.logger.Fatal().Err(err).Msg("unable to setup ice UDP mux")
2022-02-10 09:45:03 +13:00
}
}
manager.logger.Info().
2021-08-03 21:06:34 +12:00
Bool("icelite", manager.config.ICELite).
Bool("icetrickle", manager.config.ICETrickle).
Interface("iceservers-frontend", manager.config.ICEServersFrontend).
Interface("iceservers-backend", manager.config.ICEServersBackend).
2021-03-18 03:47:49 +13:00
Str("nat1to1", strings.Join(manager.config.NAT1To1IPs, ",")).
Str("epr", fmt.Sprintf("%d-%d", manager.config.EphemeralMin, manager.config.EphemeralMax)).
2022-02-10 09:45:03 +13:00
Int("tcpmux", manager.config.TCPMux).
Int("udpmux", manager.config.UDPMux).
2021-08-03 21:06:34 +12:00
Msg("webrtc starting")
}
2020-11-02 04:09:48 +13:00
func (manager *WebRTCManagerCtx) Shutdown() error {
2021-09-02 10:00:29 +12:00
manager.logger.Info().Msg("shutdown")
2021-02-07 00:59:57 +13:00
manager.curImage.Shutdown()
manager.curPosition.Shutdown()
return nil
}
2021-03-18 03:47:49 +13:00
func (manager *WebRTCManagerCtx) ICEServers() []types.ICEServer {
return manager.config.ICEServersFrontend
2021-02-03 07:21:48 +13:00
}
func (manager *WebRTCManagerCtx) newPeerConnection(logger zerolog.Logger, codecs []codec.RTPCodec) (*webrtc.PeerConnection, cc.BandwidthEstimator, error) {
// create media engine
engine := &webrtc.MediaEngine{}
for _, codec := range codecs {
if err := codec.Register(engine); err != nil {
return nil, nil, err
}
}
// create setting engine
settings := webrtc.SettingEngine{
LoggerFactory: pionlog.New(logger),
}
2023-04-17 09:34:02 +12:00
settings.DisableMediaEngineCopy(true)
settings.SetICETimeouts(disconnectedTimeout, failedTimeout, keepAliveInterval)
settings.SetNAT1To1IPs(manager.config.NAT1To1IPs, webrtc.ICECandidateTypeHost)
settings.SetLite(manager.config.ICELite)
// make sure server answer sdp setup as passive, to not force DTLS renegotiation
// otherwise iOS renegotiation fails with: Failed to set SSL role for the transport.
settings.SetAnsweringDTLSRole(webrtc.DTLSRoleServer)
var networkType []webrtc.NetworkType
// udp candidates
if manager.udpMux != nil {
settings.SetICEUDPMux(manager.udpMux)
networkType = append(networkType,
webrtc.NetworkTypeUDP4,
webrtc.NetworkTypeUDP6,
)
} else if manager.config.EphemeralMax != 0 {
_ = settings.SetEphemeralUDPPortRange(manager.config.EphemeralMin, manager.config.EphemeralMax)
networkType = append(networkType,
webrtc.NetworkTypeUDP4,
webrtc.NetworkTypeUDP6,
)
}
// tcp candidates
if manager.tcpMux != nil {
settings.SetICETCPMux(manager.tcpMux)
networkType = append(networkType,
webrtc.NetworkTypeTCP4,
webrtc.NetworkTypeTCP6,
)
}
// enable support for TCP and UDP ICE candidates
settings.SetNetworkTypes(networkType)
// create interceptor registry
registry := &interceptor.Registry{}
// create bandwidth estimator
estimatorChan := make(chan cc.BandwidthEstimator, 1)
if manager.config.Estimator.Enabled {
congestionController, err := cc.NewInterceptor(func() (cc.BandwidthEstimator, error) {
return gcc.NewSendSideBWE(
gcc.SendSideBWEInitialBitrate(manager.config.Estimator.InitialBitrate),
gcc.SendSideBWEPacer(gcc.NewNoOpPacer()),
)
})
if err != nil {
return nil, nil, err
}
congestionController.OnNewPeerConnection(func(id string, estimator cc.BandwidthEstimator) {
estimatorChan <- estimator
})
registry.Add(congestionController)
if err = webrtc.ConfigureTWCCHeaderExtensionSender(engine, registry); err != nil {
return nil, nil, err
}
} else {
// no estimator, send nil
estimatorChan <- nil
}
if err := webrtc.RegisterDefaultInterceptors(engine, registry); err != nil {
return nil, nil, err
}
// create new API
api := webrtc.NewAPI(
webrtc.WithMediaEngine(engine),
webrtc.WithSettingEngine(settings),
webrtc.WithInterceptorRegistry(registry),
)
// create new peer connection
configuration := manager.webrtcConfiguration
connection, err := api.NewPeerConnection(configuration)
return connection, <-estimatorChan, err
}
func (manager *WebRTCManagerCtx) CreatePeer(session types.Session) (*webrtc.SessionDescription, types.WebRTCPeer, error) {
2022-02-13 06:55:56 +13:00
id := atomic.AddInt32(&manager.peerId, 1)
2023-04-11 07:33:53 +12:00
// get metrics for session
metrics := manager.metrics.getBySession(session)
metrics.NewConnection()
2022-02-13 06:55:56 +13:00
2021-08-30 04:59:46 +12:00
// add session id to logger context
2022-02-13 06:55:56 +13:00
logger := manager.logger.With().Str("session_id", session.ID()).Int32("peer_id", id).Logger()
2021-08-30 04:59:46 +12:00
logger.Info().Msg("creating webrtc peer")
2021-02-03 08:43:33 +13:00
2021-09-27 11:50:49 +13:00
// all audios must have the same codec
audio := manager.capture.Audio()
audioCodec := audio.Codec()
2021-09-27 11:50:49 +13:00
2021-09-27 09:26:22 +13:00
// all videos must have the same codec
video := manager.capture.Video()
videoCodec := video.Codec()
2022-07-04 09:21:25 +12:00
connection, estimator, err := manager.newPeerConnection(
logger, []codec.RTPCodec{audioCodec, videoCodec})
if err != nil {
return nil, nil, err
}
2021-09-27 09:26:22 +13:00
// asynchronously send local ICE Candidates
2021-02-03 09:12:11 +13:00
if manager.config.ICETrickle {
connection.OnICECandidate(func(candidate *webrtc.ICECandidate) {
if candidate == nil {
logger.Debug().Msg("all local ice candidates sent")
return
}
2021-02-03 08:43:33 +13:00
2021-09-02 07:58:39 +12:00
session.Send(
event.SIGNAL_CANDIDATE,
2021-02-03 09:12:11 +13:00
message.SignalCandidate{
2021-09-02 07:58:39 +12:00
ICECandidateInit: candidate.ToJSON(),
2021-02-03 09:12:11 +13:00
})
})
}
2021-02-03 08:43:33 +13:00
2021-09-27 11:50:49 +13:00
// audio track
audioTrack, err := NewTrack(logger, audioCodec, connection)
if err != nil {
return nil, nil, err
}
// we disable audio by default manually
audioTrack.SetPaused(true)
// set stream for audio track
_, err = audioTrack.SetStream(audio)
if err != nil {
return nil, nil, err
}
2021-09-27 11:50:49 +13:00
// video track
videoRtcp := make(chan []rtcp.Packet, 1)
videoTrack, err := NewTrack(logger, videoCodec, connection, WithRtcpChan(videoRtcp))
if err != nil {
return nil, nil, err
}
//
// stream for video track will be set later
//
2021-09-27 11:50:49 +13:00
// data channel
2021-02-13 11:45:16 +13:00
dataChannel, err := connection.CreateDataChannel("data", nil)
2021-02-06 05:57:33 +13:00
if err != nil {
return nil, nil, err
2021-02-03 07:36:11 +13:00
}
2021-02-14 06:25:40 +13:00
peer := &WebRTCPeerCtx{
2023-04-11 07:21:11 +12:00
logger: logger,
2023-04-11 07:37:39 +12:00
session: session,
metrics: metrics,
2023-04-11 07:21:11 +12:00
connection: connection,
// bandwidth estimator
estimator: estimator,
estimateTrend: utils.NewTrendDetector(
utils.TrendDetectorParams{
// Probing
//RequiredSamples: 3,
//DownwardTrendThreshold: 0.0,
//CollapseValues: false,
// Non-Probing
RequiredSamples: 8,
DownwardTrendThreshold: -0.5,
CollapseValues: true,
}),
// stream selectors
video: video,
audio: audio,
2023-04-11 07:21:11 +12:00
// tracks & channels
audioTrack: audioTrack,
videoTrack: videoTrack,
dataChannel: dataChannel,
rtcpChannel: videoRtcp,
// config
iceTrickle: manager.config.ICETrickle,
estimatorConfig: manager.config.Estimator,
audioDisabled: true, // we disable audio by default manually
2023-04-11 07:44:17 +12:00
}
2021-11-29 11:16:26 +13:00
connection.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
2021-12-10 11:22:24 +13:00
logger := logger.With().
2021-12-10 09:48:02 +13:00
Str("kind", track.Kind().String()).
2021-12-03 11:06:21 +13:00
Str("mime", track.Codec().RTPCodecCapability.MimeType).
2021-12-10 11:22:24 +13:00
Logger()
2022-01-07 08:41:44 +13:00
logger.Info().Msgf("received new remote track")
2021-12-10 11:22:24 +13:00
if !session.Profile().CanShareMedia {
2023-04-12 10:36:11 +12:00
err := receiver.Stop()
logger.Warn().Err(err).Msg("media sharing is disabled for this session")
2021-12-10 11:22:24 +13:00
return
}
2021-11-29 11:16:26 +13:00
2022-01-07 08:41:44 +13:00
// parse codec from remote track
2021-12-02 10:36:45 +13:00
codec, ok := codec.ParseRTC(track.Codec())
if !ok {
2023-04-12 10:36:11 +12:00
err := receiver.Stop()
logger.Warn().Err(err).Msg("remote track with unknown codec")
2021-11-29 11:16:26 +13:00
return
}
2021-12-03 11:06:21 +13:00
var srcManager types.StreamSrcManager
2022-01-07 10:28:34 +13:00
stopped := false
stopFn := func() {
if stopped {
return
}
stopped = true
2023-04-12 10:36:11 +12:00
err := receiver.Stop()
2022-01-07 10:28:34 +13:00
srcManager.Stop()
2023-04-12 10:36:11 +12:00
logger.Err(err).Msg("remote track stopped")
2022-01-07 10:28:34 +13:00
}
2021-12-03 11:06:21 +13:00
if track.Kind() == webrtc.RTPCodecTypeAudio {
// audio -> microphone
srcManager = manager.capture.Microphone()
2022-01-07 10:28:34 +13:00
defer stopFn()
if manager.micStop != nil {
(*manager.micStop)()
}
manager.micStop = &stopFn
2021-12-03 11:06:21 +13:00
} else if track.Kind() == webrtc.RTPCodecTypeVideo {
// video -> webcam
srcManager = manager.capture.Webcam()
2022-01-07 10:28:34 +13:00
defer stopFn()
if manager.camStop != nil {
(*manager.camStop)()
}
manager.camStop = &stopFn
} else {
2023-04-12 10:36:11 +12:00
err := receiver.Stop()
logger.Warn().Err(err).Msg("remote track with unsupported codec type")
2022-01-07 10:28:34 +13:00
return
2021-12-03 11:06:21 +13:00
}
2021-12-03 11:35:38 +13:00
err := srcManager.Start(codec)
if err != nil {
logger.Err(err).Msg("failed to start pipeline")
return
}
2021-11-29 11:16:26 +13:00
2022-01-07 08:41:44 +13:00
ticker := time.NewTicker(rtcpPLIInterval)
2021-11-29 11:16:26 +13:00
defer ticker.Stop()
go func() {
for range ticker.C {
2023-04-12 10:36:11 +12:00
err := connection.WriteRTCP([]rtcp.Packet{
&rtcp.PictureLossIndication{
MediaSSRC: uint32(track.SSRC()),
},
})
2021-12-10 11:22:24 +13:00
if err != nil {
2022-01-07 08:41:44 +13:00
logger.Err(err).Msg("remote track rtcp send err")
2021-11-29 11:16:26 +13:00
}
}
}()
buf := make([]byte, 1400)
for {
i, _, err := track.Read(buf)
if err != nil {
2022-01-07 08:41:44 +13:00
logger.Warn().Err(err).Msg("failed read from remote track")
2021-11-29 11:16:26 +13:00
break
}
2021-12-03 11:06:21 +13:00
srcManager.Push(buf[:i])
2021-11-29 11:16:26 +13:00
}
2021-12-02 10:36:45 +13:00
2022-01-07 08:41:44 +13:00
logger.Info().Msg("remote track data finished")
2021-11-26 11:46:06 +13:00
})
connection.OnDataChannel(func(dc *webrtc.DataChannel) {
2022-02-13 06:55:56 +13:00
logger.Info().Interface("data_channel", dc).Msg("got remote data channel")
2021-11-26 11:46:06 +13:00
})
var once sync.Once
2021-10-08 08:41:19 +13:00
connection.OnConnectionStateChange(func(state webrtc.PeerConnectionState) {
switch state {
case webrtc.PeerConnectionStateConnected:
session.SetWebRTCConnected(peer, true)
case webrtc.PeerConnectionStateDisconnected,
webrtc.PeerConnectionStateFailed:
2023-04-18 08:38:03 +12:00
peer.Destroy()
2021-10-08 08:41:19 +13:00
case webrtc.PeerConnectionStateClosed:
// ensure we only run this once
once.Do(func() {
session.SetWebRTCConnected(peer, false)
//
// TODO: Shutdown peer?
//
audioTrack.Shutdown()
videoTrack.Shutdown()
close(videoRtcp)
})
2021-10-08 08:41:19 +13:00
}
2022-06-26 06:12:42 +12:00
2023-04-11 07:33:53 +12:00
metrics.SetState(state)
2021-10-08 08:41:19 +13:00
})
2021-02-14 06:25:40 +13:00
dataChannel.OnOpen(func() {
2023-04-17 10:42:29 +12:00
manager.curImage.AddListener(peer)
manager.curPosition.AddListener(peer)
2021-02-18 09:55:21 +13:00
// send initial cursor image
2023-04-17 10:42:29 +12:00
cur, img, err := manager.curImage.GetCurrent()
if err == nil {
2023-04-17 10:42:29 +12:00
err := peer.SendCursorImage(cur, img)
if err != nil {
logger.Err(err).Msg("failed to set cursor image")
}
} else {
2021-09-02 09:10:06 +12:00
logger.Err(err).Msg("failed to get cursor image")
}
2021-02-21 12:41:04 +13:00
2021-02-18 09:55:21 +13:00
// send initial cursor position
2021-10-22 09:53:54 +13:00
x, y := manager.desktop.GetCursorPosition()
2023-04-17 10:42:29 +12:00
err = peer.SendCursorPosition(x, y)
if err != nil {
logger.Err(err).Msg("failed to set cursor position")
}
2021-02-14 06:25:40 +13:00
})
dataChannel.OnClose(func() {
2023-04-17 10:42:29 +12:00
manager.curImage.RemoveListener(peer)
manager.curPosition.RemoveListener(peer)
})
2021-02-13 11:45:16 +13:00
dataChannel.OnMessage(func(message webrtc.DataChannelMessage) {
2023-04-17 10:47:34 +12:00
if err := manager.handle(logger, message.Data, dataChannel, session); err != nil {
2021-09-02 09:10:06 +12:00
logger.Err(err).Msg("data handle failed")
2021-02-13 11:45:16 +13:00
}
})
2021-02-14 06:25:40 +13:00
session.SetWebRTCPeer(peer)
offer, err := peer.CreateOffer(false)
if err != nil {
return nil, nil, err
}
// on negotiation needed handler must be registered after creating initial
// offer, otherwise it can fire and intercept sucessful negotiation
connection.OnNegotiationNeeded(func() {
logger.Warn().Msg("negotiation is needed")
if connection.SignalingState() != webrtc.SignalingStateStable {
logger.Warn().Msg("connection isn't stable yet; postponing...")
return
}
offer, err := peer.CreateOffer(false)
if err != nil {
logger.Err(err).Msg("sdp offer failed")
return
}
session.Send(
event.SIGNAL_OFFER,
message.SignalDescription{
SDP: offer.SDP,
})
})
2023-04-11 08:24:16 +12:00
// start metrics collectors
go metrics.rtcpReceiver(videoRtcp)
go metrics.connectionStats(connection)
2022-06-26 06:12:42 +12:00
2023-04-17 11:10:20 +12:00
// start estimator reader
go peer.estimatorReader()
return offer, peer, nil
2021-02-03 07:21:48 +13:00
}
2022-07-22 04:28:51 +12:00
func (manager *WebRTCManagerCtx) SetCursorPosition(x, y int) {
manager.curPosition.Set(x, y)
}