neko/internal/webrtc/manager.go

421 lines
11 KiB
Go
Raw Normal View History

package webrtc
import (
2021-08-03 20:47:05 +12:00
"errors"
"fmt"
"io"
2021-02-15 02:40:17 +13:00
"strings"
2021-02-15 06:16:24 +13:00
"sync"
2021-02-26 05:16:37 +13:00
"time"
2021-03-24 01:46:54 +13:00
"github.com/pion/interceptor"
2021-02-02 11:50:18 +13:00
"github.com/pion/webrtc/v3"
"github.com/pion/webrtc/v3/pkg/media"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
2021-02-15 02:40:17 +13:00
"demodesk/neko/internal/config"
2020-10-29 07:15:48 +13:00
"demodesk/neko/internal/types"
2021-02-03 08:43:33 +13:00
"demodesk/neko/internal/types/event"
"demodesk/neko/internal/types/message"
"demodesk/neko/internal/webrtc/cursor"
)
2021-02-26 05:16:37 +13:00
// the duration without network activity before a Agent is considered disconnected. Default is 5 Seconds
const disconnectedTimeout = 4 * time.Second
// the duration without network activity before a Agent is considered failed after disconnected. Default is 25 Seconds
const failedTimeout = 6 * time.Second
// how often the ICE Agent sends extra traffic if there is no activity, if media is flowing no traffic will be sent. Default is 2 seconds
const keepAliveInterval = 2 * time.Second
2020-11-02 04:09:48 +13:00
func New(desktop types.DesktopManager, capture types.CaptureManager, config *config.WebRTC) *WebRTCManagerCtx {
return &WebRTCManagerCtx{
2021-02-15 06:16:24 +13:00
logger: log.With().Str("module", "webrtc").Logger(),
desktop: desktop,
capture: capture,
config: config,
participants: 0,
curImage: cursor.NewImage(desktop),
curPosition: cursor.NewPosition(desktop),
}
}
2020-11-02 04:09:48 +13:00
type WebRTCManagerCtx struct {
2021-02-15 06:16:24 +13:00
mu sync.Mutex
logger zerolog.Logger
audioTrack *webrtc.TrackLocalStaticSample
audioStop func()
desktop types.DesktopManager
capture types.CaptureManager
config *config.WebRTC
participants uint32
curImage *cursor.ImageCtx
curPosition *cursor.PositionCtx
}
2020-11-02 04:09:48 +13:00
func (manager *WebRTCManagerCtx) Start() {
var err error
2020-11-02 04:09:48 +13:00
2021-02-02 11:50:18 +13:00
// create audio track
audio := manager.capture.Audio()
manager.audioTrack, err = webrtc.NewTrackLocalStaticSample(audio.Codec().Capability, "audio", "stream")
if err != nil {
manager.logger.Panic().Err(err).Msg("unable to create audio track")
}
2021-02-15 06:16:24 +13:00
audioListener := func(sample types.Sample) {
2021-08-30 03:09:13 +12:00
if err := manager.audioTrack.WriteSample(media.Sample(sample)); err != nil {
2021-08-03 20:47:05 +12:00
if errors.Is(err, io.ErrClosedPipe) {
// The peerConnection has been closed.
return
}
manager.logger.Warn().Err(err).Msg("audio pipeline failed to write")
}
2021-02-07 00:59:57 +13:00
}
2021-02-15 06:16:24 +13:00
audio.AddListener(&audioListener)
2021-02-15 02:40:17 +13:00
manager.audioStop = func() {
2021-02-15 06:16:24 +13:00
audio.RemoveListener(&audioListener)
2021-02-08 05:07:55 +13:00
}
manager.logger.Info().
2021-08-03 21:06:34 +12:00
Bool("icelite", manager.config.ICELite).
Bool("icetrickle", manager.config.ICETrickle).
Interface("iceservers", manager.config.ICEServers).
2021-03-18 03:47:49 +13:00
Str("nat1to1", strings.Join(manager.config.NAT1To1IPs, ",")).
Str("epr", fmt.Sprintf("%d-%d", manager.config.EphemeralMin, manager.config.EphemeralMax)).
2021-08-03 21:06:34 +12:00
Msg("webrtc starting")
2021-02-14 06:25:40 +13:00
manager.curImage.Start()
}
2020-11-02 04:09:48 +13:00
func (manager *WebRTCManagerCtx) Shutdown() error {
manager.logger.Info().Msgf("webrtc shutting down")
2021-02-07 00:59:57 +13:00
manager.curImage.Shutdown()
manager.curPosition.Shutdown()
2021-02-08 05:07:55 +13:00
manager.audioStop()
return nil
}
2021-03-18 03:47:49 +13:00
func (manager *WebRTCManagerCtx) ICEServers() []types.ICEServer {
2021-02-03 07:21:48 +13:00
return manager.config.ICEServers
}
2021-02-08 05:07:55 +13:00
func (manager *WebRTCManagerCtx) CreatePeer(session types.Session, videoID string) (*webrtc.SessionDescription, error) {
2021-03-14 11:27:28 +13:00
logger := manager.logger.With().Str("session_id", session.ID()).Logger()
2021-02-03 08:43:33 +13:00
// Create MediaEngine
2021-02-08 05:07:55 +13:00
engine, err := manager.mediaEngine(videoID)
2021-02-03 07:21:48 +13:00
if err != nil {
return nil, err
2021-02-02 11:50:18 +13:00
}
2021-02-03 07:21:48 +13:00
// Custom settings & configuration
2021-02-03 08:43:33 +13:00
settings := manager.apiSettings(logger)
2021-02-03 07:21:48 +13:00
configuration := manager.apiConfiguration()
2021-03-24 01:46:54 +13:00
registry := &interceptor.Registry{}
if err := webrtc.RegisterDefaultInterceptors(engine, registry); err != nil {
return nil, err
}
// Create NewAPI with MediaEngine and SettingEngine
2021-03-24 01:46:54 +13:00
api := webrtc.NewAPI(webrtc.WithMediaEngine(engine), webrtc.WithSettingEngine(settings), webrtc.WithInterceptorRegistry(registry))
connection, err := api.NewPeerConnection(*configuration)
if err != nil {
2021-02-03 07:21:48 +13:00
return nil, err
}
2021-02-03 08:43:33 +13:00
// Asynchronously send local ICE Candidates
2021-02-03 09:12:11 +13:00
if manager.config.ICETrickle {
connection.OnICECandidate(func(candidate *webrtc.ICECandidate) {
if candidate == nil {
logger.Debug().Msg("all local ice candidates sent")
return
}
2021-02-03 08:43:33 +13:00
2021-02-03 09:12:11 +13:00
ICECandidateInit := candidate.ToJSON()
err := session.Send(
message.SignalCandidate{
Event: event.SIGNAL_CANDIDATE,
ICECandidateInit: &ICECandidateInit,
})
2021-02-03 08:43:33 +13:00
2021-02-03 09:12:11 +13:00
if err != nil {
logger.Warn().Err(err).Msg("sending ice candidate failed")
}
})
}
2021-02-03 08:43:33 +13:00
// create video track
2021-02-08 05:07:55 +13:00
videoStream, ok := manager.capture.Video(videoID)
if !ok {
2021-02-08 05:07:55 +13:00
manager.logger.Warn().Str("videoID", videoID).Msg("video stream not found")
return nil, err
}
videoTrack, err := webrtc.NewTrackLocalStaticSample(videoStream.Codec().Capability, "video", "stream")
if err != nil {
manager.logger.Warn().Err(err).Msg("unable to create video track")
return nil, err
}
2021-02-15 06:16:24 +13:00
videoListener := func(sample types.Sample) {
2021-08-30 03:09:13 +12:00
if err := videoTrack.WriteSample(media.Sample(sample)); err != nil {
2021-08-03 20:47:05 +12:00
if errors.Is(err, io.ErrClosedPipe) {
// The peerConnection has been closed.
return
}
manager.logger.Warn().Err(err).Msg("video pipeline failed to write")
}
}
2021-02-15 06:16:24 +13:00
manager.mu.Lock()
// should be stream started
if videoStream.ListenersCount() == 0 {
if err := videoStream.Start(); err != nil {
manager.logger.Warn().Err(err).Msg("unable to start video pipeline")
return nil, err
}
}
2021-02-15 06:16:24 +13:00
videoStream.AddListener(&videoListener)
// start audio, when first participant connects
if !manager.capture.Audio().Started() {
if err := manager.capture.Audio().Start(); err != nil {
manager.logger.Panic().Err(err).Msg("unable to start audio stream")
}
}
manager.participants = manager.participants + 1
manager.mu.Unlock()
changeVideo := func(videoID string) error {
newVideoStream, ok := manager.capture.Video(videoID)
if !ok {
2021-08-30 03:09:13 +12:00
return types.ErrWebRTCVideoNotFound
}
// should be new stream started
if newVideoStream.ListenersCount() == 0 {
if err := newVideoStream.Start(); err != nil {
return err
}
}
2021-02-15 06:16:24 +13:00
// switch videoListeners
videoStream.RemoveListener(&videoListener)
newVideoStream.AddListener(&videoListener)
// should be old stream stopped
if videoStream.ListenersCount() == 0 {
videoStream.Stop()
}
2021-02-15 02:40:17 +13:00
videoStream = newVideoStream
return nil
}
2021-03-04 06:53:35 +13:00
rtpAudio, err := connection.AddTrack(manager.audioTrack)
if err != nil {
return nil, err
}
2021-03-04 06:53:35 +13:00
rtpVideo, err := connection.AddTrack(videoTrack)
if err != nil {
return nil, err
}
2021-02-13 11:45:16 +13:00
dataChannel, err := connection.CreateDataChannel("data", nil)
2021-02-06 05:57:33 +13:00
if err != nil {
2021-02-03 07:36:11 +13:00
return nil, err
}
2021-02-14 06:25:40 +13:00
peer := &WebRTCPeerCtx{
2021-02-15 02:40:17 +13:00
api: api,
connection: connection,
changeVideo: changeVideo,
dataChannel: dataChannel,
iceTrickle: manager.config.ICETrickle,
2021-02-14 06:25:40 +13:00
}
cursorImage := func(entry *cursor.ImageEntry) {
if err := peer.SendCursorImage(entry.Cursor, entry.Image); err != nil {
2021-02-14 06:25:40 +13:00
manager.logger.Warn().Err(err).Msg("could not send cursor image")
}
}
cursorPosition := func(x, y int) {
if session.IsHost() {
return
}
if err := peer.SendCursorPosition(x, y); err != nil {
manager.logger.Warn().Err(err).Msg("could not send cursor position")
}
}
connection.OnConnectionStateChange(func(state webrtc.PeerConnectionState) {
switch state {
2020-12-04 05:03:15 +13:00
case webrtc.PeerConnectionStateConnected:
session.SetWebRTCConnected(peer, true)
case webrtc.PeerConnectionStateDisconnected:
fallthrough
case webrtc.PeerConnectionStateFailed:
connection.Close()
2020-12-04 05:03:15 +13:00
case webrtc.PeerConnectionStateClosed:
2021-02-15 06:16:24 +13:00
manager.mu.Lock()
session.SetWebRTCConnected(peer, false)
2021-02-15 06:16:24 +13:00
videoStream.RemoveListener(&videoListener)
// should be stream stopped
if videoStream.ListenersCount() == 0 {
videoStream.Stop()
}
2021-02-14 06:25:40 +13:00
2021-02-15 06:16:24 +13:00
// decrease participants
manager.participants = manager.participants - 1
// stop audio, if last participant disonnects
if manager.participants <= 0 {
manager.participants = 0
if manager.capture.Audio().Started() {
manager.capture.Audio().Stop()
}
}
manager.mu.Unlock()
}
})
2021-02-14 06:25:40 +13:00
dataChannel.OnOpen(func() {
manager.curImage.AddListener(&cursorImage)
manager.curPosition.AddListener(&cursorPosition)
2021-02-18 09:55:21 +13:00
// send initial cursor image
2021-03-01 06:50:32 +13:00
entry, err := manager.curImage.Get()
if err == nil {
cursorImage(entry)
} else {
manager.logger.Warn().Err(err).Msg("failed to get cursor image")
}
2021-02-21 12:41:04 +13:00
2021-02-18 09:55:21 +13:00
// send initial cursor position
2021-03-01 06:50:32 +13:00
x, y := manager.curPosition.Get()
2021-02-18 09:55:21 +13:00
cursorPosition(x, y)
2021-02-14 06:25:40 +13:00
})
dataChannel.OnClose(func() {
manager.curImage.RemoveListener(&cursorImage)
manager.curPosition.RemoveListener(&cursorPosition)
})
2021-02-13 11:45:16 +13:00
dataChannel.OnMessage(func(message webrtc.DataChannelMessage) {
if !session.IsHost() {
return
}
2021-02-13 09:06:42 +13:00
2021-02-13 11:45:16 +13:00
if err = manager.handle(message); err != nil {
logger.Warn().Err(err).Msg("data handle failed")
}
})
2021-03-04 06:53:35 +13:00
go func() {
rtcpBuf := make([]byte, 1500)
for {
if _, _, err := rtpAudio.Read(rtcpBuf); err != nil {
return
}
}
}()
go func() {
rtcpBuf := make([]byte, 1500)
for {
if _, _, err := rtpVideo.Read(rtcpBuf); err != nil {
return
}
}
}()
2021-02-14 06:25:40 +13:00
session.SetWebRTCPeer(peer)
return peer.CreateOffer(false)
2021-02-03 07:21:48 +13:00
}
2021-02-08 05:07:55 +13:00
func (manager *WebRTCManagerCtx) mediaEngine(videoID string) (*webrtc.MediaEngine, error) {
2021-02-03 07:21:48 +13:00
engine := &webrtc.MediaEngine{}
// all videos must have the same codec
2021-02-08 05:07:55 +13:00
video, ok := manager.capture.Video(videoID)
2021-02-06 06:07:58 +13:00
if !ok {
2021-08-30 03:09:13 +12:00
return nil, types.ErrWebRTCVideoNotFound
2021-02-06 06:07:58 +13:00
}
videoCodec := video.Codec()
if err := videoCodec.Register(engine); err != nil {
2021-02-03 07:21:48 +13:00
return nil, err
}
audioCodec := manager.capture.Audio().Codec()
if err := audioCodec.Register(engine); err != nil {
2021-02-03 07:21:48 +13:00
return nil, err
}
return engine, nil
}
2021-03-24 01:46:54 +13:00
func (manager *WebRTCManagerCtx) apiSettings(logger zerolog.Logger) webrtc.SettingEngine {
settings := webrtc.SettingEngine{
2021-02-03 07:21:48 +13:00
LoggerFactory: loggerFactory{
2021-02-03 08:43:33 +13:00
logger: logger,
2021-02-03 07:21:48 +13:00
},
}
//nolint
settings.SetEphemeralUDPPortRange(manager.config.EphemeralMin, manager.config.EphemeralMax)
2021-02-26 05:16:37 +13:00
settings.SetICETimeouts(disconnectedTimeout, failedTimeout, keepAliveInterval)
2021-02-03 07:21:48 +13:00
settings.SetNAT1To1IPs(manager.config.NAT1To1IPs, webrtc.ICECandidateTypeHost)
2021-03-24 01:46:54 +13:00
//settings.SetSRTPReplayProtectionWindow(512)
2021-02-03 07:21:48 +13:00
settings.SetLite(manager.config.ICELite)
2021-02-03 09:12:11 +13:00
2021-02-03 07:21:48 +13:00
return settings
}
func (manager *WebRTCManagerCtx) apiConfiguration() *webrtc.Configuration {
if manager.config.ICELite {
return &webrtc.Configuration{
SDPSemantics: webrtc.SDPSemanticsUnifiedPlanWithFallback,
}
}
2021-03-18 03:47:49 +13:00
ICEServers := []webrtc.ICEServer{}
for _, server := range manager.config.ICEServers {
var credential interface{}
if server.Credential != "" {
credential = server.Credential
} else {
credential = false
}
ICEServers = append(ICEServers, webrtc.ICEServer{
URLs: server.URLs,
Username: server.Username,
Credential: credential,
})
}
2021-02-03 07:21:48 +13:00
return &webrtc.Configuration{
2021-03-18 03:47:49 +13:00
ICEServers: ICEServers,
2021-02-03 07:21:48 +13:00
SDPSemantics: webrtc.SDPSemanticsUnifiedPlanWithFallback,
}
}