neko/server/internal/webrtc/webrtc.go

348 lines
10 KiB
Go
Raw Normal View History

2020-01-25 04:47:37 +13:00
package webrtc
import (
2021-02-15 05:30:24 +13:00
"encoding/json"
2020-01-25 04:47:37 +13:00
"fmt"
2020-02-13 12:13:33 +13:00
"io"
"net"
2020-02-11 18:15:59 +13:00
"strings"
2021-02-16 03:42:10 +13:00
"time"
2020-01-25 04:47:37 +13:00
2021-02-15 05:30:24 +13:00
"github.com/pion/interceptor"
"github.com/pion/webrtc/v3"
"github.com/pion/webrtc/v3/pkg/media"
2020-01-25 04:47:37 +13:00
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
2022-09-13 08:18:18 +12:00
"m1k1o/neko/internal/config"
2021-10-06 09:38:24 +13:00
"m1k1o/neko/internal/types"
2022-09-13 08:24:34 +12:00
"m1k1o/neko/internal/webrtc/pionlog"
2020-01-25 04:47:37 +13:00
)
2022-09-13 08:12:47 +12:00
func New(sessions types.SessionManager, capture types.CaptureManager, desktop types.DesktopManager, config *config.WebRTC) *WebRTCManager {
2020-01-25 04:47:37 +13:00
return &WebRTCManager{
2020-04-05 19:07:45 +12:00
logger: log.With().Str("module", "webrtc").Logger(),
2022-09-13 08:12:47 +12:00
capture: capture,
desktop: desktop,
2020-01-25 04:47:37 +13:00
sessions: sessions,
config: config,
}
}
type WebRTCManager struct {
2020-04-06 10:34:51 +12:00
logger zerolog.Logger
2021-02-15 05:30:24 +13:00
videoTrack *webrtc.TrackLocalStaticSample
audioTrack *webrtc.TrackLocalStaticSample
videoCodec webrtc.RTPCodecParameters
audioCodec webrtc.RTPCodecParameters
2020-04-06 10:34:51 +12:00
sessions types.SessionManager
2022-09-13 08:12:47 +12:00
capture types.CaptureManager
desktop types.DesktopManager
2020-04-06 10:34:51 +12:00
config *config.WebRTC
api *webrtc.API
2020-01-25 04:47:37 +13:00
}
2020-04-06 10:34:51 +12:00
func (manager *WebRTCManager) Start() {
2020-02-14 16:52:43 +13:00
var err error
2022-09-13 08:12:47 +12:00
manager.audioTrack, manager.audioCodec, err = manager.createTrack(manager.capture.AudioCodec())
2020-02-13 12:13:33 +13:00
if err != nil {
2020-04-06 10:34:51 +12:00
manager.logger.Panic().Err(err).Msg("unable to create audio track")
2020-02-13 12:13:33 +13:00
}
2020-01-25 04:47:37 +13:00
2022-09-13 08:12:47 +12:00
manager.capture.OnAudioFrame(func(sample types.Sample) {
2020-04-06 10:34:51 +12:00
if err := manager.audioTrack.WriteSample(media.Sample(sample)); err != nil && err != io.ErrClosedPipe {
manager.logger.Warn().Err(err).Msg("audio pipeline failed to write")
2020-01-25 04:47:37 +13:00
}
})
2022-09-13 08:12:47 +12:00
manager.videoTrack, manager.videoCodec, err = manager.createTrack(manager.capture.VideoCodec())
2020-04-06 10:34:51 +12:00
if err != nil {
manager.logger.Panic().Err(err).Msg("unable to create video track")
}
2020-01-25 04:47:37 +13:00
2022-09-13 08:12:47 +12:00
manager.capture.OnVideoFrame(func(sample types.Sample) {
2020-04-06 10:34:51 +12:00
if err := manager.videoTrack.WriteSample(media.Sample(sample)); err != nil && err != io.ErrClosedPipe {
manager.logger.Warn().Err(err).Msg("video pipeline failed to write")
}
2020-01-25 04:47:37 +13:00
})
if err := manager.initAPI(); err != nil {
manager.logger.Panic().Err(err).Msg("failed to initialize webrtc API")
}
2020-04-06 10:34:51 +12:00
manager.logger.Info().
Str("ice_lite", fmt.Sprintf("%t", manager.config.ICELite)).
Str("ice_servers", fmt.Sprintf("%+v", manager.config.ICEServers)).
2020-04-06 10:34:51 +12:00
Str("ephemeral_port_range", fmt.Sprintf("%d-%d", manager.config.EphemeralMin, manager.config.EphemeralMax)).
Str("nat_ips", strings.Join(manager.config.NAT1To1IPs, ",")).
Msgf("webrtc starting")
2020-01-25 04:47:37 +13:00
}
2020-04-06 10:34:51 +12:00
func (manager *WebRTCManager) Shutdown() error {
manager.logger.Info().Msgf("webrtc shutting down")
2020-01-25 04:47:37 +13:00
return nil
}
func (manager *WebRTCManager) initAPI() error {
2022-09-13 08:24:34 +12:00
logger := pionlog.New(manager.logger)
2020-04-05 19:07:45 +12:00
settings := webrtc.SettingEngine{
LoggerFactory: logger,
2020-04-05 19:07:45 +12:00
}
2021-10-06 10:10:10 +13:00
_ = settings.SetEphemeralUDPPortRange(manager.config.EphemeralMin, manager.config.EphemeralMax)
2020-04-06 10:34:51 +12:00
settings.SetNAT1To1IPs(manager.config.NAT1To1IPs, webrtc.ICECandidateTypeHost)
2021-04-05 08:37:33 +12:00
settings.SetICETimeouts(6*time.Second, 6*time.Second, 3*time.Second)
2021-02-15 05:30:24 +13:00
settings.SetSRTPReplayProtectionWindow(512)
settings.SetLite(manager.config.ICELite)
2020-04-05 19:07:45 +12:00
var networkType []webrtc.NetworkType
// Add TCP Mux
if manager.config.TCPMUX > 0 {
tcpListener, err := net.ListenTCP("tcp", &net.TCPAddr{
IP: net.IP{0, 0, 0, 0},
Port: manager.config.TCPMUX,
})
2020-04-06 10:34:51 +12:00
if err != nil {
return err
}
tcpMux := webrtc.NewICETCPMux(logger.NewLogger("ice-tcp"), tcpListener, 32)
settings.SetICETCPMux(tcpMux)
networkType = append(networkType, webrtc.NetworkTypeTCP4)
manager.logger.Info().Str("listener", tcpListener.Addr().String()).Msg("using TCP MUX")
}
// Add UDP Mux
if manager.config.UDPMUX > 0 {
udpListener, err := net.ListenUDP("udp", &net.UDPAddr{
IP: net.IP{0, 0, 0, 0},
Port: manager.config.UDPMUX,
})
if err != nil {
return err
}
udpMux := webrtc.NewICEUDPMux(logger.NewLogger("ice-udp"), udpListener)
settings.SetICEUDPMux(udpMux)
networkType = append(networkType, webrtc.NetworkTypeUDP4)
manager.logger.Info().Str("listener", udpListener.LocalAddr().String()).Msg("using UDP MUX")
}
// Enable support for TCP and UDP ICE candidates
if len(networkType) > 0 {
settings.SetNetworkTypes(networkType)
}
// Create MediaEngine with selected codecs
engine := webrtc.MediaEngine{}
2021-10-06 10:10:10 +13:00
_ = engine.RegisterCodec(manager.audioCodec, webrtc.RTPCodecTypeAudio)
_ = engine.RegisterCodec(manager.videoCodec, webrtc.RTPCodecTypeVideo)
2021-02-15 05:30:24 +13:00
// Register Interceptors
2021-02-15 05:30:24 +13:00
i := &interceptor.Registry{}
if err := webrtc.RegisterDefaultInterceptors(&engine, i); err != nil {
return err
2021-02-15 05:30:24 +13:00
}
2020-04-05 19:07:45 +12:00
// Create API with MediaEngine and SettingEngine
manager.api = webrtc.NewAPI(
webrtc.WithMediaEngine(&engine),
webrtc.WithSettingEngine(settings),
webrtc.WithInterceptorRegistry(i),
)
return nil
}
func (manager *WebRTCManager) CreatePeer(id string, session types.Session) (types.Peer, error) {
configuration := webrtc.Configuration{
SDPSemantics: webrtc.SDPSemanticsUnifiedPlanWithFallback,
}
if !manager.config.ICELite {
configuration.ICEServers = manager.config.ICEServers
}
2020-04-05 19:07:45 +12:00
2020-01-26 03:29:52 +13:00
// Create new peer connection
connection, err := manager.api.NewPeerConnection(configuration)
2020-01-25 04:47:37 +13:00
if err != nil {
2021-12-03 11:43:36 +13:00
return nil, err
2020-01-25 04:47:37 +13:00
}
2021-02-15 09:39:05 +13:00
2021-02-15 05:30:24 +13:00
negotiated := true
2021-02-15 09:39:05 +13:00
_, err = connection.CreateDataChannel("data", &webrtc.DataChannelInit{
2021-02-15 05:30:24 +13:00
Negotiated: &negotiated,
})
2021-02-15 09:39:05 +13:00
if err != nil {
2021-12-03 11:43:36 +13:00
return nil, err
2021-02-15 09:39:05 +13:00
}
2021-02-15 05:30:24 +13:00
connection.OnDataChannel(func(d *webrtc.DataChannel) {
d.OnMessage(func(msg webrtc.DataChannelMessage) {
if err = manager.handle(id, msg); err != nil {
manager.logger.Warn().Err(err).Msg("data handle failed")
}
})
})
2020-01-25 04:47:37 +13:00
2021-02-15 05:30:24 +13:00
// Set the handler for ICE connection state
// This will notify you when the peer has connected/disconnected
connection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
2021-02-15 09:39:05 +13:00
manager.logger.Info().
Str("connection_state", connectionState.String()).
Msg("connection state has changed")
2021-02-15 05:30:24 +13:00
})
2021-04-05 08:37:33 +12:00
rtpVideo, err := connection.AddTrack(manager.videoTrack)
if err != nil {
2021-12-03 11:43:36 +13:00
return nil, err
2020-01-25 04:47:37 +13:00
}
2021-04-05 08:37:33 +12:00
rtpAudio, err := connection.AddTrack(manager.audioTrack)
if err != nil {
2021-12-03 11:43:36 +13:00
return nil, err
2021-02-15 05:30:24 +13:00
}
2020-01-25 04:47:37 +13:00
connection.OnConnectionStateChange(func(state webrtc.PeerConnectionState) {
switch state {
case webrtc.PeerConnectionStateDisconnected:
2021-03-02 02:31:16 +13:00
manager.logger.Info().Str("id", id).Msg("peer disconnected")
manager.sessions.Destroy(id)
2020-01-25 04:47:37 +13:00
case webrtc.PeerConnectionStateFailed:
2021-03-02 02:31:27 +13:00
manager.logger.Warn().Str("id", id).Msg("peer failed")
manager.sessions.Destroy(id)
case webrtc.PeerConnectionStateClosed:
manager.logger.Info().Str("id", id).Msg("peer closed")
2020-04-06 10:34:51 +12:00
manager.sessions.Destroy(id)
2020-01-25 04:47:37 +13:00
case webrtc.PeerConnectionStateConnected:
2020-04-06 10:34:51 +12:00
manager.logger.Info().Str("id", id).Msg("peer connected")
2020-02-13 12:13:33 +13:00
if err = session.SetConnected(true); err != nil {
2020-04-06 10:34:51 +12:00
manager.logger.Warn().Err(err).Msg("unable to set connected on peer")
manager.sessions.Destroy(id)
2020-02-13 12:13:33 +13:00
}
2020-01-25 04:47:37 +13:00
}
})
2021-12-03 11:43:36 +13:00
peer := &Peer{
id: id,
manager: manager,
connection: connection,
2021-12-03 11:43:36 +13:00
}
connection.OnNegotiationNeeded(func() {
manager.logger.Warn().Msg("negotiation is needed")
sdp, err := peer.CreateOffer()
if err != nil {
manager.logger.Err(err).Msg("creating offer failed")
return
}
err = session.SignalLocalOffer(sdp)
if err != nil {
manager.logger.Warn().Err(err).Msg("sending SignalLocalOffer failed")
return
}
})
2021-02-15 05:30:24 +13:00
connection.OnICECandidate(func(i *webrtc.ICECandidate) {
2021-02-15 09:39:05 +13:00
if i == nil {
manager.logger.Info().Msg("sent all ICECandidates")
return
}
2021-02-15 05:30:24 +13:00
2021-02-15 09:39:05 +13:00
candidateString, err := json.Marshal(i.ToJSON())
if err != nil {
manager.logger.Warn().Err(err).Msg("converting ICECandidate to json failed")
return
2021-02-15 05:30:24 +13:00
}
2021-02-15 09:39:05 +13:00
if err := session.SignalCandidate(string(candidateString)); err != nil {
manager.logger.Warn().Err(err).Msg("sending SignalCandidate failed")
return
}
})
2021-02-15 05:30:24 +13:00
2021-12-03 11:43:36 +13:00
if err := session.SetPeer(peer); err != nil {
return nil, err
2020-02-13 12:13:33 +13:00
}
go func() {
rtcpBuf := make([]byte, 1500)
for {
if _, _, rtcpErr := rtpVideo.Read(rtcpBuf); rtcpErr != nil {
return
}
}
}()
go func() {
rtcpBuf := make([]byte, 1500)
for {
if _, _, rtcpErr := rtpAudio.Read(rtcpBuf); rtcpErr != nil {
return
}
}
}()
2021-12-03 11:43:36 +13:00
return peer, nil
}
func (manager *WebRTCManager) ICELite() bool {
return manager.config.ICELite
}
func (manager *WebRTCManager) ICEServers() []webrtc.ICEServer {
return manager.config.ICEServers
2020-01-25 04:47:37 +13:00
}
2020-02-11 18:15:59 +13:00
func (manager *WebRTCManager) ImplicitControl() bool {
return manager.config.ImplicitControl
}
2021-11-28 06:04:48 +13:00
func (manager *WebRTCManager) createTrack(codecName string) (*webrtc.TrackLocalStaticSample, webrtc.RTPCodecParameters, error) {
2021-02-15 05:30:24 +13:00
var codec webrtc.RTPCodecParameters
2021-02-15 09:39:05 +13:00
2021-11-28 06:04:48 +13:00
id := ""
fb := []webrtc.RTCPFeedback{}
2021-02-15 05:30:24 +13:00
2020-04-06 10:34:51 +12:00
switch codecName {
2021-02-15 05:30:24 +13:00
case "VP8":
2021-07-23 06:58:15 +12:00
codec = webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90000, Channels: 0, SDPFmtpLine: "", RTCPFeedback: fb}, PayloadType: 96}
2021-11-28 06:04:48 +13:00
id = "video"
2021-02-15 05:30:24 +13:00
case "VP9":
2021-07-23 06:58:15 +12:00
codec = webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, Channels: 0, SDPFmtpLine: "", RTCPFeedback: fb}, PayloadType: 98}
2021-11-28 06:04:48 +13:00
id = "video"
2021-02-15 05:30:24 +13:00
case "H264":
codec = webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000, Channels: 0, SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", RTCPFeedback: fb}, PayloadType: 102}
2021-11-28 06:04:48 +13:00
id = "video"
2021-02-15 05:30:24 +13:00
case "Opus":
2022-03-25 09:48:03 +13:00
codec = webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "useinbandfec=1", RTCPFeedback: fb}, PayloadType: 111}
2021-11-28 06:04:48 +13:00
id = "audio"
2021-02-15 05:30:24 +13:00
case "G722":
2021-07-23 06:58:15 +12:00
codec = webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeG722, ClockRate: 8000, Channels: 0, SDPFmtpLine: "", RTCPFeedback: fb}, PayloadType: 9}
2021-11-28 06:04:48 +13:00
id = "audio"
2021-02-15 05:30:24 +13:00
case "PCMU":
2021-07-23 06:58:15 +12:00
codec = webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMU, ClockRate: 8000, Channels: 0, SDPFmtpLine: "", RTCPFeedback: fb}, PayloadType: 0}
2021-11-28 06:04:48 +13:00
id = "audio"
2021-02-15 05:30:24 +13:00
case "PCMA":
2021-07-23 06:58:15 +12:00
codec = webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypePCMA, ClockRate: 8000, Channels: 0, SDPFmtpLine: "", RTCPFeedback: fb}, PayloadType: 8}
2021-11-28 06:04:48 +13:00
id = "audio"
2020-04-06 10:34:51 +12:00
default:
2021-02-15 05:30:24 +13:00
return nil, codec, fmt.Errorf("unknown codec %s", codecName)
2020-02-11 18:15:59 +13:00
}
2021-11-28 06:04:48 +13:00
track, err := webrtc.NewTrackLocalStaticSample(codec.RTPCodecCapability, id, "stream")
2020-02-11 18:15:59 +13:00
if err != nil {
2021-02-15 05:30:24 +13:00
return nil, codec, err
2020-02-11 18:15:59 +13:00
}
2020-04-06 10:34:51 +12:00
return track, codec, nil
2020-02-11 18:15:59 +13:00
}