iridium/neko
2
2
Fork 0
mirror of https://github.com/m1k1o/neko.git synced 2024-07-24 14:40:50 +12:00
Code Issues Packages Projects Releases Wiki Activity

fixes #14

Browse Source
This commit is contained in:
Craig
2020-01-24 15:47:37 +00:00
parent a0866a4ab9
commit e3a73aa264
26 changed files with 1154 additions and 934 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
server/internal/webrtc/logger.go
View File

@ -1,6 +1,9 @@
package webrtc
import (
"fmt"
"strings"
"github.com/pion/logging"
"github.com/rs/zerolog"
)
@ -13,8 +16,19 @@ func (l logger) Trace(msg string) { l.logger.Trace().Ms
func (l logger) Tracef(format string, args ...interface{}) { l.logger.Trace().Msgf(format, args...) }
func (l logger) Debug(msg string) { l.logger.Debug().Msg(msg) }
func (l logger) Debugf(format string, args ...interface{}) { l.logger.Debug().Msgf(format, args...) }
func (l logger) Info(msg string) { l.logger.Info().Msg(msg) }
func (l logger) Infof(format string, args ...interface{}) { l.logger.Info().Msgf(format, args...) }
func (l logger) Info(msg string) {
if strings.Contains(msg, "packetio.Buffer is full") {
l.logger.Panic().Msg(msg)
}
l.logger.Info().Msg(msg)
}
func (l logger) Infof(format string, args ...interface{}) {
msg := fmt.Sprintf(format, args...)
if strings.Contains(msg, "packetio.Buffer is full") {
l.logger.Panic().Msg(msg)
}
l.logger.Info().Msgf(format, args...)
}
func (l logger) Warn(msg string) { l.logger.Warn().Msg(msg) }
func (l logger) Warnf(format string, args ...interface{}) { l.logger.Warn().Msgf(format, args...) }
func (l logger) Error(msg string) { l.logger.Error().Msg(msg) }

226
server/internal/webrtc/manager.go
View File

@ -1,226 +0,0 @@
package webrtc
import (
"fmt"
"time"
"github.com/pion/webrtc/v2"
"github.com/pkg/errors"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"n.eko.moe/neko/internal/config"
"n.eko.moe/neko/internal/event"
"n.eko.moe/neko/internal/gst"
"n.eko.moe/neko/internal/hid"
"n.eko.moe/neko/internal/message"
"n.eko.moe/neko/internal/session"
)
func New(sessions *session.SessionManager, conf *config.WebRTC) *WebRTCManager {
logger := log.With().Str("module", "webrtc").Logger()
engine := webrtc.MediaEngine{}
engine.RegisterDefaultCodecs()
setings := webrtc.SettingEngine{
LoggerFactory: loggerFactory{
logger: logger,
},
}
return &WebRTCManager{
logger: logger,
engine: engine,
setings: setings,
api: webrtc.NewAPI(webrtc.WithMediaEngine(engine), webrtc.WithSettingEngine(setings)),
cleanup: time.NewTicker(1 * time.Second),
shutdown: make(chan bool),
sessions: sessions,
conf: conf,
config: webrtc.Configuration{
ICEServers: []webrtc.ICEServer{
{
URLs: []string{"stun:stun.l.google.com:19302"},
},
},
SDPSemantics: webrtc.SDPSemanticsUnifiedPlanWithFallback,
},
}
}
type WebRTCManager struct {
logger zerolog.Logger
engine webrtc.MediaEngine
setings webrtc.SettingEngine
config webrtc.Configuration
sessions *session.SessionManager
api *webrtc.API
video *webrtc.Track
audio *webrtc.Track
videoPipeline *gst.Pipeline
audioPipeline *gst.Pipeline
cleanup *time.Ticker
conf *config.WebRTC
shutdown chan bool
}
func (m *WebRTCManager) Start() {
hid.Display(m.conf.Display)
switch m.conf.VideoCodec {
case "vp8":
if err := m.createVideoTrack(webrtc.DefaultPayloadTypeVP8); err != nil {
m.logger.Panic().Err(err).Msg("unable to start webrtc manager")
}
case "vp9":
if err := m.createVideoTrack(webrtc.DefaultPayloadTypeVP9); err != nil {
m.logger.Panic().Err(err).Msg("unable to start webrtc manager")
}
case "h264":
if err := m.createVideoTrack(webrtc.DefaultPayloadTypeH264); err != nil {
m.logger.Panic().Err(err).Msg("unable to start webrtc manager")
}
default:
m.logger.Panic().Err(errors.Errorf("unknown video codec %s", m.conf.AudioCodec)).Msg("unable to start webrtc manager")
}
switch m.conf.AudioCodec {
case "opus":
if err := m.createAudioTrack(webrtc.DefaultPayloadTypeOpus); err != nil {
m.logger.Panic().Err(err).Msg("unable to start webrtc manager")
}
case "g722":
if err := m.createAudioTrack(webrtc.DefaultPayloadTypeG722); err != nil {
m.logger.Panic().Err(err).Msg("unable to start webrtc manager")
}
case "pcmu":
if err := m.createAudioTrack(webrtc.DefaultPayloadTypePCMU); err != nil {
m.logger.Panic().Err(err).Msg("unable to start webrtc manager")
}
case "pcma":
if err := m.createAudioTrack(webrtc.DefaultPayloadTypePCMA); err != nil {
m.logger.Panic().Err(err).Msg("unable to start webrtc manager")
}
default:
m.logger.Panic().Err(errors.Errorf("unknown audio codec %s", m.conf.AudioCodec)).Msg("unable to start webrtc manager")
}
m.videoPipeline.Start()
m.audioPipeline.Start()
go func() {
defer func() {
m.logger.Info().Msg("shutdown")
}()
for {
select {
case <-m.shutdown:
return
case <-m.cleanup.C:
hid.Check(time.Second * 10)
}
}
}()
m.sessions.OnHostCleared(func(id string) {
hid.Reset()
})
m.sessions.OnCreated(func(id string, session *session.Session) {
m.logger.Debug().Str("id", id).Msg("session created")
})
m.sessions.OnDestroy(func(id string) {
m.logger.Debug().Str("id", id).Msg("session destroyed")
})
// TODO: log resolution, bit rate and codec parameters
m.logger.Info().
Str("video_display", m.conf.Display).
Str("video_codec", m.conf.VideoCodec).
Str("audio_device", m.conf.Device).
Str("audio_codec", m.conf.AudioCodec).
Msgf("webrtc streaming")
}
func (m *WebRTCManager) Shutdown() error {
m.logger.Info().Msgf("webrtc shutting down")
m.cleanup.Stop()
m.shutdown <- true
m.videoPipeline.Stop()
m.audioPipeline.Stop()
return nil
}
func (m *WebRTCManager) CreatePeer(id string, sdp string) error {
session, ok := m.sessions.Get(id)
if !ok {
return fmt.Errorf("invalid session id %s", id)
}
peer, err := m.api.NewPeerConnection(m.config)
if err != nil {
return err
}
if _, err := peer.AddTransceiverFromTrack(m.video, webrtc.RtpTransceiverInit{
Direction: webrtc.RTPTransceiverDirectionSendonly,
}); err != nil {
return err
}
if _, err := peer.AddTransceiverFromTrack(m.audio, webrtc.RtpTransceiverInit{
Direction: webrtc.RTPTransceiverDirectionSendonly,
}); err != nil {
return err
}
peer.SetRemoteDescription(webrtc.SessionDescription{
SDP: sdp,
Type: webrtc.SDPTypeOffer,
})
answer, err := peer.CreateAnswer(nil)
if err != nil {
return err
}
if err = peer.SetLocalDescription(answer); err != nil {
return err
}
if err := session.Send(message.Signal{
Event: event.SIGNAL_ANSWER,
SDP: answer.SDP,
}); err != nil {
return err
}
peer.OnDataChannel(func(d *webrtc.DataChannel) {
d.OnMessage(func(msg webrtc.DataChannelMessage) {
if err = m.handle(id, msg); err != nil {
m.logger.Warn().Err(err).Msg("data handle failed")
}
})
})
peer.OnConnectionStateChange(func(connectionState webrtc.PeerConnectionState) {
switch connectionState {
case webrtc.PeerConnectionStateDisconnected:
case webrtc.PeerConnectionStateFailed:
m.logger.Info().Str("id", id).Msg("peer disconnected")
m.sessions.Destroy(id)
break
case webrtc.PeerConnectionStateConnected:
m.logger.Info().Str("id", id).Msg("peer connected")
break
}
})
m.sessions.SetPeer(id, peer)
return nil
}

44
server/internal/webrtc/peer.go Normal file
View File

@ -0,0 +1,44 @@
package webrtc
import (
"github.com/pion/webrtc/v2"
"github.com/pion/webrtc/v2/pkg/media"
"n.eko.moe/neko/internal/types"
)
type Peer struct {
id string
engine webrtc.MediaEngine
api *webrtc.API
video *webrtc.Track
audio *webrtc.Track
connection *webrtc.PeerConnection
}
func (peer *Peer) WriteAudioSample(sample types.Sample) error {
if err := peer.audio.WriteSample(media.Sample(sample)); err != nil {
return err
}
return nil
}
func (peer *Peer) WriteVideoSample(sample types.Sample) error {
if err := peer.video.WriteSample(media.Sample(sample)); err != nil {
return err
}
return nil
}
func (peer *Peer) WriteData(v interface{}) error {
return nil
}
func (peer *Peer) Destroy() error {
if peer.connection != nil && peer.connection.ConnectionState() == webrtc.PeerConnectionStateConnected {
if err := peer.connection.Close(); err != nil {
return err
}
}
return nil
}

95
server/internal/webrtc/tracks.go
View File

@ -5,95 +5,36 @@ import (
"math/rand"
"github.com/pion/webrtc/v2"
"github.com/pkg/errors"
"n.eko.moe/neko/internal/gst"
)
func (m *WebRTCManager) createVideoTrack(payloadType uint8) error {
clockrate := uint32(90000)
func (m *WebRTCManager) createVideoTrack(engine webrtc.MediaEngine) (*webrtc.Track, error) {
var codec *webrtc.RTPCodec
switch payloadType {
case webrtc.DefaultPayloadTypeVP8:
codec = webrtc.NewRTPVP8Codec(payloadType, clockrate)
break
case webrtc.DefaultPayloadTypeVP9:
codec = webrtc.NewRTPVP9Codec(payloadType, clockrate)
break
case webrtc.DefaultPayloadTypeH264:
codec = webrtc.NewRTPH264Codec(payloadType, clockrate)
break
default:
return errors.Errorf("unknown video codec %s", payloadType)
for _, videoCodec := range engine.GetCodecsByKind(webrtc.RTPCodecTypeVideo) {
if videoCodec.Name == m.videoPipeline.CodecName {
codec = videoCodec
break
}
}
track, err := webrtc.NewTrack(payloadType, rand.Uint32(), "stream", "stream", codec)
if err != nil {
return err
if codec == nil || codec.PayloadType == 0 {
return nil, fmt.Errorf("remote peer does not support %s", m.videoPipeline.CodecName)
}
var pipeline *gst.Pipeline
src := fmt.Sprintf("ximagesrc xid=%s show-pointer=true use-damage=false ! video/x-raw,framerate=30/1 ! videoconvert ! queue", m.conf.Display)
switch payloadType {
case webrtc.DefaultPayloadTypeVP8:
pipeline = gst.CreatePipeline(webrtc.VP8, []*webrtc.Track{track}, src)
break
case webrtc.DefaultPayloadTypeVP9:
pipeline = gst.CreatePipeline(webrtc.VP9, []*webrtc.Track{track}, src)
break
case webrtc.DefaultPayloadTypeH264:
pipeline = gst.CreatePipeline(webrtc.H264, []*webrtc.Track{track}, src)
break
}
m.video = track
m.videoPipeline = pipeline
return nil
return webrtc.NewTrack(codec.PayloadType, rand.Uint32(), "stream", "stream", codec)
}
func (m *WebRTCManager) createAudioTrack(payloadType uint8) error {
func (m *WebRTCManager) createAudioTrack(engine webrtc.MediaEngine) (*webrtc.Track, error) {
var codec *webrtc.RTPCodec
switch payloadType {
case webrtc.DefaultPayloadTypeOpus:
codec = webrtc.NewRTPOpusCodec(payloadType, 48000)
break
case webrtc.DefaultPayloadTypeG722:
codec = webrtc.NewRTPG722Codec(payloadType, 48000)
break
case webrtc.DefaultPayloadTypePCMU:
codec = webrtc.NewRTPPCMUCodec(payloadType, 8000)
break
case webrtc.DefaultPayloadTypePCMA:
codec = webrtc.NewRTPPCMACodec(payloadType, 8000)
break
default:
return errors.Errorf("unknown audio codec %s", payloadType)
for _, videoCodec := range engine.GetCodecsByKind(webrtc.RTPCodecTypeAudio) {
if videoCodec.Name == m.audioPipeline.CodecName {
codec = videoCodec
break
}
}
track, err := webrtc.NewTrack(payloadType, rand.Uint32(), "stream", "stream", codec)
if err != nil {
return err
if codec == nil || codec.PayloadType == 0 {
return nil, fmt.Errorf("remote peer does not support %s", m.audioPipeline.CodecName)
}
var pipeline *gst.Pipeline
src := fmt.Sprintf("pulsesrc device=%s ! audioconvert", m.conf.Device)
switch payloadType {
case webrtc.DefaultPayloadTypeOpus:
pipeline = gst.CreatePipeline(webrtc.Opus, []*webrtc.Track{track}, src)
break
case webrtc.DefaultPayloadTypeG722:
pipeline = gst.CreatePipeline(webrtc.G722, []*webrtc.Track{track}, src)
break
case webrtc.DefaultPayloadTypePCMU:
pipeline = gst.CreatePipeline(webrtc.PCMU, []*webrtc.Track{track}, src)
break
case webrtc.DefaultPayloadTypePCMA:
pipeline = gst.CreatePipeline(webrtc.PCMA, []*webrtc.Track{track}, src)
break
}
m.audio = track
m.audioPipeline = pipeline
return nil
return webrtc.NewTrack(codec.PayloadType, rand.Uint32(), "stream", "stream", codec)
}

237
server/internal/webrtc/webrtc.go Normal file
View File

@ -0,0 +1,237 @@
package webrtc
import (
"fmt"
"time"
"github.com/pion/webrtc/v2"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"n.eko.moe/neko/internal/config"
"n.eko.moe/neko/internal/gst"
"n.eko.moe/neko/internal/hid"
"n.eko.moe/neko/internal/types"
)
func New(sessions types.SessionManager, config *config.WebRTC) *WebRTCManager {
logger := log.With().Str("module", "webrtc").Logger()
setings := webrtc.SettingEngine{
LoggerFactory: loggerFactory{
logger: logger,
},
}
return &WebRTCManager{
logger: logger,
setings: setings,
cleanup: time.NewTicker(1 * time.Second),
shutdown: make(chan bool),
sessions: sessions,
config: config,
configuration: &webrtc.Configuration{
ICEServers: []webrtc.ICEServer{
{
URLs: []string{"stun:stun.l.google.com:19302"},
},
},
SDPSemantics: webrtc.SDPSemanticsUnifiedPlanWithFallback,
},
}
}
type WebRTCManager struct {
logger zerolog.Logger
setings webrtc.SettingEngine
sessions types.SessionManager
videoPipeline *gst.Pipeline
audioPipeline *gst.Pipeline
cleanup *time.Ticker
config *config.WebRTC
shutdown chan bool
configuration *webrtc.Configuration
}
func (m *WebRTCManager) Start() {
hid.Display(m.config.Display)
videoPipeline, err := gst.CreatePipeline(
m.config.VideoCodec,
fmt.Sprintf("ximagesrc xid=%s show-pointer=true use-damage=false ! video/x-raw,framerate=30/1 ! videoconvert ! queue", m.config.Display),
)
if err != nil {
m.logger.Panic().Err(err).Msg("unable to start webrtc manager")
}
audioPipeline, err := gst.CreatePipeline(
m.config.AudioCodec,
fmt.Sprintf("pulsesrc device=%s ! audioconvert", m.config.Device),
)
if err != nil {
m.logger.Panic().Err(err).Msg("unable to start webrtc manager")
}
m.videoPipeline = videoPipeline
m.audioPipeline = audioPipeline
videoPipeline.Start()
audioPipeline.Start()
go func() {
defer func() {
m.logger.Info().Msg("shutdown")
}()
for {
select {
case <-m.shutdown:
return
case sample := <-videoPipeline.Sample:
if err := m.sessions.WriteVideoSample(sample); err != nil {
m.logger.Warn().Err(err).Msg("video pipeline failed")
}
case sample := <-audioPipeline.Sample:
if err := m.sessions.WriteAudioSample(sample); err != nil {
m.logger.Warn().Err(err).Msg("audio pipeline failed")
}
case <-m.cleanup.C:
hid.Check(time.Second * 10)
}
}
}()
m.sessions.OnHostCleared(func(id string) {
hid.Reset()
})
m.sessions.OnCreated(func(id string, session types.Session) {
m.logger.Debug().Str("id", id).Msg("session created")
})
m.sessions.OnDestroy(func(id string) {
m.logger.Debug().Str("id", id).Msg("session destroyed")
})
// TODO: log resolution, bit rate and codec parameters
m.logger.Info().
Str("video_display", m.config.Display).
Str("video_codec", m.config.VideoCodec).
Str("audio_device", m.config.Device).
Str("audio_codec", m.config.AudioCodec).
Msgf("webrtc streaming")
}
func (m *WebRTCManager) Shutdown() error {
m.logger.Info().Msgf("webrtc shutting down")
m.videoPipeline.Stop()
m.audioPipeline.Stop()
m.cleanup.Stop()
m.shutdown <- true
return nil
}
func (m *WebRTCManager) CreatePeer(id string, sdp string) (string, types.Peer, error) {
// create SessionDescription
description := webrtc.SessionDescription{
SDP: sdp,
Type: webrtc.SDPTypeOffer,
}
// create MediaEngine based off sdp
engine := webrtc.MediaEngine{}
engine.PopulateFromSDP(description)
// create API with MediaEngine and SettingEngine
api := webrtc.NewAPI(webrtc.WithMediaEngine(engine), webrtc.WithSettingEngine(m.setings))
// create new peer connection
connection, err := api.NewPeerConnection(*m.configuration)
if err != nil {
return "", nil, err
}
// create video track
video, err := m.createVideoTrack(engine)
if err != nil {
return "", nil, err
}
videoTransceiver, err := connection.AddTransceiverFromTrack(video, webrtc.RtpTransceiverInit{
Direction: webrtc.RTPTransceiverDirectionSendonly,
})
if err != nil {
return "", nil, err
}
// create audio track
audio, err := m.createAudioTrack(engine)
if err != nil {
return "", nil, err
}
audioTransceiver, err := connection.AddTransceiverFromTrack(audio, webrtc.RtpTransceiverInit{
Direction: webrtc.RTPTransceiverDirectionSendonly,
})
if err != nil {
return "", nil, err
}
// clear the Transceiver bufers
go func() {
for {
if _, err := audioTransceiver.Sender.ReadRTCP(); err != nil {
return
}
if _, err = videoTransceiver.Sender.ReadRTCP(); err != nil {
return
}
}
}()
// set remote description
connection.SetRemoteDescription(description)
answer, err := connection.CreateAnswer(nil)
if err != nil {
return "", nil, err
}
if err = connection.SetLocalDescription(answer); err != nil {
return "", nil, err
}
connection.OnDataChannel(func(d *webrtc.DataChannel) {
d.OnMessage(func(msg webrtc.DataChannelMessage) {
if err = m.handle(id, msg); err != nil {
m.logger.Warn().Err(err).Msg("data handle failed")
}
})
})
connection.OnConnectionStateChange(func(state webrtc.PeerConnectionState) {
switch state {
case webrtc.PeerConnectionStateDisconnected:
case webrtc.PeerConnectionStateFailed:
m.logger.Info().Str("id", id).Msg("peer disconnected")
m.sessions.Destroy(id)
break
case webrtc.PeerConnectionStateConnected:
m.logger.Info().Str("id", id).Msg("peer connected")
break
}
})
return answer.SDP, &Peer{
id: id,
api: api,
engine: engine,
video: video,
audio: audio,
connection: connection,
}, nil
}