neko/server/internal/webrtc/metrics.go

459 lines
12 KiB
Go
Raw Normal View History

2022-06-26 06:12:42 +12:00
package webrtc
import (
"sync"
2023-04-11 08:24:16 +12:00
"time"
2022-06-26 06:12:42 +12:00
"github.com/demodesk/neko/pkg/types"
2022-07-04 11:01:03 +12:00
"github.com/pion/rtcp"
2022-06-26 06:12:42 +12:00
"github.com/pion/webrtc/v3"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
2023-04-17 11:21:32 +12:00
const (
// how often to read and process webrtc connection stats
connectionStatsInterval = 5 * time.Second
)
2023-04-11 08:24:16 +12:00
2023-04-11 07:33:53 +12:00
type metricsManager struct {
2022-06-26 06:12:42 +12:00
mu sync.Mutex
2023-04-11 07:37:39 +12:00
sessions map[string]*metrics
2022-06-26 06:12:42 +12:00
}
2023-04-11 07:33:53 +12:00
func newMetricsManager() *metricsManager {
return &metricsManager{
2023-04-11 07:37:39 +12:00
sessions: map[string]*metrics{},
2022-06-26 06:12:42 +12:00
}
}
2023-04-11 07:37:39 +12:00
func (m *metricsManager) getBySession(session types.Session) *metrics {
2022-06-26 06:12:42 +12:00
m.mu.Lock()
defer m.mu.Unlock()
2023-04-11 07:33:53 +12:00
sessionId := session.ID()
met, ok := m.sessions[sessionId]
2022-06-26 06:12:42 +12:00
if ok {
return met
}
2023-04-11 07:37:39 +12:00
met = &metrics{
2023-04-11 07:33:53 +12:00
sessionId: sessionId,
2022-06-26 06:12:42 +12:00
connectionState: promauto.NewGauge(prometheus.GaugeOpts{
Name: "connection_state",
Namespace: "neko",
Subsystem: "webrtc",
Help: "Connection state of session.",
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
2022-06-26 06:12:42 +12:00
},
}),
2022-07-01 09:54:06 +12:00
connectionStateCount: promauto.NewCounter(prometheus.CounterOpts{
Name: "connection_state_count",
Namespace: "neko",
Subsystem: "webrtc",
Help: "Count of connection state changes for a session.",
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
2022-07-01 09:54:06 +12:00
},
}),
2022-06-26 06:12:42 +12:00
connectionCount: promauto.NewCounter(prometheus.CounterOpts{
Name: "connection_count",
Namespace: "neko",
Subsystem: "webrtc",
Help: "Connection count of a session.",
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
2022-06-26 06:12:42 +12:00
},
}),
2022-07-01 09:50:47 +12:00
2022-07-04 09:06:56 +12:00
iceCandidates: map[string]struct{}{},
iceCandidatesMu: &sync.Mutex{},
iceCandidatesUdpCount: promauto.NewCounter(prometheus.CounterOpts{
2022-07-01 09:50:47 +12:00
Name: "ice_candidates_count",
Namespace: "neko",
Subsystem: "webrtc",
Help: "Count of ICE candidates sent by a remote client.",
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
"protocol": "udp",
},
}),
iceCandidatesTcpCount: promauto.NewCounter(prometheus.CounterOpts{
Name: "ice_candidates_count",
Namespace: "neko",
Subsystem: "webrtc",
Help: "Count of ICE candidates sent by a remote client.",
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
"protocol": "tcp",
},
}),
iceCandidatesUsedUdp: promauto.NewGauge(prometheus.GaugeOpts{
Name: "ice_candidates_used",
Namespace: "neko",
Subsystem: "webrtc",
Help: "Used ICE candidates that are currently in use.",
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
"protocol": "udp",
},
}),
iceCandidatesUsedTcp: promauto.NewGauge(prometheus.GaugeOpts{
Name: "ice_candidates_used",
Namespace: "neko",
Subsystem: "webrtc",
Help: "Used ICE candidates that are currently in use.",
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
"protocol": "tcp",
2022-07-01 09:50:47 +12:00
},
}),
2022-07-04 09:21:25 +12:00
videoIds: map[string]prometheus.Gauge{},
videoIdsMu: &sync.Mutex{},
2022-07-04 10:38:46 +12:00
receiverEstimatedMaximumBitrate: promauto.NewGauge(prometheus.GaugeOpts{
Name: "receiver_estimated_maximum_bitrate",
Namespace: "neko",
Subsystem: "webrtc",
Help: "Receiver Estimated Maximum Bitrate from RTCP.",
2022-07-04 10:38:46 +12:00
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
2022-07-04 10:38:46 +12:00
},
}),
receiverEstimatedTargetBitrate: promauto.NewGauge(prometheus.GaugeOpts{
Name: "receiver_estimated_target_bitrate",
Namespace: "neko",
Subsystem: "webrtc",
Help: "Receiver Estimated Target Bitrate using Google's congestion control.",
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
},
}),
2022-07-04 10:38:46 +12:00
2022-07-04 11:01:03 +12:00
receiverReportDelay: promauto.NewGauge(prometheus.GaugeOpts{
Name: "receiver_report_delay",
Namespace: "neko",
Subsystem: "webrtc",
Help: "Receiver Report Delay from RTCP, expressed in units of 1/65536 seconds.",
2022-07-04 11:01:03 +12:00
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
2022-07-04 11:01:03 +12:00
},
}),
receiverReportJitter: promauto.NewGauge(prometheus.GaugeOpts{
Name: "receiver_report_jitter",
Namespace: "neko",
Subsystem: "webrtc",
Help: "Receiver Report Jitter from RTCP.",
2022-07-04 11:01:03 +12:00
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
2022-07-04 11:01:03 +12:00
},
}),
receiverReportTotalLost: promauto.NewGauge(prometheus.GaugeOpts{
Name: "receiver_report_total_lost",
Namespace: "neko",
Subsystem: "webrtc",
Help: "Receiver Report Total Lost from RTCP.",
ConstLabels: map[string]string{
"session_id": sessionId,
},
}),
transportLayerNacks: promauto.NewCounter(prometheus.CounterOpts{
Name: "transport_layer_nacks",
Namespace: "neko",
Subsystem: "webrtc",
Help: "Transport Layer NACKs from RTCP.",
2022-07-04 11:01:03 +12:00
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
2022-07-04 11:01:03 +12:00
},
}),
2022-07-01 10:16:39 +12:00
iceBytesSent: promauto.NewGauge(prometheus.GaugeOpts{
2022-07-04 10:48:29 +12:00
Name: "bytes_sent",
2022-06-26 06:12:42 +12:00
Namespace: "neko",
Subsystem: "webrtc",
2022-07-04 10:48:29 +12:00
Help: "Sent bytes to a session.",
2022-06-26 06:12:42 +12:00
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
2022-07-04 10:48:29 +12:00
"transport": "ice",
2022-06-26 06:12:42 +12:00
},
}),
2022-07-01 10:16:39 +12:00
iceBytesReceived: promauto.NewGauge(prometheus.GaugeOpts{
2022-07-04 10:48:29 +12:00
Name: "bytes_received",
2022-06-26 06:12:42 +12:00
Namespace: "neko",
Subsystem: "webrtc",
2022-07-04 10:48:29 +12:00
Help: "Received bytes from a session.",
2022-07-01 10:16:39 +12:00
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
2022-07-04 10:48:29 +12:00
"transport": "ice",
2022-07-01 10:16:39 +12:00
},
}),
sctpBytesSent: promauto.NewGauge(prometheus.GaugeOpts{
2022-07-04 10:48:29 +12:00
Name: "bytes_sent",
2022-07-01 10:16:39 +12:00
Namespace: "neko",
Subsystem: "webrtc",
2022-07-04 10:48:29 +12:00
Help: "Sent bytes to a session.",
2022-07-01 10:16:39 +12:00
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
2022-07-04 10:48:29 +12:00
"transport": "sctp",
2022-07-01 10:16:39 +12:00
},
}),
sctpBytesReceived: promauto.NewGauge(prometheus.GaugeOpts{
2022-07-04 10:48:29 +12:00
Name: "bytes_received",
2022-07-01 10:16:39 +12:00
Namespace: "neko",
Subsystem: "webrtc",
2022-07-04 10:48:29 +12:00
Help: "Received bytes from a session.",
2022-06-26 06:12:42 +12:00
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": sessionId,
2022-07-04 10:48:29 +12:00
"transport": "sctp",
2022-06-26 06:12:42 +12:00
},
}),
}
2023-04-11 07:33:53 +12:00
m.sessions[sessionId] = met
2022-06-26 06:12:42 +12:00
return met
}
2023-04-11 07:33:53 +12:00
type metrics struct {
sessionId string
connectionState prometheus.Gauge
connectionStateCount prometheus.Counter
connectionCount prometheus.Counter
iceCandidates map[string]struct{}
iceCandidatesMu *sync.Mutex
iceCandidatesUdpCount prometheus.Counter
iceCandidatesTcpCount prometheus.Counter
iceCandidatesUsedUdp prometheus.Gauge
iceCandidatesUsedTcp prometheus.Gauge
videoIds map[string]prometheus.Gauge
videoIdsMu *sync.Mutex
receiverEstimatedMaximumBitrate prometheus.Gauge
receiverEstimatedTargetBitrate prometheus.Gauge
receiverReportDelay prometheus.Gauge
receiverReportJitter prometheus.Gauge
receiverReportTotalLost prometheus.Gauge
transportLayerNacks prometheus.Counter
2023-04-11 07:33:53 +12:00
iceBytesSent prometheus.Gauge
iceBytesReceived prometheus.Gauge
sctpBytesSent prometheus.Gauge
sctpBytesReceived prometheus.Gauge
}
func (met *metrics) reset() {
2022-07-05 03:47:04 +12:00
met.videoIdsMu.Lock()
for _, entry := range met.videoIds {
entry.Set(0)
}
met.videoIdsMu.Unlock()
met.iceCandidatesUsedUdp.Set(float64(0))
met.iceCandidatesUsedTcp.Set(float64(0))
2022-07-05 03:47:04 +12:00
met.receiverEstimatedMaximumBitrate.Set(0)
met.receiverReportDelay.Set(0)
met.receiverReportJitter.Set(0)
}
2023-04-11 07:33:53 +12:00
func (met *metrics) NewConnection() {
2022-06-26 06:12:42 +12:00
met.connectionCount.Add(1)
}
2023-04-11 07:33:53 +12:00
func (met *metrics) NewICECandidate(candidate webrtc.ICECandidateStats) {
2022-07-04 09:06:56 +12:00
met.iceCandidatesMu.Lock()
defer met.iceCandidatesMu.Unlock()
if _, found := met.iceCandidates[candidate.ID]; found {
2022-07-01 09:50:47 +12:00
return
}
met.iceCandidates[candidate.ID] = struct{}{}
if candidate.Protocol == "udp" {
met.iceCandidatesUdpCount.Add(1)
} else if candidate.Protocol == "tcp" {
met.iceCandidatesTcpCount.Add(1)
}
}
2023-04-11 07:33:53 +12:00
func (met *metrics) SetICECandidatesUsed(candidates []webrtc.ICECandidateStats) {
udp, tcp := 0, 0
for _, candidate := range candidates {
if candidate.Protocol == "udp" {
udp++
} else if candidate.Protocol == "tcp" {
tcp++
}
}
met.iceCandidatesUsedUdp.Set(float64(udp))
met.iceCandidatesUsedTcp.Set(float64(tcp))
2022-07-01 09:50:47 +12:00
}
2023-04-11 07:33:53 +12:00
func (met *metrics) SetState(state webrtc.PeerConnectionState) {
2022-06-26 06:12:42 +12:00
switch state {
case webrtc.PeerConnectionStateNew:
met.connectionState.Set(0)
case webrtc.PeerConnectionStateConnecting:
met.connectionState.Set(4)
case webrtc.PeerConnectionStateConnected:
met.connectionState.Set(5)
case webrtc.PeerConnectionStateDisconnected:
met.connectionState.Set(3)
case webrtc.PeerConnectionStateFailed:
met.connectionState.Set(2)
case webrtc.PeerConnectionStateClosed:
met.connectionState.Set(1)
2023-04-11 07:33:53 +12:00
met.reset()
2022-06-26 06:12:42 +12:00
default:
met.connectionState.Set(-1)
}
2022-07-01 09:54:06 +12:00
met.connectionStateCount.Add(1)
2022-06-26 06:12:42 +12:00
}
2023-04-11 07:33:53 +12:00
func (met *metrics) SetVideoID(videoId string) {
2022-07-04 09:21:25 +12:00
met.videoIdsMu.Lock()
defer met.videoIdsMu.Unlock()
if _, found := met.videoIds[videoId]; !found {
met.videoIds[videoId] = promauto.NewGauge(prometheus.GaugeOpts{
2022-07-05 04:23:35 +12:00
Name: "video_listeners",
2022-07-04 09:21:25 +12:00
Namespace: "neko",
Subsystem: "webrtc",
2022-07-05 04:23:35 +12:00
Help: "Listeners for Video pipelines by a session.",
2022-07-04 09:21:25 +12:00
ConstLabels: map[string]string{
2023-04-11 07:33:53 +12:00
"session_id": met.sessionId,
2022-07-04 09:21:25 +12:00
"video_id": videoId,
},
})
}
for id, entry := range met.videoIds {
if id == videoId {
entry.Set(1)
} else {
entry.Set(0)
}
}
}
2023-04-11 07:33:53 +12:00
func (met *metrics) SetReceiverEstimatedMaximumBitrate(bitrate float32) {
met.receiverEstimatedMaximumBitrate.Set(float64(bitrate))
}
2023-04-11 07:33:53 +12:00
func (met *metrics) SetReceiverEstimatedTargetBitrate(bitrate float64) {
met.receiverEstimatedTargetBitrate.Set(bitrate)
2022-07-04 10:38:46 +12:00
}
2023-04-11 07:33:53 +12:00
func (met *metrics) SetReceiverReport(report rtcp.ReceptionReport) {
2022-07-04 11:01:03 +12:00
met.receiverReportDelay.Set(float64(report.Delay))
met.receiverReportJitter.Set(float64(report.Jitter))
met.receiverReportTotalLost.Set(float64(report.TotalLost))
}
2023-04-11 07:33:53 +12:00
func (met *metrics) SetIceTransportStats(data webrtc.TransportStats) {
2022-07-01 10:16:39 +12:00
met.iceBytesSent.Set(float64(data.BytesSent))
met.iceBytesReceived.Set(float64(data.BytesReceived))
}
2023-04-11 07:33:53 +12:00
func (met *metrics) SetSctpTransportStats(data webrtc.TransportStats) {
2022-07-01 10:16:39 +12:00
met.sctpBytesSent.Set(float64(data.BytesSent))
met.sctpBytesReceived.Set(float64(data.BytesReceived))
2022-06-26 06:12:42 +12:00
}
2023-04-11 08:24:16 +12:00
//
// collectors
//
func (met *metrics) rtcpReceiver(rtcpCh chan []rtcp.Packet) {
for {
packets, ok := <-rtcpCh
if !ok {
break
}
for _, p := range packets {
switch rtcpPacket := p.(type) {
case *rtcp.ReceiverEstimatedMaximumBitrate: // TODO: Deprecated.
met.SetReceiverEstimatedMaximumBitrate(rtcpPacket.Bitrate)
case *rtcp.ReceiverReport:
l := len(rtcpPacket.Reports)
if l > 0 {
// use only last report
met.SetReceiverReport(rtcpPacket.Reports[l-1])
}
case *rtcp.TransportLayerNack:
for _, pair := range rtcpPacket.Nacks {
packetList := pair.PacketList()
met.transportLayerNacks.Add(float64(len(packetList)))
}
2023-04-11 08:24:16 +12:00
}
}
}
}
func (met *metrics) connectionStats(connection *webrtc.PeerConnection) {
ticker := time.NewTicker(connectionStatsInterval)
defer ticker.Stop()
for range ticker.C {
if connection.ConnectionState() == webrtc.PeerConnectionStateClosed {
break
}
stats := connection.GetStats()
data, ok := stats["iceTransport"].(webrtc.TransportStats)
if ok {
met.SetIceTransportStats(data)
}
data, ok = stats["sctpTransport"].(webrtc.TransportStats)
if ok {
met.SetSctpTransportStats(data)
}
remoteCandidates := map[string]webrtc.ICECandidateStats{}
nominatedRemoteCandidates := map[string]struct{}{}
for _, entry := range stats {
// only remote ice candidate stats
candidate, ok := entry.(webrtc.ICECandidateStats)
if ok && candidate.Type == webrtc.StatsTypeRemoteCandidate {
met.NewICECandidate(candidate)
remoteCandidates[candidate.ID] = candidate
}
// only nominated ice candidate pair stats
pair, ok := entry.(webrtc.ICECandidatePairStats)
if ok && pair.Nominated {
nominatedRemoteCandidates[pair.RemoteCandidateID] = struct{}{}
}
}
iceCandidatesUsed := []webrtc.ICECandidateStats{}
for id := range nominatedRemoteCandidates {
if candidate, ok := remoteCandidates[id]; ok {
iceCandidatesUsed = append(iceCandidatesUsed, candidate)
}
}
met.SetICECandidatesUsed(iceCandidatesUsed)
}
}