From ebd7e7c065007c0ec0a77e3d25bddaf755025b0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Fri, 5 Feb 2021 15:10:41 +0100 Subject: [PATCH] move gst pipelines to codec. --- internal/capture/gst/gst.go | 121 --------------------------------- internal/capture/manager.go | 80 ++++++++++++++-------- internal/config/capture.go | 28 ++++---- internal/types/codec/codecs.go | 25 +++++++ 4 files changed, 92 insertions(+), 162 deletions(-) diff --git a/internal/capture/gst/gst.go b/internal/capture/gst/gst.go index c439528c..33ffe629 100644 --- a/internal/capture/gst/gst.go +++ b/internal/capture/gst/gst.go @@ -13,10 +13,8 @@ import ( "unsafe" "demodesk/neko/internal/types" - "demodesk/neko/internal/types/codec" ) -// Pipeline is a wrapper for a GStreamer Pipeline type Pipeline struct { Pipeline *C.GstElement Sample chan types.Sample @@ -28,127 +26,11 @@ var pipelines = make(map[int]*Pipeline) var pipelinesLock sync.Mutex var registry *C.GstRegistry -const ( - videoSrc = "ximagesrc display-name=%s show-pointer=false use-damage=false ! video/x-raw ! videoconvert ! queue ! " - audioSrc = "pulsesrc device=%s ! audio/x-raw,channels=2 ! audioconvert ! " - appSink = " ! appsink name=appsink" -) - func init() { C.gstreamer_init() registry = C.gst_registry_get() } -func GetRTMPPipeline(audioDevice string, videoDisplay string, pipelineSrc string) string { - video := fmt.Sprintf(videoSrc, videoDisplay) - audio := fmt.Sprintf(audioSrc, audioDevice) - - var pipelineStr string - if pipelineSrc != "" { - pipelineStr = fmt.Sprintf(pipelineSrc, audioDevice, videoDisplay) - } else { - pipelineStr = fmt.Sprintf("flvmux name=mux ! rtmpsink location='{url} live=1' %s audio/x-raw,channels=2 ! audioconvert ! voaacenc ! mux. %s x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! mux.", audio, video) - } - - return pipelineStr -} - -func GetJPEGPipeline(videoDisplay string, pipelineSrc string, rate string, quality string) string { - var pipelineStr string - if pipelineSrc != "" { - pipelineStr = fmt.Sprintf(pipelineSrc, videoDisplay) - } else { - pipelineStr = fmt.Sprintf("ximagesrc display-name=%s show-pointer=true use-damage=false ! videoconvert ! videoscale ! videorate ! video/x-raw,framerate=%s ! jpegenc quality=%s" + appSink, videoDisplay, rate, quality) - } - - return pipelineStr -} - -func GetAppPipeline(codecRTP codec.RTPCodec, pipelineDevice string, pipelineSrc string) (string, error) { - if pipelineSrc != "" { - return fmt.Sprintf(pipelineSrc + appSink, pipelineDevice), nil - } - - var pipelineStr string - - switch codecRTP.Name { - case "vp8": - // https://gstreamer.freedesktop.org/documentation/vpx/vp8enc.html - // gstreamer1.0-plugins-good - if err := CheckPlugins([]string{"ximagesrc", "vpx"}); err != nil { - return "", err - } - - pipelineStr = fmt.Sprintf(videoSrc + "vp8enc cpu-used=16 threads=4 deadline=1 error-resilient=partitions keyframe-max-dist=15 static-threshold=20" + appSink, pipelineDevice) - case "vp9": - // https://gstreamer.freedesktop.org/documentation/vpx/vp9enc.html - // gstreamer1.0-plugins-good - if err := CheckPlugins([]string{"ximagesrc", "vpx"}); err != nil { - return "", err - } - - pipelineStr = fmt.Sprintf(videoSrc + "vp9enc cpu-used=16 threads=4 deadline=1 keyframe-max-dist=15 static-threshold=20" + appSink, pipelineDevice) - case "h264": - var err error - if err = CheckPlugins([]string{"ximagesrc"}); err != nil { - return "", err - } - - // https://gstreamer.freedesktop.org/documentation/x264/index.html - // gstreamer1.0-plugins-ugly - if err = CheckPlugins([]string{"x264"}); err == nil { - pipelineStr = fmt.Sprintf(videoSrc + "video/x-raw,format=I420 ! x264enc threads=4 bitrate=4096 key-int-max=15 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream" + appSink, pipelineDevice) - break - } - - // https://gstreamer.freedesktop.org/documentation/openh264/openh264enc.html - // gstreamer1.0-plugins-bad - if err = CheckPlugins([]string{"openh264"}); err == nil { - pipelineStr = fmt.Sprintf(videoSrc + "openh264enc multi-thread=4 complexity=high bitrate=3072000 max-bitrate=4096000 ! video/x-h264,stream-format=byte-stream" + appSink, pipelineDevice) - break - } - - return "", err - case "opus": - // https://gstreamer.freedesktop.org/documentation/opus/opusenc.html - // gstreamer1.0-plugins-base - if err := CheckPlugins([]string{"pulseaudio", "opus"}); err != nil { - return "", err - } - - pipelineStr = fmt.Sprintf(audioSrc + "opusenc bitrate=128000" + appSink, pipelineDevice) - case "g722": - // https://gstreamer.freedesktop.org/documentation/libav/avenc_g722.html - // gstreamer1.0-libav - if err := CheckPlugins([]string{"pulseaudio", "libav"}); err != nil { - return "", err - } - - pipelineStr = fmt.Sprintf(audioSrc + "avenc_g722" + appSink, pipelineDevice) - case "pcmu": - // https://gstreamer.freedesktop.org/documentation/mulaw/mulawenc.html - // gstreamer1.0-plugins-good - if err := CheckPlugins([]string{"pulseaudio", "mulaw"}); err != nil { - return "", err - } - - pipelineStr = fmt.Sprintf(audioSrc + "audio/x-raw, rate=8000 ! mulawenc" + appSink, pipelineDevice) - case "pcma": - // https://gstreamer.freedesktop.org/documentation/alaw/alawenc.html - // gstreamer1.0-plugins-good - if err := CheckPlugins([]string{"pulseaudio", "alaw"}); err != nil { - return "", err - } - - pipelineStr = fmt.Sprintf(audioSrc + "audio/x-raw, rate=8000 ! alawenc" + appSink, pipelineDevice) - default: - return "", fmt.Errorf("unknown codec %s", codecRTP.Name) - } - - return pipelineStr, nil -} - -// CreatePipeline creates a GStreamer Pipeline func CreatePipeline(pipelineStr string) (*Pipeline, error) { pipelineStrUnsafe := C.CString(pipelineStr) defer C.free(unsafe.Pointer(pipelineStrUnsafe)) @@ -177,17 +59,14 @@ func CreatePipeline(pipelineStr string) (*Pipeline, error) { return p, nil } -// Start starts the GStreamer Pipeline func (p *Pipeline) Start() { C.gstreamer_send_start_pipeline(p.Pipeline, C.int(p.id)) } -// Play starts the GStreamer Pipeline func (p *Pipeline) Play() { C.gstreamer_send_play_pipeline(p.Pipeline) } -// Stop stops the GStreamer Pipeline func (p *Pipeline) Stop() { C.gstreamer_send_stop_pipeline(p.Pipeline) } diff --git a/internal/capture/manager.go b/internal/capture/manager.go index f551c641..69408572 100644 --- a/internal/capture/manager.go +++ b/internal/capture/manager.go @@ -1,6 +1,7 @@ package capture import ( + "fmt" "sync" "github.com/rs/zerolog" @@ -8,7 +9,6 @@ import ( "demodesk/neko/internal/types" "demodesk/neko/internal/config" - "demodesk/neko/internal/capture/gst" ) type CaptureManagerCtx struct { @@ -25,37 +25,59 @@ type CaptureManagerCtx struct { func New(desktop types.DesktopManager, config *config.Capture) *CaptureManagerCtx { logger := log.With().Str("module", "capture").Logger() - broadcastPipeline := gst.GetRTMPPipeline( - config.Device, - config.Display, - config.BroadcastPipeline, - ) - - screencastPipeline := gst.GetJPEGPipeline( - config.Display, - config.ScreencastPipeline, - config.ScreencastRate, - config.ScreencastQuality, - ) - - audioPipeline, err := gst.GetAppPipeline( - config.AudioCodec, - config.Device, - config.AudioParams, - ) - - if err != nil { - logger.Panic().Err(err).Msg("unable to get pipeline") + broadcastPipeline := config.BroadcastPipeline + if broadcastPipeline == "" { + broadcastPipeline = fmt.Sprintf( + "flvmux name=mux ! rtmpsink location='{url} live=1' " + + "pulsesrc device=%s " + + "! audio/x-raw,channels=2 " + + "! audioconvert " + + "! queue " + + "! voaacenc " + + "! mux. " + + "ximagesrc display-name=%s show-pointer=true use-damage=false " + + "! video/x-raw " + + "! videoconvert " + + "! queue " + + "! x264enc threads=4 bitrate=4096 key-int-max=15 byte-stream=true byte-stream=true tune=zerolatency speed-preset=veryfast " + + "! mux.", config.Device, config.Display, + ) } - videoPipeline, err := gst.GetAppPipeline( - config.VideoCodec, - config.Display, - config.VideoParams, - ) + screencastPipeline := config.ScreencastPipeline + if screencastPipeline == "" { + screencastPipeline = fmt.Sprintf( + "ximagesrc display-name=%s show-pointer=true use-damage=false " + + "! video/x-raw,framerate=%s " + + "! videoconvert " + + "! queue " + + "! jpegenc quality=%s " + + "! appsink name=appsink", config.Display, config.ScreencastRate, config.ScreencastQuality, + ) + } - if err != nil { - logger.Panic().Err(err).Msg("unable to get pipeline") + audioPipeline := config.AudioPipeline + if audioPipeline == "" { + audioPipeline = fmt.Sprintf( + "pulsesrc device=%s " + + "! audio/x-raw,channels=2 " + + "! audioconvert " + + "! queue " + + "! %s " + + "! appsink name=appsink", config.Device, config.AudioCodec.Pipeline, + ) + } + + videoPipeline := config.VideoPipeline + if videoPipeline == "" { + videoPipeline = fmt.Sprintf( + "ximagesrc display-name=%s show-pointer=false use-damage=false " + + "! video/x-raw " + + "! videoconvert " + + "! queue " + + "! %s " + + "! appsink name=appsink", config.Display, config.VideoCodec.Pipeline, + ) } return &CaptureManagerCtx{ diff --git a/internal/config/capture.go b/internal/config/capture.go index 5c1334b0..0d87af6f 100644 --- a/internal/config/capture.go +++ b/internal/config/capture.go @@ -8,17 +8,19 @@ import ( ) type Capture struct { - Display string - Device string - AudioCodec codec.RTPCodec - AudioParams string - VideoCodec codec.RTPCodec - VideoParams string + Device string + AudioCodec codec.RTPCodec + AudioPipeline string - BroadcastPipeline string - Screencast bool - ScreencastRate string - ScreencastQuality string + Display string + VideoCodec codec.RTPCodec + VideoPipeline string + + BroadcastPipeline string + + Screencast bool + ScreencastRate string + ScreencastQuality string ScreencastPipeline string } @@ -139,12 +141,14 @@ func (s *Capture) Set() { s.Device = viper.GetString("device") s.AudioCodec = audioCodec - s.AudioParams = viper.GetString("audio") + s.AudioPipeline = viper.GetString("audio") + s.Display = viper.GetString("display") s.VideoCodec = videoCodec - s.VideoParams = viper.GetString("video") + s.VideoPipeline = viper.GetString("video") s.BroadcastPipeline = viper.GetString("broadcast_pipeline") + s.Screencast = viper.GetBool("screencast") s.ScreencastRate = viper.GetString("screencast_rate") s.ScreencastQuality = viper.GetString("screencast_quality") diff --git a/internal/types/codec/codecs.go b/internal/types/codec/codecs.go index e902dd10..023cbaca 100644 --- a/internal/types/codec/codecs.go +++ b/internal/types/codec/codecs.go @@ -7,6 +7,7 @@ type RTPCodec struct { PayloadType webrtc.PayloadType Type webrtc.RTPCodecType Capability webrtc.RTPCodecCapability + Pipeline string } func (codec *RTPCodec) Register(engine *webrtc.MediaEngine) error { @@ -28,6 +29,9 @@ func VP8() RTPCodec { SDPFmtpLine: "", RTCPFeedback: nil, }, + // https://gstreamer.freedesktop.org/documentation/vpx/vp8enc.html + // gstreamer1.0-plugins-good + Pipeline: "vp8enc cpu-used=16 threads=4 deadline=1 error-resilient=partitions keyframe-max-dist=15 static-threshold=20", } } @@ -44,6 +48,9 @@ func VP9() RTPCodec { SDPFmtpLine: "profile-id=0", RTCPFeedback: nil, }, + // https://gstreamer.freedesktop.org/documentation/vpx/vp9enc.html + // gstreamer1.0-plugins-good + Pipeline: "vp9enc cpu-used=16 threads=4 deadline=1 keyframe-max-dist=15 static-threshold=20", } } @@ -60,6 +67,12 @@ func H264() RTPCodec { SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f", RTCPFeedback: nil, }, + // https://gstreamer.freedesktop.org/documentation/x264/index.html + // gstreamer1.0-plugins-ugly + Pipeline: "video/x-raw,format=I420 ! x264enc threads=4 bitrate=4096 key-int-max=15 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream", + // https://gstreamer.freedesktop.org/documentation/openh264/openh264enc.html + // gstreamer1.0-plugins-bad + //Pipeline: "openh264enc multi-thread=4 complexity=high bitrate=3072000 max-bitrate=4096000 ! video/x-h264,stream-format=byte-stream", } } @@ -75,6 +88,9 @@ func Opus() RTPCodec { SDPFmtpLine: "", RTCPFeedback: nil, }, + // https://gstreamer.freedesktop.org/documentation/opus/opusenc.html + // gstreamer1.0-plugins-base + Pipeline: "opusenc bitrate=128000", } } @@ -90,6 +106,9 @@ func G722() RTPCodec { SDPFmtpLine: "", RTCPFeedback: nil, }, + // https://gstreamer.freedesktop.org/documentation/libav/avenc_g722.html + // gstreamer1.0-libav + Pipeline: "avenc_g722", } } @@ -105,6 +124,9 @@ func PCMU() RTPCodec { SDPFmtpLine: "", RTCPFeedback: nil, }, + // https://gstreamer.freedesktop.org/documentation/mulaw/mulawenc.html + // gstreamer1.0-plugins-good + Pipeline: "audio/x-raw, rate=8000 ! mulawenc", } } @@ -120,5 +142,8 @@ func PCMA() RTPCodec { SDPFmtpLine: "", RTCPFeedback: nil, }, + // https://gstreamer.freedesktop.org/documentation/alaw/alawenc.html + // gstreamer1.0-plugins-good + Pipeline: "audio/x-raw, rate=8000 ! alawenc", } }