From deabba80ca0dbcca81096195ed4e80671edad007 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= Date: Mon, 12 Sep 2022 22:21:22 +0200 Subject: [PATCH] move pipelines from gst to capture. --- server/internal/capture/broadcast.go | 2 +- server/internal/capture/gst/gst.go | 179 -------------------------- server/internal/capture/manager.go | 6 +- server/internal/capture/pipelines.go | 185 +++++++++++++++++++++++++++ 4 files changed, 189 insertions(+), 183 deletions(-) create mode 100644 server/internal/capture/pipelines.go diff --git a/server/internal/capture/broadcast.go b/server/internal/capture/broadcast.go index fb97c27e..bec0ecfa 100644 --- a/server/internal/capture/broadcast.go +++ b/server/internal/capture/broadcast.go @@ -41,7 +41,7 @@ func (manager *BroadcastManager) Start() error { } var err error - manager.pipeline, err = gst.CreateRTMPPipeline( + manager.pipeline, err = CreateRTMPPipeline( manager.capture.Device, manager.capture.Display, manager.config.Pipeline, diff --git a/server/internal/capture/gst/gst.go b/server/internal/capture/gst/gst.go index 1dbc8ab6..4d6d8167 100644 --- a/server/internal/capture/gst/gst.go +++ b/server/internal/capture/gst/gst.go @@ -8,7 +8,6 @@ package gst import "C" import ( "fmt" - "strings" "sync" "time" "unsafe" @@ -16,26 +15,6 @@ import ( "m1k1o/neko/internal/types" ) -/* - apt-get install \ - libgstreamer1.0-0 \ - gstreamer1.0-plugins-base \ - gstreamer1.0-plugins-good \ - gstreamer1.0-plugins-bad \ - gstreamer1.0-plugins-ugly\ - gstreamer1.0-libav \ - gstreamer1.0-doc \ - gstreamer1.0-tools \ - gstreamer1.0-x \ - gstreamer1.0-alsa \ - gstreamer1.0-pulseaudio - - gst-inspect-1.0 --version - gst-inspect-1.0 plugin - gst-launch-1.0 ximagesrc show-pointer=true use-damage=false ! video/x-raw,framerate=30/1 ! videoconvert ! queue ! vp8enc error-resilient=partitions keyframe-max-dist=10 auto-alt-ref=true cpu-used=5 deadline=1 ! autovideosink - gst-launch-1.0 pulsesrc ! audioconvert ! opusenc ! autoaudiosink -*/ - // Pipeline is a wrapper for a GStreamer Pipeline type Pipeline struct { Pipeline *C.GstElement @@ -48,169 +27,11 @@ var pipelines = make(map[int]*Pipeline) var pipelinesLock sync.Mutex var registry *C.GstRegistry -const ( - videoSrc = "ximagesrc display-name=%s show-pointer=true use-damage=false ! video/x-raw,framerate=%d/1 ! videoconvert ! queue ! " - audioSrc = "pulsesrc device=%s ! audio/x-raw,channels=2 ! audioconvert ! " -) - func init() { C.gstreamer_init() registry = C.gst_registry_get() } -// CreateRTMPPipeline creates a GStreamer Pipeline -func CreateRTMPPipeline(pipelineDevice string, pipelineDisplay string, pipelineSrc string, pipelineRTMP string) (*Pipeline, error) { - video := fmt.Sprintf(videoSrc, pipelineDisplay, 25) - audio := fmt.Sprintf(audioSrc, pipelineDevice) - - var pipelineStr string - if pipelineSrc != "" { - // replace RTMP url - pipelineStr = strings.Replace(pipelineSrc, "{url}", pipelineRTMP, -1) - // replace audio device - pipelineStr = strings.Replace(pipelineStr, "{device}", pipelineDevice, -1) - // replace display - pipelineStr = strings.Replace(pipelineStr, "{display}", pipelineDisplay, -1) - } else { - pipelineStr = fmt.Sprintf("flvmux name=mux ! rtmpsink location='%s live=1' %s audio/x-raw,channels=2 ! audioconvert ! voaacenc ! mux. %s x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! mux.", pipelineRTMP, audio, video) - } - - return CreatePipeline(pipelineStr) -} - -// CreateAppPipeline creates a GStreamer Pipeline -func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc string, fps int, bitrate uint, hwenc string) (*Pipeline, error) { - pipelineStr := " ! appsink name=appsink" - - // if using custom pipeline - if pipelineSrc != "" { - pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice) - return CreatePipeline(pipelineStr) - } - - switch codecName { - case "VP8": - if hwenc == "VAAPI" { - if err := CheckPlugins([]string{"ximagesrc", "vaapi"}); err != nil { - return nil, err - } - // vp8 encode is missing from gstreamer.freedesktop.org/documentation - // note that it was removed from some recent intel CPUs: https://trac.ffmpeg.org/wiki/Hardware/QuickSync - // https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gstreamer-vaapi-plugins/html/gstreamer-vaapi-plugins-vaapivp8enc.html - pipelineStr = fmt.Sprintf(videoSrc+"video/x-raw,format=NV12 ! vaapivp8enc rate-control=vbr bitrate=%d keyframe-period=180"+pipelineStr, pipelineDevice, fps, bitrate) - } else { - // https://gstreamer.freedesktop.org/documentation/vpx/vp8enc.html?gi-language=c - // gstreamer1.0-plugins-good - // vp8enc error-resilient=partitions keyframe-max-dist=10 auto-alt-ref=true cpu-used=5 deadline=1 - if err := CheckPlugins([]string{"ximagesrc", "vpx"}); err != nil { - return nil, err - } - - pipelineStr = strings.Join([]string{ - fmt.Sprintf(videoSrc, pipelineDevice, fps), - "vp8enc", - fmt.Sprintf("target-bitrate=%d", bitrate*650), - "cpu-used=4", - "end-usage=cbr", - "threads=4", - "deadline=1", - "undershoot=95", - fmt.Sprintf("buffer-size=%d", bitrate*4), - fmt.Sprintf("buffer-initial-size=%d", bitrate*2), - fmt.Sprintf("buffer-optimal-size=%d", bitrate*3), - "keyframe-max-dist=25", - "min-quantizer=4", - "max-quantizer=20", - pipelineStr, - }, " ") - } - case "VP9": - // https://gstreamer.freedesktop.org/documentation/vpx/vp9enc.html?gi-language=c - // gstreamer1.0-plugins-good - // vp9enc - if err := CheckPlugins([]string{"ximagesrc", "vpx"}); err != nil { - return nil, err - } - - pipelineStr = fmt.Sprintf(videoSrc+"vp9enc target-bitrate=%d cpu-used=-5 threads=4 deadline=1 keyframe-max-dist=30 auto-alt-ref=true"+pipelineStr, pipelineDevice, fps, bitrate*1000) - case "H264": - if err := CheckPlugins([]string{"ximagesrc"}); err != nil { - return nil, err - } - - if hwenc == "VAAPI" { - if err := CheckPlugins([]string{"vaapi"}); err != nil { - return nil, err - } - - pipelineStr = fmt.Sprintf(videoSrc+"video/x-raw,format=NV12 ! vaapih264enc rate-control=vbr bitrate=%d keyframe-period=180 quality-level=7 ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice, fps, bitrate) - - } else { - // https://gstreamer.freedesktop.org/documentation/openh264/openh264enc.html?gi-language=c#openh264enc - // gstreamer1.0-plugins-bad - // openh264enc multi-thread=4 complexity=high bitrate=3072000 max-bitrate=4096000 - if err := CheckPlugins([]string{"openh264"}); err == nil { - pipelineStr = fmt.Sprintf(videoSrc+"openh264enc multi-thread=4 complexity=high bitrate=%d max-bitrate=%d ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice, fps, bitrate*1000, (bitrate+1024)*1000) - break - } - - // https://gstreamer.freedesktop.org/documentation/x264/index.html?gi-language=c - // gstreamer1.0-plugins-ugly - // video/x-raw,format=I420 ! x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream - if err := CheckPlugins([]string{"x264"}); err != nil { - return nil, err - } - - vbvbuf := uint(1000) - if bitrate > 1000 { - vbvbuf = bitrate - } - - pipelineStr = fmt.Sprintf(videoSrc+"video/x-raw,format=NV12 ! x264enc threads=4 bitrate=%d key-int-max=60 vbv-buf-capacity=%d byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice, fps, bitrate, vbvbuf) - } - case "Opus": - // https://gstreamer.freedesktop.org/documentation/opus/opusenc.html - // gstreamer1.0-plugins-base - // opusenc - if err := CheckPlugins([]string{"pulseaudio", "opus"}); err != nil { - return nil, err - } - - pipelineStr = fmt.Sprintf(audioSrc+"opusenc inband-fec=true bitrate=%d"+pipelineStr, pipelineDevice, bitrate*1000) - case "G722": - // https://gstreamer.freedesktop.org/documentation/libav/avenc_g722.html?gi-language=c - // gstreamer1.0-libav - // avenc_g722 - if err := CheckPlugins([]string{"pulseaudio", "libav"}); err != nil { - return nil, err - } - - pipelineStr = fmt.Sprintf(audioSrc+"avenc_g722 bitrate=%d"+pipelineStr, pipelineDevice, bitrate*1000) - case "PCMU": - // https://gstreamer.freedesktop.org/documentation/mulaw/mulawenc.html?gi-language=c - // gstreamer1.0-plugins-good - // audio/x-raw, rate=8000 ! mulawenc - if err := CheckPlugins([]string{"pulseaudio", "mulaw"}); err != nil { - return nil, err - } - - pipelineStr = fmt.Sprintf(audioSrc+"audio/x-raw, rate=8000 ! mulawenc"+pipelineStr, pipelineDevice) - case "PCMA": - // https://gstreamer.freedesktop.org/documentation/alaw/alawenc.html?gi-language=c - // gstreamer1.0-plugins-good - // audio/x-raw, rate=8000 ! alawenc - if err := CheckPlugins([]string{"pulseaudio", "alaw"}); err != nil { - return nil, err - } - - pipelineStr = fmt.Sprintf(audioSrc+"audio/x-raw, rate=8000 ! alawenc"+pipelineStr, pipelineDevice) - default: - return nil, fmt.Errorf("unknown codec %s", codecName) - } - - return CreatePipeline(pipelineStr) -} - // CreatePipeline creates a GStreamer Pipeline func CreatePipeline(pipelineStr string) (*Pipeline, error) { pipelineStrUnsafe := C.CString(pipelineStr) diff --git a/server/internal/capture/manager.go b/server/internal/capture/manager.go index c53a3ded..e64c2ac0 100644 --- a/server/internal/capture/manager.go +++ b/server/internal/capture/manager.go @@ -133,7 +133,7 @@ func (manager *CaptureManagerCtx) createPipelines() { } var err error - manager.video, err = gst.CreateAppPipeline( + manager.video, err = CreateAppPipeline( manager.config.VideoCodec, manager.config.Display, manager.config.VideoParams, @@ -145,7 +145,7 @@ func (manager *CaptureManagerCtx) createPipelines() { manager.logger.Panic().Err(err).Msg("unable to create video pipeline") } - manager.audio, err = gst.CreateAppPipeline( + manager.audio, err = CreateAppPipeline( manager.config.AudioCodec, manager.config.Device, manager.config.AudioParams, @@ -185,7 +185,7 @@ func (manager *CaptureManagerCtx) ChangeResolution(width int, height int, rate i } var err error - manager.video, err = gst.CreateAppPipeline( + manager.video, err = CreateAppPipeline( manager.config.VideoCodec, manager.config.Display, manager.config.VideoParams, diff --git a/server/internal/capture/pipelines.go b/server/internal/capture/pipelines.go new file mode 100644 index 00000000..40524793 --- /dev/null +++ b/server/internal/capture/pipelines.go @@ -0,0 +1,185 @@ +package capture + +import ( + "fmt" + "m1k1o/neko/internal/capture/gst" + "strings" +) + +/* + apt-get install \ + libgstreamer1.0-0 \ + gstreamer1.0-plugins-base \ + gstreamer1.0-plugins-good \ + gstreamer1.0-plugins-bad \ + gstreamer1.0-plugins-ugly\ + gstreamer1.0-libav \ + gstreamer1.0-doc \ + gstreamer1.0-tools \ + gstreamer1.0-x \ + gstreamer1.0-alsa \ + gstreamer1.0-pulseaudio + + gst-inspect-1.0 --version + gst-inspect-1.0 plugin + gst-launch-1.0 ximagesrc show-pointer=true use-damage=false ! video/x-raw,framerate=30/1 ! videoconvert ! queue ! vp8enc error-resilient=partitions keyframe-max-dist=10 auto-alt-ref=true cpu-used=5 deadline=1 ! autovideosink + gst-launch-1.0 pulsesrc ! audioconvert ! opusenc ! autoaudiosink +*/ + +const ( + videoSrc = "ximagesrc display-name=%s show-pointer=true use-damage=false ! video/x-raw,framerate=%d/1 ! videoconvert ! queue ! " + audioSrc = "pulsesrc device=%s ! audio/x-raw,channels=2 ! audioconvert ! " +) + +// CreateRTMPPipeline creates a GStreamer Pipeline +func CreateRTMPPipeline(pipelineDevice string, pipelineDisplay string, pipelineSrc string, pipelineRTMP string) (*gst.Pipeline, error) { + video := fmt.Sprintf(videoSrc, pipelineDisplay, 25) + audio := fmt.Sprintf(audioSrc, pipelineDevice) + + var pipelineStr string + if pipelineSrc != "" { + // replace RTMP url + pipelineStr = strings.Replace(pipelineSrc, "{url}", pipelineRTMP, -1) + // replace audio device + pipelineStr = strings.Replace(pipelineStr, "{device}", pipelineDevice, -1) + // replace display + pipelineStr = strings.Replace(pipelineStr, "{display}", pipelineDisplay, -1) + } else { + pipelineStr = fmt.Sprintf("flvmux name=mux ! rtmpsink location='%s live=1' %s audio/x-raw,channels=2 ! audioconvert ! voaacenc ! mux. %s x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! mux.", pipelineRTMP, audio, video) + } + + return gst.CreatePipeline(pipelineStr) +} + +// CreateAppPipeline creates a GStreamer Pipeline +func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc string, fps int, bitrate uint, hwenc string) (*gst.Pipeline, error) { + pipelineStr := " ! appsink name=appsink" + + // if using custom pipeline + if pipelineSrc != "" { + pipelineStr = fmt.Sprintf(pipelineSrc+pipelineStr, pipelineDevice) + return gst.CreatePipeline(pipelineStr) + } + + switch codecName { + case "VP8": + if hwenc == "VAAPI" { + if err := gst.CheckPlugins([]string{"ximagesrc", "vaapi"}); err != nil { + return nil, err + } + // vp8 encode is missing from gstreamer.freedesktop.org/documentation + // note that it was removed from some recent intel CPUs: https://trac.ffmpeg.org/wiki/Hardware/QuickSync + // https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gstreamer-vaapi-plugins/html/gstreamer-vaapi-plugins-vaapivp8enc.html + pipelineStr = fmt.Sprintf(videoSrc+"video/x-raw,format=NV12 ! vaapivp8enc rate-control=vbr bitrate=%d keyframe-period=180"+pipelineStr, pipelineDevice, fps, bitrate) + } else { + // https://gstreamer.freedesktop.org/documentation/vpx/vp8enc.html?gi-language=c + // gstreamer1.0-plugins-good + // vp8enc error-resilient=partitions keyframe-max-dist=10 auto-alt-ref=true cpu-used=5 deadline=1 + if err := gst.CheckPlugins([]string{"ximagesrc", "vpx"}); err != nil { + return nil, err + } + + pipelineStr = strings.Join([]string{ + fmt.Sprintf(videoSrc, pipelineDevice, fps), + "vp8enc", + fmt.Sprintf("target-bitrate=%d", bitrate*650), + "cpu-used=4", + "end-usage=cbr", + "threads=4", + "deadline=1", + "undershoot=95", + fmt.Sprintf("buffer-size=%d", bitrate*4), + fmt.Sprintf("buffer-initial-size=%d", bitrate*2), + fmt.Sprintf("buffer-optimal-size=%d", bitrate*3), + "keyframe-max-dist=25", + "min-quantizer=4", + "max-quantizer=20", + pipelineStr, + }, " ") + } + case "VP9": + // https://gstreamer.freedesktop.org/documentation/vpx/vp9enc.html?gi-language=c + // gstreamer1.0-plugins-good + // vp9enc + if err := gst.CheckPlugins([]string{"ximagesrc", "vpx"}); err != nil { + return nil, err + } + + pipelineStr = fmt.Sprintf(videoSrc+"vp9enc target-bitrate=%d cpu-used=-5 threads=4 deadline=1 keyframe-max-dist=30 auto-alt-ref=true"+pipelineStr, pipelineDevice, fps, bitrate*1000) + case "H264": + if err := gst.CheckPlugins([]string{"ximagesrc"}); err != nil { + return nil, err + } + + if hwenc == "VAAPI" { + if err := gst.CheckPlugins([]string{"vaapi"}); err != nil { + return nil, err + } + + pipelineStr = fmt.Sprintf(videoSrc+"video/x-raw,format=NV12 ! vaapih264enc rate-control=vbr bitrate=%d keyframe-period=180 quality-level=7 ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice, fps, bitrate) + + } else { + // https://gstreamer.freedesktop.org/documentation/openh264/openh264enc.html?gi-language=c#openh264enc + // gstreamer1.0-plugins-bad + // openh264enc multi-thread=4 complexity=high bitrate=3072000 max-bitrate=4096000 + if err := gst.CheckPlugins([]string{"openh264"}); err == nil { + pipelineStr = fmt.Sprintf(videoSrc+"openh264enc multi-thread=4 complexity=high bitrate=%d max-bitrate=%d ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice, fps, bitrate*1000, (bitrate+1024)*1000) + break + } + + // https://gstreamer.freedesktop.org/documentation/x264/index.html?gi-language=c + // gstreamer1.0-plugins-ugly + // video/x-raw,format=I420 ! x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream + if err := gst.CheckPlugins([]string{"x264"}); err != nil { + return nil, err + } + + vbvbuf := uint(1000) + if bitrate > 1000 { + vbvbuf = bitrate + } + + pipelineStr = fmt.Sprintf(videoSrc+"video/x-raw,format=NV12 ! x264enc threads=4 bitrate=%d key-int-max=60 vbv-buf-capacity=%d byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice, fps, bitrate, vbvbuf) + } + case "Opus": + // https://gstreamer.freedesktop.org/documentation/opus/opusenc.html + // gstreamer1.0-plugins-base + // opusenc + if err := gst.CheckPlugins([]string{"pulseaudio", "opus"}); err != nil { + return nil, err + } + + pipelineStr = fmt.Sprintf(audioSrc+"opusenc inband-fec=true bitrate=%d"+pipelineStr, pipelineDevice, bitrate*1000) + case "G722": + // https://gstreamer.freedesktop.org/documentation/libav/avenc_g722.html?gi-language=c + // gstreamer1.0-libav + // avenc_g722 + if err := gst.CheckPlugins([]string{"pulseaudio", "libav"}); err != nil { + return nil, err + } + + pipelineStr = fmt.Sprintf(audioSrc+"avenc_g722 bitrate=%d"+pipelineStr, pipelineDevice, bitrate*1000) + case "PCMU": + // https://gstreamer.freedesktop.org/documentation/mulaw/mulawenc.html?gi-language=c + // gstreamer1.0-plugins-good + // audio/x-raw, rate=8000 ! mulawenc + if err := gst.CheckPlugins([]string{"pulseaudio", "mulaw"}); err != nil { + return nil, err + } + + pipelineStr = fmt.Sprintf(audioSrc+"audio/x-raw, rate=8000 ! mulawenc"+pipelineStr, pipelineDevice) + case "PCMA": + // https://gstreamer.freedesktop.org/documentation/alaw/alawenc.html?gi-language=c + // gstreamer1.0-plugins-good + // audio/x-raw, rate=8000 ! alawenc + if err := gst.CheckPlugins([]string{"pulseaudio", "alaw"}); err != nil { + return nil, err + } + + pipelineStr = fmt.Sprintf(audioSrc+"audio/x-raw, rate=8000 ! alawenc"+pipelineStr, pipelineDevice) + default: + return nil, fmt.Errorf("unknown codec %s", codecName) + } + + return gst.CreatePipeline(pipelineStr) +}