From 87082bb9788f934ea82e76426842c474955a19bd Mon Sep 17 00:00:00 2001 From: Mark Lowne Date: Thu, 17 Mar 2022 20:25:17 +0100 Subject: [PATCH] Hardware accelerated encoding using Intel QuickSync via VAAPI (#151) * (nits) * add hardware encoding support for Intel QSV via VAAPI * automate RENDER_GID env var --- .docker/base/Dockerfile | 24 ++++++--- .docker/base/add-render-group.sh | 18 +++++++ .docker/base/supervisord.conf | 15 +++++- server/internal/gst/gst.go | 71 ++++++++++++++++---------- server/internal/remote/manager.go | 3 ++ server/internal/types/config/remote.go | 30 +++++++---- 6 files changed, 119 insertions(+), 42 deletions(-) create mode 100755 .docker/base/add-render-group.sh diff --git a/.docker/base/Dockerfile b/.docker/base/Dockerfile index 05113d61..80c9e503 100644 --- a/.docker/base/Dockerfile +++ b/.docker/base/Dockerfile @@ -13,7 +13,7 @@ RUN set -eux; apt-get update; \ # install libclipboard set -eux; \ cd /tmp; \ - git clone https://github.com/jtanx/libclipboard; \ + git clone --depth=1 https://github.com/jtanx/libclipboard; \ cd libclipboard; \ cmake .; \ make -j4; \ @@ -60,16 +60,26 @@ ARG USERNAME=neko ARG USER_UID=1000 ARG USER_GID=$USER_UID -# -# install dependencies -RUN set -eux; apt-get update; \ +RUN set -eux; \ + # + # add non-free repo for intel drivers + echo deb http://deb.debian.org/debian bullseye main contrib non-free > /etc/apt/sources.list; \ + echo deb http://deb.debian.org/debian-security/ bullseye-security main contrib non-free >> /etc/apt/sources.list; \ + echo deb http://deb.debian.org/debian bullseye-updates main contrib non-free >> /etc/apt/sources.list; \ + apt-get update; \ + # + # install dependencies apt-get install -y --no-install-recommends wget ca-certificates supervisor; \ apt-get install -y --no-install-recommends pulseaudio dbus-x11 xserver-xorg-video-dummy; \ apt-get install -y --no-install-recommends libcairo2 libxcb1 libxrandr2 libxv1 libopus0 libvpx6; \ # - # gst + # intel driver + vaapi + apt-get install -y --no-install-recommends intel-media-va-driver-non-free libva2 vainfo; \ + # + # gst + vaapi plugin apt-get install -y --no-install-recommends libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev \ - gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-pulseaudio; \ + gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-pulseaudio \ + gstreamer1.0-vaapi ;\ # # fonts apt-get install -y --no-install-recommends fonts-takao-mincho; \ @@ -106,6 +116,7 @@ COPY .docker/base/dbus /usr/bin/dbus COPY .docker/base/default.pa /etc/pulse/default.pa COPY .docker/base/supervisord.conf /etc/neko/supervisord.conf COPY .docker/base/xorg.conf /etc/neko/xorg.conf +COPY .docker/base/add-render-group.sh /usr/bin/add-render-group.sh # # set default envs @@ -114,6 +125,7 @@ ENV DISPLAY=:99.0 ENV NEKO_PASSWORD=neko ENV NEKO_PASSWORD_ADMIN=admin ENV NEKO_BIND=:8080 +ENV RENDER_GID= # # copy static files from previous stages diff --git a/.docker/base/add-render-group.sh b/.docker/base/add-render-group.sh new file mode 100755 index 00000000..e22326ac --- /dev/null +++ b/.docker/base/add-render-group.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# if no hwenc required, noop +[[ -z "$NEKO_HWENC" ]] && exit 0 + +if [[ -z "$RENDER_GID" ]]; then + RENDER_GID=$(stat -c "%g" /dev/dri/render* | tail -n 1) + # is /dev/dri passed to the container? + [[ -z "$RENDER_GID" ]] && exit 1 +fi + +# note that this could conceivably be a security risk... +cnt_group=$(getent group "$RENDER_GID" | cut -d: -f1) +if [[ -z "$cnt_group" ]]; then + groupadd -g "$RENDER_GID" nekorender + cnt_group=nekorender +fi +usermod -a -G "$cnt_group" "$USER" diff --git a/.docker/base/supervisord.conf b/.docker/base/supervisord.conf index 1bd7602b..37236a04 100644 --- a/.docker/base/supervisord.conf +++ b/.docker/base/supervisord.conf @@ -9,6 +9,19 @@ loglevel=debug [include] files=/etc/neko/supervisord/*.conf +[program:rendergroup-init] +environment=RENDER_GID="%(ENV_RENDER_GID)s",USER="%(ENV_USER)s" +command=/usr/bin/add-render-group.sh +startsecs=0 +startretries=0 +autorestart=false +priority=10 +user=root +stdout_logfile=/var/log/neko/rendergroup.log +stdout_logfile_maxbytes=1MB +stdout_logfile_backups=10 +redirect_stderr=true + [program:dbus] environment=HOME="/root",USER="root" command=/usr/bin/dbus @@ -33,7 +46,7 @@ redirect_stderr=true [program:pulseaudio] environment=HOME="/home/%(ENV_USER)s",USER="%(ENV_USER)s",DISPLAY="%(ENV_DISPLAY)s" -command=/usr/bin/pulseaudio --disallow-module-loading -vvvv --disallow-exit --exit-idle-time=-1 +command=/usr/bin/pulseaudio --disallow-module-loading -vvvv --disallow-exit --exit-idle-time=-1 autorestart=true priority=300 user=%(ENV_USER)s diff --git a/server/internal/gst/gst.go b/server/internal/gst/gst.go index 72837c63..a63c3f62 100644 --- a/server/internal/gst/gst.go +++ b/server/internal/gst/gst.go @@ -79,7 +79,7 @@ func CreateRTMPPipeline(pipelineDevice string, pipelineDisplay string, pipelineS } // CreateAppPipeline creates a GStreamer Pipeline -func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc string, fps int, bitrate uint) (*Pipeline, error) { +func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc string, fps int, bitrate uint, hwenc string) (*Pipeline, error) { pipelineStr := " ! appsink name=appsink" // if using custom pipeline @@ -90,14 +90,24 @@ func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc stri switch codecName { case "VP8": - // https://gstreamer.freedesktop.org/documentation/vpx/vp8enc.html?gi-language=c - // gstreamer1.0-plugins-good - // vp8enc error-resilient=partitions keyframe-max-dist=10 auto-alt-ref=true cpu-used=5 deadline=1 - if err := CheckPlugins([]string{"ximagesrc", "vpx"}); err != nil { - return nil, err - } + if hwenc == "VAAPI" { + if err := CheckPlugins([]string{"ximagesrc", "vaapi"}); err != nil { + return nil, err + } + // vp8 encode is missing from gstreamer.freedesktop.org/documentation + // note that it was removed from some recent intel CPUs: https://trac.ffmpeg.org/wiki/Hardware/QuickSync + // https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gstreamer-vaapi-plugins/html/gstreamer-vaapi-plugins-vaapivp8enc.html + pipelineStr = fmt.Sprintf(videoSrc+"video/x-raw,format=NV12 ! vaapivp8enc rate-control=vbr bitrate=%d keyframe-period=180"+pipelineStr, pipelineDevice, fps, bitrate) + } else { + // https://gstreamer.freedesktop.org/documentation/vpx/vp8enc.html?gi-language=c + // gstreamer1.0-plugins-good + // vp8enc error-resilient=partitions keyframe-max-dist=10 auto-alt-ref=true cpu-used=5 deadline=1 + if err := CheckPlugins([]string{"ximagesrc", "vpx"}); err != nil { + return nil, err + } - pipelineStr = fmt.Sprintf(videoSrc+"vp8enc target-bitrate=%d cpu-used=4 end-usage=cbr threads=4 deadline=1 undershoot=95 buffer-size=%d buffer-initial-size=%d buffer-optimal-size=%d keyframe-max-dist=180 min-quantizer=3 max-quantizer=40"+pipelineStr, pipelineDevice, fps, bitrate*1000, bitrate*6, bitrate*4, bitrate*5) + pipelineStr = fmt.Sprintf(videoSrc+"vp8enc target-bitrate=%d cpu-used=4 end-usage=cbr threads=4 deadline=1 undershoot=95 buffer-size=%d buffer-initial-size=%d buffer-optimal-size=%d keyframe-max-dist=180 min-quantizer=3 max-quantizer=40"+pipelineStr, pipelineDevice, fps, bitrate*1000, bitrate*6, bitrate*4, bitrate*5) + } case "VP9": // https://gstreamer.freedesktop.org/documentation/vpx/vp9enc.html?gi-language=c // gstreamer1.0-plugins-good @@ -112,27 +122,36 @@ func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc stri return nil, err } - // https://gstreamer.freedesktop.org/documentation/openh264/openh264enc.html?gi-language=c#openh264enc - // gstreamer1.0-plugins-bad - // openh264enc multi-thread=4 complexity=high bitrate=3072000 max-bitrate=4096000 - if err := CheckPlugins([]string{"openh264"}); err == nil { - pipelineStr = fmt.Sprintf(videoSrc+"openh264enc multi-thread=4 complexity=high bitrate=%d max-bitrate=%d ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice, fps, bitrate*1000, (bitrate+1024)*1000) - break - } + if hwenc == "VAAPI" { + if err := CheckPlugins([]string{"vaapi"}); err != nil { + return nil, err + } - // https://gstreamer.freedesktop.org/documentation/x264/index.html?gi-language=c - // gstreamer1.0-plugins-ugly - // video/x-raw,format=I420 ! x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream - if err := CheckPlugins([]string{"x264"}); err != nil { - return nil, err - } + pipelineStr = fmt.Sprintf(videoSrc+"video/x-raw,format=NV12 ! vaapih264enc rate-control=vbr bitrate=%d keyframe-period=180 quality-level=7 ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice, fps, bitrate) - vbvbuf := uint(1000) - if bitrate > 1000 { - vbvbuf = bitrate - } + } else { + // https://gstreamer.freedesktop.org/documentation/openh264/openh264enc.html?gi-language=c#openh264enc + // gstreamer1.0-plugins-bad + // openh264enc multi-thread=4 complexity=high bitrate=3072000 max-bitrate=4096000 + if err := CheckPlugins([]string{"openh264"}); err == nil { + pipelineStr = fmt.Sprintf(videoSrc+"openh264enc multi-thread=4 complexity=high bitrate=%d max-bitrate=%d ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice, fps, bitrate*1000, (bitrate+1024)*1000) + break + } - pipelineStr = fmt.Sprintf(videoSrc+"video/x-raw,format=NV12 ! x264enc threads=4 bitrate=%d key-int-max=60 vbv-buf-capacity=%d byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice, fps, bitrate, vbvbuf) + // https://gstreamer.freedesktop.org/documentation/x264/index.html?gi-language=c + // gstreamer1.0-plugins-ugly + // video/x-raw,format=I420 ! x264enc bframes=0 key-int-max=60 byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream + if err := CheckPlugins([]string{"x264"}); err != nil { + return nil, err + } + + vbvbuf := uint(1000) + if bitrate > 1000 { + vbvbuf = bitrate + } + + pipelineStr = fmt.Sprintf(videoSrc+"video/x-raw,format=NV12 ! x264enc threads=4 bitrate=%d key-int-max=60 vbv-buf-capacity=%d byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice, fps, bitrate, vbvbuf) + } case "Opus": // https://gstreamer.freedesktop.org/documentation/opus/opusenc.html // gstreamer1.0-plugins-base diff --git a/server/internal/remote/manager.go b/server/internal/remote/manager.go index af355ee2..fa733873 100644 --- a/server/internal/remote/manager.go +++ b/server/internal/remote/manager.go @@ -147,6 +147,7 @@ func (manager *RemoteManager) createPipelines() { manager.config.VideoParams, rate, manager.config.VideoBitrate, + manager.config.VideoHWEnc, ) if err != nil { manager.logger.Panic().Err(err).Msg("unable to create video pipeline") @@ -158,6 +159,7 @@ func (manager *RemoteManager) createPipelines() { manager.config.AudioParams, 0, // fps: n/a for audio manager.config.AudioBitrate, + "", // hwenc: n/a for audio ) if err != nil { manager.logger.Panic().Err(err).Msg("unable to create audio pipeline") @@ -197,6 +199,7 @@ func (manager *RemoteManager) ChangeResolution(width int, height int, rate int) manager.config.VideoParams, rate, manager.config.VideoBitrate, + manager.config.VideoHWEnc, ) if err != nil { manager.logger.Panic().Err(err).Msg("unable to create new video pipeline") diff --git a/server/internal/types/config/remote.go b/server/internal/types/config/remote.go index a48c3677..21bd6cd4 100644 --- a/server/internal/types/config/remote.go +++ b/server/internal/types/config/remote.go @@ -14,6 +14,7 @@ type Remote struct { AudioCodec string AudioParams string AudioBitrate uint + VideoHWEnc string VideoCodec string VideoParams string VideoBitrate uint @@ -64,6 +65,12 @@ func (Remote) Init(cmd *cobra.Command) error { return err } + // hw encoding + cmd.PersistentFlags().String("hwenc", "", "use hardware accelerated encoding") + if err := viper.BindPFlag("hwenc", cmd.PersistentFlags().Lookup("hwenc")); err != nil { + return err + } + // video codecs cmd.PersistentFlags().Bool("vp8", false, "use VP8 video codec") if err := viper.BindPFlag("vp8", cmd.PersistentFlags().Lookup("vp8")); err != nil { @@ -105,15 +112,6 @@ func (Remote) Init(cmd *cobra.Command) error { } func (s *Remote) Set() { - videoCodec := "VP8" - if viper.GetBool("vp8") { - videoCodec = "VP8" - } else if viper.GetBool("vp9") { - videoCodec = "VP9" - } else if viper.GetBool("h264") { - videoCodec = "H264" - } - audioCodec := "Opus" if viper.GetBool("opus") { audioCodec = "Opus" @@ -129,7 +127,21 @@ func (s *Remote) Set() { s.AudioCodec = audioCodec s.AudioParams = viper.GetString("audio") s.AudioBitrate = viper.GetUint("audio_bitrate") + + videoCodec := "VP8" + if viper.GetBool("vp8") { + videoCodec = "VP8" + } else if viper.GetBool("vp9") { + videoCodec = "VP9" + } else if viper.GetBool("h264") { + videoCodec = "H264" + } + videoHWEnc := "" + if viper.GetString("hwenc") == "VAAPI" { + videoHWEnc = "VAAPI" + } s.Display = viper.GetString("display") + s.VideoHWEnc = videoHWEnc s.VideoCodec = videoCodec s.VideoParams = viper.GetString("video") s.VideoBitrate = viper.GetUint("video_bitrate")