Hardware accelerated encoding using Intel QuickSync via VAAPI (#151)

* (nits)

* add hardware encoding support for Intel QSV via VAAPI

* automate RENDER_GID env var

.docker/base/Dockerfile

@@ -13,7 +13,7 @@ RUN set -eux; apt-get update; \
     # install libclipboard
     set -eux; \
     cd /tmp; \
-    git clone https://github.com/jtanx/libclipboard; \
+    git clone --depth=1 https://github.com/jtanx/libclipboard; \
     cd libclipboard; \
     cmake .; \
     make -j4; \
@@ -60,16 +60,26 @@ ARG USERNAME=neko
 ARG USER_UID=1000
 ARG USER_GID=$USER_UID
 
-#
+RUN set -eux; \
-# install dependencies
+    #
-RUN set -eux; apt-get update; \
+    # add non-free repo for intel drivers
+    echo deb http://deb.debian.org/debian bullseye main contrib non-free > /etc/apt/sources.list; \
+    echo deb http://deb.debian.org/debian-security/ bullseye-security main contrib non-free >> /etc/apt/sources.list; \
+    echo deb http://deb.debian.org/debian bullseye-updates main contrib non-free >> /etc/apt/sources.list; \
+    apt-get update; \
+    #
+    # install dependencies
     apt-get install -y --no-install-recommends wget ca-certificates supervisor; \
     apt-get install -y --no-install-recommends pulseaudio dbus-x11 xserver-xorg-video-dummy; \
     apt-get install -y --no-install-recommends libcairo2 libxcb1 libxrandr2 libxv1 libopus0 libvpx6; \
     #
-    # gst
+    # intel driver + vaapi
+    apt-get install -y --no-install-recommends intel-media-va-driver-non-free libva2 vainfo; \
+    #
+    # gst + vaapi plugin
     apt-get install -y --no-install-recommends libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev \
-                    gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-pulseaudio; \
+                    gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-pulseaudio \
+                    gstreamer1.0-vaapi ;\
     #
     # fonts
     apt-get install -y --no-install-recommends fonts-takao-mincho; \
@@ -106,6 +116,7 @@ COPY .docker/base/dbus /usr/bin/dbus
 COPY .docker/base/default.pa /etc/pulse/default.pa
 COPY .docker/base/supervisord.conf /etc/neko/supervisord.conf
 COPY .docker/base/xorg.conf /etc/neko/xorg.conf
+COPY .docker/base/add-render-group.sh /usr/bin/add-render-group.sh
 
 #
 # set default envs
@@ -114,6 +125,7 @@ ENV DISPLAY=:99.0
 ENV NEKO_PASSWORD=neko
 ENV NEKO_PASSWORD_ADMIN=admin
 ENV NEKO_BIND=:8080
+ENV RENDER_GID=
 
 #
 # copy static files from previous stages

.docker/base/add-render-group.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# if no hwenc required, noop
+[[ -z "$NEKO_HWENC" ]] && exit 0
+
+if [[ -z "$RENDER_GID" ]]; then
+  RENDER_GID=$(stat -c "%g" /dev/dri/render* | tail -n 1)
+  # is /dev/dri passed to the container?
+  [[ -z "$RENDER_GID" ]] && exit 1
+fi
+
+# note that this could conceivably be a security risk...
+cnt_group=$(getent group "$RENDER_GID" | cut -d: -f1)
+if [[ -z "$cnt_group" ]]; then
+  groupadd -g "$RENDER_GID" nekorender
+  cnt_group=nekorender
+fi
+usermod -a -G "$cnt_group" "$USER"

.docker/base/supervisord.conf

@@ -9,6 +9,19 @@ loglevel=debug
 [include]
 files=/etc/neko/supervisord/*.conf
 
+[program:rendergroup-init]
+environment=RENDER_GID="%(ENV_RENDER_GID)s",USER="%(ENV_USER)s"
+command=/usr/bin/add-render-group.sh
+startsecs=0
+startretries=0
+autorestart=false
+priority=10
+user=root
+stdout_logfile=/var/log/neko/rendergroup.log
+stdout_logfile_maxbytes=1MB
+stdout_logfile_backups=10
+redirect_stderr=true
+
 [program:dbus]
 environment=HOME="/root",USER="root"
 command=/usr/bin/dbus

server/internal/gst/gst.go

@@ -79,7 +79,7 @@ func CreateRTMPPipeline(pipelineDevice string, pipelineDisplay string, pipelineS
 }
 
 // CreateAppPipeline creates a GStreamer Pipeline
-func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc string, fps int, bitrate uint) (*Pipeline, error) {
+func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc string, fps int, bitrate uint, hwenc string) (*Pipeline, error) {
 	pipelineStr := " ! appsink name=appsink"
 
 	// if using custom pipeline
@@ -90,6 +90,15 @@ func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc stri
 
 	switch codecName {
 	case "VP8":
+		if hwenc == "VAAPI" {
+			if err := CheckPlugins([]string{"ximagesrc", "vaapi"}); err != nil {
+				return nil, err
+			}
+			// vp8 encode is missing from gstreamer.freedesktop.org/documentation
+			// note that it was removed from some recent intel CPUs: https://trac.ffmpeg.org/wiki/Hardware/QuickSync
+			// https://gstreamer.freedesktop.org/data/doc/gstreamer/head/gstreamer-vaapi-plugins/html/gstreamer-vaapi-plugins-vaapivp8enc.html
+			pipelineStr = fmt.Sprintf(videoSrc+"video/x-raw,format=NV12 ! vaapivp8enc rate-control=vbr bitrate=%d keyframe-period=180"+pipelineStr, pipelineDevice, fps, bitrate)
+		} else {
 			// https://gstreamer.freedesktop.org/documentation/vpx/vp8enc.html?gi-language=c
 			// gstreamer1.0-plugins-good
 			// vp8enc error-resilient=partitions keyframe-max-dist=10 auto-alt-ref=true cpu-used=5 deadline=1
@@ -98,6 +107,7 @@ func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc stri
 			}
 
 			pipelineStr = fmt.Sprintf(videoSrc+"vp8enc target-bitrate=%d cpu-used=4 end-usage=cbr threads=4 deadline=1 undershoot=95 buffer-size=%d buffer-initial-size=%d buffer-optimal-size=%d keyframe-max-dist=180 min-quantizer=3 max-quantizer=40"+pipelineStr, pipelineDevice, fps, bitrate*1000, bitrate*6, bitrate*4, bitrate*5)
+		}
 	case "VP9":
 		// https://gstreamer.freedesktop.org/documentation/vpx/vp9enc.html?gi-language=c
 		// gstreamer1.0-plugins-good
@@ -112,6 +122,14 @@ func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc stri
 			return nil, err
 		}
 
+		if hwenc == "VAAPI" {
+			if err := CheckPlugins([]string{"vaapi"}); err != nil {
+				return nil, err
+			}
+
+			pipelineStr = fmt.Sprintf(videoSrc+"video/x-raw,format=NV12 ! vaapih264enc rate-control=vbr bitrate=%d keyframe-period=180 quality-level=7 ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice, fps, bitrate)
+
+		} else {
 			// https://gstreamer.freedesktop.org/documentation/openh264/openh264enc.html?gi-language=c#openh264enc
 			// gstreamer1.0-plugins-bad
 			// openh264enc multi-thread=4 complexity=high bitrate=3072000 max-bitrate=4096000
@@ -133,6 +151,7 @@ func CreateAppPipeline(codecName string, pipelineDevice string, pipelineSrc stri
 			}
 
 			pipelineStr = fmt.Sprintf(videoSrc+"video/x-raw,format=NV12 ! x264enc threads=4 bitrate=%d key-int-max=60 vbv-buf-capacity=%d byte-stream=true tune=zerolatency speed-preset=veryfast ! video/x-h264,stream-format=byte-stream"+pipelineStr, pipelineDevice, fps, bitrate, vbvbuf)
+		}
 	case "Opus":
 		// https://gstreamer.freedesktop.org/documentation/opus/opusenc.html
 		// gstreamer1.0-plugins-base

server/internal/remote/manager.go

@@ -147,6 +147,7 @@ func (manager *RemoteManager) createPipelines() {
 		manager.config.VideoParams,
 		rate,
 		manager.config.VideoBitrate,
+		manager.config.VideoHWEnc,
 	)
 	if err != nil {
 		manager.logger.Panic().Err(err).Msg("unable to create video pipeline")
@@ -158,6 +159,7 @@ func (manager *RemoteManager) createPipelines() {
 		manager.config.AudioParams,
 		0, // fps: n/a for audio
 		manager.config.AudioBitrate,
+		"", // hwenc: n/a for audio
 	)
 	if err != nil {
 		manager.logger.Panic().Err(err).Msg("unable to create audio pipeline")
@@ -197,6 +199,7 @@ func (manager *RemoteManager) ChangeResolution(width int, height int, rate int)
 		manager.config.VideoParams,
 		rate,
 		manager.config.VideoBitrate,
+		manager.config.VideoHWEnc,
 	)
 	if err != nil {
 		manager.logger.Panic().Err(err).Msg("unable to create new video pipeline")

server/internal/types/config/remote.go

@@ -14,6 +14,7 @@ type Remote struct {
 	AudioCodec   string
 	AudioParams  string
 	AudioBitrate uint
+	VideoHWEnc   string
 	VideoCodec   string
 	VideoParams  string
 	VideoBitrate uint
@@ -64,6 +65,12 @@ func (Remote) Init(cmd *cobra.Command) error {
 		return err
 	}
 
+	// hw encoding
+	cmd.PersistentFlags().String("hwenc", "", "use hardware accelerated encoding")
+	if err := viper.BindPFlag("hwenc", cmd.PersistentFlags().Lookup("hwenc")); err != nil {
+		return err
+	}
+
 	// video codecs
 	cmd.PersistentFlags().Bool("vp8", false, "use VP8 video codec")
 	if err := viper.BindPFlag("vp8", cmd.PersistentFlags().Lookup("vp8")); err != nil {
@@ -105,15 +112,6 @@ func (Remote) Init(cmd *cobra.Command) error {
 }
 
 func (s *Remote) Set() {
-	videoCodec := "VP8"
-	if viper.GetBool("vp8") {
-		videoCodec = "VP8"
-	} else if viper.GetBool("vp9") {
-		videoCodec = "VP9"
-	} else if viper.GetBool("h264") {
-		videoCodec = "H264"
-	}
-
 	audioCodec := "Opus"
 	if viper.GetBool("opus") {
 		audioCodec = "Opus"
@@ -129,7 +127,21 @@ func (s *Remote) Set() {
 	s.AudioCodec = audioCodec
 	s.AudioParams = viper.GetString("audio")
 	s.AudioBitrate = viper.GetUint("audio_bitrate")
+
+	videoCodec := "VP8"
+	if viper.GetBool("vp8") {
+		videoCodec = "VP8"
+	} else if viper.GetBool("vp9") {
+		videoCodec = "VP9"
+	} else if viper.GetBool("h264") {
+		videoCodec = "H264"
+	}
+	videoHWEnc := ""
+	if viper.GetString("hwenc") == "VAAPI" {
+		videoHWEnc = "VAAPI"
+	}
 	s.Display = viper.GetString("display")
+	s.VideoHWEnc = videoHWEnc
 	s.VideoCodec = videoCodec
 	s.VideoParams = viper.GetString("video")
 	s.VideoBitrate = viper.GetUint("video_bitrate")