Build gstreamer in nvidia image (#36)

* build gstreamer 1.20 for nvidia dockerfile.

* add nvidia to build.

* add nvidia to start.

* add config for nvidia.

* upgrade cuda ro 11.4.3.

* add cudaconvert to comment.

* fix paths.

* add cuda to ld.

* upgrade supervisord version.

* add config to suprtvisord in rebuild.

* update nvenc params.

* update nvidia dockerfile.

* h264 profile constrained-baseline.

* h264 send SPS PPS with every IDR.

Dockerfile.nvidia

@@ -1,5 +1,55 @@
 ARG UBUNTU_RELEASE=20.04
-ARG CUDA_VERSION=11.2.2
+ARG CUDA_VERSION=11.4.3
+ARG VIRTUALGL_VERSION=3.1
+ARG GSTREAMER_VERSION=1.20
+
+#
+# Stage 0: Build gstreamer with nvidia plugins.
+#
+FROM ubuntu:${UBUNTU_RELEASE} AS gstreamer
+ARG GSTREAMER_VERSION
+
+#
+# install dependencies
+ENV DEBIAN_FRONTEND=noninteractive
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+        # Install essentials
+        curl build-essential ca-certificates git \
+        # Install pip and ninja
+        python3-pip python-gi-dev ninja-build \
+        # Install build deps
+        autopoint autoconf automake autotools-dev libtool gettext bison flex gtk-doc-tools \
+        # Install libraries
+        libtool-bin \
+        libgtk2.0-dev \
+        libgl1-mesa-dev \
+        libopus-dev \
+        libpulse-dev \
+        libssl-dev \
+        libx264-dev \
+        libvpx-dev; \
+    # Install meson
+    pip3 install meson; \
+    #
+    # clean up
+    apt-get clean -y; \
+    rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+
+#
+# build gstreamer
+RUN set -eux; \
+    git clone --depth 1 --branch $GSTREAMER_VERSION https://gitlab.freedesktop.org/gstreamer/gstreamer.git /gstreamer/src; \
+    cd /gstreamer/src; \
+    mkdir -p /opt/gstreamer; \
+    meson --prefix /opt/gstreamer \
+        -Dgpl=enabled \
+        -Dugly=enabled \
+        -Dgst-plugins-ugly:x264=enabled \
+        build; \
+    ninja -C build; \
+    meson install -C build;
 
 #
 # Stage 0: Build xserver-xorg-video-dummy 0.3.8-2 with RandR support.
@@ -66,10 +116,12 @@ RUN ./build
 #
 # Stage 2: Runtime.
 #
-FROM nvcr.io/nvidia/cudagl:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_RELEASE} as runtime
+FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_RELEASE} as runtime
+ARG UBUNTU_RELEASE
+ARG VIRTUALGL_VERSION
 
 # Make all NVIDIA GPUs visible by default
-ARG NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_VISIBLE_DEVICES all
 # All NVIDIA driver capabilities should preferably be used, check `NVIDIA_DRIVER_CAPABILITIES` inside the container if things do not work
 ENV NVIDIA_DRIVER_CAPABILITIES all
 
@@ -83,34 +135,73 @@ ARG USERNAME=neko
 ARG USER_UID=1000
 ARG USER_GID=$USER_UID
 
+#
+# install hardware accleration dependencies
+ENV DEBIAN_FRONTEND=noninteractive
+RUN set -eux; \
+    dpkg --add-architecture i386; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+        # opengl base: https://gitlab.com/nvidia/container-images/opengl/-/blob/ubuntu20.04/base/Dockerfile
+        libxau6 libxau6:i386 \
+        libxdmcp6 libxdmcp6:i386 \
+        libxcb1 libxcb1:i386 \
+        libxext6 libxext6:i386 \
+        libx11-6 libx11-6:i386 \
+        # opengl runtime: https://gitlab.com/nvidia/container-images/opengl/-/blob/ubuntu20.04/glvnd/runtime/Dockerfile
+        libglvnd0 libglvnd0:i386 \
+        libgl1 libgl1:i386 \
+        libglx0 libglx0:i386 \
+        libegl1 libegl1:i386 \
+        libgles2 libgles2:i386 \
+        # hardware accleration utilities
+        libglu1 libglu1:i386 \
+        libvulkan-dev libvulkan-dev:i386 \
+        mesa-utils mesa-utils-extra \
+        mesa-va-drivers mesa-vulkan-drivers \
+        vainfo vdpauinfo; \
+    #
+    # install vulkan-utils or vulkan-tools depending on ubuntu release
+    if [ "${UBUNTU_RELEASE}" = "18.04" ]; then \
+        apt-get install -y --no-install-recommends vulkan-utils; \
+    else \
+        apt-get install -y --no-install-recommends vulkan-tools; \
+    fi; \
+    #
+    # create symlink for libnvrtc.so (needed for cudaconvert)
+    find /usr/local/cuda/lib64/ -maxdepth 1 -type l -name "*libnvrtc.so.*" -exec sh -c 'ln -sf {} /usr/local/cuda/lib64/libnvrtc.so' \;; \
+    #
+    # clean up
+    apt-get clean -y; \
+    rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+
+#
+# add cuda to ld path, for gstreamer cuda plugins
+ENV LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/usr/lib/i386-linux-gnu${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}:/usr/local/cuda/lib:/usr/local/cuda/lib64"
+
 #
 # install dependencies
 ENV DEBIAN_FRONTEND=noninteractive
 RUN set -eux; \
     apt-get update; \
     apt-get install -y --no-install-recommends \
-        wget ca-certificates supervisor \
+        wget ca-certificates \
         pulseaudio dbus-x11 xserver-xorg-video-dummy \
-        libcairo2 libxcb1 libxrandr2 libxv1 libopus0 libvpx6 \
+        libcairo2 libxcb1 libxrandr2 libxv1 libopus0 libvpx6 libx264-155 \
-        software-properties-common cabextract aptitude vim curl \
+        libgtk-3-bin software-properties-common cabextract aptitude vim curl \
         #
         # needed for profile upload preStop hook
         zip curl \
         #
         # file chooser handler, clipboard, drop
-        xdotool xclip libgtk-3-0 \
+        xdotool xclip libgtk-3-0; \
-        #
-        # hardware acclerations utilities
-        libgtk-3-bin mesa-utils mesa-utils-extra mesa-va-drivers mesa-vulkan-drivers libvulkan-dev libvulkan-dev:i386 vdpauinfo \
-        #
-        # gst
-        gstreamer1.0-plugins-base gstreamer1.0-plugins-good \
-        gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly \
-        gstreamer1.0-pulseaudio; \
     # install libxcvt0 (not available in debian:bullseye)
     wget http://ftp.de.debian.org/debian/pool/main/libx/libxcvt/libxcvt0_0.1.2-1_amd64.deb; \
     apt-get install  --no-install-recommends ./libxcvt0_0.1.2-1_amd64.deb; \
     rm ./libxcvt0_0.1.2-1_amd64.deb; \
+    # install never version for supervisor that supports -s flag
+    apt-get install -y --no-install-recommends pip; \
+    pip install supervisor --upgrade; \
     #
     # create a non-root user
     groupadd --gid $USER_GID $USERNAME; \
@@ -149,10 +240,17 @@ RUN set -eux; \
 COPY --from=xserver-xorg-video-dummy /usr/local/lib/xorg/modules/drivers/dummy_drv.so /usr/lib/xorg/modules/drivers/dummy_drv.so
 
 #
-# Install and configure Vulkan manually
+# configure EGL and Vulkan manually
-RUN if [ "${UBUNTU_RELEASE}" = "18.04" ]; then apt-get update && apt-get install --no-install-recommends -y vulkan-utils; else apt-get update && apt-get install --no-install-recommends -y vulkan-tools; fi && \
+RUN VULKAN_API_VERSION=$(dpkg -s libvulkan1 | grep -oP 'Version: [0-9|\.]+' | grep -oP '[0-9]+(\.[0-9]+)(\.[0-9]+)') && \
-    rm -rf /var/lib/apt/lists/* && \
+    # Configure EGL manually
-    VULKAN_API_VERSION=$(dpkg -s libvulkan1 | grep -oP 'Version: [0-9|\.]+' | grep -oP '[0-9]+(\.[0-9]+)(\.[0-9]+)') && \
+    mkdir -p /usr/share/glvnd/egl_vendor.d/ && \
+    echo "{\n\
+    \"file_format_version\" : \"1.0.0\",\n\
+    \"ICD\": {\n\
+        \"library_path\": \"libEGL_nvidia.so.0\"\n\
+    }\n\
+}" > /usr/share/glvnd/egl_vendor.d/10_nvidia.json && \
+    # Configure Vulkan manually
     mkdir -p /etc/vulkan/icd.d/ && \
     echo "{\n\
     \"file_format_version\" : \"1.0.0\",\n\
@@ -162,20 +260,24 @@ RUN if [ "${UBUNTU_RELEASE}" = "18.04" ]; then apt-get update && apt-get install
     }\n\
 }" > /etc/vulkan/icd.d/nvidia_icd.json
 
-ARG VIRTUALGL_VERSION=3.1
 #
-# Install VirtualGL and make libraries available for preload
+# install VirtualGL and make libraries available for preload
-RUN curl -fsSL -O "https://sourceforge.net/projects/virtualgl/files/virtualgl_${VIRTUALGL_VERSION}_amd64.deb" && \
+RUN set -eux; \
-    curl -fsSL -O "https://sourceforge.net/projects/virtualgl/files/virtualgl32_${VIRTUALGL_VERSION}_amd64.deb" && \
+    apt-get update; \
-    apt-get update && apt-get install -y --no-install-recommends ./virtualgl_${VIRTUALGL_VERSION}_amd64.deb ./virtualgl32_${VIRTUALGL_VERSION}_amd64.deb && \
+    wget "https://sourceforge.net/projects/virtualgl/files/virtualgl_${VIRTUALGL_VERSION}_amd64.deb"; \
-    rm -f "virtualgl_${VIRTUALGL_VERSION}_amd64.deb" "virtualgl32_${VIRTUALGL_VERSION}_amd64.deb" && \
+    wget "https://sourceforge.net/projects/virtualgl/files/virtualgl32_${VIRTUALGL_VERSION}_amd64.deb"; \
-    rm -rf /var/lib/apt/lists/* && \
+    apt-get install -y --no-install-recommends ./virtualgl_${VIRTUALGL_VERSION}_amd64.deb ./virtualgl32_${VIRTUALGL_VERSION}_amd64.deb; \
-    chmod u+s /usr/lib/libvglfaker.so && \
+    rm -f "virtualgl_${VIRTUALGL_VERSION}_amd64.deb" "virtualgl32_${VIRTUALGL_VERSION}_amd64.deb"; \
-    chmod u+s /usr/lib/libdlfaker.so && \
+    chmod u+s /usr/lib/libvglfaker.so; \
-    chmod u+s /usr/lib32/libvglfaker.so && \
+    chmod u+s /usr/lib/libdlfaker.so; \
-    chmod u+s /usr/lib32/libdlfaker.so && \
+    chmod u+s /usr/lib32/libvglfaker.so; \
-    chmod u+s /usr/lib/i386-linux-gnu/libvglfaker.so && \
+    chmod u+s /usr/lib32/libdlfaker.so; \
-    chmod u+s /usr/lib/i386-linux-gnu/libdlfaker.so
+    chmod u+s /usr/lib/i386-linux-gnu/libvglfaker.so; \
+    chmod u+s /usr/lib/i386-linux-gnu/libdlfaker.so; \
+    #
+    # clean up
+    apt-get clean -y; \
+    rm -rf /var/lib/apt/lists/* /var/cache/apt/*;
 
 #
 # copy runtime configs
@@ -200,6 +302,16 @@ ENV NEKO_SERVER_BIND=:8080
 ENV NEKO_PLUGINS_ENABLED=true
 ENV NEKO_PLUGINS_DIR=/etc/neko/plugins/
 
+#
+# set gstreamer envs
+ENV PATH="/opt/gstreamer/bin:${PATH}"
+ENV LD_LIBRARY_PATH="/opt/gstreamer/lib/x86_64-linux-gnu${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+ENV PKG_CONFIG_PATH="/opt/gstreamer/lib/x86_64-linux-gnu/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}"
+
+#
+# copy gstreamer from previous stage
+COPY --from=gstreamer /opt/gstreamer /opt/gstreamer
+
 #
 # copy plugins from previous stage
 COPY --from=build /src/bin/plugins/ $NEKO_PLUGINS_DIR
@@ -215,4 +327,4 @@ HEALTHCHECK --interval=10s --timeout=5s --retries=8 \
 
 #
 # run neko
-CMD ["/usr/bin/supervisord", "-c", "/etc/neko/supervisord.conf"]
+CMD ["/usr/local/bin/supervisord", "-s", "-c", "/etc/neko/supervisord.conf"]

dev/build

@@ -8,7 +8,16 @@ set -e
 GIT_COMMIT=`git rev-parse --short HEAD`
 GIT_BRANCH=`git rev-parse --symbolic-full-name --abbrev-ref HEAD`
 
-docker build -t neko_server_build --target build --build-arg "GIT_COMMIT=$GIT_COMMIT" --build-arg "GIT_BRANCH=$GIT_BRANCH" ..
+# if first argument is nvidia, use nvidia dockerfile
-docker build -t neko_server_runtime --target runtime --build-arg "GIT_COMMIT=$GIT_COMMIT" --build-arg "GIT_BRANCH=$GIT_BRANCH" ..
+if [ "$1" = "nvidia" ]; then
+    echo "Building nvidia docker image"
+    DOCKERFILE="Dockerfile.nvidia"
+else
+    echo "Building default docker image"
+    DOCKERFILE="Dockerfile"
+fi
 
-docker build -t neko_server_app --build-arg "BASE_IMAGE=neko_server_runtime" ./runtime
+docker build -t neko_server_build --target build --build-arg "GIT_COMMIT=$GIT_COMMIT" --build-arg "GIT_BRANCH=$GIT_BRANCH" -f ../$DOCKERFILE ..
+docker build -t neko_server_runtime --target runtime --build-arg "GIT_COMMIT=$GIT_COMMIT" --build-arg "GIT_BRANCH=$GIT_BRANCH" -f ../$DOCKERFILE ..
+
+docker build -t neko_server_app --build-arg "BASE_IMAGE=neko_server_runtime" -f ./runtime/$DOCKERFILE ./runtime

dev/rebuild

@@ -29,4 +29,4 @@ fi
 
 #
 # restart server
-docker exec neko_server_dev supervisorctl restart neko
+docker exec neko_server_dev supervisorctl -c /etc/neko/supervisord.conf restart neko

dev/runtime/Dockerfile.nvidia

@@ -0,0 +1,20 @@
+ARG BASE_IMAGE=neko_server_runtime:latest
+FROM $BASE_IMAGE
+
+#
+# install xfce
+RUN set -eux; apt-get update; \
+    # nvidia docker does not have firefox only firefox-esr
+    apt-get install -y --no-install-recommends xfce4 xfce4-terminal firefox sudo; \
+    #
+    # add user to sudoers
+    usermod -aG sudo neko; \
+    echo "neko:neko" | chpasswd; \
+    echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers; \
+    # clean up
+    apt-get clean -y; \
+    rm -rf /var/lib/apt/lists/* /var/cache/apt/*
+
+#
+# copy configuation files
+COPY supervisord.conf /etc/neko/supervisord/xfce.conf

dev/runtime/config.nvidia.yml

@@ -0,0 +1,101 @@
+capture:
+  video:
+    codec: h264
+    ids:
+      - nvh264enc
+      - x264enc
+    pipelines:
+      nvh264enc:
+        fps: 25
+        bitrate: 2
+        #gst_prefix: "! cudaupload ! cudaconvert ! video/x-raw(memory:CUDAMemory),format=NV12"
+        gst_prefix: "! video/x-raw,format=NV12"
+        gst_encoder: "nvh264enc"
+        gst_params:
+          bitrate: 3000
+          rc-mode: 5 # Low-Delay CBR, High Quality
+          preset: 5 # Low Latency, High Performance
+          zerolatency: true
+          gop-size: 25
+        gst_suffix: "! h264parse config-interval=-1 ! video/x-h264,stream-format=byte-stream,profile=constrained-baseline"
+      x264enc:
+        fps: 25
+        bitrate: 1
+        gst_prefix: "! video/x-raw,format=I420"
+        gst_encoder: "x264enc"
+        gst_params:
+          threads: 4
+          bitrate: 4096
+          key-int-max: 25
+          byte-stream: true
+          tune: zerolatency
+          speed-preset: veryfast
+        gst_suffix: "! video/x-h264,stream-format=byte-stream,profile=constrained-baseline"
+
+server:
+  pprof: true
+
+desktop:
+  screen: "1920x1080@60"
+
+member:
+  provider: "object"
+  object:
+    users:
+      - username: "admin"
+        password: "admin"
+        profile:
+          name: "Administrator"
+          is_admin: true
+          can_login: true
+          can_connect: true
+          can_watch: true
+          can_host: true
+          can_share_media: true
+          can_access_clipboard: true
+          sends_inactive_cursor: true
+          can_see_inactive_cursors: true
+      - username: "user"
+        password: "neko"
+        profile:
+          name: "User"
+          is_admin: false
+          can_login: true
+          can_connect: true
+          can_watch: true
+          can_host: true
+          can_share_media: true
+          can_access_clipboard: true
+          sends_inactive_cursor: true
+          can_see_inactive_cursors: false
+  # provider: "file"
+  # file:
+  #   path: "/home/neko/members.json"
+  # provider: "multiuser"
+  # multiuser:
+  #  admin_password: "admin"
+  #  user_password: "neko"
+  # provider: "noauth"
+
+session:
+  # Allows reconnecting the websocket even if the previous
+  # connection was not closed. Can lead to session hijacking.
+  merciful_reconnect: true
+  # Show inactive cursors on the screen. Can lead to multiple
+  # data sent via WebSockets and additonal rendering cost on
+  # the clients.
+  inactive_cursors: true
+  api_token: "neko123"
+  cookie:
+    # Disabling cookies will result to use Bearer Authentication.
+    # This is less secure, because access token will be sent to
+    # client in playload and accessible via JS app.
+    enabled: false
+    secure: false
+
+webrtc:
+  icelite: true
+  iceservers:
+    - urls: [ stun:stun.l.google.com:19302 ]
+    #  username: foo
+    #  credential: bar

dev/start

@@ -27,6 +27,17 @@ if [ -z $NEKO_NAT1TO1 ]; then
     fi
 fi
 
+# if first argument is nvidia, start with nvidia runtime
+if [ "$1" = "nvidia" ]; then
+    echo "Starting nvidia docker image"
+    EXTRAOPTS="--gpus all"
+    CONFIG="config.nvidia.yml"
+else
+    echo "Starting default docker image"
+    EXTRAOPTS=""
+    CONFIG="config.yml"
+fi
+
 echo "Using app port: ${NEKO_PORT}"
 echo "Using mux port: ${NEKO_MUX}"
 echo "Using IP address: ${NEKO_NAT1TO1}"
@@ -41,6 +52,8 @@ docker run --rm -it \
     -e "NEKO_WEBRTC_TCPMUX=${NEKO_MUX}" \
     -e "NEKO_WEBRTC_NAT1TO1=${NEKO_NAT1TO1}" \
     -e "NEKO_SESSION_FILE=/home/neko/sessions.txt" \
-    -v "${PWD}/runtime/config.yml:/etc/neko/neko.yml" \
+    -v "${PWD}/runtime/$CONFIG:/etc/neko/neko.yml" \
     -e "NEKO_DEBUG=1" \
+    --shm-size=2G \
+    $EXTRAOPTS \
     neko_server_app:latest;