diff --git a/Dockerfile.nvidia b/Dockerfile.nvidia index df8396c2..1bccff48 100644 --- a/Dockerfile.nvidia +++ b/Dockerfile.nvidia @@ -1,5 +1,55 @@ ARG UBUNTU_RELEASE=20.04 -ARG CUDA_VERSION=11.2.2 +ARG CUDA_VERSION=11.4.3 +ARG VIRTUALGL_VERSION=3.1 +ARG GSTREAMER_VERSION=1.20 + +# +# Stage 0: Build gstreamer with nvidia plugins. +# +FROM ubuntu:${UBUNTU_RELEASE} AS gstreamer +ARG GSTREAMER_VERSION + +# +# install dependencies +ENV DEBIAN_FRONTEND=noninteractive +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + # Install essentials + curl build-essential ca-certificates git \ + # Install pip and ninja + python3-pip python-gi-dev ninja-build \ + # Install build deps + autopoint autoconf automake autotools-dev libtool gettext bison flex gtk-doc-tools \ + # Install libraries + libtool-bin \ + libgtk2.0-dev \ + libgl1-mesa-dev \ + libopus-dev \ + libpulse-dev \ + libssl-dev \ + libx264-dev \ + libvpx-dev; \ + # Install meson + pip3 install meson; \ + # + # clean up + apt-get clean -y; \ + rm -rf /var/lib/apt/lists/* /var/cache/apt/* + +# +# build gstreamer +RUN set -eux; \ + git clone --depth 1 --branch $GSTREAMER_VERSION https://gitlab.freedesktop.org/gstreamer/gstreamer.git /gstreamer/src; \ + cd /gstreamer/src; \ + mkdir -p /opt/gstreamer; \ + meson --prefix /opt/gstreamer \ + -Dgpl=enabled \ + -Dugly=enabled \ + -Dgst-plugins-ugly:x264=enabled \ + build; \ + ninja -C build; \ + meson install -C build; # # Stage 0: Build xserver-xorg-video-dummy 0.3.8-2 with RandR support. @@ -66,10 +116,12 @@ RUN ./build # # Stage 2: Runtime. # -FROM nvcr.io/nvidia/cudagl:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_RELEASE} as runtime +FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_RELEASE} as runtime +ARG UBUNTU_RELEASE +ARG VIRTUALGL_VERSION # Make all NVIDIA GPUs visible by default -ARG NVIDIA_VISIBLE_DEVICES=all +ENV NVIDIA_VISIBLE_DEVICES all # All NVIDIA driver capabilities should preferably be used, check `NVIDIA_DRIVER_CAPABILITIES` inside the container if things do not work ENV NVIDIA_DRIVER_CAPABILITIES all @@ -83,34 +135,73 @@ ARG USERNAME=neko ARG USER_UID=1000 ARG USER_GID=$USER_UID +# +# install hardware accleration dependencies +ENV DEBIAN_FRONTEND=noninteractive +RUN set -eux; \ + dpkg --add-architecture i386; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + # opengl base: https://gitlab.com/nvidia/container-images/opengl/-/blob/ubuntu20.04/base/Dockerfile + libxau6 libxau6:i386 \ + libxdmcp6 libxdmcp6:i386 \ + libxcb1 libxcb1:i386 \ + libxext6 libxext6:i386 \ + libx11-6 libx11-6:i386 \ + # opengl runtime: https://gitlab.com/nvidia/container-images/opengl/-/blob/ubuntu20.04/glvnd/runtime/Dockerfile + libglvnd0 libglvnd0:i386 \ + libgl1 libgl1:i386 \ + libglx0 libglx0:i386 \ + libegl1 libegl1:i386 \ + libgles2 libgles2:i386 \ + # hardware accleration utilities + libglu1 libglu1:i386 \ + libvulkan-dev libvulkan-dev:i386 \ + mesa-utils mesa-utils-extra \ + mesa-va-drivers mesa-vulkan-drivers \ + vainfo vdpauinfo; \ + # + # install vulkan-utils or vulkan-tools depending on ubuntu release + if [ "${UBUNTU_RELEASE}" = "18.04" ]; then \ + apt-get install -y --no-install-recommends vulkan-utils; \ + else \ + apt-get install -y --no-install-recommends vulkan-tools; \ + fi; \ + # + # create symlink for libnvrtc.so (needed for cudaconvert) + find /usr/local/cuda/lib64/ -maxdepth 1 -type l -name "*libnvrtc.so.*" -exec sh -c 'ln -sf {} /usr/local/cuda/lib64/libnvrtc.so' \;; \ + # + # clean up + apt-get clean -y; \ + rm -rf /var/lib/apt/lists/* /var/cache/apt/* + +# +# add cuda to ld path, for gstreamer cuda plugins +ENV LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:/usr/lib/i386-linux-gnu${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}:/usr/local/cuda/lib:/usr/local/cuda/lib64" + # # install dependencies ENV DEBIAN_FRONTEND=noninteractive RUN set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends \ - wget ca-certificates supervisor \ + wget ca-certificates \ pulseaudio dbus-x11 xserver-xorg-video-dummy \ - libcairo2 libxcb1 libxrandr2 libxv1 libopus0 libvpx6 \ - software-properties-common cabextract aptitude vim curl \ + libcairo2 libxcb1 libxrandr2 libxv1 libopus0 libvpx6 libx264-155 \ + libgtk-3-bin software-properties-common cabextract aptitude vim curl \ # # needed for profile upload preStop hook zip curl \ # # file chooser handler, clipboard, drop - xdotool xclip libgtk-3-0 \ - # - # hardware acclerations utilities - libgtk-3-bin mesa-utils mesa-utils-extra mesa-va-drivers mesa-vulkan-drivers libvulkan-dev libvulkan-dev:i386 vdpauinfo \ - # - # gst - gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ - gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly \ - gstreamer1.0-pulseaudio; \ + xdotool xclip libgtk-3-0; \ # install libxcvt0 (not available in debian:bullseye) wget http://ftp.de.debian.org/debian/pool/main/libx/libxcvt/libxcvt0_0.1.2-1_amd64.deb; \ apt-get install --no-install-recommends ./libxcvt0_0.1.2-1_amd64.deb; \ rm ./libxcvt0_0.1.2-1_amd64.deb; \ + # install never version for supervisor that supports -s flag + apt-get install -y --no-install-recommends pip; \ + pip install supervisor --upgrade; \ # # create a non-root user groupadd --gid $USER_GID $USERNAME; \ @@ -149,10 +240,17 @@ RUN set -eux; \ COPY --from=xserver-xorg-video-dummy /usr/local/lib/xorg/modules/drivers/dummy_drv.so /usr/lib/xorg/modules/drivers/dummy_drv.so # -# Install and configure Vulkan manually -RUN if [ "${UBUNTU_RELEASE}" = "18.04" ]; then apt-get update && apt-get install --no-install-recommends -y vulkan-utils; else apt-get update && apt-get install --no-install-recommends -y vulkan-tools; fi && \ - rm -rf /var/lib/apt/lists/* && \ - VULKAN_API_VERSION=$(dpkg -s libvulkan1 | grep -oP 'Version: [0-9|\.]+' | grep -oP '[0-9]+(\.[0-9]+)(\.[0-9]+)') && \ +# configure EGL and Vulkan manually +RUN VULKAN_API_VERSION=$(dpkg -s libvulkan1 | grep -oP 'Version: [0-9|\.]+' | grep -oP '[0-9]+(\.[0-9]+)(\.[0-9]+)') && \ + # Configure EGL manually + mkdir -p /usr/share/glvnd/egl_vendor.d/ && \ + echo "{\n\ + \"file_format_version\" : \"1.0.0\",\n\ + \"ICD\": {\n\ + \"library_path\": \"libEGL_nvidia.so.0\"\n\ + }\n\ +}" > /usr/share/glvnd/egl_vendor.d/10_nvidia.json && \ + # Configure Vulkan manually mkdir -p /etc/vulkan/icd.d/ && \ echo "{\n\ \"file_format_version\" : \"1.0.0\",\n\ @@ -162,20 +260,24 @@ RUN if [ "${UBUNTU_RELEASE}" = "18.04" ]; then apt-get update && apt-get install }\n\ }" > /etc/vulkan/icd.d/nvidia_icd.json -ARG VIRTUALGL_VERSION=3.1 # -# Install VirtualGL and make libraries available for preload -RUN curl -fsSL -O "https://sourceforge.net/projects/virtualgl/files/virtualgl_${VIRTUALGL_VERSION}_amd64.deb" && \ - curl -fsSL -O "https://sourceforge.net/projects/virtualgl/files/virtualgl32_${VIRTUALGL_VERSION}_amd64.deb" && \ - apt-get update && apt-get install -y --no-install-recommends ./virtualgl_${VIRTUALGL_VERSION}_amd64.deb ./virtualgl32_${VIRTUALGL_VERSION}_amd64.deb && \ - rm -f "virtualgl_${VIRTUALGL_VERSION}_amd64.deb" "virtualgl32_${VIRTUALGL_VERSION}_amd64.deb" && \ - rm -rf /var/lib/apt/lists/* && \ - chmod u+s /usr/lib/libvglfaker.so && \ - chmod u+s /usr/lib/libdlfaker.so && \ - chmod u+s /usr/lib32/libvglfaker.so && \ - chmod u+s /usr/lib32/libdlfaker.so && \ - chmod u+s /usr/lib/i386-linux-gnu/libvglfaker.so && \ - chmod u+s /usr/lib/i386-linux-gnu/libdlfaker.so +# install VirtualGL and make libraries available for preload +RUN set -eux; \ + apt-get update; \ + wget "https://sourceforge.net/projects/virtualgl/files/virtualgl_${VIRTUALGL_VERSION}_amd64.deb"; \ + wget "https://sourceforge.net/projects/virtualgl/files/virtualgl32_${VIRTUALGL_VERSION}_amd64.deb"; \ + apt-get install -y --no-install-recommends ./virtualgl_${VIRTUALGL_VERSION}_amd64.deb ./virtualgl32_${VIRTUALGL_VERSION}_amd64.deb; \ + rm -f "virtualgl_${VIRTUALGL_VERSION}_amd64.deb" "virtualgl32_${VIRTUALGL_VERSION}_amd64.deb"; \ + chmod u+s /usr/lib/libvglfaker.so; \ + chmod u+s /usr/lib/libdlfaker.so; \ + chmod u+s /usr/lib32/libvglfaker.so; \ + chmod u+s /usr/lib32/libdlfaker.so; \ + chmod u+s /usr/lib/i386-linux-gnu/libvglfaker.so; \ + chmod u+s /usr/lib/i386-linux-gnu/libdlfaker.so; \ + # + # clean up + apt-get clean -y; \ + rm -rf /var/lib/apt/lists/* /var/cache/apt/*; # # copy runtime configs @@ -200,6 +302,16 @@ ENV NEKO_SERVER_BIND=:8080 ENV NEKO_PLUGINS_ENABLED=true ENV NEKO_PLUGINS_DIR=/etc/neko/plugins/ +# +# set gstreamer envs +ENV PATH="/opt/gstreamer/bin:${PATH}" +ENV LD_LIBRARY_PATH="/opt/gstreamer/lib/x86_64-linux-gnu${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" +ENV PKG_CONFIG_PATH="/opt/gstreamer/lib/x86_64-linux-gnu/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" + +# +# copy gstreamer from previous stage +COPY --from=gstreamer /opt/gstreamer /opt/gstreamer + # # copy plugins from previous stage COPY --from=build /src/bin/plugins/ $NEKO_PLUGINS_DIR @@ -215,4 +327,4 @@ HEALTHCHECK --interval=10s --timeout=5s --retries=8 \ # # run neko -CMD ["/usr/bin/supervisord", "-c", "/etc/neko/supervisord.conf"] +CMD ["/usr/local/bin/supervisord", "-s", "-c", "/etc/neko/supervisord.conf"] diff --git a/dev/build b/dev/build index 3a6ef04b..f3ae34cd 100755 --- a/dev/build +++ b/dev/build @@ -8,7 +8,16 @@ set -e GIT_COMMIT=`git rev-parse --short HEAD` GIT_BRANCH=`git rev-parse --symbolic-full-name --abbrev-ref HEAD` -docker build -t neko_server_build --target build --build-arg "GIT_COMMIT=$GIT_COMMIT" --build-arg "GIT_BRANCH=$GIT_BRANCH" .. -docker build -t neko_server_runtime --target runtime --build-arg "GIT_COMMIT=$GIT_COMMIT" --build-arg "GIT_BRANCH=$GIT_BRANCH" .. +# if first argument is nvidia, use nvidia dockerfile +if [ "$1" = "nvidia" ]; then + echo "Building nvidia docker image" + DOCKERFILE="Dockerfile.nvidia" +else + echo "Building default docker image" + DOCKERFILE="Dockerfile" +fi -docker build -t neko_server_app --build-arg "BASE_IMAGE=neko_server_runtime" ./runtime +docker build -t neko_server_build --target build --build-arg "GIT_COMMIT=$GIT_COMMIT" --build-arg "GIT_BRANCH=$GIT_BRANCH" -f ../$DOCKERFILE .. +docker build -t neko_server_runtime --target runtime --build-arg "GIT_COMMIT=$GIT_COMMIT" --build-arg "GIT_BRANCH=$GIT_BRANCH" -f ../$DOCKERFILE .. + +docker build -t neko_server_app --build-arg "BASE_IMAGE=neko_server_runtime" -f ./runtime/$DOCKERFILE ./runtime diff --git a/dev/rebuild b/dev/rebuild index 34ca4328..ffa57bf3 100755 --- a/dev/rebuild +++ b/dev/rebuild @@ -29,4 +29,4 @@ fi # # restart server -docker exec neko_server_dev supervisorctl restart neko +docker exec neko_server_dev supervisorctl -c /etc/neko/supervisord.conf restart neko diff --git a/dev/runtime/Dockerfile.nvidia b/dev/runtime/Dockerfile.nvidia new file mode 100644 index 00000000..e01cbc6d --- /dev/null +++ b/dev/runtime/Dockerfile.nvidia @@ -0,0 +1,20 @@ +ARG BASE_IMAGE=neko_server_runtime:latest +FROM $BASE_IMAGE + +# +# install xfce +RUN set -eux; apt-get update; \ + # nvidia docker does not have firefox only firefox-esr + apt-get install -y --no-install-recommends xfce4 xfce4-terminal firefox sudo; \ + # + # add user to sudoers + usermod -aG sudo neko; \ + echo "neko:neko" | chpasswd; \ + echo "%sudo ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers; \ + # clean up + apt-get clean -y; \ + rm -rf /var/lib/apt/lists/* /var/cache/apt/* + +# +# copy configuation files +COPY supervisord.conf /etc/neko/supervisord/xfce.conf diff --git a/dev/runtime/config.nvidia.yml b/dev/runtime/config.nvidia.yml new file mode 100644 index 00000000..0f193a5d --- /dev/null +++ b/dev/runtime/config.nvidia.yml @@ -0,0 +1,101 @@ +capture: + video: + codec: h264 + ids: + - nvh264enc + - x264enc + pipelines: + nvh264enc: + fps: 25 + bitrate: 2 + #gst_prefix: "! cudaupload ! cudaconvert ! video/x-raw(memory:CUDAMemory),format=NV12" + gst_prefix: "! video/x-raw,format=NV12" + gst_encoder: "nvh264enc" + gst_params: + bitrate: 3000 + rc-mode: 5 # Low-Delay CBR, High Quality + preset: 5 # Low Latency, High Performance + zerolatency: true + gop-size: 25 + gst_suffix: "! h264parse config-interval=-1 ! video/x-h264,stream-format=byte-stream,profile=constrained-baseline" + x264enc: + fps: 25 + bitrate: 1 + gst_prefix: "! video/x-raw,format=I420" + gst_encoder: "x264enc" + gst_params: + threads: 4 + bitrate: 4096 + key-int-max: 25 + byte-stream: true + tune: zerolatency + speed-preset: veryfast + gst_suffix: "! video/x-h264,stream-format=byte-stream,profile=constrained-baseline" + +server: + pprof: true + +desktop: + screen: "1920x1080@60" + +member: + provider: "object" + object: + users: + - username: "admin" + password: "admin" + profile: + name: "Administrator" + is_admin: true + can_login: true + can_connect: true + can_watch: true + can_host: true + can_share_media: true + can_access_clipboard: true + sends_inactive_cursor: true + can_see_inactive_cursors: true + - username: "user" + password: "neko" + profile: + name: "User" + is_admin: false + can_login: true + can_connect: true + can_watch: true + can_host: true + can_share_media: true + can_access_clipboard: true + sends_inactive_cursor: true + can_see_inactive_cursors: false + # provider: "file" + # file: + # path: "/home/neko/members.json" + # provider: "multiuser" + # multiuser: + # admin_password: "admin" + # user_password: "neko" + # provider: "noauth" + +session: + # Allows reconnecting the websocket even if the previous + # connection was not closed. Can lead to session hijacking. + merciful_reconnect: true + # Show inactive cursors on the screen. Can lead to multiple + # data sent via WebSockets and additonal rendering cost on + # the clients. + inactive_cursors: true + api_token: "neko123" + cookie: + # Disabling cookies will result to use Bearer Authentication. + # This is less secure, because access token will be sent to + # client in playload and accessible via JS app. + enabled: false + secure: false + +webrtc: + icelite: true + iceservers: + - urls: [ stun:stun.l.google.com:19302 ] + # username: foo + # credential: bar diff --git a/dev/start b/dev/start index 9aaf5c60..ad65b3cd 100755 --- a/dev/start +++ b/dev/start @@ -27,6 +27,17 @@ if [ -z $NEKO_NAT1TO1 ]; then fi fi +# if first argument is nvidia, start with nvidia runtime +if [ "$1" = "nvidia" ]; then + echo "Starting nvidia docker image" + EXTRAOPTS="--gpus all" + CONFIG="config.nvidia.yml" +else + echo "Starting default docker image" + EXTRAOPTS="" + CONFIG="config.yml" +fi + echo "Using app port: ${NEKO_PORT}" echo "Using mux port: ${NEKO_MUX}" echo "Using IP address: ${NEKO_NAT1TO1}" @@ -41,6 +52,8 @@ docker run --rm -it \ -e "NEKO_WEBRTC_TCPMUX=${NEKO_MUX}" \ -e "NEKO_WEBRTC_NAT1TO1=${NEKO_NAT1TO1}" \ -e "NEKO_SESSION_FILE=/home/neko/sessions.txt" \ - -v "${PWD}/runtime/config.yml:/etc/neko/neko.yml" \ + -v "${PWD}/runtime/$CONFIG:/etc/neko/neko.yml" \ -e "NEKO_DEBUG=1" \ + --shm-size=2G \ + $EXTRAOPTS \ neko_server_app:latest;